feat: move to federated cluster

ai4papi/nomad/common.py

@@ -17,6 +17,7 @@
 from nomad.api import exceptions
 import requests
 
+import ai4papi.conf as papiconf
 import ai4papi.nomad.patches as nomad_patches
 
 
@@ -42,13 +43,14 @@
 def get_deployments(
     namespace: str,
     owner: str,
+    prefix: str = "",
     ):
     """
     Returns a list of all deployments belonging to a user, in a given namespace.
     """
     job_filter = \
         'Status != "dead" and ' + \
-        'Name matches "^userjob" and ' + \
+        f'Name matches "^{prefix}" and ' + \
         'Meta is not empty and ' + \
         f'Meta.owner == "{owner}"'
     jobs = Nomad.jobs.get_jobs(namespace=namespace, filter_=job_filter)
@@ -95,6 +97,7 @@ def get_deployment(
     # Create job info dict
     info = {
         'job_ID': j['ID'],
+        'name': j['Name'],
         'status': '',  # do not use j['Status'] as misleading
         'owner': j['Meta']['owner'],
         'title': j['Meta']['title'],
@@ -114,7 +117,7 @@ def get_deployment(
 
     # Retrieve tasks
     tasks = j['TaskGroups'][0]['Tasks']
-    usertask = [t for t in tasks if t['Name'] == 'usertask'][0]
+    usertask = [t for t in tasks if t['Name'] == 'main'][0]
 
     # Retrieve Docker image
     info['docker_image'] = usertask['Config']['image']
@@ -165,17 +168,6 @@ def get_deployment(
     except Exception:  # return first endpoint
         info['main_endpoint'] = list(info['endpoints'].values())[0]
 
-    # Add active endpoints
-    if full_info:
-        info['active_endpoints'] = []
-        for k, v in info['endpoints'].items():
-            try:
-                r = session.get(v, timeout=2)
-                if r.status_code == 200:
-                    info['active_endpoints'].append(k)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError):
-                continue
-
     # Only fill resources if the job is allocated
     allocs = Nomad.job.get_allocations(
         id_=j['ID'],
@@ -229,7 +221,7 @@ def get_deployment(
 
         # Add error messages if needed
         if info['status'] == 'failed':
-            info['error_msg'] = a['TaskStates']['usertask']['Events'][0]['Message']
+            info['error_msg'] = a['TaskStates']['main']['Events'][0]['Message']
 
             # Replace with clearer message
             if info['error_msg'] == 'Docker container exited with non-zero exit code: 1':
@@ -245,12 +237,8 @@ def get_deployment(
                 "the network is restored and you should be able to fully recover " \
                 "your deployment."
 
-        # Disable access to endpoints if there is a network cut
-        if info['status'] == 'down' and info['active_endpoints']:
-            info['active_endpoints'] = []
-
         # Add resources
-        res = a['AllocatedResources']['Tasks']['usertask']
+        res = a['AllocatedResources']['Tasks']['main']
         gpu = [d for d in res['Devices'] if d['Type'] == 'gpu'][0] if res['Devices'] else None
         cpu_cores = res['Cpu']['ReservedCores']
         info['resources'] = {
@@ -261,6 +249,26 @@ def get_deployment(
             'disk_MB': a['AllocatedResources']['Shared']['DiskMB'],
         }
 
+        # Retrieve the node the jobs landed at in order to properly fill the endpoints
+        n = Nomad.node.get_node(a['NodeID'])
+        for k, v in info['endpoints'].items():
+            info['endpoints'][k] = v.replace('${meta.domain}', n['Meta']['domain'])
+
+        # Add active endpoints
+        if full_info:
+            info['active_endpoints'] = []
+            for k, v in info['endpoints'].items():
+                try:
+                    r = session.get(v, timeout=2)
+                    if r.status_code == 200:
+                        info['active_endpoints'].append(k)
+                except (requests.exceptions.Timeout, requests.exceptions.ConnectionError):
+                    continue
+
+        # Disable access to endpoints if there is a network cut
+        if info['status'] == 'down' and info['active_endpoints']:
+            info['active_endpoints'] = []
+
     elif evals:
         # Something happened, job didn't deploy (eg. job needs port that's currently being used)
         # We have to return `placement failures message`.
@@ -357,13 +365,19 @@ def delete_deployment(
     return {'status': 'success'}
 
 
-def get_gpu_models():
+@cached(cache=TTLCache(maxsize=1024, ttl=1*60*60))
+def get_gpu_models(vo):
     """
-    Retrieve available GPU models in the cluster.
+    Retrieve available GPU models in the cluster, filtering nodes by VO.
     """
     gpu_models = set()
     nodes = Nomad.nodes.get_nodes(resources=True)
     for node in nodes:
+        # Discard nodes that don't belong to the requested VO
+        meta = Nomad.node.get_node(node['ID'])['Meta']
+        if papiconf.MAIN_CONF['nomad']['namespaces'][vo] not in meta['namespace']:
+            continue
+
         # Discard GPU models of nodes that are not eligible
         if node['SchedulingEligibility'] != 'eligible':
             continue

ai4papi/routers/v1/catalog/modules.py

@@ -87,7 +87,7 @@ def get_config(
     )
 
     # Fill with available GPU models in the cluster
-    models = nomad.common.get_gpu_models()
+    models = nomad.common.get_gpu_models(vo)
     if models:
         conf["hardware"]["gpu_type"]["options"] += models
 

ai4papi/routers/v1/deployments/modules.py

@@ -57,6 +57,7 @@ def get_deployments(
         jobs = nomad.get_deployments(
             namespace=papiconf.MAIN_CONF['nomad']['namespaces'][vo],
             owner=auth_info['id'],
+            prefix='module',
         )
 
         # Retrieve info for jobs in namespace
@@ -119,15 +120,10 @@ def get_deployment(
     )
 
     # Check the deployment is indeed a module
-    tool_list = papiconf.TOOLS.keys()
-    module_name = re.search(
-            '/(.*):',  # remove dockerhub account and tag
-            job['docker_image'],
-            ).group(1)
-    if module_name in tool_list:
+    if not job['name'].startswith('module'):
         raise HTTPException(
             status_code=400,
-            detail="This deployment is a tool, not a module.",
+            detail="This deployment is not a module.",
             )
 
     return job
@@ -207,19 +203,23 @@ def create_deployment(
     else:
         priority = 50
 
-    # Generate a domain for user-app and check nothing is running there
-    domain = utils.generate_domain(
+    # Remove non-compliant characters from hostname
+    base_domain = papiconf.MAIN_CONF['lb']['domain'][vo]
+    hostname = utils.safe_hostname(
         hostname=user_conf['general']['hostname'],
-        base_domain=papiconf.MAIN_CONF['lb']['domain'][vo],
         job_uuid=job_uuid,
     )
-    utils.check_domain(domain)
 
-    #TODO: remove when we solve disk issues
-    # For now on we fix disk here because, if not fixed, jobs are not being deployed
-    # (ie. "resource disk exhausted").
-    # In any case, this limit is useless because it has not yet been passed to docker
-    user_conf['hardware']['disk'] = 500
+    #TODO: reenable custom hostname, when we are able to parse all node metadata
+    # (domain key) to build the true domain
+    hostname = job_uuid
+
+    # # Check the hostname is available in all data-centers
+    # # (we don't know beforehand where the job will land)
+    # #TODO: make sure this does not break if the datacenter is unavailable
+    # #TODO: disallow custom hostname, pain in the ass, slower deploys
+    # for datacenter in papiconf.MAIN_CONF['nomad']['datacenters']:
+    #     utils.check_domain(f"{hostname}.{datacenter}-{base_domain}")
 
     # Replace the Nomad job template
     nomad_conf = nomad_conf.safe_substitute(
@@ -232,7 +232,8 @@ def create_deployment(
             'OWNER_EMAIL': auth_info['email'],
             'TITLE': user_conf['general']['title'][:45],  # keep only 45 first characters
             'DESCRIPTION': user_conf['general']['desc'][:1000],  # limit to 1K characters
-            'DOMAIN': domain,
+            'BASE_DOMAIN': base_domain,
+            'HOSTNAME': hostname,
             'DOCKER_IMAGE': user_conf['general']['docker_image'],
             'DOCKER_TAG': user_conf['general']['docker_tag'],
             'SERVICE': user_conf['general']['service'],
@@ -256,7 +257,7 @@ def create_deployment(
     nomad_conf = nomad.load_job_conf(nomad_conf)
 
     tasks = nomad_conf['TaskGroups'][0]['Tasks']
-    usertask = [t for t in tasks if t['Name']=='usertask'][0]
+    usertask = [t for t in tasks if t['Name']=='main'][0]
 
     # Apply patches if needed
     usertask = module_patches.patch_nextcloud_mount(

ai4papi/routers/v1/deployments/tools.py

@@ -60,6 +60,7 @@ def get_deployments(
         jobs = nomad.get_deployments(
             namespace=papiconf.MAIN_CONF['nomad']['namespaces'][vo],
             owner=auth_info['id'],
+            prefix='tool',
         )
 
         # Retrieve info for jobs in namespace
@@ -122,15 +123,10 @@ def get_deployment(
     )
 
     # Check the deployment is indeed a tool
-    tool_list = papiconf.TOOLS.keys()
-    tool_name = re.search(
-            '/(.*):',  # remove dockerhub account and tag
-            job['docker_image'],
-            ).group(1)
-    if tool_name not in tool_list:
+    if not job['name'].startswith('tool'):
         raise HTTPException(
             status_code=400,
-            detail="This deployment is a module, not a tool.",
+            detail="This deployment is not a tool.",
             )
 
     return job
@@ -201,13 +197,23 @@ def create_deployment(
     else:
         priority = 50
 
-    # Generate a domain for user-app and check nothing is running there
-    domain = utils.generate_domain(
+    # Remove non-compliant characters from hostname
+    base_domain = papiconf.MAIN_CONF['lb']['domain'][vo]
+    hostname = utils.safe_hostname(
         hostname=user_conf['general']['hostname'],
-        base_domain=papiconf.MAIN_CONF['lb']['domain'][vo],
         job_uuid=job_uuid,
     )
-    utils.check_domain(domain)
+
+    #TODO: reenable custom hostname, when we are able to parse all node metadata
+    # (domain key) to build the true domain
+    hostname = job_uuid
+
+    # # Check the hostname is available in all data-centers
+    # # (we don't know beforehand where the job will land)
+    # #TODO: make sure this does not break if the datacenter is unavailable
+    # #TODO: disallow custom hostname, pain in the ass, slower deploys
+    # for datacenter in papiconf.MAIN_CONF['nomad']['datacenters']:
+    #     utils.check_domain(f"{hostname}.{datacenter}-{base_domain}")
 
     # Create a default secret for the Federated Server
     _ = ai4secrets.create_secret(
@@ -237,7 +243,8 @@ def create_deployment(
             'OWNER_EMAIL': auth_info['email'],
             'TITLE': user_conf['general']['title'][:45],  # keep only 45 first characters
             'DESCRIPTION': user_conf['general']['desc'][:1000],  # limit to 1K characters
-            'DOMAIN': domain,
+            'BASE_DOMAIN': base_domain,
+            'HOSTNAME': hostname,
             'DOCKER_IMAGE': user_conf['general']['docker_image'],
             'DOCKER_TAG': user_conf['general']['docker_tag'],
             'CPU_NUM': user_conf['hardware']['cpu_num'],
@@ -258,7 +265,7 @@ def create_deployment(
     nomad_conf = nomad.load_job_conf(nomad_conf)
 
     tasks = nomad_conf['TaskGroups'][0]['Tasks']
-    usertask = [t for t in tasks if t['Name']=='usertask'][0]
+    usertask = [t for t in tasks if t['Name']=='main'][0]
 
     # Launch `deep-start` compatible service if needed
     service = user_conf['general']['service']