Merge branch 'master' into cvat

README.md

@@ -199,6 +199,9 @@ More details can be found in the [API docs](https://api.cloud.ai4eosc.eu/docs).
   **Notes**: The catalog caches results for up to 6 hours to improve UX (see
   [doctring](./ai4papi/routers/v1/modules.py)).
 
+* `/v1/try_me/`:
+   endpoint where anyone can deploy a short-lived container to try a module
+
 * `/v1/deployments/`: (🔒)
    deploy modules/tools in the platform to perform trainings
 
@@ -262,7 +265,7 @@ These are the configuration files the API uses:
 * `etc/main_conf.yaml`: main configuration file of the API
 * `etc/modules`: configuration files for standard modules
 * `etc/tools`: configuration files for tools
-  - `deep-oc-federated-server`: federated server
+  - `ai4os-federated-server`: federated server
 
 The pattern for the subfolders follows:
   - `user.yaml`: user customizable configuration to make a deployment in Nomad.

ai4papi/auth.py

@@ -51,41 +51,17 @@ def get_user_info(token):
             detail="Invalid token",
             )
 
-    # Check scopes
-    # Scope can appear if non existent if user doesn't belong to any VO,
-    # even if scope was requested in token.
-    # VO do not need to be one of the project's (this is next check), but we can still
-    # add the project VOs in the project detail.
-    if user_infos.get('eduperson_entitlement') is None:
-        raise HTTPException(
-            status_code=401,
-            detail="Check that (1) you enabled the `eduperson_entitlement` scope for" \
-                   "your token, and (2) you belong to at least one Virtual " \
-                   f"Organization supported by the project: {MAIN_CONF['auth']['VO']}",
-            )
-
-    # Parse Virtual Organizations manually from URNs
-    # If more complexity is need in the future, check https://github.com/oarepo/urnparse
+    # Retrieve VOs the user belongs to
+    # VOs can be empty if the user does not belong to any VO, or the
+    # 'eduperson_entitlement wasn't correctly retrieved from the token
     vos = []
-    for i in user_infos.get('eduperson_entitlement'):
+    for i in user_infos.get('eduperson_entitlement', []):
+        # Parse Virtual Organizations manually from URNs
+        # If more complexity is need in the future, check https://github.com/oarepo/urnparse
         ent_i = re.search(r"group:(.+?):", i)
         if ent_i:  # your entitlement has indeed a group `tag`
             vos.append(ent_i.group(1))
 
-    # Filter VOs to keep only the ones relevant to us
-    vos = set(vos).intersection(
-        set(MAIN_CONF['auth']['VO'])
-    )
-    vos = sorted(vos)
-
-    # Check if VOs is empty after filtering
-    if not vos:
-        raise HTTPException(
-            status_code=401,
-            detail="You should belong to at least one of the Virtual Organizations " \
-                   f"supported by the project: {MAIN_CONF['auth']['VO']}.",
-            )
-
     # Generate user info dict
     for k in ['sub', 'iss', 'name', 'email']:
         if user_infos.get(k) is None:
@@ -114,5 +90,5 @@ def check_vo_membership(
     if requested_vo not in user_vos:
         raise HTTPException(
             status_code=401,
-            detail=f"The provided Virtual Organization does not match with any of your available VOs: {user_vos}."
+            detail=f"The requested Virtual Organization ({requested_vo}) does not match with any of your available VOs: {user_vos}."
             )

ai4papi/conf.py

@@ -4,6 +4,7 @@
 
 from pathlib import Path
 from string import Template
+import subprocess
 
 import yaml
 
@@ -88,3 +89,28 @@ def load_yaml_conf(fpath):
 for tool in TOOLS.keys():
     if tool not in tools_nomad2id.values():
         raise Exception(f"The tool {tool} is missing from the mapping dictionary.")
+
+# OSCAR template
+with open(paths['conf'] / 'oscar.yaml', 'r') as f:
+    OSCAR_TMPL = Template(f.read())
+
+# Try-me endpoints
+nmd = load_nomad_job(paths['conf'] / 'try_me' / 'nomad.hcl')
+TRY_ME = {
+    'nomad': nmd,
+}
+
+# Retrieve git info from PAPI, to show current version in the docs
+papi_commit = subprocess.run(
+    ['git', 'log', '-1', '--format=%H'],
+    stdout=subprocess.PIPE,
+    text=True,
+    cwd=main_path,
+    ).stdout.strip()
+papi_branch = subprocess.run(
+    ['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'],
+    stdout=subprocess.PIPE,
+    text=True,
+    cwd=main_path,
+    ).stdout.strip()
+papi_branch = papi_branch.split('/')[-1]  # remove the "origin/" part

ai4papi/main.py

@@ -6,7 +6,7 @@
 import fastapi
 import uvicorn
 
-from ai4papi.conf import MAIN_CONF, paths
+from ai4papi.conf import MAIN_CONF, paths, papi_branch, papi_commit
 from fastapi.responses import FileResponse
 from ai4papi.routers import v1
 from ai4papi.routers.v1.stats.deployments import get_cluster_stats_bg
@@ -39,7 +39,11 @@
     "This work is co-funded by [AI4EOSC](https://ai4eosc.eu/) project that has "
     "received funding from the European Union's Horizon Europe 2022 research and "
     "innovation programme under agreement No 101058593"
+    "<br><br>"
 
+    "PAPI version:"
+    f"[`ai4-papi/{papi_branch}@{papi_commit[:5]}`]"
+    f"(https://github.com/ai4os/ai4-papi/tree/{papi_commit})"
 )
 
 @asynccontextmanager

ai4papi/nomad/common.py

@@ -217,6 +217,7 @@ def get_deployment(
         elif a['ClientStatus'] == 'unknown':
             info['status'] = 'down'
         else:
+            # This status can be for example: "complete", "failed"
             info['status'] = a['ClientStatus']
 
         # Add error messages if needed
@@ -336,30 +337,29 @@ def delete_deployment(
 
     Returns a dict with status
     """
-    # Check the deployment exists
-    try:
-        j = Nomad.job.get_job(
-            id_=deployment_uuid,
-            namespace=namespace,
-            )
-    except exceptions.URLNotFoundNomadException:
-        raise HTTPException(
-            status_code=400,
-            detail="No deployment exists with this uuid.",
-            )
+    # Retrieve the deployment information. Under-the-hood it checks that:
+    # - the job indeed exists
+    # - the owner does indeed own the job
+    info = get_deployment(
+        deployment_uuid=deployment_uuid,
+        namespace=namespace,
+        owner=owner,
+        full_info=False,
+        )
 
-    # Check job does belong to owner
-    if j['Meta'] and owner != j['Meta'].get('owner', ''):
-        raise HTTPException(
-            status_code=400,
-            detail="You are not the owner of that deployment.",
-            )
+    # If job is in stuck status, allow deleting with purge.
+    # Most of the time, when a job is in this status, it is due to a platform error.
+    # It gets stuck and cannot be deleted without purge
+    if info['status'] in ['queued', 'complete', 'failed', 'error', 'down'] :
+        purge = True
+    else:
+        purge = False
 
     # Delete deployment
     Nomad.job.deregister_job(
         id_=deployment_uuid,
         namespace=namespace,
-        purge=False,
+        purge=purge,
         )
 
     return {'status': 'success'}

ai4papi/routers/v1/__init__.py

@@ -1,12 +1,15 @@
 import fastapi
 
-from . import catalog, deployments, secrets, stats
+from . import catalog, deployments, inference, secrets, stats, try_me
+
 
 app = fastapi.APIRouter()
 app.include_router(catalog.app)
 app.include_router(deployments.app)
+app.include_router(inference.app)
 app.include_router(secrets.router)
 app.include_router(stats.app)
+app.include_router(try_me.app)
 
 
 @app.get(

ai4papi/routers/v1/catalog/common.py

@@ -30,6 +30,7 @@
 from fastapi import HTTPException, Query
 import requests
 
+from ai4papi import utils
 import ai4papi.conf as papiconf
 
 
@@ -245,7 +246,7 @@ def get_metadata(
         items = self.get_items()
         if item_name not in items.keys():
             raise HTTPException(
-                status_code=400,
+                status_code=404,
                 detail=f"Item {item_name} not in catalog: {list(items.keys())}",
                 )
 
@@ -280,6 +281,19 @@ def get_metadata(
         # Format "description" field nicely for the Dashboards Markdown parser
         metadata["description"] = "\n".join(metadata["description"])
 
+        # Replace some fields with the info gathered from Github
+        pattern = r'github\.com/([^/]+)/([^/]+?)(?:\.git|/)?$'
+        match = re.search(pattern, items[item_name]['url'])
+        if match:
+            owner, repo = match.group(1), match.group(2)
+            gh_info = utils.get_github_info(owner, repo)
+
+            metadata['date_creation'] = gh_info.get('created', '')
+            # metadata['updated'] = gh_info.get('updated', '')
+            metadata['license'] = gh_info.get('license', '')
+        else:
+            print(f"Failed to parse owner/repo in {items[item_name]['url']}")
+
         return metadata
 
     def get_config(

ai4papi/routers/v1/catalog/modules.py

@@ -43,10 +43,7 @@ def get_config(
     conf["general"]["docker_tag"]["value"] = tags[0]
 
     # Custom conf for development environment
-    if item_name == 'ai4os-dev-env' or item_name == 'deep-oc-generic-dev':
-        #TODO: remove second condition when 'deep-oc-generic-dev' is removed from the
-        # modules catalog
-
+    if item_name == 'ai4os-dev-env':
         # For dev-env, order the tags in "Z-A" order instead of "newest"
         # This is done because builds are done in parallel, so "newest" is meaningless
         # (Z-A + natsort) allows to show more recent semver first

ai4papi/routers/v1/deployments/tools.py

@@ -200,6 +200,9 @@ def create_deployment(
             reference=user_conf,
         )
 
+    # Utils validate conf
+    user_conf = utils.validate_conf(user_conf)
+
     # Check if the provided configuration is within the job quotas
     # Skip this check with CVAT because it does not have a "hardware" section in the conf
     if tool_name not in ['ai4os-cvat']:

ai4papi/routers/v1/inference/__init__.py

@@ -0,0 +1,10 @@
+import fastapi
+
+from . import oscar
+
+
+app = fastapi.APIRouter()
+app.include_router(
+    router=oscar.router,
+    prefix='/inference',
+    )