style: start formatting with Ruff (#78)

* style: start formatting with Ruff * style: add vscode support for Ruff * style: start using Ruff linter * build: start using pre-commit
ai4os · Nov 27, 2024 · 5270d1d · 5270d1d
1 parent 649f284
commit 5270d1d
Show file tree

Hide file tree

Showing 49 changed files with 1,347 additions and 1,290 deletions.
diff --git a/.gitignore b/.gitignore
@@ -67,7 +67,7 @@ target/
 .idea
 
 # VS Code
-.vscode/
+# .vscode/
 
 # Spyder
 .spyproject/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.8.0
+    hooks:
+      # Run the linter.
+      - id: ruff
+        types_or: [ python, pyi ]
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+        types_or: [ python, pyi ]
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "ms-python.python",
+        "charliermarsh.ruff",
+        "bdsoftware.format-on-auto-save"
+    ]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,10 @@
+{
+    "[python]": {
+        "editor.defaultFormatter": "charliermarsh.ruff",
+        "editor.formatOnSave": true
+    },
+    "files.autoSave": "afterDelay",
+    "editor.rulers": [
+        88
+    ],
+}
diff --git a/README.md b/README.md
@@ -5,6 +5,9 @@
 # AI4EOSC - Platform API
 
 [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-%23FE5196?logo=conventionalcommits&logoColor=white)](https://conventionalcommits.org)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Build Docker](https://github.com/ai4os/ai4-papi/actions/workflows/build-docker-prod.yml/badge.svg)](https://github.com/ai4os/ai4-papi/actions/workflows/build-docker-prod.yml)
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/ai4os/ai4-papi/master.svg)](https://results.pre-commit.ci/latest/github/ai4os/ai4-papi/master)
 
 [//]: # ([![GitHub license]&#40;https://img.shields.io/github/license/ai4papi/ai4papi.svg&#41;]&#40;https://github.com/ai4papi/ai4papi/blob/master/LICENSE&#41;)
 [//]: # ([![GitHub release]&#40;https://img.shields.io/github/release/ai4papi/ai4papi.svg&#41;]&#40;https://github.com/ai4papi/ai4papi/releases&#41;)
@@ -271,3 +274,10 @@ The pattern for the subfolders follows:
   - `user.yaml`: user customizable configuration to make a deployment in Nomad.
     Also contains the generic quotas for hardware (see `range` parameter).
   - `nomad.hcl`: additional non-customizable values (eg. ports)
+
+### Implementation notes
+
+This repository is formatted with [Ruff](https://docs.astral.sh/ruff/).
+We use [Ruff](https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff) and [FormatOnSave](https://marketplace.visualstudio.com/items?itemName=BdSoftware.format-on-auto-save) ([issue](https://github.com/microsoft/vscode/issues/45997#issuecomment-950405496)) VScode extensions to make the development workflow smoother.
+
+We use [Precommit](https://pre-commit.com/) locally to enforce format in commits. Then use [Precommit.CI](https://pre-commit.ci/) to enforce it at the Github level.
diff --git a/ai4papi/auth.py b/ai4papi/auth.py
@@ -35,46 +35,45 @@
 
 
 def get_user_info(token):
-
     try:
         user_infos = flaat.get_user_infos_from_access_token(token)
     except Exception as e:
         raise HTTPException(
             status_code=401,
             detail=str(e),
-            )
+        )
 
     # Check output
     if user_infos is None:
         raise HTTPException(
             status_code=401,
             detail="Invalid token",
-            )
+        )
 
     # Retrieve VOs the user belongs to
     # VOs can be empty if the user does not belong to any VO, or the
     # 'eduperson_entitlement wasn't correctly retrieved from the token
     vos = []
-    for i in user_infos.get('eduperson_entitlement', []):
+    for i in user_infos.get("eduperson_entitlement", []):
         # Parse Virtual Organizations manually from URNs
         # If more complexity is need in the future, check https://github.com/oarepo/urnparse
         ent_i = re.search(r"group:(.+?):", i)
         if ent_i:  # your entitlement has indeed a group `tag`
             vos.append(ent_i.group(1))
 
     # Generate user info dict
-    for k in ['sub', 'iss', 'name', 'email']:
+    for k in ["sub", "iss", "name", "email"]:
         if user_infos.get(k) is None:
             raise HTTPException(
                 status_code=401,
                 detail=f"You token should have scopes for {k}.",
-                )
+            )
     out = {
-        'id': user_infos.get('sub'),  # subject, user-ID
-        'issuer': user_infos.get('iss'),  # URL of the access token issuer
-        'name': user_infos.get('name'),
-        'email': user_infos.get('email'),
-        'vos': vos,
+        "id": user_infos.get("sub"),  # subject, user-ID
+        "issuer": user_infos.get("iss"),  # URL of the access token issuer
+        "name": user_infos.get("name"),
+        "email": user_infos.get("email"),
+        "vos": vos,
     }
 
     return out
@@ -90,5 +89,5 @@ def check_vo_membership(
     if requested_vo not in user_vos:
         raise HTTPException(
             status_code=401,
-            detail=f"The requested Virtual Organization ({requested_vo}) does not match with any of your available VOs: {user_vos}."
-            )
+            detail=f"The requested Virtual Organization ({requested_vo}) does not match with any of your available VOs: {user_vos}.",
+        )
diff --git a/ai4papi/conf.py b/ai4papi/conf.py
@@ -14,17 +14,19 @@
 # intensive (eg. disables calls to Github API)
 # The variables 'FORWARDED_ALLOW_IPS' serves as proxy for this, as it is only defined
 # when running from the Docker container
-IS_DEV = False if os.getenv('FORWARDED_ALLOW_IPS') else True
+IS_DEV = False if os.getenv("FORWARDED_ALLOW_IPS") else True
 
 # Harbor token is kind of mandatory in production, otherwise snapshots won't work.
 HARBOR_USER = "robot$user-snapshots+snapshot-api"
-HARBOR_PASS = os.environ.get('HARBOR_ROBOT_PASSWORD')
+HARBOR_PASS = os.environ.get("HARBOR_ROBOT_PASSWORD")
 if not HARBOR_PASS:
     if IS_DEV:
         # Not enforce this for developers
-        print("You should define the variable \"HARBOR_ROBOT_PASSWORD\" to use the \"/snapshots\" endpoint.")
+        print(
+            'You should define the variable "HARBOR_ROBOT_PASSWORD" to use the "/snapshots" endpoint.'
+        )
     else:
-        raise Exception("You need to define the variable \"HARBOR_ROBOT_PASSWORD\".")
+        raise Exception('You need to define the variable "HARBOR_ROBOT_PASSWORD".')
 
 # Paths
 main_path = Path(__file__).parent.absolute()
@@ -34,15 +36,15 @@
 }
 
 # Load main API configuration
-with open(paths['conf'] / 'main.yaml', 'r') as f:
+with open(paths["conf"] / "main.yaml", "r") as f:
     MAIN_CONF = yaml.safe_load(f)
 
 
 def load_nomad_job(fpath):
     """
     Load default Nomad job configuration
     """
-    with open(fpath, 'r') as f:
+    with open(fpath, "r") as f:
         raw_job = f.read()
         job_template = Template(raw_job)
     return job_template
@@ -52,84 +54,84 @@ def load_yaml_conf(fpath):
     """
     Load user customizable parameters
     """
-    with open(fpath, 'r') as f:
+    with open(fpath, "r") as f:
         conf_full = yaml.safe_load(f)
 
     conf_values = {}
     for group_name, params in conf_full.items():
         conf_values[group_name] = {}
         for k, v in params.items():
-            if 'name' not in v.keys():
+            if "name" not in v.keys():
                 raise Exception(f"Parameter {k} needs to have a name.")
-            if 'value' not in v.keys():
+            if "value" not in v.keys():
                 raise Exception(f"Parameter {k} needs to have a value.")
-            conf_values[group_name][k] = v['value']
+            conf_values[group_name][k] = v["value"]
 
     return conf_full, conf_values
 
 
 # Standard modules
-nmd = load_nomad_job(paths['conf'] / 'modules' / 'nomad.hcl')
-yml = load_yaml_conf(paths['conf'] / 'modules' / 'user.yaml')
+nmd = load_nomad_job(paths["conf"] / "modules" / "nomad.hcl")
+yml = load_yaml_conf(paths["conf"] / "modules" / "user.yaml")
 MODULES = {
-    'nomad': nmd,
-    'user': {
-        'full': yml[0],
-        'values': yml[1],
+    "nomad": nmd,
+    "user": {
+        "full": yml[0],
+        "values": yml[1],
     },
 }
 
 # Tools
-tool_dir = paths['conf'] / 'tools'
+tool_dir = paths["conf"] / "tools"
 tool_list = [f for f in tool_dir.iterdir() if f.is_dir()]
 TOOLS = {}
 for tool_path in tool_list:
-    nmd = load_nomad_job(tool_path / 'nomad.hcl')
-    yml = load_yaml_conf(tool_path / 'user.yaml')
+    nmd = load_nomad_job(tool_path / "nomad.hcl")
+    yml = load_yaml_conf(tool_path / "user.yaml")
     TOOLS[tool_path.name] = {
-        'nomad': nmd,
-        'user': {
-            'full': yml[0],
-            'values': yml[1],
+        "nomad": nmd,
+        "user": {
+            "full": yml[0],
+            "values": yml[1],
         },
     }
 
 # For tools, map the Nomad job name prefixes to tool IDs
 tools_nomad2id = {
-    'fl': 'ai4os-federated-server',
-    'cvat': 'ai4os-cvat',
+    "fl": "ai4os-federated-server",
+    "cvat": "ai4os-cvat",
 }
 for tool in TOOLS.keys():
     if tool not in tools_nomad2id.values():
         raise Exception(f"The tool {tool} is missing from the mapping dictionary.")
 
 # OSCAR template
-with open(paths['conf'] / 'oscar.yaml', 'r') as f:
+with open(paths["conf"] / "oscar.yaml", "r") as f:
     OSCAR_TMPL = Template(f.read())
 
 # Try-me endpoints
-nmd = load_nomad_job(paths['conf'] / 'try_me' / 'nomad.hcl')
+nmd = load_nomad_job(paths["conf"] / "try_me" / "nomad.hcl")
 TRY_ME = {
-    'nomad': nmd,
+    "nomad": nmd,
 }
 
 # Snapshot endpoints
-nmd = load_nomad_job(paths['conf'] / 'snapshots' / 'nomad.hcl')
+nmd = load_nomad_job(paths["conf"] / "snapshots" / "nomad.hcl")
 SNAPSHOTS = {
-    'nomad': nmd,
+    "nomad": nmd,
 }
 
 # Retrieve git info from PAPI, to show current version in the docs
 papi_commit = subprocess.run(
-    ['git', 'log', '-1', '--format=%H'],
+    ["git", "log", "-1", "--format=%H"],
     stdout=subprocess.PIPE,
     text=True,
     cwd=main_path,
 ).stdout.strip()
 papi_branch = subprocess.run(
-    ['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'],
+    ["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"],
     stdout=subprocess.PIPE,
     text=True,
     cwd=main_path,
 ).stdout.strip()
-papi_branch = papi_branch.split('/')[-1]  # remove the "origin/" part
+papi_branch = papi_branch.split("/")[-1]  # remove the "origin/" part
diff --git a/ai4papi/main.py b/ai4papi/main.py
@@ -19,33 +19,29 @@
     " src='https://ai4eosc.eu/wp-content/uploads/sites/10/2023/01/horizontal-bg-dark.png'"
     " width=200 alt='' />"
     "<br><br>"
-
     "This is the Platform API for interacting with the AI4EOSC services. "
     "It aims at providing a stable UI, effectively decoupling the services offered by "
     "the project from the underlying tools we use to provide them (ie. Nomad)."
     "<br><br>"
-
     "You can also access the functionalities of the API through our dashboards: <br>"
     "- [AIEOSC Dashboard](https://dashboard.cloud.ai4eosc.eu/) <br>"
     "- [iMagine Dashboard](https://dashboard.cloud.imagine-ai.eu/)"
     "<br><br>"
-
     "For more information, please visit: <br>"
     "- [AI4EOSC Homepage](https://ai4eosc.eu) <br>"
     "- [API Github repository](https://github.com/AI4EOSC/ai4-papi)"
     "<br><br>"
-
     "**Acknowledgements** <br>"
     "This work is co-funded by [AI4EOSC](https://ai4eosc.eu/) project that has "
     "received funding from the European Union's Horizon Europe 2022 research and "
     "innovation programme under agreement No 101058593"
     "<br><br>"
-
     "PAPI version:"
     f"[`ai4-papi/{papi_branch}@{papi_commit[:5]}`]"
     f"(https://github.com/ai4os/ai4-papi/tree/{papi_commit})"
 )
 
+
 @asynccontextmanager
 async def lifespan(app: fastapi.FastAPI):
     # on startup
@@ -114,11 +110,11 @@ async def favicon():
 
 
 def run(
-        host:str = "0.0.0.0",
-        port:int = 8080,
-        ssl_keyfile:str = None,
-        ssl_certfile:str = None,
-    ):
+    host: str = "0.0.0.0",
+    port: int = 8080,
+    ssl_keyfile: str = None,
+    ssl_certfile: str = None,
+):
     uvicorn.run(
         app,
         host=host,

diff --git a/ai4papi/module_patches.py b/ai4papi/module_patches.py
@@ -3,10 +3,11 @@
 fix/rebuild them.
 """
 
+
 def patch_nextcloud_mount(
     docker_image: str,
     task: dict,
-    ):
+):
     """
     Some module are blocked when running deepaas.
 
@@ -37,10 +38,10 @@ def patch_nextcloud_mount(
         "DEEP-OC-image-classification-tf-dicom",
         "DEEP-OC-speech-to-text-tf",
     ]
-    modules = [f'deephdc/{m.lower()}' for m in modules]
+    modules = [f"deephdc/{m.lower()}" for m in modules]
     # TODO: this will need to be updated to ai4os-hub
 
     if docker_image in modules:
-        task['Env']['RCLONE_CONTIMEOUT'] = '1s'
+        task["Env"]["RCLONE_CONTIMEOUT"] = "1s"
 
     return task