feat: support refreshing metadata from CI/CD (#70)

* feat: cache refresh

* test: add tests

* feat: add token authentication and refactor code

* fix: update catalog index before refreshing

---------

Co-authored-by: Ignacio Heredia <[email protected]>

ai4papi/routers/v1/catalog/common.py

@@ -23,27 +23,36 @@
 """
 
 import configparser
+import os
 import re
 from typing import Tuple, Union
 import yaml
 
 import ai4_metadata.validate
 from cachetools import cached, TTLCache
-from fastapi import HTTPException, Query
+from fastapi import Depends, HTTPException, Query
+from fastapi.security import HTTPBearer
 import requests
 
 from ai4papi import utils
 import ai4papi.conf as papiconf
 
 
+security = HTTPBearer()
+
+JENKINS_TOKEN = os.getenv('PAPI_JENKINS_TOKEN')
+
+
 class Catalog:
 
-    def __init__(self, repo: str) -> None:
+    def __init__(self, repo:str, item_type:str='item') -> None:
         """
         Parameters:
         * repo: Github repo where the catalog is hosted (via git submodules)
+        * item_type: Name to display in messages (eg. "module", "tool")
         """
         self.repo = repo
+        self.item_type = item_type
 
 
     @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
@@ -104,6 +113,7 @@ def get_filtered_list(
         # ValueError: [ValueError('dictionary update sequence element #0 has length 1; 2 is required'), TypeError('vars() argument must have __dict__ attribute')]
         return modules
 
+
     @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
     def get_summary(
         self,
@@ -145,7 +155,7 @@ def get_tags(
         return []
 
 
-    @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
+    @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60), key=lambda self, item_name: item_name,)
     def get_metadata(
         self,
         item_name: str,
@@ -273,6 +283,42 @@ def get_metadata(
 
         return metadata
 
+
+    def refresh_metadata_cache_entry(
+        self,
+        item_name: str,
+        authorization=Depends(security),
+    ):
+        """
+        Expire the metadata cache of a given item and recompute new cache value.
+        """
+        # Check if token is valid
+        if authorization.credentials != JENKINS_TOKEN:
+            raise HTTPException(
+                status_code=401,
+                detail="Invalid authorization token.",
+            )
+
+        # First refresh the items in the catalog, because this item might be a
+        # new addition to the catalog (ie. not present since last parsing the catalog)
+        self.get_items.cache_clear()
+
+        # Check if the item is indeed valid
+        if item_name not in self.get_items().keys():
+            raise HTTPException(
+                status_code=400,
+                detail=f"{item_name} is not an available {self.item_type}.",
+            )
+
+        # Refresh cache
+        try:
+            self.get_metadata.cache.pop(item_name, None)
+            self.get_metadata(item_name)
+            return {"message": "Cache refreshed successfully"}
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=str(e))
+
+
     def get_config(
         self,
         ):

ai4papi/routers/v1/catalog/modules.py

@@ -72,6 +72,7 @@ def get_config(
 
 Modules = Catalog(
     repo='ai4os-hub/modules-catalog',
+    item_type='module',
 )
 Modules.get_config = types.MethodType(get_config, Modules)
 
@@ -107,3 +108,9 @@ def get_config(
     Modules.get_config,
     methods=["GET"],
     )
+
+router.add_api_route(
+    "/{item_name}/refresh",
+    Modules.refresh_metadata_cache_entry,
+    methods=["PUT"],
+)

ai4papi/routers/v1/catalog/tools.py

@@ -61,6 +61,7 @@ def get_config(
 
 Tools = Catalog(
     repo='ai4os/tools-catalog',
+    item_type='tool',
 )
 Tools.get_config = types.MethodType(get_config, Tools)
 
@@ -96,3 +97,8 @@ def get_config(
     Tools.get_config,
     methods=["GET"],
     )
+router.add_api_route(
+    "/{item_name}/refresh",
+    Tools.refresh_metadata_cache_entry,
+    methods=["PUT"],
+)

tests/catalog/modules.py

@@ -1,3 +1,6 @@
+from types import SimpleNamespace
+
+from ai4papi.routers.v1.catalog import common
 from ai4papi.routers.v1.catalog.modules import Modules
 
 
@@ -50,6 +53,16 @@
 assert isinstance(module_meta, dict)
 assert 'title' in module_meta.keys()
 
+# Refresh metadata cache
+common.JENKINS_TOKEN = '1234'
+module_meta = Modules.refresh_metadata_cache_entry(
+    item_name=module_name,
+    authorization=SimpleNamespace(
+        credentials='1234',
+    ),
+)
+assert isinstance(module_meta, dict)
+
 #TODO: we should not be able to get config or metadata for a tool_name
 
 print('Catalog (modules) tests passed!')

tests/catalog/tools.py

@@ -1,7 +1,7 @@
-
 import os
 from types import SimpleNamespace
 
+from ai4papi.routers.v1.catalog import common
 from ai4papi.routers.v1.catalog.tools import Tools
 
 
@@ -70,6 +70,16 @@
     assert isinstance(tool_meta, dict)
     assert 'title' in tool_meta.keys()
 
+# Refresh metadata cache
+common.JENKINS_TOKEN = '1234'
+module_meta = Tools.refresh_metadata_cache_entry(
+    item_name=tool_name,
+    authorization=SimpleNamespace(
+        credentials='1234',
+    ),
+)
+assert isinstance(module_meta, dict)
+
 #TODO: we should not be able to get config or metadata for a module_name
 
 print('Catalog (tools) tests passed!')