From 233b38ffc67ba1e2873c072afbd8e11fafe5aa8d Mon Sep 17 00:00:00 2001 From: Sftobias <154964601+Sftobias@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:08:50 +0100 Subject: [PATCH] feat: support refreshing metadata from CI/CD (#70) * feat: cache refresh * test: add tests * feat: add token authentication and refactor code * fix: update catalog index before refreshing --------- Co-authored-by: Ignacio Heredia --- ai4papi/routers/v1/catalog/common.py | 52 +++++++++++++++++++++++++-- ai4papi/routers/v1/catalog/modules.py | 7 ++++ ai4papi/routers/v1/catalog/tools.py | 6 ++++ tests/catalog/modules.py | 13 +++++++ tests/catalog/tools.py | 12 ++++++- 5 files changed, 86 insertions(+), 4 deletions(-) diff --git a/ai4papi/routers/v1/catalog/common.py b/ai4papi/routers/v1/catalog/common.py index f886284..ae048e7 100644 --- a/ai4papi/routers/v1/catalog/common.py +++ b/ai4papi/routers/v1/catalog/common.py @@ -23,27 +23,36 @@ """ import configparser +import os import re from typing import Tuple, Union import yaml import ai4_metadata.validate from cachetools import cached, TTLCache -from fastapi import HTTPException, Query +from fastapi import Depends, HTTPException, Query +from fastapi.security import HTTPBearer import requests from ai4papi import utils import ai4papi.conf as papiconf +security = HTTPBearer() + +JENKINS_TOKEN = os.getenv('PAPI_JENKINS_TOKEN') + + class Catalog: - def __init__(self, repo: str) -> None: + def __init__(self, repo:str, item_type:str='item') -> None: """ Parameters: * repo: Github repo where the catalog is hosted (via git submodules) + * item_type: Name to display in messages (eg. "module", "tool") """ self.repo = repo + self.item_type = item_type @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60)) @@ -104,6 +113,7 @@ def get_filtered_list( # ValueError: [ValueError('dictionary update sequence element #0 has length 1; 2 is required'), TypeError('vars() argument must have __dict__ attribute')] return modules + @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60)) def get_summary( self, @@ -145,7 +155,7 @@ def get_tags( return [] - @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60)) + @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60), key=lambda self, item_name: item_name,) def get_metadata( self, item_name: str, @@ -273,6 +283,42 @@ def get_metadata( return metadata + + def refresh_metadata_cache_entry( + self, + item_name: str, + authorization=Depends(security), + ): + """ + Expire the metadata cache of a given item and recompute new cache value. + """ + # Check if token is valid + if authorization.credentials != JENKINS_TOKEN: + raise HTTPException( + status_code=401, + detail="Invalid authorization token.", + ) + + # First refresh the items in the catalog, because this item might be a + # new addition to the catalog (ie. not present since last parsing the catalog) + self.get_items.cache_clear() + + # Check if the item is indeed valid + if item_name not in self.get_items().keys(): + raise HTTPException( + status_code=400, + detail=f"{item_name} is not an available {self.item_type}.", + ) + + # Refresh cache + try: + self.get_metadata.cache.pop(item_name, None) + self.get_metadata(item_name) + return {"message": "Cache refreshed successfully"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + def get_config( self, ): diff --git a/ai4papi/routers/v1/catalog/modules.py b/ai4papi/routers/v1/catalog/modules.py index 339f48d..20b5c58 100644 --- a/ai4papi/routers/v1/catalog/modules.py +++ b/ai4papi/routers/v1/catalog/modules.py @@ -72,6 +72,7 @@ def get_config( Modules = Catalog( repo='ai4os-hub/modules-catalog', + item_type='module', ) Modules.get_config = types.MethodType(get_config, Modules) @@ -107,3 +108,9 @@ def get_config( Modules.get_config, methods=["GET"], ) + +router.add_api_route( + "/{item_name}/refresh", + Modules.refresh_metadata_cache_entry, + methods=["PUT"], +) diff --git a/ai4papi/routers/v1/catalog/tools.py b/ai4papi/routers/v1/catalog/tools.py index 22a0446..c5d5dab 100644 --- a/ai4papi/routers/v1/catalog/tools.py +++ b/ai4papi/routers/v1/catalog/tools.py @@ -61,6 +61,7 @@ def get_config( Tools = Catalog( repo='ai4os/tools-catalog', + item_type='tool', ) Tools.get_config = types.MethodType(get_config, Tools) @@ -96,3 +97,8 @@ def get_config( Tools.get_config, methods=["GET"], ) +router.add_api_route( + "/{item_name}/refresh", + Tools.refresh_metadata_cache_entry, + methods=["PUT"], +) diff --git a/tests/catalog/modules.py b/tests/catalog/modules.py index 8862cc2..83f15e0 100644 --- a/tests/catalog/modules.py +++ b/tests/catalog/modules.py @@ -1,3 +1,6 @@ +from types import SimpleNamespace + +from ai4papi.routers.v1.catalog import common from ai4papi.routers.v1.catalog.modules import Modules @@ -50,6 +53,16 @@ assert isinstance(module_meta, dict) assert 'title' in module_meta.keys() +# Refresh metadata cache +common.JENKINS_TOKEN = '1234' +module_meta = Modules.refresh_metadata_cache_entry( + item_name=module_name, + authorization=SimpleNamespace( + credentials='1234', + ), +) +assert isinstance(module_meta, dict) + #TODO: we should not be able to get config or metadata for a tool_name print('Catalog (modules) tests passed!') diff --git a/tests/catalog/tools.py b/tests/catalog/tools.py index 78f8e6e..0666c05 100644 --- a/tests/catalog/tools.py +++ b/tests/catalog/tools.py @@ -1,7 +1,7 @@ - import os from types import SimpleNamespace +from ai4papi.routers.v1.catalog import common from ai4papi.routers.v1.catalog.tools import Tools @@ -70,6 +70,16 @@ assert isinstance(tool_meta, dict) assert 'title' in tool_meta.keys() +# Refresh metadata cache +common.JENKINS_TOKEN = '1234' +module_meta = Tools.refresh_metadata_cache_entry( + item_name=tool_name, + authorization=SimpleNamespace( + credentials='1234', + ), +) +assert isinstance(module_meta, dict) + #TODO: we should not be able to get config or metadata for a module_name print('Catalog (tools) tests passed!')