From bd4412a23c98b300f45f3959eafb39c32c6a0d3f Mon Sep 17 00:00:00 2001 From: Drew Bollinger Date: Tue, 25 Aug 2020 12:14:45 -0400 Subject: [PATCH 1/5] Add dataset memcache util --- covid_api/db/memcache.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/covid_api/db/memcache.py b/covid_api/db/memcache.py index 5c7b181..d5861db 100644 --- a/covid_api/db/memcache.py +++ b/covid_api/db/memcache.py @@ -1,6 +1,6 @@ """covid_api.cache.memcache: memcached layer.""" -from typing import Optional, Tuple +from typing import Optional, Tuple, Dict from bmemcached import Client @@ -61,3 +61,14 @@ def set_image_cache( return self.client.set(img_hash, body, time=timeout) except Exception: return False + + def get_dataset_from_cache(self, ds_hash) -> Dict: + """Get dataset response from cache layer""" + return self.client.get(ds_hash) + + def set_dataset_cache(self, ds_hash: str, body: Dict, timeout: int = 3600) -> bool: + """Set dataset response in cache layer""" + try: + return self.client.set(ds_hash, body, time=timeout) + except Exception: + return False From c7fea7c05da27f93859b6ffefcafdb0de87b2ddd Mon Sep 17 00:00:00 2001 From: Drew Bollinger Date: Tue, 25 Aug 2020 12:42:06 -0400 Subject: [PATCH 2/5] use cached dataset in api --- covid_api/api/api_v1/endpoints/datasets.py | 50 +++++++++++++++++++--- covid_api/api/utils.py | 2 +- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/covid_api/api/api_v1/endpoints/datasets.py b/covid_api/api/api_v1/endpoints/datasets.py index bafe01e..1a5a284 100644 --- a/covid_api/api/api_v1/endpoints/datasets.py +++ b/covid_api/api/api_v1/endpoints/datasets.py @@ -1,11 +1,13 @@ """Dataset endpoints.""" +from typing import Dict from covid_api.db.static.errors import InvalidIdentifier -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, HTTPException, Depends, Response from covid_api.models.static import Datasets from covid_api.db.static.datasets import datasets - +from covid_api.db.memcache import CacheLayer +from covid_api.api import utils router = APIRouter() @@ -15,9 +17,27 @@ responses={200: dict(description="return a list of all available datasets")}, response_model=Datasets, ) -def get_datasets(): +def get_datasets( + response: Response, + cache_client: CacheLayer = Depends(utils.get_cache), +): """Return a list of datasets.""" - return datasets.get_all() + dataset_hash = utils.get_hash(spotlight_id='all') + content = None + + if cache_client: + try: + content = cache_client.get_image_from_cache(dataset_hash) + response.headers = headers["X-Cache"] = "HIT" + except Exception: + content = None + if not content: + content = datasets.get_all() + if cache_client and content: + cache_client.set_dataset_cache(dataset_hash, content) + + + return content @router.get( @@ -27,10 +47,28 @@ def get_datasets(): }, response_model=Datasets, ) -def get_dataset(spotlight_id: str): +def get_dataset( + spotlight_id: str, + response: Response, + cache_client: CacheLayer = Depends(utils.get_cache), +): """Return dataset info for all datasets available for a given spotlight""" try: - return datasets.get(spotlight_id) + dataset_hash = utils.get_hash(spotlight_id=spotlight_id) + content = None + + if cache_client: + try: + content = cache_client.get_image_from_cache(dataset_hash) + response.headers["X-Cache"] = "HIT" + except Exception: + content = None + if not content: + content = datasets.get(spotlight_id) + if cache_client and content: + cache_client.set_dataset_cache(dataset_hash, content) + + return content except InvalidIdentifier: raise HTTPException( status_code=404, detail=f"Invalid spotlight identifier: {spotlight_id}" diff --git a/covid_api/api/utils.py b/covid_api/api/utils.py index 0003da4..2200213 100644 --- a/covid_api/api/utils.py +++ b/covid_api/api/utils.py @@ -44,7 +44,7 @@ def get_cache(request: Request) -> CacheLayer: def get_hash(**kwargs: Any) -> str: - """Create hash from a dict.""" + """Create hash from kwargs.""" return hashlib.sha224(json.dumps(kwargs, sort_keys=True).encode()).hexdigest() From 46577a37f3a21f50a7e3e8b37e533c9cb2ae4257 Mon Sep 17 00:00:00 2001 From: Drew Bollinger Date: Tue, 25 Aug 2020 12:47:07 -0400 Subject: [PATCH 3/5] skipped precommit somehow --- covid_api/api/api_v1/endpoints/datasets.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/covid_api/api/api_v1/endpoints/datasets.py b/covid_api/api/api_v1/endpoints/datasets.py index 1a5a284..9c18607 100644 --- a/covid_api/api/api_v1/endpoints/datasets.py +++ b/covid_api/api/api_v1/endpoints/datasets.py @@ -1,6 +1,4 @@ """Dataset endpoints.""" -from typing import Dict - from covid_api.db.static.errors import InvalidIdentifier from fastapi import APIRouter, HTTPException, Depends, Response @@ -18,17 +16,16 @@ response_model=Datasets, ) def get_datasets( - response: Response, - cache_client: CacheLayer = Depends(utils.get_cache), + response: Response, cache_client: CacheLayer = Depends(utils.get_cache), ): """Return a list of datasets.""" - dataset_hash = utils.get_hash(spotlight_id='all') + dataset_hash = utils.get_hash(spotlight_id="all") content = None if cache_client: try: content = cache_client.get_image_from_cache(dataset_hash) - response.headers = headers["X-Cache"] = "HIT" + response.headers["X-Cache"] = "HIT" except Exception: content = None if not content: @@ -36,7 +33,6 @@ def get_datasets( if cache_client and content: cache_client.set_dataset_cache(dataset_hash, content) - return content From 56e67c6feccfb2a1291bf9f4effb9c3909835161 Mon Sep 17 00:00:00 2001 From: Drew Bollinger Date: Tue, 25 Aug 2020 13:52:36 -0400 Subject: [PATCH 4/5] Update covid_api/db/memcache.py Co-authored-by: Leo Thomas --- covid_api/db/memcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/covid_api/db/memcache.py b/covid_api/db/memcache.py index d5861db..eaa268e 100644 --- a/covid_api/db/memcache.py +++ b/covid_api/db/memcache.py @@ -62,7 +62,7 @@ def set_image_cache( except Exception: return False - def get_dataset_from_cache(self, ds_hash) -> Dict: + def get_dataset_from_cache(self, ds_hash: str ) -> Dict: """Get dataset response from cache layer""" return self.client.get(ds_hash) From fdd27c8611a458bbbc8853e3c6a56bf21854320d Mon Sep 17 00:00:00 2001 From: Drew Bollinger Date: Tue, 25 Aug 2020 13:58:07 -0400 Subject: [PATCH 5/5] extra space in github comment commit --- covid_api/db/memcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/covid_api/db/memcache.py b/covid_api/db/memcache.py index eaa268e..eaadeb0 100644 --- a/covid_api/db/memcache.py +++ b/covid_api/db/memcache.py @@ -62,7 +62,7 @@ def set_image_cache( except Exception: return False - def get_dataset_from_cache(self, ds_hash: str ) -> Dict: + def get_dataset_from_cache(self, ds_hash: str) -> Dict: """Get dataset response from cache layer""" return self.client.get(ds_hash)