From 40d1da213171223c28afb0ebe59cc9639e6269fa Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 09:37:36 +0100 Subject: [PATCH 1/9] Improve cache code --- app/__init__.py | 19 ++++++++++--------- app/utils.py | 17 ++++++++++++----- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index b7d14061..7c0b0300 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -26,6 +26,7 @@ import logging import copy import requests +import datetime from requests.exceptions import Timeout from werkzeug.middleware.proxy_fix import ProxyFix from flask_dance.consumer import OAuth2ConsumerBlueprint @@ -245,6 +246,9 @@ def home(): flash("Error getting User info: \n" + account_info.text, 'error') return render_template('home.html', oidc_name=settings.oidcName) + scheduler.add_job(func=utils.get_cache_creds, trigger='date', run_date=datetime.datetime.now(), + misfire_grace_time=20, args=[cred, get_cred_id()], id='get_cache_creds') + # if there are any next url, redirect to it if "next" in session and session["next"]: next_url = session.pop("next") @@ -491,7 +495,7 @@ def showinfrastructures(): app.logger.exception("Error getting vm info: %s" % ex) radl_json = [] try: - creds = utils.get_cache_creds(session, cred, get_cred_id()) + creds = utils.get_cache_creds(cred, get_cred_id()) except Exception as ex: app.logger.exception("Error getting user credentials: %s" % ex) creds = [] @@ -761,7 +765,7 @@ def configure(): app.logger.debug("Template: " + json.dumps(toscaInfo[selected_tosca])) try: - creds = utils.get_cache_creds(session, cred, get_cred_id(), 1) + creds = utils.get_cache_creds(cred, get_cred_id(), 1) except Exception as ex: flash("Error getting user credentials: %s" % ex, "error") creds = [] @@ -1115,7 +1119,7 @@ def manage_creds(): creds = {} try: - creds = utils.get_cache_creds(session, cred, get_cred_id()) + creds = utils.get_cache_creds(cred, get_cred_id()) # Get the project_id in case it has changed utils.get_project_ids(creds) except Exception as e: @@ -1170,8 +1174,7 @@ def write_creds(): # Get project_id to save it to de DB utils.get_project_ids([creds]) # delete cached credentials - if 'creds' in session: - del session['creds'] + utils.clear_cache_creds(get_cred_id()) cred.write_creds(creds["id"], get_cred_id(), creds, cred_id in [None, '']) if val_res == 0: flash("Credentials successfully written!", 'success') @@ -1189,8 +1192,7 @@ def delete_creds(): cred_id = request.args.get('cred_id', "") try: # delete cached credentials - if 'creds' in session: - del session['creds'] + utils.clear_cache_creds(get_cred_id()) cred.delete_cred(cred_id, get_cred_id()) flash("Credentials successfully deleted!", 'success') except Exception as ex: @@ -1209,8 +1211,7 @@ def enable_creds(): if val_res == 2: flash(val_msg, 'warning') # delete cached credentials - if 'creds' in session: - del session['creds'] + utils.clear_cache_creds(get_cred_id()) cred.enable_cred(cred_id, get_cred_id(), enable) except Exception as ex: flash("Error updating credentials %s!" % ex, 'error') diff --git a/app/utils.py b/app/utils.py index f6bb7587..c6472dc5 100644 --- a/app/utils.py +++ b/app/utils.py @@ -46,7 +46,7 @@ SITE_LIST = {} LAST_UPDATE = 0 PORT_SPECT_TYPES = ["PortSpec", "tosca.datatypes.network.PortSpec", "tosca.datatypes.indigo.network.PortSpec"] - +CREDS_CACHE = {} def _getStaticSitesInfo(force=False): # Remove cache if force is True @@ -984,13 +984,20 @@ def merge_templates(template, new_template): return template -def get_cache_creds(session, cred, userid, enabled=None): - if "creds" not in session: - session["creds"] = cred.get_creds(userid) +def get_cache_creds(cred, userid, enabled=None): + global CREDS_CACHE + if userid not in CREDS_CACHE: + CREDS_CACHE[userid] = cred.get_creds(userid) res = [] - for cred in session["creds"]: + for cred in CREDS_CACHE[userid]: if enabled is None or enabled == cred['enabled']: res.append(cred) return res + + +def clear_cache_creds(userid): + global CREDS_CACHE + if userid in CREDS_CACHE: + del CREDS_CACHE[userid] From 5749e8cb4c4d8068aa23a2e8ddaf667f99082ee1 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 09:40:41 +0100 Subject: [PATCH 2/9] Fix style --- app/__init__.py | 1 + app/utils.py | 1 + 2 files changed, 2 insertions(+) diff --git a/app/__init__.py b/app/__init__.py index 7c0b0300..86d9347c 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -246,6 +246,7 @@ def home(): flash("Error getting User info: \n" + account_info.text, 'error') return render_template('home.html', oidc_name=settings.oidcName) + # Force to get the user credentials to cache them scheduler.add_job(func=utils.get_cache_creds, trigger='date', run_date=datetime.datetime.now(), misfire_grace_time=20, args=[cred, get_cred_id()], id='get_cache_creds') diff --git a/app/utils.py b/app/utils.py index c6472dc5..bb202afa 100644 --- a/app/utils.py +++ b/app/utils.py @@ -48,6 +48,7 @@ PORT_SPECT_TYPES = ["PortSpec", "tosca.datatypes.network.PortSpec", "tosca.datatypes.indigo.network.PortSpec"] CREDS_CACHE = {} + def _getStaticSitesInfo(force=False): # Remove cache if force is True if force and g.settings.static_sites_url: From dff9cb5c037a42817c93817824f648aa4fdb6bf9 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 09:56:58 +0100 Subject: [PATCH 3/9] Fix test --- app/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index 86d9347c..77cd764d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -247,7 +247,8 @@ def home(): return render_template('home.html', oidc_name=settings.oidcName) # Force to get the user credentials to cache them - scheduler.add_job(func=utils.get_cache_creds, trigger='date', run_date=datetime.datetime.now(), + ndate = datetime.datetime.now() + datetime.timedelta(0,2) + scheduler.add_job(func=utils.get_cache_creds, trigger='date', run_date=ndate, misfire_grace_time=20, args=[cred, get_cred_id()], id='get_cache_creds') # if there are any next url, redirect to it From 6dd735d13d8d8936d6c7a5a1d97ffa548c119f52 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 09:58:40 +0100 Subject: [PATCH 4/9] Fix style --- app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index 77cd764d..eb0562aa 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -247,7 +247,7 @@ def home(): return render_template('home.html', oidc_name=settings.oidcName) # Force to get the user credentials to cache them - ndate = datetime.datetime.now() + datetime.timedelta(0,2) + ndate = datetime.datetime.now() + datetime.timedelta(0, 2) scheduler.add_job(func=utils.get_cache_creds, trigger='date', run_date=ndate, misfire_grace_time=20, args=[cred, get_cred_id()], id='get_cache_creds') From 13d08924f1659b4c60d586a6b840a32bad09d693 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 10:25:01 +0100 Subject: [PATCH 5/9] Fix tests --- .github/workflows/main.yaml | 1 + app/__init__.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dd5b70e3..65ea7f35 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -37,6 +37,7 @@ jobs: wget -P tosca-templates https://raw.githubusercontent.com/grycap/tosca/main/templates/simple-node-disk.yml sed -i -e 's|/opt|'${GITHUB_WORKSPACE}'|g' app/config.json sed -i -e 's|creds.db|tmp/creds.db|g' app/config.json + sed -i -e 's|"EXTRA_AUTH": {},|"DISABLE_APPSCHEDULER": true,|g' app/config.json - name: Unit tests run: python -m coverage run --source=. -m unittest discover -s app/tests -p 'test*.py' diff --git a/app/__init__.py b/app/__init__.py index eb0562aa..c444f290 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -74,11 +74,12 @@ def create_app(oidc_blueprint=None): vault_info = VaultInfo(settings.db_url) ott = OneTimeTokenData(settings.vault_url) - # To Reload internally the site cache scheduler = APScheduler() - scheduler.api_enabled = False - scheduler.init_app(app) - scheduler.start() + # To Reload internally the site cache + if 'DISABLE_APPSCHEDULER' not in app.config: + scheduler.api_enabled = False + scheduler.init_app(app) + scheduler.start() toscaTemplates = utils.loadToscaTemplates(settings.toscaDir) toscaInfo = utils.extractToscaInfo(settings.toscaDir, toscaTemplates, settings.hide_tosca_tags) From 9040dd73e6a871299b0740d962c719123f053066 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 11:24:31 +0100 Subject: [PATCH 6/9] Fix tests --- .github/workflows/main.yaml | 1 - app/__init__.py | 14 ++++++-------- app/tests/test_app.py | 3 ++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 65ea7f35..dd5b70e3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -37,7 +37,6 @@ jobs: wget -P tosca-templates https://raw.githubusercontent.com/grycap/tosca/main/templates/simple-node-disk.yml sed -i -e 's|/opt|'${GITHUB_WORKSPACE}'|g' app/config.json sed -i -e 's|creds.db|tmp/creds.db|g' app/config.json - sed -i -e 's|"EXTRA_AUTH": {},|"DISABLE_APPSCHEDULER": true,|g' app/config.json - name: Unit tests run: python -m coverage run --source=. -m unittest discover -s app/tests -p 'test*.py' diff --git a/app/__init__.py b/app/__init__.py index c444f290..b2b633fd 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -73,13 +73,12 @@ def create_app(oidc_blueprint=None): ssh_key = SSHKey(settings.db_url) vault_info = VaultInfo(settings.db_url) ott = OneTimeTokenData(settings.vault_url) - - scheduler = APScheduler() + # To Reload internally the site cache - if 'DISABLE_APPSCHEDULER' not in app.config: - scheduler.api_enabled = False - scheduler.init_app(app) - scheduler.start() + scheduler = APScheduler() + scheduler.api_enabled = False + scheduler.init_app(app) + scheduler.start() toscaTemplates = utils.loadToscaTemplates(settings.toscaDir) toscaInfo = utils.extractToscaInfo(settings.toscaDir, toscaTemplates, settings.hide_tosca_tags) @@ -248,8 +247,7 @@ def home(): return render_template('home.html', oidc_name=settings.oidcName) # Force to get the user credentials to cache them - ndate = datetime.datetime.now() + datetime.timedelta(0, 2) - scheduler.add_job(func=utils.get_cache_creds, trigger='date', run_date=ndate, + scheduler.add_job(func=utils.get_cache_creds, trigger='date', run_date=datetime.datetime.now(), misfire_grace_time=20, args=[cred, get_cred_id()], id='get_cache_creds') # if there are any next url, redirect to it diff --git a/app/tests/test_app.py b/app/tests/test_app.py index cc9fa0a6..28b67c4e 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -6,7 +6,7 @@ import unittest import json import defusedxml.ElementTree as etree -from app import create_app +from app import create_app, utils from urllib.parse import urlparse from mock import patch, MagicMock @@ -569,6 +569,7 @@ def test_manage_creds(self, get_project_ids, get_sites, get_creds, avatar): get_sites.return_value = {"SITE_NAME": {"url": "URL", "state": "", "id": ""}, "SITE2": {"url": "URL2", "state": "CRITICAL", "id": ""}} get_creds.return_value = [{"id": "credid", "type": "fedcloud", "host": "site_url", "project_id": "project"}] + utils.CREDS_CACHE = {} res = self.client.get('/manage_creds') self.assertEqual(200, res.status_code) self.assertIn(b'credid', res.data) From 261315901a2e3eef5dd397f17ea2d9273cecd15e Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 12:15:57 +0100 Subject: [PATCH 7/9] Fix error --- app/__init__.py | 14 +++++++------- app/utils.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index b2b633fd..b59af8d5 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -248,7 +248,7 @@ def home(): # Force to get the user credentials to cache them scheduler.add_job(func=utils.get_cache_creds, trigger='date', run_date=datetime.datetime.now(), - misfire_grace_time=20, args=[cred, get_cred_id()], id='get_cache_creds') + misfire_grace_time=20, args=[cred, session['userid'], get_cred_id()], id='get_cache_creds') # if there are any next url, redirect to it if "next" in session and session["next"]: @@ -496,7 +496,7 @@ def showinfrastructures(): app.logger.exception("Error getting vm info: %s" % ex) radl_json = [] try: - creds = utils.get_cache_creds(cred, get_cred_id()) + creds = utils.get_cache_creds(cred, session['userid'], get_cred_id()) except Exception as ex: app.logger.exception("Error getting user credentials: %s" % ex) creds = [] @@ -766,7 +766,7 @@ def configure(): app.logger.debug("Template: " + json.dumps(toscaInfo[selected_tosca])) try: - creds = utils.get_cache_creds(cred, get_cred_id(), 1) + creds = utils.get_cache_creds(cred, session['userid'], get_cred_id(), 1) except Exception as ex: flash("Error getting user credentials: %s" % ex, "error") creds = [] @@ -1120,7 +1120,7 @@ def manage_creds(): creds = {} try: - creds = utils.get_cache_creds(cred, get_cred_id()) + creds = utils.get_cache_creds(cred, session['userid'], get_cred_id()) # Get the project_id in case it has changed utils.get_project_ids(creds) except Exception as e: @@ -1175,7 +1175,7 @@ def write_creds(): # Get project_id to save it to de DB utils.get_project_ids([creds]) # delete cached credentials - utils.clear_cache_creds(get_cred_id()) + utils.clear_cache_creds(session['userid']) cred.write_creds(creds["id"], get_cred_id(), creds, cred_id in [None, '']) if val_res == 0: flash("Credentials successfully written!", 'success') @@ -1193,7 +1193,7 @@ def delete_creds(): cred_id = request.args.get('cred_id', "") try: # delete cached credentials - utils.clear_cache_creds(get_cred_id()) + utils.clear_cache_creds(session['userid']) cred.delete_cred(cred_id, get_cred_id()) flash("Credentials successfully deleted!", 'success') except Exception as ex: @@ -1212,7 +1212,7 @@ def enable_creds(): if val_res == 2: flash(val_msg, 'warning') # delete cached credentials - utils.clear_cache_creds(get_cred_id()) + utils.clear_cache_creds(session['userid']) cred.enable_cred(cred_id, get_cred_id(), enable) except Exception as ex: flash("Error updating credentials %s!" % ex, 'error') diff --git a/app/utils.py b/app/utils.py index bb202afa..75622f52 100644 --- a/app/utils.py +++ b/app/utils.py @@ -985,10 +985,10 @@ def merge_templates(template, new_template): return template -def get_cache_creds(cred, userid, enabled=None): +def get_cache_creds(cred, userid, creduserid, enabled=None): global CREDS_CACHE if userid not in CREDS_CACHE: - CREDS_CACHE[userid] = cred.get_creds(userid) + CREDS_CACHE[userid] = cred.get_creds(creduserid) res = [] for cred in CREDS_CACHE[userid]: From c83b602ebcf67da91ce67632f15871c9b8227831 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 11:25:54 +0100 Subject: [PATCH 8/9] =?UTF-8?q?Fix=20sty=C3=B1e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__init__.py b/app/__init__.py index b59af8d5..3b8fd49c 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -73,7 +73,7 @@ def create_app(oidc_blueprint=None): ssh_key = SSHKey(settings.db_url) vault_info = VaultInfo(settings.db_url) ott = OneTimeTokenData(settings.vault_url) - + # To Reload internally the site cache scheduler = APScheduler() scheduler.api_enabled = False From 3425e297497f2c170048a03e69d7f59274527f6d Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Fri, 13 Dec 2024 13:44:03 +0100 Subject: [PATCH 9/9] Fix test --- app/tests/test_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 28b67c4e..4c682b49 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -431,6 +431,7 @@ def test_configure(self, get, get_creds, avatar): self.assertEqual(200, res.status_code) self.assertIn(b"Select Optional Features:", res.data) + utils.CREDS_CACHE = {} get_creds.return_value = [{"id": "credid", "type": "fedcloud", "host": "site_url", "vo": "voname", "enabled": True}, {"id": "credid1", "type": "OpenStack", "host": "site_url1",