diff --git a/README.md b/README.md index 944448c45..ee7b24fec 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Create the `config.json` file (see the [example](app/config-sample.json)) settin | ANALYTICS_TAG | Google Analytic Tag | N | "" | | STATIC_SITES | List of static sites added to the AppDB ones ([{"name": "static_site_name", "url": "static_site_url", "id": "static_id", "vos": {"vo": "stprojectid"}}]) | N | [] | | STATIC_SITES_URL | URL of a JSON file with the list of static sites added to the AppDB ones | N | "" | -| SITES_CACHE_TIMEOUT | AppDB cache TTL | N | 3600 | +| APPDB_CACHE_TIMEOUT | AppDB cache TTL | N | 3600 | | CHECK_TOSCA_CHANGES_TIME | Interval to look for changes in TOSCA templates | N | 120 | | VAULT_URL | Vault service URL to store Cloud credentials | N | None | diff --git a/app/__init__.py b/app/__init__.py index 9bcda149a..76f71acb8 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -36,8 +36,7 @@ from app.im import InfrastructureManager from app.ssh_key import SSHKey from app.ott import OneTimeTokenData -from app import utils, db -from app import appdb as sitesInfo +from app import utils, appdb, db from app.vault_info import VaultInfo from oauthlib.oauth2.rfc6749.errors import InvalidTokenError, TokenExpiredError, InvalidGrantError from werkzeug.exceptions import Forbidden @@ -778,8 +777,8 @@ def getsites(vo=None): for site_name, site in static_sites.items(): res += '' % (site['url'], site_name) - sites = utils.getSiteInfoProvider().get_sites(vo) - for site_name, site in sites.items(): + appdb_sites = appdb.get_sites(vo) + for site_name, site in appdb_sites.items(): # avoid site duplication if site_name not in static_sites: if site["state"]: @@ -808,7 +807,7 @@ def getimages(cred_id=None): else: site, _, vo = utils.get_site_info(cred_id, cred, get_cred_id()) - for image_name, image_id in utils.getSiteInfoProvider().get_images(site['id'], vo): + for image_name, image_id in appdb.get_images(site['id'], vo): res += '' % (image_id, image_name) return res @@ -1540,7 +1539,7 @@ def handle_csrf_error(e): # Reload internally the site cache @scheduler.task('interval', id='reload_sites', seconds=5) def reload_sites(): - scheduler.modify_job('reload_sites', trigger='interval', seconds=settings.sites_cache_timeout - 30) + scheduler.modify_job('reload_sites', trigger='interval', seconds=settings.appdb_cache_timeout - 30) with app.app_context(): app.logger.debug('Reload Site List.') g.settings = settings diff --git a/app/config-sample.json b/app/config-sample.json index 182db0dad..5878afaa2 100644 --- a/app/config-sample.json +++ b/app/config-sample.json @@ -16,7 +16,7 @@ "MOTOMO_INFO": {"url": "", "siteid": ""}, "STATIC_SITES": [{"name": "static_site_name", "url": "static_site_url", "id": "static_id", "vos": {"vo": "stprojectid"}, "api_version": "1.1"}], "STATIC_SITES_URL": "", - "SITES_CACHE_TIMEOUT": 3600, + "APPDB_CACHE_TIMEOUT": 3600, "CHECK_TOSCA_CHANGES_TIME": 120, "IM_TIMEOUT": 60, "VAULT_URL": "", diff --git a/app/egi_catch_all.py b/app/egi_catch_all.py deleted file mode 100644 index e61f1a72a..000000000 --- a/app/egi_catch_all.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# IM - Infrastructure Manager Dashboard -# Copyright (C) 2020 - GRyCAP - Universitat Politecnica de Valencia -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Function to get sites info from https://github.com/EGI-Federation/fedcloud-catchall-operations/tree/main/sites.""" -import requests -import yaml -from urllib.parse import urlparse - - -GIT_REPO = "EGI-Federation/fedcloud-catchall-operations" -GIT_BRANCH = "main" -REQUESTS_TIMEOUT = 10 -RAW_URL = "https://raw.githubusercontent.com/%s/%s/sites/" % (GIT_REPO, GIT_BRANCH) -SITES_CACHE = {} - - -def get_sites(vo=None): - global SITES_CACHE - sites = {} - url = "https://api.github.com/repos/%s/contents/sites?branch%s" % (GIT_REPO, GIT_BRANCH) - headers = {"Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28"} - resp = requests.get(url, headers=headers, timeout=REQUESTS_TIMEOUT) - if resp.status_code == 200: - for file_info in resp.json(): - if file_info["type"] == "file": - name = file_info["name"] - if (name in SITES_CACHE and SITES_CACHE[name] and SITES_CACHE[name]["sha"] == file_info["sha"] and - SITES_CACHE[name]["size"] == file_info["size"]): - site_info = SITES_CACHE[name]["info"] - else: - site_info = get_site_info(name) - SITES_CACHE[name] = {"sha": file_info["sha"], - "size": file_info["size"], - "info": site_info} - if vo is None or vo in site_info["vos"]: - sites[site_info["name"]] = site_info - - return sites - - -def get_site_info(site_name): - site_file = "%s%s" % (RAW_URL, site_name if site_name.endswith(".yaml") else site_name + ".yaml") - resp = requests.get(site_file, timeout=REQUESTS_TIMEOUT) - if resp.status_code == 200: - site_info = yaml.safe_load(resp.text) - site_info["gocdb"] - site_info["endpoint"] - vos = {} - for vo in site_info["vos"]: - vos[vo["name"]] = vo["auth"]["project_id"] - - url = urlparse(site_info["endpoint"]) - return {"url": "%s://%s" % url[0:2], - "state": "", - "id": site_info["gocdb"], - "name": site_info["gocdb"], - "vos": vos} - - -def get_images(site_id, vo): - return [] - - -def get_project_ids(site_name): - site_info = get_site_info(site_name) - return site_info["vos"] diff --git a/app/settings.py b/app/settings.py index 1ac535bfb..f3adf5b41 100644 --- a/app/settings.py +++ b/app/settings.py @@ -41,7 +41,7 @@ def __init__(self, config): self.motomo_info = config.get('MOTOMO_INFO') self.static_sites = config.get('STATIC_SITES', []) self.static_sites_url = config.get('STATIC_SITES_URL', "") - self.sites_cache_timeout = config.get('SITES_CACHE_TIMEOUT', 3600) + self.appdb_cache_timeout = config.get('APPDB_CACHE_TIMEOUT', 3600) self.debug_oidc_token = config.get('DEBUG_OIDC_TOKEN', None) self.imTimeout = config.get('IM_TIMEOUT', 60) self.checkToscaChangesTime = config.get('CHECK_TOSCA_CHANGES_TIME', 120) @@ -52,4 +52,3 @@ def __init__(self, config): self.vos_user_role = config.get('VOS_USER_ROLE') self.enable_external_vault = config.get('ENABLE_EXTERNAL_VAULT', False) self.hide_tosca_tags = config.get('HIDE_TOSCA_TAGS', []) - self.site_info_provider = config.get('SITE_INFO_PROVIDER', 'AppDB') diff --git a/app/tests/test_egi_catch_all.py b/app/tests/test_egi_catch_all.py deleted file mode 100644 index 57e4924a4..000000000 --- a/app/tests/test_egi_catch_all.py +++ /dev/null @@ -1,103 +0,0 @@ -#! /usr/bin/env python -# -# IM - Infrastructure Manager -# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import unittest -import os -import xmltodict - -from app import egi_catch_all -from mock import patch, MagicMock -from urllib.parse import urlparse - - -class TestEGICatchAll(unittest.TestCase): - """Class to test the EGI Catch all functions.""" - - @staticmethod - def requests_response(url, **kwargs): - resp = MagicMock() - parts = urlparse(url) - url = parts[2] - - resp.status_code = 404 - resp.ok = False - - if url == "/repos/EGI-Federation/fedcloud-catchall-operations/contents/sites": - resp.status_code = 200 - resp.json.return_value = [{ - "name": "UPV-GRyCAP.yaml", - "sha": "f9688bafd4d5645611b2905cdffd8e1b9cd1676a", - "size": 390, - "type": "file", - }] - elif url == "/EGI-Federation/fedcloud-catchall-operations/main/sites/UPV-GRyCAP.yaml": - resp.status_code = 200 - resp.text = """--- -gocdb: UPV-GRyCAP -endpoint: https://menoscloud.i3m.upv.es:5000/v3 -vos: -- name: eosc-synergy.eu - auth: - project_id: 6f84e31391024330b16d29d6ccd26932 -- name: fedcloud.egi.eu - auth: - project_id: db929e9034f04d1698c1a0d58283366e -- name: ops - auth: - project_id: 292568ead7454709a17f19189d5a840a -- name: saps-vo.i3m.upv.es - auth: - project_id: e7608e969cfd4f49907cff17d1774898""" - - return resp - - @patch('requests.get') - def test_get_sites(self, requests): - requests.side_effect = self.requests_response - res = egi_catch_all.get_sites() - expected = { - "UPV-GRyCAP": { - "id": "UPV-GRyCAP", - "name": "UPV-GRyCAP", - "state": "", - "url": "https://menoscloud.i3m.upv.es:5000", - "vos": { - "eosc-synergy.eu": "6f84e31391024330b16d29d6ccd26932", - "fedcloud.egi.eu": "db929e9034f04d1698c1a0d58283366e", - "ops": "292568ead7454709a17f19189d5a840a", - "saps-vo.i3m.upv.es": "e7608e969cfd4f49907cff17d1774898", - }, - } - } - self.assertEquals(res, expected) - - @patch('requests.get') - def test_get_project_ids(self, requests): - requests.side_effect = self.requests_response - res = egi_catch_all.get_project_ids('UPV-GRyCAP') - expected = { - "eosc-synergy.eu": "6f84e31391024330b16d29d6ccd26932", - "fedcloud.egi.eu": "db929e9034f04d1698c1a0d58283366e", - "ops": "292568ead7454709a17f19189d5a840a", - "saps-vo.i3m.upv.es": "e7608e969cfd4f49907cff17d1774898", - } - self.assertEquals(res, expected) - - -if __name__ == '__main__': - unittest.main() diff --git a/app/utils.py b/app/utils.py index b593ae61c..72a1567a9 100644 --- a/app/utils.py +++ b/app/utils.py @@ -39,7 +39,6 @@ from requests.packages.urllib3.exceptions import InsecureRequestWarning from app import appdb -from app import egi_catch_all requests.packages.urllib3.disable_warnings(InsecureRequestWarning) urllib3.disable_warnings(InsecureRequestWarning) @@ -74,13 +73,6 @@ def _getStaticSitesInfo(force=False): return [] -def getSiteInfoProvider(): - if g.settings.site_info_provider == "AppDB": - return appdb - elif g.settings.site_info_provider == "EGI Catch All": - return egi_catch_all - - def getCachedProjectIDs(site_id): res = {} for site in getCachedSiteList().values(): @@ -89,7 +81,7 @@ def getCachedProjectIDs(site_id): site["vos"] = {} if "vos_updated" not in site or not site["vos_updated"]: try: - site["vos"].update(getSiteInfoProvider().get_project_ids(site_id)) + site["vos"].update(appdb.get_project_ids(site_id)) site["vos_updated"] = True except Exception as ex: print("Error loading project IDs from AppDB: %s" % ex, file=sys.stderr) @@ -105,8 +97,6 @@ def getStaticSites(vo=None, force=False): if vo is None or ("vos" in site and site["vos"] and vo in site["vos"]): res[site["name"]] = site site["state"] = "" - if g.settings.site_info_provider == "EGI Catch All": - site["id"] = site["name"] return res @@ -148,11 +138,11 @@ def getCachedSiteList(force=False): global LAST_UPDATE now = int(time.time()) - if force or not SITE_LIST or now - LAST_UPDATE > g.settings.sites_cache_timeout: + if force or not SITE_LIST or now - LAST_UPDATE > g.settings.appdb_cache_timeout: try: - sites = getSiteInfoProvider().get_sites() + sites = appdb.get_sites() if sites: - SITE_LIST = getSiteInfoProvider().get_sites() + SITE_LIST = appdb.get_sites() # in case of error do not update time LAST_UPDATE = now except Exception as ex: