GitHub info

README.md

@@ -58,7 +58,7 @@ Create the `config.json` file (see the [example](app/config-sample.json)) settin
 | ANALYTICS_TAG | Google Analytic Tag | N | "" |
 | STATIC_SITES | List of static sites added to the AppDB ones ([{"name": "static_site_name", "url": "static_site_url", "id": "static_id", "vos": {"vo": "stprojectid"}}]) | N | [] |
 | STATIC_SITES_URL | URL of a JSON file with the list of static sites added to the AppDB ones | N | "" |
-| APPDB_CACHE_TIMEOUT | AppDB cache TTL | N | 3600 |
+| SITES_CACHE_TIMEOUT | AppDB cache TTL | N | 3600 |
 | CHECK_TOSCA_CHANGES_TIME | Interval to look for changes in TOSCA templates | N | 120 |
 | VAULT_URL | Vault service URL to store Cloud credentials | N | None |
 

app/__init__.py

@@ -36,7 +36,7 @@
 from app.im import InfrastructureManager
 from app.ssh_key import SSHKey
 from app.ott import OneTimeTokenData
-from app import utils, appdb, db
+from app import utils, db
 from app.vault_info import VaultInfo
 from oauthlib.oauth2.rfc6749.errors import InvalidTokenError, TokenExpiredError, InvalidGrantError
 from werkzeug.exceptions import Forbidden
@@ -777,8 +777,8 @@ def getsites(vo=None):
         for site_name, site in static_sites.items():
             res += '<option name="selectedSite" value=%s>%s</option>' % (site['url'], site_name)
 
-        appdb_sites = appdb.get_sites(vo)
-        for site_name, site in appdb_sites.items():
+        sites = utils.getSiteInfoProvider().get_sites(vo)
+        for site_name, site in sites.items():
             # avoid site duplication
             if site_name not in static_sites:
                 if site["state"]:
@@ -807,7 +807,7 @@ def getimages(cred_id=None):
 
         else:
             site, _, vo = utils.get_site_info(cred_id, cred, get_cred_id())
-            for image_name, image_id in appdb.get_images(site['id'], vo):
+            for image_name, image_id in utils.getSiteInfoProvider().get_images(site['id'], vo):
                 res += '<option name="selectedImage" value=%s>%s</option>' % (image_id, image_name)
         return res
 
@@ -1539,7 +1539,7 @@ def handle_csrf_error(e):
     # Reload internally the site cache
     @scheduler.task('interval', id='reload_sites', seconds=5)
     def reload_sites():
-        scheduler.modify_job('reload_sites', trigger='interval', seconds=settings.appdb_cache_timeout - 30)
+        scheduler.modify_job('reload_sites', trigger='interval', seconds=settings.sites_cache_timeout - 30)
         with app.app_context():
             app.logger.debug('Reload Site List.')
             g.settings = settings

app/config-sample.json

@@ -16,7 +16,7 @@
     "MOTOMO_INFO": {"url": "", "siteid": ""},
     "STATIC_SITES": [{"name": "static_site_name", "url": "static_site_url", "id": "static_id", "vos": {"vo": "stprojectid"}, "api_version": "1.1"}],
     "STATIC_SITES_URL": "",
-    "APPDB_CACHE_TIMEOUT": 3600,
+    "SITES_CACHE_TIMEOUT": 3600,
     "CHECK_TOSCA_CHANGES_TIME": 120,
     "IM_TIMEOUT": 60,
     "VAULT_URL": "",

app/egi_catch_all.py

@@ -0,0 +1,82 @@
+#
+# IM - Infrastructure Manager Dashboard
+# Copyright (C) 2020 - GRyCAP - Universitat Politecnica de Valencia
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Function to get sites info from https://github.com/EGI-Federation/fedcloud-catchall-operations/tree/main/sites."""
+import requests
+import yaml
+from urllib.parse import urlparse
+
+
+GIT_REPO = "EGI-Federation/fedcloud-catchall-operations"
+GIT_BRANCH = "main"
+REQUESTS_TIMEOUT = 10
+RAW_URL = "https://raw.githubusercontent.com/%s/%s/sites/" % (GIT_REPO, GIT_BRANCH)
+SITES_CACHE = {}
+
+
+def get_sites(vo=None):
+    global SITES_CACHE
+    sites = {}
+    url = "https://api.github.com/repos/%s/contents/sites?branch%s" % (GIT_REPO, GIT_BRANCH)
+    headers = {"Accept": "application/vnd.github+json",
+               "X-GitHub-Api-Version": "2022-11-28"}
+    resp = requests.get(url, headers=headers, timeout=REQUESTS_TIMEOUT)
+    if resp.status_code == 200:
+        for file_info in resp.json():
+            if file_info["type"] == "file":
+                name = file_info["name"]
+                if (name in SITES_CACHE and SITES_CACHE[name] and SITES_CACHE[name]["sha"] == file_info["sha"] and
+                        SITES_CACHE[name]["size"] == file_info["size"]):
+                    site_info = SITES_CACHE[name]["info"]
+                else:
+                    site_info = get_site_info(name)
+                    SITES_CACHE[name] = {"sha": file_info["sha"],
+                                         "size": file_info["size"],
+                                         "info": site_info}
+                if vo is None or vo in site_info["vos"]:
+                    sites[site_info["name"]] = site_info
+
+    return sites
+
+
+def get_site_info(site_name):
+    site_file = "%s%s" % (RAW_URL, site_name if site_name.endswith(".yaml") else site_name + ".yaml")
+    resp = requests.get(site_file, timeout=REQUESTS_TIMEOUT)
+    if resp.status_code == 200:
+        site_info = yaml.safe_load(resp.text)
+        vos = {}
+        for vo in site_info["vos"]:
+            vos[vo["name"]] = vo["auth"]["project_id"]
+
+        url = urlparse(site_info["endpoint"])
+        return {"url": "%s://%s" % url[0:2],
+                "state": "",
+                "id": site_info["gocdb"],
+                "name": site_info["gocdb"],
+                "vos": vos}
+
+
+def get_images(site_id, vo):
+    return []
+
+
+def get_project_ids(site_name):
+    site_info = get_site_info(site_name)
+    return site_info["vos"]

app/settings.py

@@ -41,7 +41,7 @@ def __init__(self, config):
         self.motomo_info = config.get('MOTOMO_INFO')
         self.static_sites = config.get('STATIC_SITES', [])
         self.static_sites_url = config.get('STATIC_SITES_URL', "")
-        self.appdb_cache_timeout = config.get('APPDB_CACHE_TIMEOUT', 3600)
+        self.sites_cache_timeout = config.get('SITES_CACHE_TIMEOUT', 3600)
         self.debug_oidc_token = config.get('DEBUG_OIDC_TOKEN', None)
         self.imTimeout = config.get('IM_TIMEOUT', 60)
         self.checkToscaChangesTime = config.get('CHECK_TOSCA_CHANGES_TIME', 120)
@@ -52,3 +52,4 @@ def __init__(self, config):
         self.vos_user_role = config.get('VOS_USER_ROLE')
         self.enable_external_vault = config.get('ENABLE_EXTERNAL_VAULT', False)
         self.hide_tosca_tags = config.get('HIDE_TOSCA_TAGS', [])
+        self.site_info_provider = config.get('SITE_INFO_PROVIDER', 'AppDB')

app/tests/test_egi_catch_all.py

@@ -0,0 +1,101 @@
+#! /usr/bin/env python
+#
+# IM - Infrastructure Manager
+# Copyright (C) 2011 - GRyCAP - Universitat Politecnica de Valencia
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+
+from app import egi_catch_all
+from mock import patch, MagicMock
+from urllib.parse import urlparse
+
+
+class TestEGICatchAll(unittest.TestCase):
+    """Class to test the EGI Catch all functions."""
+
+    @staticmethod
+    def requests_response(url, **kwargs):
+        resp = MagicMock()
+        parts = urlparse(url)
+        url = parts[2]
+
+        resp.status_code = 404
+        resp.ok = False
+
+        if url == "/repos/EGI-Federation/fedcloud-catchall-operations/contents/sites":
+            resp.status_code = 200
+            resp.json.return_value = [{
+                "name": "UPV-GRyCAP.yaml",
+                "sha": "f9688bafd4d5645611b2905cdffd8e1b9cd1676a",
+                "size": 390,
+                "type": "file",
+            }]
+        elif url == "/EGI-Federation/fedcloud-catchall-operations/main/sites/UPV-GRyCAP.yaml":
+            resp.status_code = 200
+            resp.text = """---
+gocdb: UPV-GRyCAP
+endpoint: https://menoscloud.i3m.upv.es:5000/v3
+vos:
+- name: eosc-synergy.eu
+  auth:
+    project_id: 6f84e31391024330b16d29d6ccd26932
+- name: fedcloud.egi.eu
+  auth:
+    project_id: db929e9034f04d1698c1a0d58283366e
+- name: ops
+  auth:
+    project_id: 292568ead7454709a17f19189d5a840a
+- name: saps-vo.i3m.upv.es
+  auth:
+    project_id: e7608e969cfd4f49907cff17d1774898"""
+
+        return resp
+
+    @patch('requests.get')
+    def test_get_sites(self, requests):
+        requests.side_effect = self.requests_response
+        res = egi_catch_all.get_sites()
+        expected = {
+            "UPV-GRyCAP": {
+                "id": "UPV-GRyCAP",
+                "name": "UPV-GRyCAP",
+                "state": "",
+                "url": "https://menoscloud.i3m.upv.es:5000",
+                "vos": {
+                    "eosc-synergy.eu": "6f84e31391024330b16d29d6ccd26932",
+                    "fedcloud.egi.eu": "db929e9034f04d1698c1a0d58283366e",
+                    "ops": "292568ead7454709a17f19189d5a840a",
+                    "saps-vo.i3m.upv.es": "e7608e969cfd4f49907cff17d1774898",
+                },
+            }
+        }
+        self.assertEquals(res, expected)
+
+    @patch('requests.get')
+    def test_get_project_ids(self, requests):
+        requests.side_effect = self.requests_response
+        res = egi_catch_all.get_project_ids('UPV-GRyCAP')
+        expected = {
+            "eosc-synergy.eu": "6f84e31391024330b16d29d6ccd26932",
+            "fedcloud.egi.eu": "db929e9034f04d1698c1a0d58283366e",
+            "ops": "292568ead7454709a17f19189d5a840a",
+            "saps-vo.i3m.upv.es": "e7608e969cfd4f49907cff17d1774898",
+        }
+        self.assertEquals(res, expected)
+
+
+if __name__ == '__main__':
+    unittest.main()

app/utils.py

@@ -39,6 +39,7 @@
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
 from app import appdb
+from app import egi_catch_all
 
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 urllib3.disable_warnings(InsecureRequestWarning)
@@ -73,6 +74,13 @@ def _getStaticSitesInfo(force=False):
         return []
 
 
+def getSiteInfoProvider():
+    if g.settings.site_info_provider == "AppDB":
+        return appdb
+    elif g.settings.site_info_provider == "EGI Catch All":
+        return egi_catch_all
+
+
 def getCachedProjectIDs(site_id):
     res = {}
     for site in getCachedSiteList().values():
@@ -81,7 +89,7 @@ def getCachedProjectIDs(site_id):
                 site["vos"] = {}
             if "vos_updated" not in site or not site["vos_updated"]:
                 try:
-                    site["vos"].update(appdb.get_project_ids(site_id))
+                    site["vos"].update(getSiteInfoProvider().get_project_ids(site_id))
                     site["vos_updated"] = True
                 except Exception as ex:
                     print("Error loading project IDs from AppDB: %s" % ex, file=sys.stderr)
@@ -97,6 +105,8 @@ def getStaticSites(vo=None, force=False):
         if vo is None or ("vos" in site and site["vos"] and vo in site["vos"]):
             res[site["name"]] = site
             site["state"] = ""
+            if g.settings.site_info_provider == "EGI Catch All":
+                site["id"] = site["name"]
 
     return res
 
@@ -138,11 +148,11 @@ def getCachedSiteList(force=False):
     global LAST_UPDATE
 
     now = int(time.time())
-    if force or not SITE_LIST or now - LAST_UPDATE > g.settings.appdb_cache_timeout:
+    if force or not SITE_LIST or now - LAST_UPDATE > g.settings.sites_cache_timeout:
         try:
-            sites = appdb.get_sites()
+            sites = getSiteInfoProvider().get_sites()
             if sites:
-                SITE_LIST = appdb.get_sites()
+                SITE_LIST = getSiteInfoProvider().get_sites()
             # in case of error do not update time
             LAST_UPDATE = now
         except Exception as ex: