Remove the AppDB dependency

cloud-info/Dockerfile

@@ -10,13 +10,18 @@ RUN curl -s https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGrid
     && apt-get install -y ca-policy-egi-core \
     && rm -rf /var/lib/apt/lists/*
 
+# Fedcloud client is pinning dependencies strictly so it does not play
+# very well with the rest of the available venv. Installing on its own
+RUN python -m venv /fedcloud && \
+    /fedcloud/bin/pip install --no-cache-dir fedcloudclient
+
 WORKDIR /cloud-info
 
+COPY requirements.txt .
+
 RUN python -m venv /cloud-info/venv
 ENV PATH="/cloud-info/venv/bin:$PATH"
 
-COPY requirements.txt .
-
 RUN pip install --no-cache-dir -r requirements.txt \
     && cat /etc/grid-security/certificates/*.pem >> "$(python -m requests.certs)"
 
@@ -37,7 +42,6 @@ RUN apt-get update \
        jq rclone \
     && rm -rf /var/lib/apt/lists/*
 
-
 RUN mkdir /cloud-info \
     && groupadd -g 1999 python \
     && useradd -r -u 1999 -g python python \
@@ -47,6 +51,7 @@ WORKDIR /cloud-info
 
 # All the python code from the build image above
 COPY --chown=python:python --from=build /cloud-info/venv ./venv
+COPY --chown=python:python --from=build /fedcloud /fedcloud
 # Add the scripts that call the cloud-info-provider as needed for the site
 # these create the configuration for the site by discovering the available
 # projects for the credentials and will send the output to the AMS queue and

cloud-info/ams-wrapper.sh

@@ -96,16 +96,17 @@ if test -s cloud-info.json; then
 		OIDC_ACCESS_TOKEN=$(yq -r '.checkin.access_token' <"$ACCESS_TOKEN_FILE")
 		export OIDC_ACCESS_TOKEN
 		export EGI_VO="$SWIFT_VO_NAME"
-		SWIFT_URL=$(fedcloud openstack \
+		SWIFT_URL=$(/fedcloud/bin/fedcloud openstack \
 			--site "$SWIFT_SITE_NAME" \
 			catalog show swift -f json |
 			jq -r '(.endpoints[] | select(.interface=="public")).url')
 		export RCLONE_CONFIG_REMOTE_TYPE="swift"
 		export RCLONE_CONFIG_REMOTE_ENV_AUTH="false"
 		export RCLONE_CONFIG_REMOTE_STORAGE_URL="$SWIFT_URL"
-		eval "$(fedcloud site env --site "$SWIFT_SITE_NAME")"
+		eval "$(/fedcloud/bin/fedcloud site env --site "$SWIFT_SITE_NAME")"
 		export RCLONE_CONFIG_REMOTE_AUTH_URL="$OS_AUTH_URL"
-		OS_AUTH_TOKEN=$(fedcloud openstack --site "$SWIFT_SITE_NAME" token issue -c id -f value)
+		OS_AUTH_TOKEN=$(/fedcloud/bin/fedcloud openstack \
+			--site "$SWIFT_SITE_NAME" token issue -c id -f value)
 		export RCLONE_CONFIG_REMOTE_AUTH_TOKEN="$OS_AUTH_TOKEN"
 		rclone mkdir "remote:$SWIFT_CONTAINER_NAME"
 		rclone copy cloud-info.json "remote:$SWIFT_CONTAINER_NAME/$SITE_NAME"

cloud-info/requirements.txt

@@ -1,6 +1,6 @@
-# Cloud info version is 43cefc204b3e07211c6c37df2ee20eab845c3428
-# 43cefc204b3e07211c6c37df2ee20eab845c3428 includes json glue support
-git+https://github.com/EGI-Federation/cloud-info-provider.git@43cefc204b3e07211c6c37df2ee20eab845c3428
+# Cloud info version is 6fbbf1c24bd32c21d5b8de783d7face6c6c5987e
+# 6fbbf1c24bd32c21d5b8de783d7face6c6c5987e includes json glue support and fixes for it
+git+https://github.com/EGI-Federation/cloud-info-provider.git@6fbbf1c24bd32c21d5b8de783d7face6c6c5987e
 git+https://github.com/ARGOeu/argo-ams-library@devel
 python-glanceclient
 python-novaclient

image-sync/image_sync/sync.py

@@ -6,7 +6,7 @@
 import sys
 import tempfile
 
-import requests
+import httpx
 import yaml
 from oslo_config import cfg
 
@@ -15,6 +15,13 @@
 CONF.register_opts(
     [
         cfg.StrOpt("site_config_dir", default="."),
+        cfg.StrOpt(
+            "cloud_info_url",
+            default=(
+                "https://stratus-stor.ncg.ingrid.pt:8080/swift/"
+                "v1/AUTH_bd5a81e1670b48f18af33b05512a9d77/cloud-info/"
+            ),
+        ),
         cfg.StrOpt("graphql_url", default="https://is.appdb.egi.eu/graphql"),
         cfg.ListOpt("formats", default=[]),
         cfg.StrOpt("appdb_token"),
@@ -38,7 +45,15 @@
 )
 
 
-def fetch_site_info():
+def get_share_vo(share, site_info):
+    for policy in site_info["MappingPolicy"]:
+        for assoc, share_id in policy["Associations"].items():
+            if assoc == "Share" and share_id == share["ID"]:
+                return policy["Rule"][0].removeprefix("VO:")
+    return ""
+
+
+def fetch_site_info_appdb():
     logging.debug("Fetching site info from AppDB")
     query = """
         {
@@ -58,14 +73,55 @@ def fetch_site_info():
         }
     """
     params = {"query": query}
-    r = requests.get(
+    r = httpx.get(
         CONF.sync.graphql_url, params=params, headers={"accept": "application/json"}
     )
     r.raise_for_status()
     data = r.json()["data"]["siteCloudComputingEndpoints"]["items"]
     return data
 
 
+def fetch_site_info_cloud_info():
+    logging.debug("Fetching site info from cloud-info")
+    sites = []
+    # 1 - Get all sites listing
+    r = httpx.get(CONF.sync.cloud_info_url, headers={"Accept": "application/json"})
+    r.raise_for_status()
+    # 2 - Go one by one getting the shares
+    for file in r.json():
+        try:
+            r = httpx.get(os.path.join(CONF.sync.cloud_info_url, file["name"]))
+            r.raise_for_status()
+        except httpx.HTTPError as e:
+            logging.warning(f"Exception while trying to get {file['name']}: {e}")
+            continue
+        full_site = r.json()
+        admin_domain = full_site["CloudComputingService"][0]["Associations"].get(
+            "AdminDomain", None
+        )
+        if not admin_domain:
+            continue
+        # Some associations are a list, other directly strings, cloud-info should harmonise
+        site = admin_domain[0]
+        shares = []
+        for share in full_site["Share"]:
+            shares.append(
+                {"projectID": share["ProjectID"], "VO": get_share_vo(share, full_site)}
+            )
+        sites.append(
+            {
+                "site": {"name": site},
+                "endpointURL": full_site["CloudComputingEndpoint"][0]["URL"],
+                "shares": shares,
+            }
+        )
+    return sites
+
+
+def fetch_site_info():
+    return fetch_site_info_cloud_info()
+
+
 def dump_atrope_config(site, share, hepix_file):
     config_template = """
 [DEFAULT]

image-sync/requirements.txt

@@ -1,4 +1,4 @@
 git+https://github.com/EGI-Federation/atrope@catchall
-requests
+httpx
 oslo.config
 PyYAML