diff --git a/cloud-info/Dockerfile b/cloud-info/Dockerfile index 1bea3bdc..1a0dc22e 100644 --- a/cloud-info/Dockerfile +++ b/cloud-info/Dockerfile @@ -10,13 +10,18 @@ RUN curl -s https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGrid && apt-get install -y ca-policy-egi-core \ && rm -rf /var/lib/apt/lists/* +# Fedcloud client is pinning dependencies strictly so it does not play +# very well with the rest of the available venv. Installing on its own +RUN python -m venv /fedcloud && \ + /fedcloud/bin/pip install --no-cache-dir fedcloudclient + WORKDIR /cloud-info +COPY requirements.txt . + RUN python -m venv /cloud-info/venv ENV PATH="/cloud-info/venv/bin:$PATH" -COPY requirements.txt . - RUN pip install --no-cache-dir -r requirements.txt \ && cat /etc/grid-security/certificates/*.pem >> "$(python -m requests.certs)" @@ -37,7 +42,6 @@ RUN apt-get update \ jq rclone \ && rm -rf /var/lib/apt/lists/* - RUN mkdir /cloud-info \ && groupadd -g 1999 python \ && useradd -r -u 1999 -g python python \ @@ -47,6 +51,7 @@ WORKDIR /cloud-info # All the python code from the build image above COPY --chown=python:python --from=build /cloud-info/venv ./venv +COPY --chown=python:python --from=build /fedcloud /fedcloud # Add the scripts that call the cloud-info-provider as needed for the site # these create the configuration for the site by discovering the available # projects for the credentials and will send the output to the AMS queue and diff --git a/cloud-info/ams-wrapper.sh b/cloud-info/ams-wrapper.sh index 36e8596a..36693e95 100755 --- a/cloud-info/ams-wrapper.sh +++ b/cloud-info/ams-wrapper.sh @@ -96,16 +96,17 @@ if test -s cloud-info.json; then OIDC_ACCESS_TOKEN=$(yq -r '.checkin.access_token' <"$ACCESS_TOKEN_FILE") export OIDC_ACCESS_TOKEN export EGI_VO="$SWIFT_VO_NAME" - SWIFT_URL=$(fedcloud openstack \ + SWIFT_URL=$(/fedcloud/bin/fedcloud openstack \ --site "$SWIFT_SITE_NAME" \ catalog show swift -f json | jq -r '(.endpoints[] | select(.interface=="public")).url') export RCLONE_CONFIG_REMOTE_TYPE="swift" export RCLONE_CONFIG_REMOTE_ENV_AUTH="false" export RCLONE_CONFIG_REMOTE_STORAGE_URL="$SWIFT_URL" - eval "$(fedcloud site env --site "$SWIFT_SITE_NAME")" + eval "$(/fedcloud/bin/fedcloud site env --site "$SWIFT_SITE_NAME")" export RCLONE_CONFIG_REMOTE_AUTH_URL="$OS_AUTH_URL" - OS_AUTH_TOKEN=$(fedcloud openstack --site "$SWIFT_SITE_NAME" token issue -c id -f value) + OS_AUTH_TOKEN=$(/fedcloud/bin/fedcloud openstack \ + --site "$SWIFT_SITE_NAME" token issue -c id -f value) export RCLONE_CONFIG_REMOTE_AUTH_TOKEN="$OS_AUTH_TOKEN" rclone mkdir "remote:$SWIFT_CONTAINER_NAME" rclone copy cloud-info.json "remote:$SWIFT_CONTAINER_NAME/$SITE_NAME" diff --git a/cloud-info/requirements.txt b/cloud-info/requirements.txt index bfd6f90f..0197d769 100644 --- a/cloud-info/requirements.txt +++ b/cloud-info/requirements.txt @@ -1,6 +1,6 @@ -# Cloud info version is 43cefc204b3e07211c6c37df2ee20eab845c3428 -# 43cefc204b3e07211c6c37df2ee20eab845c3428 includes json glue support -git+https://github.com/EGI-Federation/cloud-info-provider.git@43cefc204b3e07211c6c37df2ee20eab845c3428 +# Cloud info version is 6fbbf1c24bd32c21d5b8de783d7face6c6c5987e +# 6fbbf1c24bd32c21d5b8de783d7face6c6c5987e includes json glue support and fixes for it +git+https://github.com/EGI-Federation/cloud-info-provider.git@6fbbf1c24bd32c21d5b8de783d7face6c6c5987e git+https://github.com/ARGOeu/argo-ams-library@devel python-glanceclient python-novaclient diff --git a/image-sync/image_sync/sync.py b/image-sync/image_sync/sync.py index 7e4aea71..78b4cfd6 100644 --- a/image-sync/image_sync/sync.py +++ b/image-sync/image_sync/sync.py @@ -6,7 +6,7 @@ import sys import tempfile -import requests +import httpx import yaml from oslo_config import cfg @@ -15,6 +15,13 @@ CONF.register_opts( [ cfg.StrOpt("site_config_dir", default="."), + cfg.StrOpt( + "cloud_info_url", + default=( + "https://stratus-stor.ncg.ingrid.pt:8080/swift/" + "v1/AUTH_bd5a81e1670b48f18af33b05512a9d77/cloud-info/" + ), + ), cfg.StrOpt("graphql_url", default="https://is.appdb.egi.eu/graphql"), cfg.ListOpt("formats", default=[]), cfg.StrOpt("appdb_token"), @@ -38,7 +45,15 @@ ) -def fetch_site_info(): +def get_share_vo(share, site_info): + for policy in site_info["MappingPolicy"]: + for assoc, share_id in policy["Associations"].items(): + if assoc == "Share" and share_id == share["ID"]: + return policy["Rule"][0].removeprefix("VO:") + return "" + + +def fetch_site_info_appdb(): logging.debug("Fetching site info from AppDB") query = """ { @@ -58,7 +73,7 @@ def fetch_site_info(): } """ params = {"query": query} - r = requests.get( + r = httpx.get( CONF.sync.graphql_url, params=params, headers={"accept": "application/json"} ) r.raise_for_status() @@ -66,6 +81,47 @@ def fetch_site_info(): return data +def fetch_site_info_cloud_info(): + logging.debug("Fetching site info from cloud-info") + sites = [] + # 1 - Get all sites listing + r = httpx.get(CONF.sync.cloud_info_url, headers={"Accept": "application/json"}) + r.raise_for_status() + # 2 - Go one by one getting the shares + for file in r.json(): + try: + r = httpx.get(os.path.join(CONF.sync.cloud_info_url, file["name"])) + r.raise_for_status() + except httpx.HTTPError as e: + logging.warning(f"Exception while trying to get {file['name']}: {e}") + continue + full_site = r.json() + admin_domain = full_site["CloudComputingService"][0]["Associations"].get( + "AdminDomain", None + ) + if not admin_domain: + continue + # Some associations are a list, other directly strings, cloud-info should harmonise + site = admin_domain[0] + shares = [] + for share in full_site["Share"]: + shares.append( + {"projectID": share["ProjectID"], "VO": get_share_vo(share, full_site)} + ) + sites.append( + { + "site": {"name": site}, + "endpointURL": full_site["CloudComputingEndpoint"][0]["URL"], + "shares": shares, + } + ) + return sites + + +def fetch_site_info(): + return fetch_site_info_cloud_info() + + def dump_atrope_config(site, share, hepix_file): config_template = """ [DEFAULT] diff --git a/image-sync/requirements.txt b/image-sync/requirements.txt index d8d15cf0..7b6da39f 100644 --- a/image-sync/requirements.txt +++ b/image-sync/requirements.txt @@ -1,4 +1,4 @@ git+https://github.com/EGI-Federation/atrope@catchall -requests +httpx oslo.config PyYAML