Develop into main (#144)

base_images/isce3/environment.yml

@@ -52,4 +52,4 @@ dependencies:
   - pip:
     - git+https://github.com/opera-adt/[email protected]
     - rio-tiler==6.6.1
-    - git+https://github.com/MAAP-Project/maap-py.git@v4.0.0
+    - git+https://github.com/MAAP-Project/maap-py.git@v4.1.0

base_images/pangeo/environment.yml

@@ -121,4 +121,4 @@ dependencies:
   - pip:
     - morecantile==5.3.0
     - rio-tiler==6.6.1
-    - git+https://github.com/MAAP-Project/maap-py.git@v4.0.0
+    - git+https://github.com/MAAP-Project/maap-py.git@v4.1.0

base_images/python/environment.yml

@@ -52,4 +52,4 @@ dependencies:
   - pip:
     - morecantile==5.3.0
     - rio-tiler==6.6.1
-    - git+https://github.com/MAAP-Project/maap-py.git@v4.0.0
+    - git+https://github.com/MAAP-Project/maap-py.git@v4.1.0

base_images/r/docker/Dockerfile

@@ -21,7 +21,7 @@ RUN apt-get clean && apt-get update && \
     libhdf4-alt-dev=4.2.15-3 \
     libhdf5-dev=1.10.6+repack-4+deb11u1 \
     libjq-dev=1.6-2.1 \
-    libpq-dev=13.14-0+deb11u1 \
+    libpq-dev=13.16-0+deb11u1 \
     libproj-dev=7.2.1-1 \
     libprotobuf-dev=3.12.4-1+deb11u1 \
     libnetcdf-dev=1:4.7.4-1 \

base_images/r/environment.yml

@@ -82,6 +82,7 @@ dependencies:
   - r-ncdf4=1.21
   - r-nlme=3.1_164
   - r-patchwork=1.2.0
+  - r-paws=0.6.0
   - r-plogr=0.2.0
   - r-plyr=1.8.9
   - r-png=0.1_8
@@ -151,4 +152,4 @@ dependencies:
     - grid==0.7.1
     - morecantile==5.3.0
     - rio-tiler==6.6.1
-    - git+https://github.com/MAAP-Project/maap-py.git@v4.0.0
+    - git+https://github.com/MAAP-Project/maap-py.git@v4.1.0

custom_images/build-image.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -ex
+base_image_dir=$(dirname "$0")
+# Check if on a branch or in a detached HEAD state get commit sha
+BRANCH=$(basename "$(git symbolic-ref -q --short HEAD || git rev-parse --short HEAD)")
+if [ "$#" -eq 0 ]; then
+  DIRS_LIST=( "maap_base" )
+else
+  DIRS_LIST=( "$@" )
+fi
+for dir in "${DIRS_LIST[@]}"; do
+    pushd "$base_image_dir"/"$dir"
+    IMAGE_NAME=$(basename "$dir")
+    IMAGE_PATH=$(basename "$base_image_dir")
+    IMAGE_REF=${CI_REGISTRY_IMAGE}/${IMAGE_PATH}/${IMAGE_NAME}:${BRANCH}
+    docker build -t "${IMAGE_REF}" --build-arg IMAGE_REF="${IMAGE_REF}" -f docker/Dockerfile .
+    docker push "${IMAGE_REF}"
+    popd
+    echo "$IMAGE_REF" >> built_images.txt
+done
+

custom_images/maap_base/docker/Dockerfile

@@ -0,0 +1,20 @@
+FROM ubuntu:24.10
+
+ARG MINICONDA_INSTALLER="https://repo.anaconda.com/miniconda/Miniconda3-py311_24.7.1-0-Linux-x86_64.sh"
+ARG MINICONDA_INSTALLER_SHA256="a098a5b1581d8fd078c430b82e27106602223e335efef708a124e723814d120c"
+
+RUN apt-get update -y && apt-get install -y vim wget git && \
+    apt-get clean all
+
+RUN groupadd conda-users
+
+RUN wget -O Miniconda3-installer.sh ${MINICONDA_INSTALLER} && \
+    echo "${MINICONDA_INSTALLER_SHA256} Miniconda3-installer.sh" | sha256sum --check --status && \
+    bash Miniconda3-installer.sh -b -p /opt/conda && \
+    chgrp -R conda-users /opt/conda/ && chmod -R 775 /opt/conda/ && \
+    find /opt/conda/ -follow -type f -name '*.a' -delete && \
+    find /opt/conda/ -follow -type f -name '*.js.map' -delete && \
+    /opt/conda/bin/conda clean -afy && \
+    /opt/conda/bin/conda init --all --system
+
+ENV PATH /opt/conda/bin:$PATH

devfiles/isce3/devfile/devfile.yaml

@@ -8,7 +8,9 @@ components:
       - attributes:
           type: ide
           discoverable: 'false'
+          secure: 'true'
           path: /
+          cookiesAuthEnabled: 'true'
           public: 'true'
           protocol: http
         name: jupyter
@@ -37,7 +39,7 @@ components:
                 - name: DOCKERIMAGE_PATH
                   value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/isce3:develop'
                 - name: MAAP_API_HOST
-                  value: ade.dit.maap-project.org
+                  value: api.dit.maap-project.org
                 resources:
                   limits:
                     memory: 8096Mi

devfiles/isce3/devfile/meta.yaml

@@ -1,6 +1,6 @@
 ---
 displayName: "ISCE3"
-description: "ISCE3 workspace version: 3.1.5"
+description: "ISCE3 MAAP workspace (version 4.1.0)"
 tags: ["JupyterLab", "Python", "MAAP", "ISCE3"]
 icon: /devfiles/isce3/devfile/isce.png
 links: 

devfiles/pangeo/devfile/devfile.yaml

@@ -8,7 +8,9 @@ components:
       - attributes:
           type: ide
           discoverable: 'false'
+          secure: 'true'
           path: /
+          cookiesAuthEnabled: 'true'
           public: 'true'
           protocol: http
         name: jupyter
@@ -37,7 +39,7 @@ components:
                 - name: DOCKERIMAGE_PATH
                   value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/pangeo:develop'
                 - name: MAAP_API_HOST
-                  value: ade.dit.maap-project.org
+                  value: api.dit.maap-project.org
                 resources:
                   limits:
                     memory: 8096Mi

devfiles/pangeo/devfile/meta.yaml

@@ -1,6 +1,6 @@
 ---
 displayName: "Pangeo"
-description: "Pangeo workspace version: 3.1.5"
+description: "Pangeo MAAP workspace (version 4.1.0)"
 tags: ["Pangeo", "JupyterLab", "MAAP"]
 icon: /devfiles/pangeo/devfile/pangeo_simple_logo.svg
 links: 

devfiles/python/devfile/devfile.yaml

@@ -8,7 +8,9 @@ components:
       - attributes:
           type: ide
           discoverable: 'false'
+          secure: 'true'
           path: /
+          cookiesAuthEnabled: 'true'
           public: 'true'
           protocol: http
         name: jupyter
@@ -35,9 +37,9 @@ components:
                 imagePullPolicy: Always
                 env:
                 - name: DOCKERIMAGE_PATH
-                  value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/vanilla:develop'
+                  value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/python:develop'
                 - name: MAAP_API_HOST
-                  value: ade.dit.maap-project.org
+                  value: api.dit.maap-project.org
                 resources:
                   limits:
                     memory: 8096Mi

devfiles/python/devfile/meta.yaml

@@ -1,6 +1,6 @@
 ---
 displayName: "Python (default)"
-description: "Python workspace version: 3.1.5"
+description: "Basic MAAP python workspace (version 4.1.0)"
 tags: ["JupyterLab", "Python", "MAAP"]
 icon: /devfiles/python/devfile/python-logo-generic.svg
 links: 

devfiles/r/devfile/devfile.yaml

@@ -8,7 +8,9 @@ components:
       - attributes:
           type: ide
           discoverable: 'false'
+          secure: 'true'
           path: /
+          cookiesAuthEnabled: 'true'
           public: 'true'
           protocol: http
         name: jupyter
@@ -37,7 +39,7 @@ components:
                 - name: DOCKERIMAGE_PATH
                   value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/r:develop'
                 - name: MAAP_API_HOST
-                  value: ade.dit.maap-project.org
+                  value: api.dit.maap-project.org
                 resources:
                   limits:
                     memory: 8096Mi

devfiles/r/devfile/meta.yaml

@@ -1,6 +1,6 @@
 ---
 displayName: "R/Python"
-description: "R/Python workspace version: 3.1.5"
+description: "R/Python MAAP workspace (version 4.1.0)"
 tags: ["Python", "R", "JupyterLab", "MAAP"]
 icon: /devfiles/r/devfile/r.png
 links: 

gitlab-ci/custom-image-build.yml.tmpl

@@ -0,0 +1,8 @@
+${BASE_IMAGE_TYPE}:
+  script:
+    - docker login -u "$CI_DEPLOY_USER" -p "$CI_DEPLOY_PASSWORD" "$CI_REGISTRY"
+    - git clone https://github.com/MAAP-Project/maap-workspaces.git
+    - pushd maap-workspaces
+    - git checkout ${LATEST_COMMIT}
+    - bash custom_images/build-image.sh ${BASE_IMAGE_TYPE}
+    - cat built_images.txt

gitlab-ci/generate_configs.sh

@@ -1,14 +1,49 @@
 #!/bin/bash
 set -e
 basedir=$( cd "$(dirname "$0")" ; pwd -P )
+
+# Get webhook payload
+echo TRIGGER_PAYLOAD=$TRIGGER_PAYLOAD
+if [[ ! -z $TRIGGER_PAYLOAD ]]; then
+    PAYLOAD_COMMIT=$(cat $TRIGGER_PAYLOAD | python3 -c "import sys, json; print(json.loads(json.load(sys.stdin)['payload'])['after'])")
+    REF=$(cat $TRIGGER_PAYLOAD | python3 -c "import sys, json; print(json.loads(json.load(sys.stdin)['payload'])['ref'])")
+fi
 git clone https://github.com/MAAP-Project/maap-workspaces.git
 pushd maap-workspaces
-LATEST_COMMIT=$(git log -n 1 --all --format='%h')
+LATEST_COMMIT=$PAYLOAD_COMMIT
+TAG=$LATEST_COMMIT
+if [[ -z ${PAYLOAD_COMMIT} ]]; then
+    # If no payload commit was set, find the latest commit on the repo
+    LATEST_COMMIT=$(git log -n 1 --all --format='%h')
+fi
 if [[ ! -z ${FORCE_REF_BUILD} ]]; then
     LATEST_COMMIT=${FORCE_REF_BUILD}
 fi
 git checkout ${LATEST_COMMIT}
 
+# Set LATEST_COMMIT to what we want as image tag
+# Default set it to latest commit hash, branch name if commit is HEAD of branch
+TAG=$(basename $(git symbolic-ref -q --short HEAD || git rev-parse --short HEAD))
+
+# If REF (branch name) in payload use that
+if [[ ! -z ${REF} ]]; then
+    TAG=$(basename ${REF})
+fi
+
+# If FORCE_CUSTOM_TAGNAME then use that as tag, eg. nightly
+if [[ ! -z ${FORCE_CUSTOM_TAGNAME} ]]; then
+    TAG=${FORCE_CUSTOM_TAGNAME}
+fi
+
+echo "Using ${TAG} as tag for images"
+if [[ "$TAG" == "develop" || "$TAG" == "main" ]]; then
+    echo "Building all images as the tag is ${TAG}"
+    LATEST_COMMIT=${TAG}
+    BUILD_ALL_BASE_IMAGES=1
+    BUILD_ALL_CUSTOM_IMAGES=1
+fi
+
+
 if [[ ! -z ${BUILD_ALL_BASE_IMAGES} ]]; then
     ls -d base_images/*/* > ${basedir}/files_changed.txt
 elif [[ ! -z ${BUILD_SPECIFIC_BASE_IMAGES} ]]; then
@@ -22,23 +57,49 @@ else
     echo "Getting files changed in last commit"
     git diff --name-only HEAD HEAD~1 > ${basedir}/files_changed.txt
 fi
+
+if [[ ! -z ${BUILD_ALL_CUSTOM_IMAGES} ]]; then
+    ls -d custom_images/*/* > ${basedir}/files_changed.txt
+elif [[ ! -z ${BUILD_SPECIFIC_CUSTOM_IMAGES} ]]; then
+    custom_image_array=(${BUILD_SPECIFIC_CUSTOM_IMAGES})
+    for path in ${custom_image_array[@]}; do
+        echo "custom_images/${path}/docker" >> ${basedir}/files_changed.txt
+    done
+else
+    # Find files changed in the latest commit
+    echo "BUILD_ALL_CUSTOM_IMAGES and BUILD_SPECIFIC_CUSTOM_IMAGES unset"
+    echo "Getting files changed in last commit"
+    git diff --name-only HEAD HEAD~1 > ${basedir}/files_changed.txt
+fi
+
 cat ${basedir}/files_changed.txt
 export LATEST_COMMIT
 popd
 rm -rf maap-workspaces
-template="${basedir}/stage.yml.tmpl"
+touch stages.yml
 # For each file changed, check if its in one of the base images directory
 cat ${basedir}/files_changed.txt | while read path
 do
   if [[ "$path" == base_images/*/* ]]; then
+    template="${basedir}/stage.yml.tmpl"
     second_dir=$(echo "$path" | cut -d'/' -f2)
     export BASE_IMAGE_TYPE="$second_dir"
     echo "Adding stage for $path"
     # Add the base image changed file to build stage for downstream pipeline
     cat ${template} | CI_JOB_TOKEN='$CI_JOB_TOKEN' CI_REGISTRY='$CI_REGISTRY' envsubst >> stages.yml
   else
-    echo "Path does not begin with base_images or does not have a second directory, will not do anything"
-    touch stages.yml
+    echo "Path does not begin with base_images or does not have a second level directory, will not do anything"
+  fi
+  # This is not an elif because we can have both custom and base image repos with changes
+  if [[ "$path" == custom_images/*/* ]]; then
+    template="${basedir}/custom-image-build.yml.tmpl"
+    second_dir=$(echo "$path" | cut -d'/' -f2)
+    export BASE_IMAGE_TYPE="$second_dir"
+    echo "Adding stage for $path"
+    # Add the base image changed file to build stage for downstream pipeline
+    cat "${template}" | CI_JOB_TOKEN='$CI_JOB_TOKEN' CI_REGISTRY='$CI_REGISTRY' envsubst >> stages.yml
+  else
+    echo "Path does not begin with base_images or custom_images or does not have a second level directory, will not do anything"
   fi
 done
 

gitlab-ci/stage.yml.tmpl

@@ -1,10 +1,14 @@
 ${BASE_IMAGE_TYPE}:
   script:
-    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+    - docker login -u "$CI_DEPLOY_USER" -p "$CI_DEPLOY_PASSWORD" "$CI_REGISTRY"
     - git clone https://github.com/MAAP-Project/maap-workspaces.git
     - pushd maap-workspaces
     - git checkout ${LATEST_COMMIT}
+    - cp -r ../overlays/ ./
     - bash base_images/build-image.sh ${BASE_IMAGE_TYPE}
     - cat built_images.txt
     - export BASE_IMAGE_NAME=$(tail -1 built_images.txt)
     - bash jupyterlab/build-image.sh
+    - scp -vr devfiles/${BASE_IMAGE_TYPE}/devfile/ ${DIT_API_MACHINE}:/var/www/devfile-registry/devfiles/${BASE_IMAGE_TYPE}/
+    - scp -vr devfiles/devfile_compile.py ${DIT_API_MACHINE}:/var/www/devfile-registry/devfiles/devfile_compile.py
+    - ssh ${DIT_API_MACHINE} python3 /var/www/devfile-registry/devfiles/devfile_compile.py

jupyterlab/entrypoint.sh

@@ -43,7 +43,9 @@ data=$(get_request_data "$1" "endpoints")
 
 import os
 che_workspace_id = os.environ.get('CHE_WORKSPACE_ID')
-che_machine_name = os.environ.get('CHE_MACHINE_NAME').lower().replace('/', '-')
+
+# Disallow insecure workspaces. Assume that the workspace runs behind a jwtproxy container.
+che_machine_name = '-jwtproxy' # os.environ.get('CHE_MACHINE_NAME').lower().replace('/', '-')
 
 endpoints = data['items']
 for endpoint in endpoints: