diff --git a/base_images/isce3/environment.yml b/base_images/isce3/environment.yml index 4a6e5d3..6d38fba 100644 --- a/base_images/isce3/environment.yml +++ b/base_images/isce3/environment.yml @@ -52,4 +52,4 @@ dependencies: - pip: - git+https://github.com/opera-adt/RTC.git@v1.0.2 - rio-tiler==6.6.1 - - git+https://github.com/MAAP-Project/maap-py.git@v4.0.0 \ No newline at end of file + - git+https://github.com/MAAP-Project/maap-py.git@v4.1.0 \ No newline at end of file diff --git a/base_images/pangeo/environment.yml b/base_images/pangeo/environment.yml index 806861a..7af1c15 100644 --- a/base_images/pangeo/environment.yml +++ b/base_images/pangeo/environment.yml @@ -121,4 +121,4 @@ dependencies: - pip: - morecantile==5.3.0 - rio-tiler==6.6.1 - - git+https://github.com/MAAP-Project/maap-py.git@v4.0.0 \ No newline at end of file + - git+https://github.com/MAAP-Project/maap-py.git@v4.1.0 \ No newline at end of file diff --git a/base_images/python/environment.yml b/base_images/python/environment.yml index a10a34a..ca9d7b6 100644 --- a/base_images/python/environment.yml +++ b/base_images/python/environment.yml @@ -52,4 +52,4 @@ dependencies: - pip: - morecantile==5.3.0 - rio-tiler==6.6.1 - - git+https://github.com/MAAP-Project/maap-py.git@v4.0.0 \ No newline at end of file + - git+https://github.com/MAAP-Project/maap-py.git@v4.1.0 \ No newline at end of file diff --git a/base_images/r/docker/Dockerfile b/base_images/r/docker/Dockerfile index 1d56198..39f365c 100644 --- a/base_images/r/docker/Dockerfile +++ b/base_images/r/docker/Dockerfile @@ -21,7 +21,7 @@ RUN apt-get clean && apt-get update && \ libhdf4-alt-dev=4.2.15-3 \ libhdf5-dev=1.10.6+repack-4+deb11u1 \ libjq-dev=1.6-2.1 \ - libpq-dev=13.14-0+deb11u1 \ + libpq-dev=13.16-0+deb11u1 \ libproj-dev=7.2.1-1 \ libprotobuf-dev=3.12.4-1+deb11u1 \ libnetcdf-dev=1:4.7.4-1 \ diff --git a/base_images/r/environment.yml b/base_images/r/environment.yml index f5197a4..4f0d030 100644 --- a/base_images/r/environment.yml +++ b/base_images/r/environment.yml @@ -82,6 +82,7 @@ dependencies: - r-ncdf4=1.21 - r-nlme=3.1_164 - r-patchwork=1.2.0 + - r-paws=0.6.0 - r-plogr=0.2.0 - r-plyr=1.8.9 - r-png=0.1_8 @@ -151,4 +152,4 @@ dependencies: - grid==0.7.1 - morecantile==5.3.0 - rio-tiler==6.6.1 - - git+https://github.com/MAAP-Project/maap-py.git@v4.0.0 \ No newline at end of file + - git+https://github.com/MAAP-Project/maap-py.git@v4.1.0 \ No newline at end of file diff --git a/base_images/vanilla/environment.yml b/base_images/vanilla/environment.yml deleted file mode 100644 index fddb0d5..0000000 --- a/base_images/vanilla/environment.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: vanilla -channels: - - conda-forge - - nodefaults -dependencies: - - awscli=2.14.1 - - backoff=2.2.1 - - basemap=1.3.7 - - boto3=1.34.41 - - cython=3.0.7 - - earthengine-api=0.1.384 - - gdal=3.7.0 - - geocube=0.4.2 - - geopandas=0.14.2 - - groff=1.22.4 - - h5py=3.9.0 - - hdf5=1.14.0 - - httpx=0.26.0 - - mapclassify=2.6.1 - - matplotlib=3.7.3 - - mizani=0.10.0 - - mpl-scatter-density=0.7 - - numba=0.58.1 - - numpy=1.26.3 - - pandas=2.1.4 - - pandarallel=1.6.5 - - pycurl=7.45.1 - - pygeos=0.14 - - pyogrio=0.6.0 - - pyproj=3.5.0 - - pystac-client=0.7.5 - - python=3.10.13 - - rasterio=1.3.7 - - rasterstats=0.19.0 - - requests=2.31.0 - - rio-cogeo=5.1.1 - - rtree=1.1.0 - - s3fs=2023.12.2 - - scikit-learn=1.3.2 - - scipy=1.11.4 - - seaborn=0.13.1 - - shapely=2.0.1 - - sliderule=4.1.0 - - statsmodels=0.14.1 - - tqdm=4.66.1 - - unidecode=1.3.7 - - xmltodict=0.13.0 - - pip=23.3.2 - - pip: - - morecantile==5.1.0 - - rio-tiler==6.2.8 -variables: - MAAP_CONF: '/maap-py/' - - diff --git a/custom_images/build-image.sh b/custom_images/build-image.sh new file mode 100755 index 0000000..8b0edc9 --- /dev/null +++ b/custom_images/build-image.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -ex +base_image_dir=$(dirname "$0") +# Check if on a branch or in a detached HEAD state get commit sha +BRANCH=$(basename "$(git symbolic-ref -q --short HEAD || git rev-parse --short HEAD)") +if [ "$#" -eq 0 ]; then + DIRS_LIST=( "maap_base" ) +else + DIRS_LIST=( "$@" ) +fi +for dir in "${DIRS_LIST[@]}"; do + pushd "$base_image_dir"/"$dir" + IMAGE_NAME=$(basename "$dir") + IMAGE_PATH=$(basename "$base_image_dir") + IMAGE_REF=${CI_REGISTRY_IMAGE}/${IMAGE_PATH}/${IMAGE_NAME}:${BRANCH} + docker build -t "${IMAGE_REF}" --build-arg IMAGE_REF="${IMAGE_REF}" -f docker/Dockerfile . + docker push "${IMAGE_REF}" + popd + echo "$IMAGE_REF" >> built_images.txt +done + diff --git a/custom_images/maap_base/docker/Dockerfile b/custom_images/maap_base/docker/Dockerfile new file mode 100644 index 0000000..cadbdc7 --- /dev/null +++ b/custom_images/maap_base/docker/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:24.10 + +ARG MINICONDA_INSTALLER="https://repo.anaconda.com/miniconda/Miniconda3-py311_24.7.1-0-Linux-x86_64.sh" +ARG MINICONDA_INSTALLER_SHA256="a098a5b1581d8fd078c430b82e27106602223e335efef708a124e723814d120c" + +RUN apt-get update -y && apt-get install -y vim wget git && \ + apt-get clean all + +RUN groupadd conda-users + +RUN wget -O Miniconda3-installer.sh ${MINICONDA_INSTALLER} && \ + echo "${MINICONDA_INSTALLER_SHA256} Miniconda3-installer.sh" | sha256sum --check --status && \ + bash Miniconda3-installer.sh -b -p /opt/conda && \ + chgrp -R conda-users /opt/conda/ && chmod -R 775 /opt/conda/ && \ + find /opt/conda/ -follow -type f -name '*.a' -delete && \ + find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ + /opt/conda/bin/conda clean -afy && \ + /opt/conda/bin/conda init --all --system + +ENV PATH /opt/conda/bin:$PATH diff --git a/devfiles/isce3/devfile/devfile.yaml b/devfiles/isce3/devfile/devfile.yaml index 2b8f66d..df3b020 100644 --- a/devfiles/isce3/devfile/devfile.yaml +++ b/devfiles/isce3/devfile/devfile.yaml @@ -8,7 +8,9 @@ components: - attributes: type: ide discoverable: 'false' + secure: 'true' path: / + cookiesAuthEnabled: 'true' public: 'true' protocol: http name: jupyter @@ -37,7 +39,7 @@ components: - name: DOCKERIMAGE_PATH value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/isce3:develop' - name: MAAP_API_HOST - value: ade.dit.maap-project.org + value: api.dit.maap-project.org resources: limits: memory: 8096Mi diff --git a/devfiles/isce3/devfile/meta.yaml b/devfiles/isce3/devfile/meta.yaml index a57442d..a380c31 100644 --- a/devfiles/isce3/devfile/meta.yaml +++ b/devfiles/isce3/devfile/meta.yaml @@ -1,6 +1,6 @@ --- displayName: "ISCE3" -description: "ISCE3 workspace version: 3.1.5" +description: "ISCE3 MAAP workspace (version 4.1.0)" tags: ["JupyterLab", "Python", "MAAP", "ISCE3"] icon: /devfiles/isce3/devfile/isce.png links: diff --git a/devfiles/pangeo/devfile/devfile.yaml b/devfiles/pangeo/devfile/devfile.yaml index 18cfff7..2352e06 100644 --- a/devfiles/pangeo/devfile/devfile.yaml +++ b/devfiles/pangeo/devfile/devfile.yaml @@ -8,7 +8,9 @@ components: - attributes: type: ide discoverable: 'false' + secure: 'true' path: / + cookiesAuthEnabled: 'true' public: 'true' protocol: http name: jupyter @@ -37,7 +39,7 @@ components: - name: DOCKERIMAGE_PATH value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/pangeo:develop' - name: MAAP_API_HOST - value: ade.dit.maap-project.org + value: api.dit.maap-project.org resources: limits: memory: 8096Mi diff --git a/devfiles/pangeo/devfile/meta.yaml b/devfiles/pangeo/devfile/meta.yaml index 6560907..a32260a 100644 --- a/devfiles/pangeo/devfile/meta.yaml +++ b/devfiles/pangeo/devfile/meta.yaml @@ -1,6 +1,6 @@ --- displayName: "Pangeo" -description: "Pangeo workspace version: 3.1.5" +description: "Pangeo MAAP workspace (version 4.1.0)" tags: ["Pangeo", "JupyterLab", "MAAP"] icon: /devfiles/pangeo/devfile/pangeo_simple_logo.svg links: diff --git a/devfiles/python/devfile/devfile.yaml b/devfiles/python/devfile/devfile.yaml index f63c1ad..aabc081 100644 --- a/devfiles/python/devfile/devfile.yaml +++ b/devfiles/python/devfile/devfile.yaml @@ -8,7 +8,9 @@ components: - attributes: type: ide discoverable: 'false' + secure: 'true' path: / + cookiesAuthEnabled: 'true' public: 'true' protocol: http name: jupyter @@ -35,9 +37,9 @@ components: imagePullPolicy: Always env: - name: DOCKERIMAGE_PATH - value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/vanilla:develop' + value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/python:develop' - name: MAAP_API_HOST - value: ade.dit.maap-project.org + value: api.dit.maap-project.org resources: limits: memory: 8096Mi diff --git a/devfiles/python/devfile/meta.yaml b/devfiles/python/devfile/meta.yaml index c1e833a..ab39fe8 100644 --- a/devfiles/python/devfile/meta.yaml +++ b/devfiles/python/devfile/meta.yaml @@ -1,6 +1,6 @@ --- displayName: "Python (default)" -description: "Python workspace version: 3.1.5" +description: "Basic MAAP python workspace (version 4.1.0)" tags: ["JupyterLab", "Python", "MAAP"] icon: /devfiles/python/devfile/python-logo-generic.svg links: diff --git a/devfiles/r/devfile/devfile.yaml b/devfiles/r/devfile/devfile.yaml index 510ddb0..c8bbc03 100644 --- a/devfiles/r/devfile/devfile.yaml +++ b/devfiles/r/devfile/devfile.yaml @@ -8,7 +8,9 @@ components: - attributes: type: ide discoverable: 'false' + secure: 'true' path: / + cookiesAuthEnabled: 'true' public: 'true' protocol: http name: jupyter @@ -37,7 +39,7 @@ components: - name: DOCKERIMAGE_PATH value: 'mas.dit.maap-project.org/root/maap-workspaces/base_images/r:develop' - name: MAAP_API_HOST - value: ade.dit.maap-project.org + value: api.dit.maap-project.org resources: limits: memory: 8096Mi diff --git a/devfiles/r/devfile/meta.yaml b/devfiles/r/devfile/meta.yaml index 61d3ffa..7696bfb 100644 --- a/devfiles/r/devfile/meta.yaml +++ b/devfiles/r/devfile/meta.yaml @@ -1,6 +1,6 @@ --- displayName: "R/Python" -description: "R/Python workspace version: 3.1.5" +description: "R/Python MAAP workspace (version 4.1.0)" tags: ["Python", "R", "JupyterLab", "MAAP"] icon: /devfiles/r/devfile/r.png links: diff --git a/gitlab-ci/custom-image-build.yml.tmpl b/gitlab-ci/custom-image-build.yml.tmpl new file mode 100644 index 0000000..c6c7195 --- /dev/null +++ b/gitlab-ci/custom-image-build.yml.tmpl @@ -0,0 +1,8 @@ +${BASE_IMAGE_TYPE}: + script: + - docker login -u "$CI_DEPLOY_USER" -p "$CI_DEPLOY_PASSWORD" "$CI_REGISTRY" + - git clone https://github.com/MAAP-Project/maap-workspaces.git + - pushd maap-workspaces + - git checkout ${LATEST_COMMIT} + - bash custom_images/build-image.sh ${BASE_IMAGE_TYPE} + - cat built_images.txt diff --git a/gitlab-ci/generate_configs.sh b/gitlab-ci/generate_configs.sh index 7aa6b30..cccb134 100644 --- a/gitlab-ci/generate_configs.sh +++ b/gitlab-ci/generate_configs.sh @@ -1,14 +1,49 @@ #!/bin/bash set -e basedir=$( cd "$(dirname "$0")" ; pwd -P ) + +# Get webhook payload +echo TRIGGER_PAYLOAD=$TRIGGER_PAYLOAD +if [[ ! -z $TRIGGER_PAYLOAD ]]; then + PAYLOAD_COMMIT=$(cat $TRIGGER_PAYLOAD | python3 -c "import sys, json; print(json.loads(json.load(sys.stdin)['payload'])['after'])") + REF=$(cat $TRIGGER_PAYLOAD | python3 -c "import sys, json; print(json.loads(json.load(sys.stdin)['payload'])['ref'])") +fi git clone https://github.com/MAAP-Project/maap-workspaces.git pushd maap-workspaces -LATEST_COMMIT=$(git log -n 1 --all --format='%h') +LATEST_COMMIT=$PAYLOAD_COMMIT +TAG=$LATEST_COMMIT +if [[ -z ${PAYLOAD_COMMIT} ]]; then + # If no payload commit was set, find the latest commit on the repo + LATEST_COMMIT=$(git log -n 1 --all --format='%h') +fi if [[ ! -z ${FORCE_REF_BUILD} ]]; then LATEST_COMMIT=${FORCE_REF_BUILD} fi git checkout ${LATEST_COMMIT} +# Set LATEST_COMMIT to what we want as image tag +# Default set it to latest commit hash, branch name if commit is HEAD of branch +TAG=$(basename $(git symbolic-ref -q --short HEAD || git rev-parse --short HEAD)) + +# If REF (branch name) in payload use that +if [[ ! -z ${REF} ]]; then + TAG=$(basename ${REF}) +fi + +# If FORCE_CUSTOM_TAGNAME then use that as tag, eg. nightly +if [[ ! -z ${FORCE_CUSTOM_TAGNAME} ]]; then + TAG=${FORCE_CUSTOM_TAGNAME} +fi + +echo "Using ${TAG} as tag for images" +if [[ "$TAG" == "develop" || "$TAG" == "main" ]]; then + echo "Building all images as the tag is ${TAG}" + LATEST_COMMIT=${TAG} + BUILD_ALL_BASE_IMAGES=1 + BUILD_ALL_CUSTOM_IMAGES=1 +fi + + if [[ ! -z ${BUILD_ALL_BASE_IMAGES} ]]; then ls -d base_images/*/* > ${basedir}/files_changed.txt elif [[ ! -z ${BUILD_SPECIFIC_BASE_IMAGES} ]]; then @@ -22,23 +57,49 @@ else echo "Getting files changed in last commit" git diff --name-only HEAD HEAD~1 > ${basedir}/files_changed.txt fi + +if [[ ! -z ${BUILD_ALL_CUSTOM_IMAGES} ]]; then + ls -d custom_images/*/* > ${basedir}/files_changed.txt +elif [[ ! -z ${BUILD_SPECIFIC_CUSTOM_IMAGES} ]]; then + custom_image_array=(${BUILD_SPECIFIC_CUSTOM_IMAGES}) + for path in ${custom_image_array[@]}; do + echo "custom_images/${path}/docker" >> ${basedir}/files_changed.txt + done +else + # Find files changed in the latest commit + echo "BUILD_ALL_CUSTOM_IMAGES and BUILD_SPECIFIC_CUSTOM_IMAGES unset" + echo "Getting files changed in last commit" + git diff --name-only HEAD HEAD~1 > ${basedir}/files_changed.txt +fi + cat ${basedir}/files_changed.txt export LATEST_COMMIT popd rm -rf maap-workspaces -template="${basedir}/stage.yml.tmpl" +touch stages.yml # For each file changed, check if its in one of the base images directory cat ${basedir}/files_changed.txt | while read path do if [[ "$path" == base_images/*/* ]]; then + template="${basedir}/stage.yml.tmpl" second_dir=$(echo "$path" | cut -d'/' -f2) export BASE_IMAGE_TYPE="$second_dir" echo "Adding stage for $path" # Add the base image changed file to build stage for downstream pipeline cat ${template} | CI_JOB_TOKEN='$CI_JOB_TOKEN' CI_REGISTRY='$CI_REGISTRY' envsubst >> stages.yml else - echo "Path does not begin with base_images or does not have a second directory, will not do anything" - touch stages.yml + echo "Path does not begin with base_images or does not have a second level directory, will not do anything" + fi + # This is not an elif because we can have both custom and base image repos with changes + if [[ "$path" == custom_images/*/* ]]; then + template="${basedir}/custom-image-build.yml.tmpl" + second_dir=$(echo "$path" | cut -d'/' -f2) + export BASE_IMAGE_TYPE="$second_dir" + echo "Adding stage for $path" + # Add the base image changed file to build stage for downstream pipeline + cat "${template}" | CI_JOB_TOKEN='$CI_JOB_TOKEN' CI_REGISTRY='$CI_REGISTRY' envsubst >> stages.yml + else + echo "Path does not begin with base_images or custom_images or does not have a second level directory, will not do anything" fi done diff --git a/gitlab-ci/stage.yml.tmpl b/gitlab-ci/stage.yml.tmpl index 69f457f..0a088b1 100644 --- a/gitlab-ci/stage.yml.tmpl +++ b/gitlab-ci/stage.yml.tmpl @@ -1,10 +1,14 @@ ${BASE_IMAGE_TYPE}: script: - - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" + - docker login -u "$CI_DEPLOY_USER" -p "$CI_DEPLOY_PASSWORD" "$CI_REGISTRY" - git clone https://github.com/MAAP-Project/maap-workspaces.git - pushd maap-workspaces - git checkout ${LATEST_COMMIT} + - cp -r ../overlays/ ./ - bash base_images/build-image.sh ${BASE_IMAGE_TYPE} - cat built_images.txt - export BASE_IMAGE_NAME=$(tail -1 built_images.txt) - bash jupyterlab/build-image.sh + - scp -vr devfiles/${BASE_IMAGE_TYPE}/devfile/ ${DIT_API_MACHINE}:/var/www/devfile-registry/devfiles/${BASE_IMAGE_TYPE}/ + - scp -vr devfiles/devfile_compile.py ${DIT_API_MACHINE}:/var/www/devfile-registry/devfiles/devfile_compile.py + - ssh ${DIT_API_MACHINE} python3 /var/www/devfile-registry/devfiles/devfile_compile.py diff --git a/jupyterlab/entrypoint.sh b/jupyterlab/entrypoint.sh index 9cc2476..68170a0 100644 --- a/jupyterlab/entrypoint.sh +++ b/jupyterlab/entrypoint.sh @@ -43,7 +43,9 @@ data=$(get_request_data "$1" "endpoints") import os che_workspace_id = os.environ.get('CHE_WORKSPACE_ID') -che_machine_name = os.environ.get('CHE_MACHINE_NAME').lower().replace('/', '-') + +# Disallow insecure workspaces. Assume that the workspace runs behind a jwtproxy container. +che_machine_name = '-jwtproxy' # os.environ.get('CHE_MACHINE_NAME').lower().replace('/', '-') endpoints = data['items'] for endpoint in endpoints: diff --git a/jupyterlab/pangeo/environment.yml b/jupyterlab/pangeo/environment.yml index 11219d0..52f4001 100644 --- a/jupyterlab/pangeo/environment.yml +++ b/jupyterlab/pangeo/environment.yml @@ -17,7 +17,6 @@ dependencies: - jupyter-panel-proxy=0.1.0 - jupyterlab_widgets=3.0.10 - jupyterlab-git=0.50.0 - - jupyterlab-s3-browser=0.12.0 - jupyterlab=4.2.0 - nb_conda=2.2.1 - nb_conda_kernels=2.5.1 @@ -37,14 +36,14 @@ dependencies: - jupyter-resource-usage==1.0.2 - git+https://github.com/MAAP-Project/stac_ipyleaflet.git@0.3.6#egg-info=stac_ipyleaflet - rio-tiler==6.2.8 - - maap-algorithms-jupyter-extension==0.3.1 + - maap-algorithms-jupyter-extension==0.3.4 - maap-che-sidebar-visibility-jupyter-extension==1.1.2 - maap-dps-jupyter-extension==0.7.1 - maap-edsc-jupyter-extension==1.1.1 - maap-help-jupyter-extension==2.0.0 - maap-jupyter-server-extension==2.0.7 - - maap-libs-jupyter-extension==1.2.3 - - maap-user-workspace-management-jupyter-extension==0.1.2 + - maap-libs-jupyter-extension==1.2.4 + - maap-user-workspace-management-jupyter-extension==0.1.3 variables: TITILER_STAC_ENDPOINT: 'https://titiler-stac.maap-project.org/' TITILER_ENDPOINT: 'https://titiler.maap-project.org/' diff --git a/jupyterlab/shared/environment.yml b/jupyterlab/shared/environment.yml index 412b178..95c447f 100644 --- a/jupyterlab/shared/environment.yml +++ b/jupyterlab/shared/environment.yml @@ -18,14 +18,14 @@ dependencies: - pip: - jupyter-resource-usage==1.0.2 - git+https://github.com/MAAP-Project/stac_ipyleaflet.git@0.3.6#egg-info=stac_ipyleaflet - - maap-algorithms-jupyter-extension==0.3.1 + - maap-algorithms-jupyter-extension==0.3.6 - maap-che-sidebar-visibility-jupyter-extension==1.1.2 - maap-dps-jupyter-extension==0.7.1 - maap-edsc-jupyter-extension==1.1.1 - maap-help-jupyter-extension==2.0.0 - - maap-jupyter-server-extension==2.0.7 - - maap-libs-jupyter-extension==1.2.3 - - maap-user-workspace-management-jupyter-extension==0.1.2 + - maap-jupyter-server-extension==2.0.8 + - maap-libs-jupyter-extension==1.2.4 + - maap-user-workspace-management-jupyter-extension==0.1.3 variables: TITILER_STAC_ENDPOINT: 'https://titiler-pgstac.maap-project.org/' TITILER_ENDPOINT: 'https://titiler.maap-project.org/'