fixes for demo cluster

.dockerignore

@@ -2,3 +2,4 @@ jupyterhub
 values.yaml
 config.yml
 config-generator
+skaffold.yaml

.gitignore

@@ -5,3 +5,4 @@ values.yaml
 build
 _README.md
 dist
+.env-config-generator

application_hub_context/app_hub_context.py

@@ -544,9 +544,7 @@ def apply_manifest(self, manifest):
 
     def unapply_manifests(self, manifest_content):
 
-        manifests = yaml.safe_load_all(manifest_content)
-
-        for k8_object in manifests:
+        for k8_object in manifest_content:
             kind = k8_object.get("kind")
             self.spawner.log.info(
                 f"Deleting {kind} {k8_object.get('metadata', {}).get('name')}"
@@ -567,6 +565,14 @@ def unapply_manifests(self, manifest_content):
                     self.batch_v1_api.delete_namespaced_job(name, namespace)
                 elif kind == "Pod":
                     self.core_v1_api.delete_namespaced_pod(name, namespace)
+                elif kind == "Role":
+                    self.rbac_authorization_v1_api.delete_namespaced_role(name, namespace)
+                elif kind == "RoleBinding":
+                    self.rbac_authorization_v1_api.delete_namespaced_role_binding(name, namespace)
+                elif kind == "ServiceAccount":
+                    self.core_v1_api.delete_namespaced_service_account(name, namespace)
+
+
                 # Add other kinds as needed
                 else:
                     self.spawner.log.error(f"Unsupported kind: {kind}")
@@ -848,18 +854,25 @@ def initialise(self):
                 for manifest in manifests:
                     self.spawner.log.info(f"Apply manifest {manifest.name}")
 
-                    try:
-                        ms = yaml.safe_load_all(manifest.content)
-                        for k8_object in ms:
-                            self.spawner.log.info(
-                                f"Apply manifest kind {k8_object['kind']}"
+
+                    for k8_object in manifest.content:
+                        try:
+                            # Log the object and its type
+                            self.spawner.log.info(f"K8 Object: {k8_object}")
+                            self.spawner.log.info(f"Object Type: {type(k8_object)}")
+
+                            # Check and log the 'kind' of the Kubernetes object
+                            if 'kind' in k8_object:
+                                self.spawner.log.info(f"Applying manifest of kind: {k8_object['kind']}")
+                                self.apply_manifest(k8_object)  # Apply the manifest
+                            else:
+                                self.spawner.log.warning(f"Manifest does not contain a 'kind': {k8_object}")
+
+                        except Exception as err:
+                            self.spawner.log.error(f"Unexpected {err}, {type(err)}")
+                            self.spawner.log.error(
+                                f"Skipping creation of manifest {manifest.name}"
                             )
-                            self.apply_manifest(k8_object)
-                    except Exception as err:
-                        self.spawner.log.error(f"Unexpected {err}, {type(err)}")
-                        self.spawner.log.error(
-                            f"Skipping creation of manifest {manifest.name}"
-                        )
 
     def dispose(self):
         profile_id = self.config_parser.get_profile_by_slug(slug=self.profile_slug).id

application_hub_context/models.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Union
 
 from pydantic import BaseModel, Field
 
@@ -40,7 +40,7 @@ class defining a volume object:
 class Manifest(BaseModel):
     name: str
     key: str
-    content: Optional[str] = None
+    content: Optional[List[Dict]] = None
     persist: Optional[bool] = True
 
 

config-generator/README.md

@@ -0,0 +1,5 @@
+## ApplicationHub configuration generator
+
+This folder contains a notebook and the python modules to support the generation of ApplicationHub configurations.
+
+Create a Python environment with the dependencies listed in the file `requirements.txt` and open the notebook `config-generator.ipynb`

config-generator/config-generator.ipynb

@@ -408,7 +408,7 @@
     "image_pull_secret = ImagePullSecret(\n",
     "    name=\"cr-config\",\n",
     "    persist=False,\n",
-    "    data=\"\",\n",
+    "    data=\"ewogICAgImF1dGhzIjogewogICAgICAgICJjci50ZXJyYWR1ZS5jb20iOiB7CiAgICAgICAgICAgICJ1c2VybmFtZSI6ICJyb2JvdCRlb2VwY2EtcGx1cy1ybyIsCiAgICAgICAgICAgICJwYXNzd29yZCI6ICJQMlE4TnkyZ0lHODhkZkxveXlLN05QVUZVbHJOekFZSiIsCiAgICAgICAgICAgICJlbWFpbCI6ICJlb2VwY2EtcGx1c0B0ZXJyYWR1ZS5jb20iLAogICAgICAgICAgICAiYXV0aCI6ICJjbTlpYjNRa1pXOWxjR05oTFhCc2RYTXRjbTg2VURKUk9FNTVNbWRKUnpnNFpHWk1iM2w1U3pkT1VGVkdWV3h5VG5wQldVbz0iCiAgICAgICAgfQogICAgfQp9\",\n",
     ")"
    ]
   },
@@ -470,7 +470,7 @@
     ")\n",
     "\n",
     "with open(\n",
-    "    \"../jupyterhub/files/hub/config.yml\", \"w\"\n",
+    "    \"../files/hub/config.yml\", \"w\"\n",
     ") as file:\n",
     "    yaml.dump(config.dict(), file)"
    ]
@@ -487,7 +487,7 @@
        " Profile(id='profile_studio_coder2', groups=['group-a', 'group-b'], definition=ProfileDefinition(display_name='Code Server Medium', description=None, slug='ellip_studio_coder_slug_m', default=False, kubespawner_override=KubespawnerOverride(cpu_limit=4, cpu_guarantee=None, mem_limit='12G', mem_guarantee=None, image='eoepca/pde-code-server:develop', extra_resource_limits={}, extra_resource_guarantees={})), config_maps=[ConfigMap(name='bash-rc', key='bash-rc', mount_path='/workspace/.bashrc', default_mode=None, readonly=True, content='alias ll=\"ls -l\"\\nalias calrissian=\"/opt/conda/bin/calrissian --pod-nodeselectors /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram 16G --max-cores \"8\" --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/\"\\nalias cwltool=\"/opt/conda/bin/cwltool --podman\"\\n. /home/jovyan/.bashrc\\n# >>> conda initialize >>>\\n# !! Contents within this block are managed by \\'conda init\\' !!\\n__conda_setup=\"$(\\'/opt/conda/bin/conda\\' \\'shell.bash\\' \\'hook\\' 2> /dev/null)\"\\nif [ $? -eq 0 ]; then\\n    eval \"$__conda_setup\"\\nelse\\n    if [ -f \"/opt/conda/etc/profile.d/conda.sh\" ]; then\\n        . \"/opt/conda/etc/profile.d/conda.sh\"\\n    else\\n        export PATH=\"/srv/conda/bin:$PATH\"\\n    fi\\nfi\\nunset __conda_setup\\n\\nif [ -f \"/opt/conda/etc/profile.d/mamba.sh\" ]; then\\n    . \"/opt/conda/etc/profile.d/mamba.sh\"\\nfi\\n# <<< conda initialize <<<\\n', persist=True)], volumes=[Volume(name='calrissian-volume', claim_name='calrissian-claim', size='50Gi', storage_class='standard', access_modes=['ReadWriteMany'], volume_mount=VolumeMount(name='calrissian-volume', mount_path='/calrissian'), persist=False), Volume(name='workspace-volume', claim_name='workspace-claim', size='50Gi', storage_class='standard', access_modes=['ReadWriteOnce'], volume_mount=VolumeMount(name='workspace-volume', mount_path='/workspace'), persist=True)], pod_env_vars={'HOME': '/workspace', 'CONDA_ENVS_PATH': ' /workspace/.envs'}, default_url=None, node_selector={}, role_bindings=None, image_pull_secrets=[], init_containers=[], manifests=None),\n",
        " Profile(id='profile_demo_init_script', groups=['group-a', 'group-b'], definition=ProfileDefinition(display_name='Coder demo init script', description='This profile is used to demonstrate the use of an init script', slug='eoepca_demo_init_script', default=False, kubespawner_override=KubespawnerOverride(cpu_limit=2, cpu_guarantee=1, mem_limit='6G', mem_guarantee='4G', image='eoepca/pde-code-server:develop', extra_resource_limits={}, extra_resource_guarantees={})), config_maps=[ConfigMap(name='init', key='init', mount_path='/opt/init/.init.sh', default_mode='0660', readonly=True, content=\"set -x\\n\\ncd /workspace\\n\\ngit clone 'https://github.com/eoap/mastering-app-package.git'\\n\\ncode-server --install-extension ms-python.python\\ncode-server --install-extension redhat.vscode-yaml\\ncode-server --install-extension sbg-rabix.benten-cwl\\ncode-server --install-extension ms-toolsai.jupyter\\n\\nln -s /workspace/.local/share/code-server/extensions /workspace/extensions\\n\\nexit 0\\n\", persist=False)], volumes=[Volume(name='calrissian-volume', claim_name='calrissian-claim', size='50Gi', storage_class='standard', access_modes=['ReadWriteMany'], volume_mount=VolumeMount(name='calrissian-volume', mount_path='/calrissian'), persist=False), Volume(name='workspace-volume', claim_name='workspace-claim', size='50Gi', storage_class='standard', access_modes=['ReadWriteOnce'], volume_mount=VolumeMount(name='workspace-volume', mount_path='/workspace'), persist=True)], pod_env_vars={'HOME': '/workspace', 'CONDA_ENVS_PATH': '/workspace/.envs', 'CONDARC': '/workspace/.condarc', 'XDG_RUNTIME_DIR': '/workspace/.local', 'CODE_SERVER_WS': '/workspace/mastering-app-package'}, default_url=None, node_selector={}, role_bindings=None, image_pull_secrets=[], init_containers=[InitContainer(name='init-file-on-volume', image='eoepca/pde-code-server:develop', command=['sh', '-c', 'sh /opt/init/.init.sh'], volume_mounts=[VolumeMount(name='workspace-volume', mount_path='/workspace'), InitContainerVolumeMount(name='init', mount_path='/opt/init/.init.sh', sub_path='init')])], manifests=None),\n",
        " Profile(id='profile_jupyter_lab', groups=['group-c'], definition=ProfileDefinition(display_name='Jupyter Lab', description='Jupyter Lab with Python 3.11', slug='eoepca_jupyter_lab', default=False, kubespawner_override=KubespawnerOverride(cpu_limit=2, cpu_guarantee=1, mem_limit='6G', mem_guarantee='4G', image='jupyter/scipy-notebook', extra_resource_limits={}, extra_resource_guarantees={})), config_maps=[], volumes=[Volume(name='workspace-volume', claim_name='workspace-claim', size='50Gi', storage_class='standard', access_modes=['ReadWriteOnce'], volume_mount=VolumeMount(name='workspace-volume', mount_path='/workspace'), persist=True)], pod_env_vars={'HOME': '/workspace', 'XDG_RUNTIME_DIR': '/workspace/.local', 'XDG_CONFIG_HOME': '/workspace/.config'}, default_url=None, node_selector={}, role_bindings=None, image_pull_secrets=[], init_containers=[], manifests=None),\n",
-       " Profile(id='profile_jupyter_lab_2', groups=['group-c'], definition=ProfileDefinition(display_name='Jupyter Lab - profile 2', description='Jupyter Lab with Python 3.11 - demoes the use of an image pull secret', slug='eoepca_jupyter_lab_2', default=False, kubespawner_override=KubespawnerOverride(cpu_limit=2, cpu_guarantee=1, mem_limit='6G', mem_guarantee='4G', image='jupyter/scipy-notebook', extra_resource_limits={}, extra_resource_guarantees={})), config_maps=[], volumes=[Volume(name='workspace-volume', claim_name='workspace-claim', size='50Gi', storage_class='standard', access_modes=['ReadWriteOnce'], volume_mount=VolumeMount(name='workspace-volume', mount_path='/workspace'), persist=True)], pod_env_vars={'HOME': '/workspace', 'XDG_RUNTIME_DIR': '/workspace/.local', 'XDG_CONFIG_HOME': '/workspace/.config'}, default_url=None, node_selector={}, role_bindings=None, image_pull_secrets=[ImagePullSecret(name='cr-config', persist=False, data='')], init_containers=[], manifests=None)]"
+       " Profile(id='profile_jupyter_lab_2', groups=['group-c'], definition=ProfileDefinition(display_name='Jupyter Lab - profile 2', description='Jupyter Lab with Python 3.11 - demoes the use of an image pull secret', slug='eoepca_jupyter_lab_2', default=False, kubespawner_override=KubespawnerOverride(cpu_limit=2, cpu_guarantee=1, mem_limit='6G', mem_guarantee='4G', image='jupyter/scipy-notebook', extra_resource_limits={}, extra_resource_guarantees={})), config_maps=[], volumes=[Volume(name='workspace-volume', claim_name='workspace-claim', size='50Gi', storage_class='standard', access_modes=['ReadWriteOnce'], volume_mount=VolumeMount(name='workspace-volume', mount_path='/workspace'), persist=True)], pod_env_vars={'HOME': '/workspace', 'XDG_RUNTIME_DIR': '/workspace/.local', 'XDG_CONFIG_HOME': '/workspace/.config'}, default_url=None, node_selector={}, role_bindings=None, image_pull_secrets=[ImagePullSecret(name='cr-config', persist=False, data='ewogICAgImF1dGhzIjogewogICAgICAgICJjci50ZXJyYWR1ZS5jb20iOiB7CiAgICAgICAgICAgICJ1c2VybmFtZSI6ICJyb2JvdCRlb2VwY2EtcGx1cy1ybyIsCiAgICAgICAgICAgICJwYXNzd29yZCI6ICJQMlE4TnkyZ0lHODhkZkxveXlLN05QVUZVbHJOekFZSiIsCiAgICAgICAgICAgICJlbWFpbCI6ICJlb2VwY2EtcGx1c0B0ZXJyYWR1ZS5jb20iLAogICAgICAgICAgICAiYXV0aCI6ICJjbTlpYjNRa1pXOWxjR05oTFhCc2RYTXRjbTg2VURKUk9FNTVNbWRKUnpnNFpHWk1iM2w1U3pkT1VGVkdWV3h5VG5wQldVbz0iCiAgICAgICAgfQogICAgfQp9')], init_containers=[], manifests=None)]"
       ]
      },
      "execution_count": 15,
@@ -537,7 +537,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.10.14"
   },
   "orig_nbformat": 4
  },

config-generator/config-maps/bash-rc

@@ -2,6 +2,9 @@ alias ll="ls -l"
 alias calrissian="/opt/conda/bin/calrissian --pod-nodeselectors /etc/calrissian/pod-node-selector.yml --stdout /calrissian/results.json --max-ram 16G --max-cores "8" --tmp-outdir-prefix /calrissian/tmp/ --outdir /calrissian/"
 alias cwltool="/opt/conda/bin/cwltool --podman"
 . /home/jovyan/.bashrc
+
+alias aws="aws --endpoint-url=http://localstack:4566"
+
 # >>> conda initialize >>>
 # !! Contents within this block are managed by 'conda init' !!
 __conda_setup="$('/opt/conda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"

config-generator/config-maps/init-stac.sh

@@ -0,0 +1,31 @@
+set -x 
+
+cd /workspace
+
+git clone 'https://github.com/eoap/stac-eoap.git'
+
+code-server --install-extension ms-python.python 
+code-server --install-extension redhat.vscode-yaml
+code-server --install-extension sbg-rabix.benten-cwl
+code-server --install-extension ms-toolsai.jupyter
+
+ln -s /workspace/.local/share/code-server/extensions /workspace/extensions
+
+mkdir -p /workspace/User/
+
+echo '{"workbench.colorTheme": "Visual Studio Dark"}' > /workspace/User/settings.json
+
+python -m venv /workspace/.venv
+source /workspace/.venv/bin/activate
+/workspace/.venv/bin/python -m pip install --no-cache-dir stactools rasterio requests stac-asset click-logging tabulate tqdm pystac-client ipykernel loguru scikit-image rio_stac boto3==1.35.23
+
+/workspace/.venv/bin/python -m pip install --index-url https://test.pypi.org/simple cwl-wrapper
+
+/workspace/.venv/bin/python -m ipykernel install --user --name stac_env --display-name "Python (STAC)"
+
+export AWS_DEFAULT_REGION="us-east-1"
+export AWS_ACCESS_KEY_ID="test"
+export AWS_SECRET_ACCESS_KEY="test"
+aws s3 mb s3://results --endpoint-url=http://localstack:4566
+
+exit 0