Merge pull request #187 from grycap/devel

Devel

artifacts/jupyter/jupyterhub_k8s.yml

@@ -8,6 +8,37 @@
     jup_admin_user: "{{ admin_user | default('admin') }}"
     dns_name: "{{ kube_public_dns_name | default('') }}"
     jup_storage_size: "{{ storage_size | default('10Gi') }}"
+    default_profile_list:
+      - display_name: '1CPU 1GB environment'
+        description: '1CPU 1GB environment'
+        default: true
+      - display_name: '2CPU 2GB environment'
+        description: '2CPU 2GB environment'
+        kubespawner_override:
+          cpu_limit: 2
+          mem_limit: '2G'
+          cpu_guarantee: 2
+          mem_guarantee': '2G'
+      - display_name: '2CPU 2GB ItwinAI environment'
+        description: '2CPU 2GB ItwinAI environment'
+        kubespawner_override:
+          image: 'grycap/jupyterhub-k8s-itwinai'
+          cpu_limit: 2
+          mem_limit: '2G'
+          cpu_guarantee: 2
+          mem_guarantee': '2G'
+      - display_name: '2CPU 4GB 1GPU environment'
+        description: '2CPU 4GB 1GPU environment (only use it if some of the K8s WNs has GPU support)'
+        kubespawner_override:
+          image: 'grycap/k8s-singleuser-sample-gpu:latest'
+          cpu_limit: 2
+          mem_limit: '4G'
+          cpu_guarantee: 2
+          mem_guarantee': '4G'
+          extra_resource_limits:
+            nvidia.com/gpu: 1
+          extra_resource_guarantees:
+            nvidia.com/gpu: 1
   tasks:
     - name: Add jupyterhub helm repo
       command: helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
@@ -66,6 +97,10 @@
       environment:
         KUBECONFIG: "/etc/kubernetes/admin.conf"
 
+    - name: Set profile list
+      set_fact:
+        jup_profile_list: "{{ profile_list | default(default_profile_list) }}"
+
     - name: "Create values file"
       copy:
         dest: /tmp/config.yaml
@@ -106,37 +141,7 @@
                   - name: vol-data
                     mountPath: /home/jovyan/data
                     readOnly: True
-              profileList:
-                - display_name: '1CPU 1GB environment'
-                  description: '1CPU 1GB environment'
-                  default: true
-                - display_name: '2CPU 2GB environment'
-                  description: '2CPU 2GB environment'
-                  kubespawner_override:
-                    cpu_limit: 2
-                    mem_limit: '2G'
-                    cpu_guarantee: 2
-                    mem_guarantee': '2G'
-                - display_name: '2CPU 2GB ItwinAI environment'
-                  description: '2CPU 2GB ItwinAI environment'
-                  kubespawner_override:
-                    image: 'grycap/jupyterhub-k8s-itwinai'
-                    cpu_limit: 2
-                    mem_limit: '2G'
-                    cpu_guarantee: 2
-                    mem_guarantee': '2G'
-                - display_name: '2CPU 4GB 1GPU environment'
-                  description: '2CPU 4GB 1GPU environment (only use it if some of the K8s WNs has GPU support)'
-                  kubespawner_override:
-                    image: 'grycap/k8s-singleuser-sample-gpu:latest'
-                    cpu_limit: 2
-                    mem_limit: '4G'
-                    cpu_guarantee: 2
-                    mem_guarantee': '4G'
-                    extra_resource_limits:
-                      nvidia.com/gpu: 1
-                    extra_resource_guarantees:
-                      nvidia.com/gpu: 1
+              profileList: {{ jup_profile_list }}
 
     - name: Install (or upgrade) the chart
       command: helm upgrade --install jupyterhub jupyterhub/jupyterhub --namespace jupyter --create-namespace --values /tmp/config.yaml --timeout 10m

artifacts/mlflow_auth_compose.yml

@@ -0,0 +1,89 @@
+---
+- hosts: localhost
+  connection: local
+  vars:
+    # defalt pass is "operatorpass"
+    GOACCESS_PASSWORD: "{{ mlflow_operator_pass | default('$$2y$$05$$lPcMyqoHliTPF5QeAHoPWOGyLFZxGQg.pSfJlHssQJ9Ny7OFcSI3i') }}"
+    MLFLOW_USERNAME: "{{ mlflow_admin_user | default('admin') }}"
+    MLFLOW_PASSWORD: "{{ mlflow_admin_password | default('password') }}"
+    OIDC_AUTHORITY: "{{ mlflow_oidc_auth | default('https://aai-demo.egi.eu/auth/realms/egi') }}"
+    OIDC_CLIENT_ID: "{{ mlflow_oidc_client_id | default('oidc-client') }}"
+    OAUTH_USERINFO_ENDPOINT: "{{ mlflow_auth_userinfo_endpoint | default(OIDC_AUTHORITY + '/protocol/openid-connect/userinfo') }}"
+    SUPPORTED_VO: "{{ mlflow_oidc_vo | default('vo.ai4eosc.eu') }}"
+    REQUIRED_ENTITLEMENT: "{{ mlflow_required_entitlement | default('urn:mace:egi.eu:group:' + SUPPORTED_VO + ':role=member#aai.egi.eu') }}"
+  roles:
+    - role: 'grycap.docker'
+  tasks:
+    - name: Clone git repository
+      git:
+        repo: https://github.com/m-team-kit/mlflow-auth-gui
+        dest: /opt/mlflow-auth-gui
+        version: main
+      ignore_errors: true
+
+    - name: Set default DNS name (nip.io)
+      set_fact:
+        dns_name: "mlflow.{{ public_ip_address }}.nip.io"
+      when: mlflow_dns_name is not defined or mlflow_dns_name == ""
+    - name: Set custom DNS name
+      set_fact:
+        dns_name: "{{ mlflow_dns_name }}"
+      when: mlflow_dns_name is defined and mlflow_dns_name != ""
+
+    - name: Set admin username and password in ini file
+      ini_file:
+        path: /opt/mlflow-auth-gui/backend/srv/auth_config.ini
+        section: "mlflow"
+        option: "{{ item.option }}"
+        value: "{{ item.value }}"
+        mode: '0644'
+      loop:
+        - { "option": "admin_username", "value": "{{ MLFLOW_USERNAME }}" }
+        - { "option": "admin_password", "value": "{{ MLFLOW_PASSWORD }}" }
+
+    - name: Create .env file
+      copy:
+        dest: /opt/mlflow-auth-gui/.env
+        mode: '0644'
+        content: |
+          # Domain where MLFLOW is hosted
+          DOMAIN={{ dns_name }}
+          # POSTGRES CONFIG
+          DATABASE_NAME=mlflowdb
+          DATABASE_USER=postgres
+          DATABASE_PASSWORD=dummypassword
+          DATABASE_HOST=database
+          DATABASE_PORT=5432
+          # MLFLOW Authentication Database File
+          AUTH_DB_FILE=basic_auth.db
+          # Local (host) BASE PATH to store MLFLOW data, e.g. app-data, user-data, artifacts, backup_db.
+          # Pay attention about subdirectories ownership, may need to set uid:gid of "docker"
+          MLFLOW_BASE_PATH_LOCAL=/opt/mlflow-auth-gui
+          # MLflow user registration service
+          MLFLOW_USERNAME={{ MLFLOW_USERNAME }}
+          MLFLOW_PASSWORD={{ MLFLOW_PASSWORD }}
+          MLFLOW_HOSTNAME=http://backend:5000
+          OIDC_AUTHORITY={{ OIDC_AUTHORITY }}
+          OIDC_CLIENT_ID={{ OIDC_CLIENT_ID }}
+          OAUTH_USERINFO_ENDPOINT={{ OAUTH_USERINFO_ENDPOINT }}
+          REQUIRED_ENTITLEMENT={{ REQUIRED_ENTITLEMENT }}
+          PRIVACY_POLICY_URL=https://confluence.egi.eu/display/IMPAIP/Privacy+Policy
+          TERMS_OF_USE_URL=https://confluence.egi.eu/display/IMPAIP/Acceptable+Use+Policy
+          # can be left blank, but prefer putting one
+          LETSENCRYPT_EMAIL={{ mlflow_cert_email }}
+          CORS_ORIGINS=http://localhost,http://foo.bar
+          # Monitoring based on goaccess
+          GOACCESS_ETC_PATH_LOCAL=${MLFLOW_BASE_PATH_LOCAL}/goaccess-etc
+          GOACCESS_OUT_PATH_LOCAL=${MLFLOW_BASE_PATH_LOCAL}/goaccess-out
+          GOACCESS_WEB_ROUTE=goaccess
+          GOACCESS_USER=operator
+          GOACCESS_PASSWORD={{ GOACCESS_PASSWORD }}     # bcrypt encrypted password, use "htpasswd -B -n operator" to generate. use $$ instead of $ in GOACCESS_PASSWORD value
+          GOACCESS_ACCESSLOG_FORMAT=TRAEFIKCLF
+
+    - name: Exec docker-compose up
+      community.docker.docker_compose_v2:
+        project_src: /opt/mlflow-auth-gui
+        state: present
+        files:
+          - compose.yml
+          - compose.prod.yml

templates/mlflow_authvm.yaml

@@ -0,0 +1,136 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+imports:
+  - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/main/custom_types.yaml
+
+metadata:
+  template_version: "1.0.0"
+  template_name: MLFlow Auth VM
+  display_name: Deploy MLFlow Auth GUI on a VM
+  icon: images/mlflow.png
+  tabs:
+    MLFlow: mlflow_.*
+  parents:
+    - simple-node-disk.yml
+
+description: Deploy MLFlow on a VM with the MLFlow Auth GUI.
+
+topology_template:
+
+  inputs:
+
+    mlflow_admin_user:
+      type: string
+      description: MLFlow Admin server Username
+      default: admin
+      required: yes
+
+    mlflow_admin_password:
+      type: string
+      description: MLFlow Admin server Password
+      default: password
+      required: yes
+
+    mlflow_operator_pass:
+      type: string
+      description: MLFlow Operator Password
+      default: 'operatorpass'
+      required: no
+
+    mlflow_oidc_auth:
+      type: string
+      description: MLFlow OIDC Authority
+      default: 'https://aai-demo.egi.eu/auth/realms/egi'
+      required: no
+
+    mlflow_oidc_client_id:
+      type: string
+      description: MLFlow OIDC Client ID
+      default: 'oidc-client'
+      required: no
+
+    mlflow_oidc_vo:
+      type: string
+      description: MLFlow OIDC VO name
+      default: 'vo.ai4eosc.eu'
+      required: no
+
+    mlflow_dns_name:
+      type: string
+      description: MLFlow DNS name (leave empty to use mlflow.<node_ip>.nip.io)
+      default: ''
+      required: no
+
+    mlflow_cert_email:
+      type: string
+      description: MLFlow LetsEncrypt certificate email
+      default: '[email protected]'
+      required: no
+
+    # Restrict some simple-node-disk input values
+
+    storage_size:
+      type: scalar-unit.size
+      description: Size of the extra HD added to the instance (Set 0 if disk is not needed)
+      default: 20 GB
+      constraints:
+        - valid_values: [ 20 GB, 50 GB, 100 GB, 200 GB, 500 GB, 1 TB, 2 TB, 10 TB, 20 TB, 40 TB, 100 TB ]
+
+    mount_path:
+      type: string
+      description: Path to mount the extra disk
+      default: /opt/mlflow-auth-gui
+
+  node_templates:
+
+    mlflow:
+      type: tosca.nodes.ec3.Application
+      artifacts:
+        docker_role:
+          file: grycap.docker
+          type: tosca.artifacts.AnsibleGalaxy.role
+        docker_collection:
+          file: community.docker,3.12.2
+          type: tosca.artifacts.AnsibleGalaxy.collection
+      capabilities:
+        endpoint:
+          properties:
+            ports:
+              https:
+                protocol: tcp
+                source: 443
+              http:
+                protocol: tcp
+                source: 80
+      interfaces:
+        Standard:
+          configure:
+            implementation: https://raw.githubusercontent.com/grycap/tosca/main/artifacts/mlflow_compose.yml
+            inputs:
+              public_ip_address:
+                get_attribute: [ simple_node, public_address, 0 ]
+              mlflow_admin_user:
+                get_input: mlflow_admin_user
+              mlflow_admin_password:
+                get_input: mlflow_admin_password
+              mlflow_cert_email:
+                get_input: mlflow_cert_email
+              mlflow_dns_name:
+                get_input: mlflow_dns_name
+              mlflow_oidc_auth:
+                get_input: mlflow_oidc_auth
+              mlflow_oidc_client_id:
+                get_input: mlflow_oidc_client_id
+              mlflow_oidc_vo:
+                get_input: mlflow_oidc_vo
+              mlflow_operator_pass:
+                get_input: mlflow_operator_pass
+
+      requirements:
+        - host: simple_node
+
+  outputs:
+    mlflow_nip_endpoint:
+      value: { concat: [ 'https://mlflow.', get_attribute: [ simple_node, public_address, 0 ], 'nip.io/signup' ] }
+    mlflow_dns_endpoint:
+      value: { concat: [ 'https://', get_input: mlflow_dns_name, '/signup' ] }

templates/simple-node-disk.yml

@@ -41,6 +41,7 @@ metadata:
     - image-service.yaml
     - ai4eoscvm.yaml
     - mlflowvm.yaml
+    - mlflow_authvm.yaml
     - wget.yml
     - sgde.yaml
     - dydns_egi_update_vm.yml

tests/requirements.yaml

@@ -32,3 +32,5 @@ roles:
 
 collections:
   - name: community.crypto
+  - name: community.docker
+    version: 3.12.2