From c42494f725d671698ae631bc89995b76736a086d Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 18 Sep 2024 09:40:04 +0200 Subject: [PATCH 1/7] Improve Jupyterhub recipe --- artifacts/jupyter/jupyterhub_k8s.yml | 67 +++++++++++++++------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/artifacts/jupyter/jupyterhub_k8s.yml b/artifacts/jupyter/jupyterhub_k8s.yml index d01189f..d544605 100644 --- a/artifacts/jupyter/jupyterhub_k8s.yml +++ b/artifacts/jupyter/jupyterhub_k8s.yml @@ -8,6 +8,37 @@ jup_admin_user: "{{ admin_user | default('admin') }}" dns_name: "{{ kube_public_dns_name | default('') }}" jup_storage_size: "{{ storage_size | default('10Gi') }}" + default_profile_list: + - display_name: '1CPU 1GB environment' + description: '1CPU 1GB environment' + default: true + - display_name: '2CPU 2GB environment' + description: '2CPU 2GB environment' + kubespawner_override: + cpu_limit: 2 + mem_limit: '2G' + cpu_guarantee: 2 + mem_guarantee': '2G' + - display_name: '2CPU 2GB ItwinAI environment' + description: '2CPU 2GB ItwinAI environment' + kubespawner_override: + image: 'grycap/jupyterhub-k8s-itwinai' + cpu_limit: 2 + mem_limit: '2G' + cpu_guarantee: 2 + mem_guarantee': '2G' + - display_name: '2CPU 4GB 1GPU environment' + description: '2CPU 4GB 1GPU environment (only use it if some of the K8s WNs has GPU support)' + kubespawner_override: + image: 'grycap/k8s-singleuser-sample-gpu:latest' + cpu_limit: 2 + mem_limit: '4G' + cpu_guarantee: 2 + mem_guarantee': '4G' + extra_resource_limits: + nvidia.com/gpu: 1 + extra_resource_guarantees: + nvidia.com/gpu: 1 tasks: - name: Add jupyterhub helm repo command: helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ @@ -66,6 +97,10 @@ environment: KUBECONFIG: "/etc/kubernetes/admin.conf" + - name: Set profile list + set_fact: + jup_profile_list: "{{ profile_list | default(default_profile_list) }}" + - name: "Create values file" copy: dest: /tmp/config.yaml @@ -106,37 +141,7 @@ - name: vol-data mountPath: /home/jovyan/data readOnly: True - profileList: - - display_name: '1CPU 1GB environment' - description: '1CPU 1GB environment' - default: true - - display_name: '2CPU 2GB environment' - description: '2CPU 2GB environment' - kubespawner_override: - cpu_limit: 2 - mem_limit: '2G' - cpu_guarantee: 2 - mem_guarantee': '2G' - - display_name: '2CPU 2GB ItwinAI environment' - description: '2CPU 2GB ItwinAI environment' - kubespawner_override: - image: 'grycap/jupyterhub-k8s-itwinai' - cpu_limit: 2 - mem_limit: '2G' - cpu_guarantee: 2 - mem_guarantee': '2G' - - display_name: '2CPU 4GB 1GPU environment' - description: '2CPU 4GB 1GPU environment (only use it if some of the K8s WNs has GPU support)' - kubespawner_override: - image: 'grycap/k8s-singleuser-sample-gpu:latest' - cpu_limit: 2 - mem_limit: '4G' - cpu_guarantee: 2 - mem_guarantee': '4G' - extra_resource_limits: - nvidia.com/gpu: 1 - extra_resource_guarantees: - nvidia.com/gpu: 1 + profileList: {{ jup_profile_list | to_yaml }} - name: Install (or upgrade) the chart command: helm upgrade --install jupyterhub jupyterhub/jupyterhub --namespace jupyter --create-namespace --values /tmp/config.yaml --timeout 10m From 013190a2cefcfbd165c860a304aeb33685a2caa1 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 18 Sep 2024 10:38:22 +0200 Subject: [PATCH 2/7] Fix error --- artifacts/jupyter/jupyterhub_k8s.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/jupyter/jupyterhub_k8s.yml b/artifacts/jupyter/jupyterhub_k8s.yml index d544605..b9f043c 100644 --- a/artifacts/jupyter/jupyterhub_k8s.yml +++ b/artifacts/jupyter/jupyterhub_k8s.yml @@ -141,7 +141,7 @@ - name: vol-data mountPath: /home/jovyan/data readOnly: True - profileList: {{ jup_profile_list | to_yaml }} + profileList: {{ jup_profile_list }} - name: Install (or upgrade) the chart command: helm upgrade --install jupyterhub jupyterhub/jupyterhub --namespace jupyter --create-namespace --values /tmp/config.yaml --timeout 10m From 140b1c31586cad253b1caa6e406a87249a5325f5 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 19 Sep 2024 10:27:06 +0200 Subject: [PATCH 3/7] Add MLFlow GUI Auth template --- artifacts/mlflow_auth_compose.yml | 87 +++++++++++++++++++ templates/mlflow_authvm.yaml | 133 ++++++++++++++++++++++++++++++ templates/simple-node-disk.yml | 1 + 3 files changed, 221 insertions(+) create mode 100644 artifacts/mlflow_auth_compose.yml create mode 100644 templates/mlflow_authvm.yaml diff --git a/artifacts/mlflow_auth_compose.yml b/artifacts/mlflow_auth_compose.yml new file mode 100644 index 0000000..e4bf3c4 --- /dev/null +++ b/artifacts/mlflow_auth_compose.yml @@ -0,0 +1,87 @@ +--- +- hosts: localhost + connection: local + vars: + # defalt pass is "operatorpass" + GOACCESS_PASSWORD: "{{ mlflow_operator_pass | default('$$2y$$05$$lPcMyqoHliTPF5QeAHoPWOGyLFZxGQg.pSfJlHssQJ9Ny7OFcSI3i') }}" + MLFLOW_USERNAME: "{{ mlflow_admin_user | default('admin') }}" + MLFLOW_PASSWORD: "{{ mlflow_admin_password | default('password') }}" + OIDC_AUTHORITY: "{{ mlflow_oidc_auth | default('https://aai-demo.egi.eu/auth/realms/egi') }}" + OIDC_CLIENT_ID: "{{ mlflow_oidc_client_id | default('oidc-client') }}" + OAUTH_USERINFO_ENDPOINT: "{{ mlflow_auth_userinfo_endpoint | default(OIDC_AUTHORITY + '/protocol/openid-connect/userinfo') }}" + SUPPORTED_VO: "{{ mlflow_oidc_vo | default('vo.ai4eosc.eu') }}" + REQUIRED_ENTITLEMENT: "{{ mlflow_required_entitlement | default('urn:mace:egi.eu:group:' + SUPPORTED_VO + ':role=member#aai.egi.eu') }}" + roles: + - role: 'grycap.docker' + tasks: + - name: Clone git repository + git: + repo: https://github.com/m-team-kit/mlflow-auth-gui + dest: /opt/mlflow-auth-gui + version: main + ignore_errors: yes + + - name: Set default DNS name (nip.io) + set_fact: + dns_name: "mlflow.{{ public_ip_address }}.nip.io" + when: mlflow_dns_name is not defined or mlflow_dns_name == "" + - name: Set custom DNS name + set_fact: + dns_name: "{{ mlflow_dns_name }}" + when: mlflow_dns_name is defined and mlflow_dns_name != "" + + - name: Set admin username and password in ini file + ini_file: + path: /opt/mlflow-auth-gui/backend/srv/auth_config.ini + section: "mlflow" + option: "{{ item.option }}" + value: "{{ item.value }}" + loop: + - { "option": "admin_username", "value" "{{ MLFLOW_USERNAME }}" } + - { "option": "admin_password", "value" "{{ MLFLOW_PASSWORD }}" } + + - name: Create .env file + copy: + dest: /opt/mlflow-auth-gui/.env + content: | + # Domain where MLFLOW is hosted + DOMAIN={{ dns_name }} + # POSTGRES CONFIG + DATABASE_NAME=mlflowdb + DATABASE_USER=postgres + DATABASE_PASSWORD=dummypassword + DATABASE_HOST=database + DATABASE_PORT=5432 + # MLFLOW Authentication Database File + AUTH_DB_FILE=basic_auth.db + # Local (host) BASE PATH to store MLFLOW data, e.g. app-data, user-data, artifacts, backup_db. + # Pay attention about subdirectories ownership, may need to set uid:gid of "docker" + MLFLOW_BASE_PATH_LOCAL=. + # MLflow user registration service + MLFLOW_USERNAME={{ MLFLOW_USERNAME }} + MLFLOW_PASSWORD={{ MLFLOW_PASSWORD }} + MLFLOW_HOSTNAME=http://backend:5000 + OIDC_AUTHORITY={{ OIDC_AUTHORITY }} + OIDC_CLIENT_ID={{ OIDC_CLIENT_ID }} + OAUTH_USERINFO_ENDPOINT={{ OAUTH_USERINFO_ENDPOINT }} + REQUIRED_ENTITLEMENT={{ REQUIRED_ENTITLEMENT }} + PRIVACY_POLICY_URL=https://confluence.egi.eu/display/IMPAIP/Privacy+Policy + TERMS_OF_USE_URL=https://confluence.egi.eu/display/IMPAIP/Acceptable+Use+Policy + # can be left blank, but prefer putting one + LETSENCRYPT_EMAIL={{ mlflow_cert_email}} + CORS_ORIGINS=http://localhost,http://foo.bar + # Monitoring based on goaccess + GOACCESS_ETC_PATH_LOCAL=$MLFLOW_BASE_PATH_LOCAL/goaccess-etc + GOACCESS_OUT_PATH_LOCAL=$MLFLOW_BASE_PATH_LOCAL/goaccess-out + GOACCESS_WEB_ROUTE=goaccess + GOACCESS_USER=operator + GOACCESS_PASSWORD={{ GOACCESS_PASSWORD }} # bcrypt encrypted password, use "htpasswd -B -n operator" to generate. use $$ instead of $ in GOACCESS_PASSWORD value + GOACCESS_ACCESSLOG_FORMAT=TRAEFIKCLF + + - name: Exec docker-compose up + docker_compose: + project_src: /opt/mlflow-auth-gui + state: present + files: + - compose.yml + - compose.prod.yml \ No newline at end of file diff --git a/templates/mlflow_authvm.yaml b/templates/mlflow_authvm.yaml new file mode 100644 index 0000000..97faf30 --- /dev/null +++ b/templates/mlflow_authvm.yaml @@ -0,0 +1,133 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +imports: + - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/main/custom_types.yaml + +metadata: + template_version: "1.0.0" + template_name: MLFlow Auth VM + display_name: Deploy MLFlow Auth GUI on a VM + icon: images/mlflow.png + tabs: + MLFlow: mlflow_.* + parents: + - simple-node-disk.yml + +description: Deploy MLFlow on a VM with the MLFlow Auth GUI. + +topology_template: + + inputs: + + mlflow_admin_user: + type: string + description: MLFlow Admin server Username + default: admin + required: yes + + mlflow_admin_password: + type: string + description: MLFlow Admin server Password + default: password + required: yes + + mlflow_operator_pass: + type: string + description: MLFlow Operator Password + default: 'operatorpass' + required: no + + mlflow_oidc_auth: + type: string + description: MLFlow OIDC Authority + default: 'https://aai-demo.egi.eu/auth/realms/egi' + required: no + + mlflow_oidc_client_id: + type: string + description: MLFlow OIDC Client ID + default: 'oidc-client' + required: no + + mlflow_oidc_vo: + type: string + description: MLFlow OIDC VO name + default: 'vo.ai4eosc.eu' + required: no + + mlflow_dns_name: + type: string + description: MLFlow DNS name (leave empty to use mlflow..nip.io) + default: '' + required: no + + mlflow_cert_email: + type: string + description: MLFlow LetsEncrypt certificate email + default: 'johndoe@server.com' + required: no + + # Restrict some simple-node-disk input values + + storage_size: + type: scalar-unit.size + description: Size of the extra HD added to the instance (Set 0 if disk is not needed) + default: 20 GB + constraints: + - valid_values: [ 20 GB, 50 GB, 100 GB, 200 GB, 500 GB, 1 TB, 2 TB, 10 TB, 20 TB, 40 TB, 100 TB ] + + mount_path: + type: string + description: Path to mount the extra disk + default: /opt/mlflow-auth-gui + + node_templates: + + mlflow: + type: tosca.nodes.ec3.Application + artifacts: + docker_role: + file: grycap.docker + type: tosca.artifacts.AnsibleGalaxy.role + capabilities: + endpoint: + properties: + ports: + https: + protocol: tcp + source: 443 + http: + protocol: tcp + source: 80 + interfaces: + Standard: + configure: + implementation: https://raw.githubusercontent.com/grycap/tosca/main/artifacts/mlflow_compose.yml + inputs: + public_ip_address: + get_attribute: [ simple_node, public_address, 0 ] + mlflow_admin_user: + get_input: mlflow_admin_user + mlflow_admin_password: + get_input: mlflow_admin_password + mlflow_cert_email: + get_input: mlflow_cert_email + mlflow_dns_name: + get_input: mlflow_dns_name + mlflow_oidc_auth: + get_input: mlflow_oidc_auth + mlflow_oidc_client_id: + get_input: mlflow_oidc_client_id + mlflow_oidc_vo: + get_input: mlflow_oidc_vo + mlflow_operator_pass: + get_input: mlflow_operator_pass + + requirements: + - host: simple_node + + outputs: + mlflow_nip_endpoint: + value: { concat: [ 'https://mlflow.', get_attribute: [ simple_node, public_address, 0 ], 'nip.io/signup' ] } + mlflow_dns_endpoint: + value: { concat: [ 'https://', get_input: mlflow_dns_name, '/signup' ] } diff --git a/templates/simple-node-disk.yml b/templates/simple-node-disk.yml index 771321c..55cc616 100644 --- a/templates/simple-node-disk.yml +++ b/templates/simple-node-disk.yml @@ -41,6 +41,7 @@ metadata: - image-service.yaml - ai4eoscvm.yaml - mlflowvm.yaml + - mlflow_authvm.yaml - wget.yml - sgde.yaml - dydns_egi_update_vm.yml From 83bdc08d078389bce4d6709c0f53c16dd1b6feba Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 19 Sep 2024 10:34:30 +0200 Subject: [PATCH 4/7] Fix lint issues --- artifacts/mlflow_auth_compose.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/artifacts/mlflow_auth_compose.yml b/artifacts/mlflow_auth_compose.yml index e4bf3c4..b3c3ca2 100644 --- a/artifacts/mlflow_auth_compose.yml +++ b/artifacts/mlflow_auth_compose.yml @@ -19,8 +19,8 @@ repo: https://github.com/m-team-kit/mlflow-auth-gui dest: /opt/mlflow-auth-gui version: main - ignore_errors: yes - + ignore_errors: true + - name: Set default DNS name (nip.io) set_fact: dns_name: "mlflow.{{ public_ip_address }}.nip.io" @@ -36,13 +36,15 @@ section: "mlflow" option: "{{ item.option }}" value: "{{ item.value }}" + mode: '0644' loop: - - { "option": "admin_username", "value" "{{ MLFLOW_USERNAME }}" } - - { "option": "admin_password", "value" "{{ MLFLOW_PASSWORD }}" } + - { "option": "admin_username", "value": "{{ MLFLOW_USERNAME }}" } + - { "option": "admin_password", "value": "{{ MLFLOW_PASSWORD }}" } - name: Create .env file copy: dest: /opt/mlflow-auth-gui/.env + mode: '0644' content: | # Domain where MLFLOW is hosted DOMAIN={{ dns_name }} @@ -68,7 +70,7 @@ PRIVACY_POLICY_URL=https://confluence.egi.eu/display/IMPAIP/Privacy+Policy TERMS_OF_USE_URL=https://confluence.egi.eu/display/IMPAIP/Acceptable+Use+Policy # can be left blank, but prefer putting one - LETSENCRYPT_EMAIL={{ mlflow_cert_email}} + LETSENCRYPT_EMAIL={{ mlflow_cert_email }} CORS_ORIGINS=http://localhost,http://foo.bar # Monitoring based on goaccess GOACCESS_ETC_PATH_LOCAL=$MLFLOW_BASE_PATH_LOCAL/goaccess-etc @@ -84,4 +86,4 @@ state: present files: - compose.yml - - compose.prod.yml \ No newline at end of file + - compose.prod.yml From beeacbe55f05802ee2a0932b029a9ea85212e242 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 19 Sep 2024 10:57:40 +0200 Subject: [PATCH 5/7] Fix conf errors --- artifacts/mlflow_auth_compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/artifacts/mlflow_auth_compose.yml b/artifacts/mlflow_auth_compose.yml index b3c3ca2..385601f 100644 --- a/artifacts/mlflow_auth_compose.yml +++ b/artifacts/mlflow_auth_compose.yml @@ -19,7 +19,7 @@ repo: https://github.com/m-team-kit/mlflow-auth-gui dest: /opt/mlflow-auth-gui version: main - ignore_errors: true + ignore_errors: true - name: Set default DNS name (nip.io) set_fact: @@ -58,7 +58,7 @@ AUTH_DB_FILE=basic_auth.db # Local (host) BASE PATH to store MLFLOW data, e.g. app-data, user-data, artifacts, backup_db. # Pay attention about subdirectories ownership, may need to set uid:gid of "docker" - MLFLOW_BASE_PATH_LOCAL=. + MLFLOW_BASE_PATH_LOCAL=/opt/mlflow-auth-gui # MLflow user registration service MLFLOW_USERNAME={{ MLFLOW_USERNAME }} MLFLOW_PASSWORD={{ MLFLOW_PASSWORD }} @@ -73,8 +73,8 @@ LETSENCRYPT_EMAIL={{ mlflow_cert_email }} CORS_ORIGINS=http://localhost,http://foo.bar # Monitoring based on goaccess - GOACCESS_ETC_PATH_LOCAL=$MLFLOW_BASE_PATH_LOCAL/goaccess-etc - GOACCESS_OUT_PATH_LOCAL=$MLFLOW_BASE_PATH_LOCAL/goaccess-out + GOACCESS_ETC_PATH_LOCAL=${MLFLOW_BASE_PATH_LOCAL}/goaccess-etc + GOACCESS_OUT_PATH_LOCAL=${MLFLOW_BASE_PATH_LOCAL}/goaccess-out GOACCESS_WEB_ROUTE=goaccess GOACCESS_USER=operator GOACCESS_PASSWORD={{ GOACCESS_PASSWORD }} # bcrypt encrypted password, use "htpasswd -B -n operator" to generate. use $$ instead of $ in GOACCESS_PASSWORD value From ca810086e463ae60438916852f0c79972b6a290b Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 19 Sep 2024 11:33:31 +0200 Subject: [PATCH 6/7] Fix conf errors --- artifacts/mlflow_auth_compose.yml | 2 +- templates/mlflow_authvm.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/artifacts/mlflow_auth_compose.yml b/artifacts/mlflow_auth_compose.yml index 385601f..3227fa5 100644 --- a/artifacts/mlflow_auth_compose.yml +++ b/artifacts/mlflow_auth_compose.yml @@ -81,7 +81,7 @@ GOACCESS_ACCESSLOG_FORMAT=TRAEFIKCLF - name: Exec docker-compose up - docker_compose: + community.docker.docker_compose_v2: project_src: /opt/mlflow-auth-gui state: present files: diff --git a/templates/mlflow_authvm.yaml b/templates/mlflow_authvm.yaml index 97faf30..4b12ce4 100644 --- a/templates/mlflow_authvm.yaml +++ b/templates/mlflow_authvm.yaml @@ -89,6 +89,9 @@ topology_template: docker_role: file: grycap.docker type: tosca.artifacts.AnsibleGalaxy.role + docker_collection: + file: community.docker,3.12.2 + type: tosca.artifacts.AnsibleGalaxy.collection capabilities: endpoint: properties: From 79493766912f40db03b0017b5d4dc520b8ac8ce9 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 19 Sep 2024 11:45:02 +0200 Subject: [PATCH 7/7] fix test --- tests/requirements.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/requirements.yaml b/tests/requirements.yaml index e6581be..3a1cd9a 100644 --- a/tests/requirements.yaml +++ b/tests/requirements.yaml @@ -32,3 +32,5 @@ roles: collections: - name: community.crypto + - name: community.docker + version: 3.12.2