From ff54557e79f8b3abfe352d1560cbfb856395f05d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Leroy <82111151+leroyjb@users.noreply.github.com> Date: Tue, 11 Feb 2025 19:56:47 +0000 Subject: [PATCH] add dws multiclusters example folder (#949) * add dws multiclusters example folder * add terraform support * update README Terraform command --- .../dws-multiclusters-example/.gitignore | 1 + .../dws-multiclusters-example/README.md | 80 ++++++ .../create-clusters.sh | 70 ++++++ .../create-multikueue-kubeconfig.sh | 232 ++++++++++++++++++ .../deploy-multikueue.sh | 70 ++++++ .../dws-multi-worker.yaml | 54 ++++ .../dws-multiclusters-example/dws-multi.yaml | 94 +++++++ .../job-multi-dws-autopilot.yaml | 39 +++ .../dws-multiclusters-example/prom.yaml | 16 ++ .../dws-multiclusters-example/tf/clusters.tf | 67 +++++ .../dws-multiclusters-example/tf/outputs.tf | 10 + .../dws-multiclusters-example/tf/variables.tf | 41 ++++ .../dws-multiclusters-example/tf/versions.tf | 22 ++ 13 files changed, 796 insertions(+) create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/.gitignore create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/README.md create mode 100755 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-clusters.sh create mode 100755 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-multikueue-kubeconfig.sh create mode 100755 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/deploy-multikueue.sh create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/dws-multi-worker.yaml create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/dws-multi.yaml create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/job-multi-dws-autopilot.yaml create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/prom.yaml create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/clusters.tf create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/outputs.tf create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/variables.tf create mode 100644 tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/versions.tf diff --git a/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/.gitignore b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/.gitignore new file mode 100644 index 000000000..7cc402c7a --- /dev/null +++ b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/.gitignore @@ -0,0 +1 @@ +*.kubeconfig diff --git a/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/README.md b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/README.md new file mode 100644 index 000000000..ca1f65387 --- /dev/null +++ b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/README.md @@ -0,0 +1,80 @@ +# Multikueue-dws-integration + +This repository provides the files needed to demonstrate how to use MultiKueue with Dynamic Workload Scheduler (DWS) GKE Autopilot. This setup allows you to run workloads across multiple GKE clusters in different regions, automatically leveraging available GPU resources thanks to DWS. + +## Repository Contents + +This repository contains the following files: + +* `create-clusters.sh`: Script to create the required GKE clusters (one manager and three workers). +* `tf folder`: contains the terraform script to create the required GKE clusters (one manager and three workers). You can use it instead of the bash script. +* `deploy-multikueue.sh`: Script to install and configure Kueue and MultiKueue on the clusters. +* `dws-multi-worker.yaml`: Kueue configuration for the worker clusters, including manager configuration. +* `job-multi-dws-autopilot.yaml`: Example job definition to be submitted to the MultiKueue setup. + +## Setup and Usage + +### Create Clusters + +``` +terraform -chdir=tf init +terraform -chdir=tf plan +terraform -chdir=tf apply -var project_id= +``` + +### Install Kueue + +After creating the GKE clusters and updating your kubeconfig files, install the Kueue components: + +``` +./deploy-multikueue.sh +``` + +### Validate installation + +Verify the Kueue installation and the connection between the manager and worker clusters: + +``` +kubectl get clusterqueues dws-cluster-queue -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}CQ - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}" +kubectl get admissionchecks sample-dws-multikueue -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}AC - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}" +kubectl get multikueuecluster multikueue-dws-worker-asia -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}MC-ASIA - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}" +kubectl get multikueuecluster multikueue-dws-worker-us -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}MC-US - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}" +kubectl get multikueuecluster multikueue-dws-worker-eu -o jsonpath="{range .status.conditions[?(@.type == \"Active\")]}MC-EU - Active: {@.status} Reason: {@.reason} Message: {@.message}{'\n'}{end}" +``` + +A successful output should look like this: + +``` +CQ - Active: True Reason: Ready Message: Can admit new workloads +AC - Active: True Reason: Active Message: The admission check is active +MC-ASIA - Active: True Reason: Active Message: Connected +MC-US - Active: True Reason: Active Message: Connected +MC-EU - Active: True Reason: Active Message: Connected +``` + +### Launch job + +Submit your job to the Kueue controller, which will run it on a worker cluster with available resources: + +``` +kubectl create -f job-multi-dws-autopilot.yaml +``` + +### Get the status of the job + +To check the job status and see where it's scheduled: + +``` +kubectl get workloads.kueue.x-k8s.io -o jsonpath='{range .items[*]}{.status.admissionChecks}{"\n"}{end}' +``` + +In the output message, you can find where the job is scheduled# + +### Destroy resources + + +``` +terraform -chdir=tf destroy -var project_id= +``` + + diff --git a/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-clusters.sh b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-clusters.sh new file mode 100755 index 000000000..c9d46e52a --- /dev/null +++ b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-clusters.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +echo 'Create GKE Autopilot clusters' + +KUEUE_VERSION=v0.8.1 +regions=("europe-west4" "asia-southeast1" "us-east4" "europe-west4") +kubeconfigs=("manager-europe-west4" "worker-asia-southeast1" "worker-us-east4" "worker-eu-west4") +PROJECT_ID=$(gcloud config get-value project) +PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)") +PREFIX_MANAGER="man" +PREFIX_WORKER="w" +JOBSET_VERSION=v0.6.0 + +# Loop through the regions +for i in "${!regions[@]}"; do + region="${regions[$i]}" + echo "$region" + # Construct the cluster name, adding "manager" if it's the first region + if [[ $i -eq 0 ]]; then + cluster_name="$PREFIX_MANAGER-$region" + else + cluster_name="$PREFIX_WORKER-$region" + fi + + #Create the cluster + gcloud container clusters create-auto "$cluster_name" \ + --project "$PROJECT_ID" \ + --region "$region" \ + --release-channel "regular" \ + --async +done +for i in "${!regions[@]}"; do + region="${regions[$i]}" + if [[ $i -eq 0 ]]; then + cluster_name="$PREFIX_MANAGER-$region" + else + cluster_name="$PREFIX_WORKER-$region" + fi + + opId=$(gcloud container operations list --filter "TARGET=https://container.googleapis.com/v1/projects/$PROJECT_NUMBER/locations/$region/clusters/$cluster_name" --format="value(name)") + gcloud container operations wait "$opId" --project "$PROJECT_ID" --region "$region" + set +e + until gcloud -q container clusters get-credentials "$cluster_name" \ + --project "$PROJECT_ID" \ + --region "$region"; do + echo "GKE Cluster is provisioning. Retrying in 15 seconds..." + sleep 15 + done + set -e + configname="${kubeconfigs[$i]}" + kubectl config rename-context "gke_$PROJECT_ID"_"$region"_"$cluster_name" "$configname" +done diff --git a/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-multikueue-kubeconfig.sh b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-multikueue-kubeconfig.sh new file mode 100755 index 000000000..75c392b16 --- /dev/null +++ b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/create-multikueue-kubeconfig.sh @@ -0,0 +1,232 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBECONFIG_OUT=${1:-kubeconfig} +MULTIKUEUE_SA=multikueue-sa +NAMESPACE=kueue-system + +# Creating a restricted MultiKueue role, service account and role binding" +kubectl apply -f - <"${KUBECONFIG_OUT}" < { + region = region + name = "${var.cluster_worker_names_prefix}-${region}" # Use prefix and region + } + } + + name = each.value.name + location = each.value.region + network = google_compute_network.network.id # Reference the SINGLE network + enable_autopilot = true + deletion_protection = false + +} + +# Get the kubeconfig for each cluster and update the context +resource "null_resource" "update_kubeconfig" { + for_each = google_container_cluster.autopilot_worker_clusters + + provisioner "local-exec" { + command = < { + name = cluster.name + region = cluster.location + network = cluster.network + subnet = cluster.subnetwork + } + } +} diff --git a/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/variables.tf b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/variables.tf new file mode 100644 index 000000000..65f6a03dd --- /dev/null +++ b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/variables.tf @@ -0,0 +1,41 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "project_id" { + type = string + description = "GCP project ID" +} + +variable "location_manager" { + type = string + description = "Location of GKE cluster" + default = "europe-west4" +} +variable "cluster_manager_name_prefix" { + type = string + description = "Prefix of MultiKueue Manager GKE cluster" + default = "manager" +} + +variable "regions_workers" { + type = list(string) + default = ["europe-west4", "asia-southeast1", "us-east4"] +} + +variable "cluster_worker_names_prefix" { + type = string + description = "Prefix of MultiKueue Workers GKE cluster" + default = "worker" +} + diff --git a/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/versions.tf b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/versions.tf new file mode 100644 index 000000000..4daa3eca5 --- /dev/null +++ b/tutorials-and-examples/workflow-orchestration/dws-multiclusters-example/tf/versions.tf @@ -0,0 +1,22 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + + } +}