forked from apache/airflow
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement KubernetesInstallKueueOperator +
KubernetesStartKueueJobOperator and refactor GKE operatores
- Loading branch information
1 parent
35000c9
commit 1967094
Showing
11 changed files
with
1,716 additions
and
2,440 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
105 changes: 105 additions & 0 deletions
105
providers/src/airflow/providers/cncf/kubernetes/operators/kueue.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
# Licensed to the Apache Software Foundation (ASF) under one | ||
# or more contributor license agreements. See the NOTICE file | ||
# distributed with this work for additional information | ||
# regarding copyright ownership. The ASF licenses this file | ||
# to you under the Apache License, Version 2.0 (the | ||
# "License"); you may not use this file except in compliance | ||
# with the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, | ||
# software distributed under the License is distributed on an | ||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
# KIND, either express or implied. See the License for the | ||
# specific language governing permissions and limitations | ||
# under the License. | ||
"""Manage a Kubernetes Kueue.""" | ||
|
||
from __future__ import annotations | ||
|
||
import json | ||
import warnings | ||
from collections.abc import Sequence | ||
from functools import cached_property | ||
|
||
from kubernetes.utils import FailToCreateError | ||
|
||
from airflow.exceptions import AirflowException | ||
from airflow.models import BaseOperator | ||
from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook | ||
from airflow.providers.cncf.kubernetes.operators.job import KubernetesJobOperator | ||
|
||
|
||
class KubernetesInstallKueueOperator(BaseOperator): | ||
""" | ||
Installs a Kubernetes Kueue. | ||
:param kueue_version: The Kubernetes Kueue version to install. | ||
:param kubernetes_conn_id: The :ref:`kubernetes connection id <howto/connection:kubernetes>` | ||
for the Kubernetes cluster. | ||
""" | ||
|
||
template_fields: Sequence[str] = ( | ||
"kueue_version", | ||
"kubernetes_conn_id", | ||
) | ||
|
||
def __init__(self, kueue_version: str, kubernetes_conn_id: str = "kubernetes_default", *args, **kwargs): | ||
super().__init__(*args, **kwargs) | ||
self.kubernetes_conn_id = kubernetes_conn_id | ||
self.kueue_version = kueue_version | ||
self._kueue_yaml_url = ( | ||
f"https://github.com/kubernetes-sigs/kueue/releases/download/{self.kueue_version}/manifests.yaml" | ||
) | ||
|
||
@cached_property | ||
def hook(self) -> KubernetesHook: | ||
return KubernetesHook(conn_id=self.kubernetes_conn_id) | ||
|
||
def execute(self, context): | ||
yaml_objects = self.hook.get_yaml_content_from_file(kueue_yaml_url=self._kueue_yaml_url) | ||
try: | ||
self.hook.apply_from_yaml_file(yaml_objects=yaml_objects) | ||
except FailToCreateError as ex: | ||
error_bodies = [json.loads(e.body) for e in ex.api_exceptions] | ||
if next((e for e in error_bodies if e.get("reason") == "AlreadyExists"), None): | ||
self.log.info("Kueue is already enabled for the cluster") | ||
|
||
if errors := [e for e in error_bodies if e.get("reason") != "AlreadyExists"]: | ||
error_message = "\n".join(e.get("body") for e in errors) | ||
raise AirflowException(error_message) | ||
return | ||
|
||
self.hook.check_kueue_deployment_running(name="kueue-controller-manager", namespace="kueue-system") | ||
self.log.info("Kueue installed successfully!") | ||
|
||
|
||
class KubernetesStartKueueJobOperator(KubernetesJobOperator): | ||
""" | ||
Executes a Kubernetes Job in Kueue. | ||
:param queue_name: The name of the Queue in the cluster | ||
""" | ||
|
||
template_fields = tuple({"queue_name"} | set(KubernetesJobOperator.template_fields)) | ||
|
||
def __init__(self, queue_name: str, *args, **kwargs) -> None: | ||
super().__init__(*args, **kwargs) | ||
self.queue_name = queue_name | ||
|
||
if self.suspend is False: | ||
raise AirflowException( | ||
"The `suspend` parameter can't be False. If you want to use Kueue for running Job" | ||
" in a Kubernetes cluster, set the `suspend` parameter to True.", | ||
) | ||
elif self.suspend is None: | ||
warnings.warn( | ||
f"You have not set parameter `suspend` in class {self.__class__.__name__}. " | ||
"For running a Job in Kueue the `suspend` parameter should set to True.", | ||
UserWarning, | ||
stacklevel=2, | ||
) | ||
self.suspend = True | ||
self.labels.update({"kueue.x-k8s.io/queue-name": self.queue_name}) | ||
self.annotations.update({"kueue.x-k8s.io/queue-name": self.queue_name}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.