From 92037010c7b4944b6893decde7ab4f605678fee5 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 22 Oct 2024 11:14:56 -0400 Subject: [PATCH 1/3] Fix lint warning and import error in data_types_and_io tf example Signed-off-by: Eduardo Apolinario --- examples/data_types_and_io/data_types_and_io/tensorflow_type.py | 2 +- examples/kfmpi_plugin/README.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/data_types_and_io/data_types_and_io/tensorflow_type.py b/examples/data_types_and_io/data_types_and_io/tensorflow_type.py index 349f34b67..5cc9622d8 100644 --- a/examples/data_types_and_io/data_types_and_io/tensorflow_type.py +++ b/examples/data_types_and_io/data_types_and_io/tensorflow_type.py @@ -1,6 +1,6 @@ # Import necessary libraries and modules -from flytekit import task, workflow +from flytekit import ImageSpec, task, workflow from flytekit.types.directory import TFRecordsDirectory from flytekit.types.file import TFRecordFile diff --git a/examples/kfmpi_plugin/README.md b/examples/kfmpi_plugin/README.md index 0a43ff0ba..1eeeac68b 100644 --- a/examples/kfmpi_plugin/README.md +++ b/examples/kfmpi_plugin/README.md @@ -88,4 +88,3 @@ If your MPI workflow hangs or times out, it may be caused by an incorrect workfl 1. Verify Registration Method: When using a custom image, refer to the Flyte documentation on [Registering workflows](https://docs.flyte.org/en/latest/user_guide/flyte_fundamentals/registering_workflows.html#registration-patterns) to ensure you're following the correct registration method. - From f93908211e05f7bd6d262e5e1513fb40f5d3ce45 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 22 Oct 2024 11:40:28 -0400 Subject: [PATCH 2/3] Remove use of is_container in tensorflow_type.py example Signed-off-by: Eduardo Apolinario --- .../data_types_and_io/tensorflow_type.py | 89 +++++++++---------- examples/data_types_and_io/requirements.in | 1 + 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/examples/data_types_and_io/data_types_and_io/tensorflow_type.py b/examples/data_types_and_io/data_types_and_io/tensorflow_type.py index 5cc9622d8..fe364315f 100644 --- a/examples/data_types_and_io/data_types_and_io/tensorflow_type.py +++ b/examples/data_types_and_io/data_types_and_io/tensorflow_type.py @@ -9,48 +9,47 @@ registry="ghcr.io/flyteorg", ) -if custom_image.is_container(): - import tensorflow as tf - - # TensorFlow Model - @task - def train_model() -> tf.keras.Model: - model = tf.keras.Sequential( - [tf.keras.layers.Dense(128, activation="relu"), tf.keras.layers.Dense(10, activation="softmax")] - ) - model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"]) - return model - - @task - def evaluate_model(model: tf.keras.Model, x: tf.Tensor, y: tf.Tensor) -> float: - loss, accuracy = model.evaluate(x, y) - return accuracy - - @workflow - def training_workflow(x: tf.Tensor, y: tf.Tensor) -> float: - model = train_model() - return evaluate_model(model=model, x=x, y=y) - - # TFRecord Files - @task - def process_tfrecord(file: TFRecordFile) -> int: - count = 0 - for record in tf.data.TFRecordDataset(file): - count += 1 - return count - - @workflow - def tfrecord_workflow(file: TFRecordFile) -> int: - return process_tfrecord(file=file) - - # TFRecord Directories - @task - def process_tfrecords_dir(dir: TFRecordsDirectory) -> int: - count = 0 - for record in tf.data.TFRecordDataset(dir.path): - count += 1 - return count - - @workflow - def tfrecords_dir_workflow(dir: TFRecordsDirectory) -> int: - return process_tfrecords_dir(dir=dir) +import tensorflow as tf + +# TensorFlow Model +@task +def train_model() -> tf.keras.Model: + model = tf.keras.Sequential( + [tf.keras.layers.Dense(128, activation="relu"), tf.keras.layers.Dense(10, activation="softmax")] + ) + model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"]) + return model + +@task +def evaluate_model(model: tf.keras.Model, x: tf.Tensor, y: tf.Tensor) -> float: + loss, accuracy = model.evaluate(x, y) + return accuracy + +@workflow +def training_workflow(x: tf.Tensor, y: tf.Tensor) -> float: + model = train_model() + return evaluate_model(model=model, x=x, y=y) + +# TFRecord Files +@task +def process_tfrecord(file: TFRecordFile) -> int: + count = 0 + for record in tf.data.TFRecordDataset(file): + count += 1 + return count + +@workflow +def tfrecord_workflow(file: TFRecordFile) -> int: + return process_tfrecord(file=file) + +# TFRecord Directories +@task +def process_tfrecords_dir(dir: TFRecordsDirectory) -> int: + count = 0 + for record in tf.data.TFRecordDataset(dir.path): + count += 1 + return count + +@workflow +def tfrecords_dir_workflow(dir: TFRecordsDirectory) -> int: + return process_tfrecords_dir(dir=dir) diff --git a/examples/data_types_and_io/requirements.in b/examples/data_types_and_io/requirements.in index 79bd303e5..2bcce8b12 100644 --- a/examples/data_types_and_io/requirements.in +++ b/examples/data_types_and_io/requirements.in @@ -1,4 +1,5 @@ pandas torch tabulate +tensorflow pyarrow From d12fff1ed549adf22a0682ae43d67467bfbca95f Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 22 Oct 2024 17:21:06 -0400 Subject: [PATCH 3/3] Fix lint warning Signed-off-by: Eduardo Apolinario --- .../data_types_and_io/data_types_and_io/tensorflow_type.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/data_types_and_io/data_types_and_io/tensorflow_type.py b/examples/data_types_and_io/data_types_and_io/tensorflow_type.py index fe364315f..3ec8aea71 100644 --- a/examples/data_types_and_io/data_types_and_io/tensorflow_type.py +++ b/examples/data_types_and_io/data_types_and_io/tensorflow_type.py @@ -11,6 +11,7 @@ import tensorflow as tf + # TensorFlow Model @task def train_model() -> tf.keras.Model: @@ -20,16 +21,19 @@ def train_model() -> tf.keras.Model: model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"]) return model + @task def evaluate_model(model: tf.keras.Model, x: tf.Tensor, y: tf.Tensor) -> float: loss, accuracy = model.evaluate(x, y) return accuracy + @workflow def training_workflow(x: tf.Tensor, y: tf.Tensor) -> float: model = train_model() return evaluate_model(model=model, x=x, y=y) + # TFRecord Files @task def process_tfrecord(file: TFRecordFile) -> int: @@ -38,10 +42,12 @@ def process_tfrecord(file: TFRecordFile) -> int: count += 1 return count + @workflow def tfrecord_workflow(file: TFRecordFile) -> int: return process_tfrecord(file=file) + # TFRecord Directories @task def process_tfrecords_dir(dir: TFRecordsDirectory) -> int: @@ -50,6 +56,7 @@ def process_tfrecords_dir(dir: TFRecordsDirectory) -> int: count += 1 return count + @workflow def tfrecords_dir_workflow(dir: TFRecordsDirectory) -> int: return process_tfrecords_dir(dir=dir)