From 3acc761f606a3cd92806419032cfba90e9b72b42 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Thu, 15 Feb 2024 11:14:47 -0800 Subject: [PATCH] add back PTransform --- pangeo_forge_runner/commands/bake.py | 11 +++++++---- tests/unit/test_bake.py | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pangeo_forge_runner/commands/bake.py b/pangeo_forge_runner/commands/bake.py index 942a2b2..c2024d3 100644 --- a/pangeo_forge_runner/commands/bake.py +++ b/pangeo_forge_runner/commands/bake.py @@ -11,7 +11,7 @@ from pathlib import Path import escapism -from apache_beam import Pipeline +from apache_beam import Pipeline, PTransform from traitlets import Bool, Type, Unicode, validate from .. import Feedstock @@ -266,10 +266,13 @@ def start(self): # Set argv explicitly to empty so Apache Beam doesn't try to parse the commandline # for pipeline options - we have traitlets doing that for us. pipeline = Pipeline(options=pipeline_options, argv=[]) + # Chain our recipe to the pipeline. This mutates the `pipeline` object! - # We expect `recipe` to either be a beam PTransform, or an object with a 'to_beam' - # method that returns a transform. - pipeline | recipe + # We expect `recipe` to be 1) a beam PTransform or 2) or a a string that leverages the + # `dict_object:` see `tests/test-data/gpcp-from-gcs/feedstock-0.10.x-dictobj/meta.yaml` + # as an example + if isinstance(recipe, PTransform): + pipeline | recipe # Some bakeries are blocking - if Beam is configured to use them, calling # pipeline.run() blocks. Some are not. We handle that here, and provide diff --git a/tests/unit/test_bake.py b/tests/unit/test_bake.py index dfb4cad..210f0c5 100644 --- a/tests/unit/test_bake.py +++ b/tests/unit/test_bake.py @@ -144,10 +144,9 @@ def test_gpcp_bake( no_input_cache, recipes_version_ref, ): - if recipes_version_ref == "0.9.x-dictobj" or ( + if ( recipes_version_ref == "0.10.x-dictobj" and recipe_id ): - # TODO: clarify fixturing story to avoid this hackiness pytest.skip( "We only test dictobjs for recipes >0.10.0, and without recipe_id's" )