Skip to content

Commit

Permalink
add back PTransform
Browse files Browse the repository at this point in the history
  • Loading branch information
ranchodeluxe committed Feb 15, 2024
1 parent e9df873 commit 3acc761
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 6 deletions.
11 changes: 7 additions & 4 deletions pangeo_forge_runner/commands/bake.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from pathlib import Path

import escapism
from apache_beam import Pipeline
from apache_beam import Pipeline, PTransform
from traitlets import Bool, Type, Unicode, validate

from .. import Feedstock
Expand Down Expand Up @@ -266,10 +266,13 @@ def start(self):
# Set argv explicitly to empty so Apache Beam doesn't try to parse the commandline
# for pipeline options - we have traitlets doing that for us.
pipeline = Pipeline(options=pipeline_options, argv=[])

# Chain our recipe to the pipeline. This mutates the `pipeline` object!
# We expect `recipe` to either be a beam PTransform, or an object with a 'to_beam'
# method that returns a transform.
pipeline | recipe
# We expect `recipe` to be 1) a beam PTransform or 2) or a a string that leverages the
# `dict_object:` see `tests/test-data/gpcp-from-gcs/feedstock-0.10.x-dictobj/meta.yaml`
# as an example
if isinstance(recipe, PTransform):
pipeline | recipe

# Some bakeries are blocking - if Beam is configured to use them, calling
# pipeline.run() blocks. Some are not. We handle that here, and provide
Expand Down
3 changes: 1 addition & 2 deletions tests/unit/test_bake.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,9 @@ def test_gpcp_bake(
no_input_cache,
recipes_version_ref,
):
if recipes_version_ref == "0.9.x-dictobj" or (
if (
recipes_version_ref == "0.10.x-dictobj" and recipe_id
):
# TODO: clarify fixturing story to avoid this hackiness
pytest.skip(
"We only test dictobjs for recipes >0.10.0, and without recipe_id's"
)
Expand Down

0 comments on commit 3acc761

Please sign in to comment.