Add wait loop after write_assets(). Open-EO/openeo-geotrellis-extensi…

…ons#329

openeogeotrellis/deploy/batch_job.py

@@ -49,6 +49,7 @@
     add_permissions,
     json_default,
     to_jsonable,
+    wait_till_path_available,
 )
 
 logger = logging.getLogger('openeogeotrellis.deploy.batch_job')
@@ -354,6 +355,7 @@ def run_job(
                 ml_model_metadata = result.get_model_metadata(str(output_file))
                 logger.info("Extracted ml model metadata from %s" % output_file)
             for name, asset in the_assets_metadata.items():
+                wait_till_path_available(Path(asset["href"]))
                 add_permissions(Path(asset["href"]), stat.S_IWGRP)
             logger.info(f"wrote {len(the_assets_metadata)} assets to {output_file}")
             assets_metadata.append(the_assets_metadata)

openeogeotrellis/utils.py

@@ -13,6 +13,7 @@
 import resource
 import stat
 import tempfile
+import time
 from functools import partial
 from pathlib import Path
 from typing import Callable, Iterable, Optional, Tuple, Union, Dict, Any, TypeVar
@@ -768,3 +769,16 @@ def to_jsonable(x):
         return [to_jsonable(elem) for elem in x]
 
     return x
+
+
+def wait_till_path_available(path: Path):
+    retry = 0
+    max_tries = 5
+    while not os.path.exists(path):
+        if retry < max_tries:
+            retry += 1
+            time.sleep(10)
+            logger.info(f"Waiting for path to be available. Try {retry}/{max_tries}: {path}")
+        else:
+            logger.warning(f"Path is not available after {max_tries} tries: {path}")
+            return  # TODO: Throw error instead