Try dask again

developmentseed · Nov 7, 2023 · 1c44089 · 1c44089
1 parent 83378e1
commit 1c44089
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 8 deletions.
diff --git a/infrastructure/aws/lambda/Dockerfile b/infrastructure/aws/lambda/Dockerfile
@@ -26,6 +26,8 @@ RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
 RUN rm -rdf /asset/numpy/doc/ /asset/bin /asset/geos_license /asset/Misc
 RUN rm -rdf /asset/boto3*
 RUN rm -rdf /asset/botocore*
+RUN rm -rdf /asset/dask/tests*
+RUN rm -rdf /asset/dask/dataframe*
 
 COPY infrastructure/aws/lambda/handler.py /asset/handler.py
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -26,6 +26,7 @@ classifiers = [
 dynamic = ["version"]
 dependencies = [
     "cftime",
+    "dask",
     "h5netcdf",
     "xarray",
     "rioxarray",

diff --git a/titiler/xarray/reader.py b/titiler/xarray/reader.py
@@ -37,10 +37,7 @@ def get_cache_args(protocol: str) -> Dict[str, Any]:
     """
     Get the cache arguments for the given protocol.
     """
-    if protocol == "s3":
-        directory = f"{api_settings.fsspec_cache_directory}/blockcache"
-    else:
-        directory = f"{api_settings.fsspec_cache_directory}/filecache"
+    directory = f"{api_settings.fsspec_cache_directory}/filecache"
 
     return {
         "target_protocol": protocol,
@@ -75,7 +72,7 @@ def get_filesystem(
     """
     if protocol == "s3":
         s3_filesystem = (
-            fsspec.filesystem("blockcache", **get_cache_args(protocol))
+            fsspec.filesystem("filecache", **get_cache_args(protocol))
             if enable_fsspec_cache
             else s3fs.S3FileSystem()
         )
@@ -137,6 +134,7 @@ def xarray_open_dataset(
         "decode_coords": "all",
         "decode_times": decode_times,
         "engine": xr_engine,
+        "chunks": 'auto'
     }
 
     # Argument if we're opening a datatree
@@ -155,7 +153,6 @@ def xarray_open_dataset(
     if reference:
         xr_open_args["consolidated"] = False
         xr_open_args["backend_kwargs"] = {"consolidated": False}
-
     ds = xarray.open_dataset(file_handler, **xr_open_args)
     return ds
 
@@ -261,8 +258,7 @@ class ZarrReader(XarrayReader):
     def __attrs_post_init__(self):
         """Set bounds and CRS."""
         self.ds = self._ctx_stack.enter_context(
-            xarray_open_dataset(
-                self.src_path,
+            xarray_open_dataset(self.src_path,
                 group=self.group,
                 reference=self.reference,
                 consolidated=self.consolidated,