From ee2d32f727681ecdb01e3a543f87df614c03534f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 6 Nov 2023 18:34:21 -0800 Subject: [PATCH] Try using blockcache --- titiler/xarray/factory.py | 1 + titiler/xarray/main.py | 2 -- titiler/xarray/reader.py | 11 +++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/titiler/xarray/factory.py b/titiler/xarray/factory.py index 92f75dd..d82800e 100644 --- a/titiler/xarray/factory.py +++ b/titiler/xarray/factory.py @@ -255,6 +255,7 @@ def tiles_endpoint( # type: ignore x, y, z, tilesize=scale * 256, nodata=src_dst.input.rio.nodata ) + if post_process: image = post_process(image) diff --git a/titiler/xarray/main.py b/titiler/xarray/main.py index 1b684e8..6319a69 100644 --- a/titiler/xarray/main.py +++ b/titiler/xarray/main.py @@ -3,8 +3,6 @@ import logging import os import shutil -import sys -sys.path.appned('/mnt/efs/libraries') import rioxarray import zarr diff --git a/titiler/xarray/reader.py b/titiler/xarray/reader.py index 61019d7..4258592 100644 --- a/titiler/xarray/reader.py +++ b/titiler/xarray/reader.py @@ -37,9 +37,14 @@ def get_cache_args(protocol: str) -> Dict[str, Any]: """ Get the cache arguments for the given protocol. """ + if protocol == "s3": + directory = f"{api_settings.fsspec_cache_directory}/blockcache" + else: + directory = f"{api_settings.fsspec_cache_directory}/filecache" + return { "target_protocol": protocol, - "cache_storage": api_settings.fsspec_cache_directory, + "cache_storage": directory, "remote_options": {"anon": True}, } @@ -70,7 +75,7 @@ def get_filesystem( """ if protocol == "s3": s3_filesystem = ( - fsspec.filesystem("filecache", **get_cache_args(protocol)) + fsspec.filesystem("blockcache", **get_cache_args(protocol)) if enable_fsspec_cache else s3fs.S3FileSystem() ) @@ -132,8 +137,6 @@ def xarray_open_dataset( "decode_coords": "all", "decode_times": decode_times, "engine": xr_engine, - "cache": False, - "chunks": {}, # loads the dataset with dask using engine preferred chunks if exposed by the backend } # Argument if we're opening a datatree