Disable fsspec caching (#3)

* Remove storage options parameter

* Create store module

virtualize_nex_gddp_cmip6/run.py

@@ -2,15 +2,9 @@
 
 import xarray as xr
 from distributed import Client
-from icechunk import (
-    IcechunkStore,
-    StorageConfig,
-    StoreConfig,
-    VirtualRefConfig,
-)
-from virtualizarr.writers.icechunk import dataset_to_icechunk
 
 from virtualize_nex_gddp_cmip6.inventory import get_uris
+from virtualize_nex_gddp_cmip6.store import store_virtual_dataset
 from virtualize_nex_gddp_cmip6.virtualize import combine_virtual_datasets, execute_tasks, generate_tasks
 
 
@@ -29,25 +23,6 @@ def execute_dataset_virtualization(uris: list[str]) -> xr.Dataset:
     return vds
 
 
-def store_virtual_dataset(vds: xr.Dataset, name: tuple[str, str, str, str]) -> None:
-    """Store virtual dataset in a icechunk virtual reference store
-
-    Args:
-        vds (xr.Dataset): virtual dataset
-        name (tuple[str, str, str, str]): group name in the form of (GCM, scenario, model, variable)
-    """
-    storage = StorageConfig.s3_from_env(
-        bucket="nasa-veda-scratch",
-        prefix=f"virtualizarr-test/maxrjones/{name[0]}/{name[1]}/{name[2]}/{name[3]}",
-    )
-    config = StoreConfig(
-        virtual_ref_config=VirtualRefConfig.s3_anonymous(),
-    )
-    virtual_store = IcechunkStore.open_or_create(storage=storage, config=config, mode="w")
-    dataset_to_icechunk(vds, virtual_store)
-    virtual_store.commit("Create refenence dataset")
-
-
 def virtualize():
     """Virtualize the NASA NEX-GDDP-CMIP6 datasets and store in icechunk"""
     df = get_uris()

virtualize_nex_gddp_cmip6/store.py

@@ -0,0 +1,27 @@
+import xarray as xr
+from icechunk import (
+    IcechunkStore,
+    StorageConfig,
+    StoreConfig,
+    VirtualRefConfig,
+)
+from virtualizarr.writers.icechunk import dataset_to_icechunk
+
+
+def store_virtual_dataset(vds: xr.Dataset, name: tuple[str, str, str, str]) -> None:
+    """Store virtual dataset in a icechunk virtual reference store
+
+    Args:
+        vds (xr.Dataset): virtual dataset
+        name (tuple[str, str, str, str]): group name in the form of (GCM, scenario, model, variable)
+    """
+    storage = StorageConfig.s3_from_env(
+        bucket="nasa-veda-scratch",
+        prefix=f"virtualizarr-test/maxrjones/{name[0]}/{name[1]}/{name[2]}/{name[3]}",
+    )
+    config = StoreConfig(
+        virtual_ref_config=VirtualRefConfig.s3_anonymous(),
+    )
+    virtual_store = IcechunkStore.open_or_create(storage=storage, config=config, mode="w")
+    dataset_to_icechunk(vds, virtual_store)
+    virtual_store.commit("Create refenence dataset")

virtualize_nex_gddp_cmip6/virtualize.py

@@ -4,16 +4,15 @@
 from virtualizarr import open_virtual_dataset
 
 
-def generate_virtual_dataset(file: str, storage_options: dict) -> xr.Dataset:
+def generate_virtual_dataset(file: str) -> xr.Dataset:
     """Generate a virtual dataset for a NetCDF file
 
     Args:
         file (str): Dataset URI
-        storage_options (dict): Options to pass through to fsspec for reading the dataset
-
     Returns:
         xr.Dataset: virtual dataset containing metadata and data references
     """
+    storage_options = {"anon": True, "default_fill_cache": False, "default_cache_type": "none"}
     return open_virtual_dataset(file, indexes={}, reader_options={"storage_options": storage_options})
 
 
@@ -26,8 +25,7 @@ def generate_tasks(uris: list[str]) -> list[Delayed]:
     Returns:
         list[Delayed]: List of dask delayed objects for virtualized datasets
     """
-    storage_options = {"anon": True, "default_fill_cache": False, "default_cache_type": "first"}
-    tasks = [dask.delayed(generate_virtual_dataset)(file, storage_options) for file in uris]
+    tasks = [dask.delayed(generate_virtual_dataset)(file) for file in uris]
     return tasks