Skip to content

Commit

Permalink
fix zarr pyramid tests
Browse files Browse the repository at this point in the history
  • Loading branch information
vincentsarago committed Nov 6, 2024
1 parent 021f609 commit c9bbdd5
Show file tree
Hide file tree
Showing 17 changed files with 35 additions and 19 deletions.
26 changes: 21 additions & 5 deletions src/titiler/xarray/tests/fixtures/generate_fixtures.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -107,9 +107,25 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 21,
"metadata": {},
"outputs": [],
"outputs": [
{
"ename": "FileNotFoundError",
"evalue": "[Errno 2] No such file or directory",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/var/folders/5m/9w3sxz2n1d12vyk8k_cz76w40000gn/T/ipykernel_72652/1958665632.py\u001b[0m in \u001b[0;36m<cell line: 23>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdec\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_zarr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"./pyramid.zarr\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"w\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m~/Dev/venv/py39/lib/python3.9/site-packages/xarray/core/dataset.py\u001b[0m in \u001b[0;36mto_zarr\u001b[0;34m(self, store, chunk_store, mode, synchronizer, group, encoding, compute, consolidated, append_dim, region, safe_chunks, storage_options, zarr_version, write_empty_chunks, chunkmanager_store_kwargs)\u001b[0m\n\u001b[1;32m 2551\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mxarray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackends\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapi\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mto_zarr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2552\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2553\u001b[0;31m return to_zarr( # type: ignore[call-overload,misc]\n\u001b[0m\u001b[1;32m 2554\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2555\u001b[0m \u001b[0mstore\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Dev/venv/py39/lib/python3.9/site-packages/xarray/backends/api.py\u001b[0m in \u001b[0;36mto_zarr\u001b[0;34m(dataset, store, chunk_store, mode, synchronizer, group, encoding, compute, consolidated, append_dim, region, safe_chunks, storage_options, zarr_version, write_empty_chunks, chunkmanager_store_kwargs)\u001b[0m\n\u001b[1;32m 1622\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1623\u001b[0m \u001b[0;31m# expand str and path-like arguments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1624\u001b[0;31m \u001b[0mstore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_normalize_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1625\u001b[0m \u001b[0mchunk_store\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_normalize_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk_store\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1626\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/Dev/venv/py39/lib/python3.9/site-packages/xarray/backends/common.py\u001b[0m in \u001b[0;36m_normalize_path\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_remote_uri\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabspath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpanduser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/homebrew/Cellar/[email protected]/3.9.19/Frameworks/Python.framework/Versions/3.9/lib/python3.9/posixpath.py\u001b[0m in \u001b[0;36mabspath\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m 378\u001b[0m \u001b[0mcwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetcwdb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 379\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 380\u001b[0;31m \u001b[0mcwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetcwd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 381\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 382\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mnormpath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory"
]
}
],
"source": [
"# Zarr Pyramid\n",
"def create_dataset(decimation: int = 0):\n",
Expand All @@ -120,7 +136,7 @@
" x_res = abs(-170 - 170) / width\n",
" y_res = abs(-80 - 80) / height\n",
"\n",
" arr = numpy.zeros((height, width), dtype=\"uint8\") + decimation\n",
" arr = numpy.zeros((height, width), dtype=\"uint8\") + 1 + decimation\n",
"\n",
" data = xarray.DataArray(\n",
" arr,\n",
Expand All @@ -135,7 +151,7 @@
"\n",
"for ix, dec in enumerate([0, 2, 4]):\n",
" ds = create_dataset(dec)\n",
" ds.to_zarr(store=f\"pyramid.zarr\", mode=\"w\", group=ix)"
" ds.to_zarr(store=\"pyramid.zarr\", mode=\"w\", group=ix)"
]
},
{
Expand Down
12 changes: 6 additions & 6 deletions src/titiler/xarray/tests/fixtures/pyramid.zarr/.zmetadata
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@
"x"
],
"fill_value": 0,
"valid_max": 0,
"valid_min": 0
"valid_max": 1,
"valid_min": 1
},
"0/x/.zarray": {
"chunks": [
Expand Down Expand Up @@ -120,8 +120,8 @@
"x"
],
"fill_value": 0,
"valid_max": 2,
"valid_min": 2
"valid_max": 3,
"valid_min": 3
},
"1/x/.zarray": {
"chunks": [
Expand Down Expand Up @@ -205,8 +205,8 @@
"x"
],
"fill_value": 0,
"valid_max": 4,
"valid_min": 4
"valid_max": 5,
"valid_min": 5
},
"2/x/.zarray": {
"chunks": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@
"x"
],
"fill_value": 0,
"valid_max": 0,
"valid_min": 0
"valid_max": 1,
"valid_min": 1
}
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/0/dataset/0.0
Binary file not shown.
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/0/dataset/0.1
Binary file not shown.
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/0/dataset/1.0
Binary file not shown.
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/0/dataset/1.1
Binary file not shown.
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/0/dataset/2.0
Binary file not shown.
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/0/dataset/2.1
Binary file not shown.
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/0/dataset/3.0
Binary file not shown.
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/0/dataset/3.1
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@
"x"
],
"fill_value": 0,
"valid_max": 2,
"valid_min": 2
"valid_max": 3,
"valid_min": 3
}
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/1/dataset/0.0
Binary file not shown.
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/1/dataset/1.0
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@
"x"
],
"fill_value": 0,
"valid_max": 4,
"valid_min": 4
"valid_max": 5,
"valid_min": 5
}
Binary file modified src/titiler/xarray/tests/fixtures/pyramid.zarr/2/dataset/0.0
Binary file not shown.
4 changes: 2 additions & 2 deletions src/titiler/xarray/tests/test_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,10 +319,10 @@ def test_zarr_group(group, app):
with MemoryFile(resp.content) as mem:
with mem.open() as dst:
arr = dst.read(1)
assert arr.max() == group * 2
assert arr.max() == group * 2 + 1

resp = app.get(
"/md/point/0,0",
params={"url": zarr_pyramid, "variable": "dataset", "group": group},
)
assert resp.json()["values"] == [group * 2]
assert resp.json()["values"] == [group * 2 + 1]

0 comments on commit c9bbdd5

Please sign in to comment.