Add wagl package test for Sentinel

eodatasets3/images.py

@@ -325,10 +325,14 @@ def _as_named_grids(self) -> Dict[str, Tuple[GridSpec, _Measurements]]:
         named_grids = {"default": default_grid}
         for grid, measurements in grids_by_frequency:
             res_y, res_x = grid.resolution_yx
+            if res_x > 1:
+                res_x = int(res_x)
+            grid_name = f"{res_x}"
             if grid_name in named_grids:
                 # Clash of names! This strategy wont work.
                 break
-            named_grids[str(res_x)] = (grid, measurements)
+
+            named_grids[grid_name] = (grid, measurements)
         else:
             # We finished without a clash.
             return named_grids

eodatasets3/scripts/packagewagl.py

@@ -6,7 +6,7 @@
 for files.
 """
 from pathlib import Path
-from typing import Sequence
+from typing import Sequence, Optional
 
 import click
 import rasterio
@@ -45,14 +45,37 @@
     is_flag=True,
     default=True,
 )
+@click.option(
+    "--with-oa/--no-oa",
+    "with_oa",
+    help="Include observation attributes (default: true)",
+    is_flag=True,
+    default=True,
+)
+@click.option(
+    "--oa-resolution",
+    help="Resolution choice for observation attributes "
+    "(default: automatic based on sensor)",
+    type=float,
+    default=None,
+)
 @click.argument("h5_file", type=PathPath(exists=True, readable=True, writable=False))
 def run(
-    level1: Path, output: Path, h5_file: Path, products: Sequence[str], with_oa: bool
+    level1: Path,
+    output: Path,
+    h5_file: Path,
+    products: Sequence[str],
+    with_oa: bool,
+    oa_resolution: Optional[float],
 ):
     if products:
         products = set(p.lower() for p in products)
     else:
         products = wagl.DEFAULT_PRODUCTS
+
+    if oa_resolution is not None:
+        oa_resolution = (oa_resolution, oa_resolution)
+
     with rasterio.Env():
         for granule in wagl.Granule.for_path(h5_file, level1_metadata_path=level1):
             with wagl.do(
@@ -68,6 +91,7 @@ def run(
                     granule=granule,
                     included_products=products,
                     include_oa=with_oa,
+                    oa_resolution=oa_resolution,
                 )
                 secho(f"Created folder {click.style(str(dataset_path), fg='green')}")
 

eodatasets3/wagl.py

@@ -164,6 +164,7 @@ def _unpack_observation_attributes(
     product_list: Iterable[str],
     h5group: h5py.Group,
     infer_datetime_range=False,
+    oa_resolution: Optional[Tuple[float, float]] = None,
 ):
     """
     Unpack the angles + other supplementary datasets produced by wagl.
@@ -180,17 +181,7 @@ def _unpack_observation_attributes(
         del p.properties["eo:gsd"]
     p.properties["eo:gsd"] = min(min(resolution_groups.keys()))
 
-    if p.platform.startswith("landsat"):
-        # For Landsat, we only cared about packaging OA data for the "common"
-        # bands (not panchromatic). So we always pick the higher resolution.
-        oa_resolution = max(resolution_groups.keys())
-    elif p.platform.startswith("sentinel"):
-        oa_resolution = (20.0, 20.0)
-    else:
-        raise NotImplementedError(
-            f"Don't know how to choose OA resolution for platform {p.platform!r}"
-        )
-    res_grp = resolution_groups[oa_resolution]
+    res_grp = choose_resolution_group(resolution_groups, p.platform, oa_resolution)
 
     def _write(section: str, dataset_names: Sequence[str]):
         """
@@ -238,6 +229,36 @@ def _write(section: str, dataset_names: Sequence[str]):
         )
 
 
+def choose_resolution_group(
+    resolution_groups: Dict[tuple, h5py.Group],
+    platform: str,
+    oa_resolution=Optional[Tuple[float, float]],
+) -> h5py.Group:
+
+    # None specified? Figure out a default.
+
+    if oa_resolution is None:
+        # For Landsat, we only cared about packaging OA data for the "common"
+        # bands (not panchromatic). So we always pick the higher resolution.
+        if platform.startswith("landsat"):
+            oa_resolution = max(resolution_groups.keys())
+        elif platform.startswith("sentinel"):
+            oa_resolution = (20.0, 20.0)
+        else:
+            raise NotImplementedError(
+                f"Don't know how to choose a default OA resolution for platform {platform !r}"
+            )
+
+    res_grp = resolution_groups.get(oa_resolution)
+    if res_grp is None:
+        raise RuntimeError(
+            f"Resolution {oa_resolution} not found in input. "
+            f"Have resolutions {tuple(resolution_groups.keys())}"
+        )
+
+    return res_grp
+
+
 def _create_contiguity(
     p: DatasetAssembler,
     product_list: Iterable[str],
@@ -495,6 +516,7 @@ def package(
     granule: Granule,
     included_products: Iterable[str] = DEFAULT_PRODUCTS,
     include_oa: bool = True,
+    oa_resolution=Optional[Tuple[float, float]],
 ) -> Tuple[UUID, Path]:
     """
     Package an L2 product.
@@ -578,6 +600,7 @@ def package(
                         included_products,
                         granule_group,
                         infer_datetime_range=p.platform.startswith("landsat"),
+                        oa_resolution=oa_resolution,
                     )
                 if granule.fmask_image:
                     with do(f"Writing fmask from {granule.fmask_image} "):
@@ -643,17 +666,19 @@ def find_a_granule_name(wagl_hdf5: Path) -> str:
 
     >>> find_a_granule_name(Path('LT50910841993188ASA00.wagl.h5'))
     'LT50910841993188ASA00'
+    >>> find_a_granule_name(Path('S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09.wagl.h5'))
+    'S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09'
     >>> find_a_granule_name(Path('my-test-granule.h5'))
     Traceback (most recent call last):
     ...
-    ValueError: No granule specified, and cannot find it on input filename 'my-test-granule'.
+    ValueError: Does not appear to be a wagl output filename? 'my-test-granule.h5'.
     """
-    granule_name = wagl_hdf5.stem.split(".")[0]
-    if not granule_name.startswith("L"):
+    if not wagl_hdf5.name.endswith(".wagl.h5"):
         raise ValueError(
-            f"No granule specified, and cannot find it on input filename {wagl_hdf5.stem!r}."
+            f"Does not appear to be a wagl output filename? {wagl_hdf5.name!r}."
         )
-    return granule_name
+
+    return wagl_hdf5.name[: -len(".wagl.h5")]
 
 
 def _apply_wagl_metadata(p: DatasetAssembler, granule_group: h5py.Group) -> Dict:

tests/integration/__init__.py

@@ -30,14 +30,18 @@ def assert_image(
         assert d.count == bands, f"Expected {bands} band{'s' if bands > 1 else ''}"
 
         if overviews is not allow_anything:
-            assert d.overviews(1) == overviews
+            assert (
+                d.overviews(1) == overviews
+            ), f"Unexpected overview: {d.overviews(1)!r} != {overviews!r}"
         if nodata is not allow_anything:
-            assert d.nodata == nodata
+            assert d.nodata == nodata, f"Unexpected nodata: {d.nodata!r} != {nodata!r}"
 
         if unique_pixel_counts is not allow_anything:
             array = d.read(1)
             value_counts = dict(zip(*numpy.unique(array, return_counts=True)))
-            assert value_counts == unique_pixel_counts
+            assert (
+                value_counts == unique_pixel_counts
+            ), f"Unexpected pixel counts: {value_counts!r} != {unique_pixel_counts!r}"
 
         if shape:
             assert shape == d.shape, f"Unexpected shape: {shape!r} != {d.shape!r}"

tests/integration/data/wagl-input/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09.fmask.img-luigi-tmp-3206354868.aux.xml

@@ -0,0 +1,31 @@
+<PAMDataset>
+  <PAMRasterBand band="1">
+    <Description>Layer_1</Description>
+    <Histograms>
+      <HistItem>
+        <HistMin>-0.5</HistMin>
+        <HistMax>255.5</HistMax>
+        <BucketCount>256</BucketCount>
+        <IncludeOutOfRange>0</IncludeOutOfRange>
+        <Approximate>0</Approximate>
+        <HistCounts>0|12456675|1871098|118102|2466507|79|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0</HistCounts>
+      </HistItem>
+    </Histograms>
+    <Metadata domain="IMAGE_STRUCTURE">
+      <MDI key="COMPRESSION">RLE</MDI>
+    </Metadata>
+    <Metadata>
+      <MDI key="LAYER_TYPE">thematic</MDI>
+      <MDI key="STATISTICS_EXCLUDEDVALUES">0</MDI>
+      <MDI key="STATISTICS_HISTOBINFUNCTION">direct</MDI>
+      <MDI key="STATISTICS_MAXIMUM">5.0</MDI>
+      <MDI key="STATISTICS_MEAN">1.5621381181603315</MDI>
+      <MDI key="STATISTICS_MEDIAN">1</MDI>
+      <MDI key="STATISTICS_MINIMUM">1.0</MDI>
+      <MDI key="STATISTICS_MODE">1</MDI>
+      <MDI key="STATISTICS_SKIPFACTORX">1</MDI>
+      <MDI key="STATISTICS_SKIPFACTORY">1</MDI>
+      <MDI key="STATISTICS_STDDEV">1.0654571499740102</MDI>
+    </Metadata>
+  </PAMRasterBand>
+</PAMDataset>

tests/integration/data/wagl-input/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09.fmask.tmp..img.aux.xml

@@ -0,0 +1,8 @@
+<PAMDataset>
+  <PAMRasterBand band="1">
+    <Description>Layer_1</Description>
+    <Metadata>
+      <MDI key="LAYER_TYPE">thematic</MDI>
+    </Metadata>
+  </PAMRasterBand>
+</PAMDataset>

tests/integration/data/wagl-input/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09.fmask.yaml

@@ -0,0 +1,17 @@
+parameters:
+    cloud_buffer_distance_metres: 0.0
+    cloud_shadow_buffer_distance_metres: 0.0
+    frantz_parallax_sentinel_2: false
+percent_class_distribution:
+    clear: 73.65382838133374
+    cloud: 11.063428320692061
+    cloud_shadow: 0.6983135097842945
+    snow: 14.583962676987106
+    water: 0.0004671112027989303
+software_versions:
+    eugl:
+        repo_url: https://github.com/OpenDataCubePipelines/eugl.git
+        version: 0.2.1
+    fmask:
+        repo_url: https://bitbucket.org/chchrsc/python-fmask
+        version: 0.5.4

tests/integration/data/wagl-input/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09.gqa.yaml

@@ -0,0 +1,44 @@
+colors:
+    blue: 75.0
+    green: 347.0
+    red: 50.0
+    teal: 312.0
+    yellow: 53.0
+error_message: no errors
+final_qa_count: 726
+granule: S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09
+ref_date: '2013-08-15T00:00:00+00:00'
+ref_source: GQA_v3
+ref_source_path: /g/data/v10/eoancillarydata/GCP/GQA_v3/wrs2/100/079/LC81000792013227LGN00_B6.TIF
+residual:
+    abs:
+        x: 0.69
+        xy: 1.07
+        y: 0.82
+    abs_iterative_mean:
+        x: 0.42
+        xy: 0.53
+        y: 0.32
+    cep90: 0.97
+    iterative_mean:
+        x: 0.4
+        xy: 0.4
+        y: 0.04
+    iterative_stddev:
+        x: 0.29
+        xy: 0.53
+        y: 0.44
+    mean:
+        x: 0.38
+        xy: 0.39
+        y: -0.07
+    stddev:
+        x: 1.18
+        xy: 2.24
+        y: 1.9
+software_versions:
+    eugl:
+        repo_url: https://github.com/OpenDataCubePipelines/eugl.git
+        version: 0.2.1
+    gverify:
+        version: v0.25c

tests/integration/data/wagl-input/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09/S2A_OPER_MSI_L1C_TL_EPAE_20201031T022859_A027984_T53JQJ_N02.09.tesp.yaml

@@ -0,0 +1,2 @@
+software_versions:
+  tesp: {repo_url: 'https://github.com/OpenDataCubePipelines/tesp.git', version: 0.6.2}