Merge pull request #33 from sertit/0.14.0

0.14.0

CI/SCRIPTS/scripts_utils.py

@@ -26,18 +26,6 @@
 CI_EOREADER_S3 = "CI_EOREADER_USE_S3"
 
 
-def reduce_verbosity() -> None:
-    """ Reduce verbosity for other loggers """
-    logging.getLogger("boto3").setLevel(logging.WARNING)
-    logging.getLogger("botocore").setLevel(logging.WARNING)
-    logging.getLogger("shapely").setLevel(logging.WARNING)
-    logging.getLogger("fiona").setLevel(logging.WARNING)
-    logging.getLogger("rasterio").setLevel(logging.WARNING)
-    logging.getLogger("urllib3").setLevel(logging.WARNING)
-    logging.getLogger("s3transfer").setLevel(logging.WARNING)
-    logging.getLogger("pyproj").setLevel(logging.WARNING)
-
-
 def get_ci_dir() -> Union[CloudPath, Path]:
     """
     Get CI DATA directory

CI/SCRIPTS/test_custom.py

@@ -15,6 +15,8 @@
 
 LOGGER = logging.getLogger(EOREADER_NAME)
 
+ci.reduce_verbosity()
+
 
 @s3_env
 @dask_env
@@ -26,17 +28,17 @@ def test_custom():
         opt_stack,
         custom=True,
         name="20200310T030415_WV02_Ortho",
-        acquisition_datetime="20200310T030415",
+        datetime="20200310T030415",
         sensor_type=SensorType.OPTICAL,
         platform="WV02",
-        default_resolution=2.0,
+        resolution=2.0,
         product_type="Ortho",
         band_map={BLUE: 1, GREEN: 2, RED: 3, NIR: 4, SWIR_1: 5},
     )
     LOGGER.info(prod_all)
-    extent_all = prod_all.extent
-    footprint_all = prod_all.footprint
-    crs_all = prod_all.crs
+    extent_all = prod_all.extent()
+    footprint_all = prod_all.footprint()
+    crs_all = prod_all.crs()
     stack = prod_all.stack([BLUE, GREEN, RED, NIR])
 
     # Check attributes
@@ -57,9 +59,9 @@ def test_custom():
         band_map={"BLUE": 1, "GREEN": 2, "RED": 3, "NIR": 4, SWIR_1: 5},
     )
     LOGGER.info(prod_min)
-    extent_min = prod_min.extent
-    footprint_min = prod_min.footprint
-    crs_min = prod_min.crs
+    extent_min = prod_min.extent()
+    footprint_min = prod_min.footprint()
+    crs_min = prod_min.crs()
     bands = prod_min.load([BLUE, NIR])
 
     # Check attributes
@@ -87,17 +89,17 @@ def test_custom():
     prod_some = READER.open(
         opt_stack,
         custom=True,
-        acquisition_datetime="20200310T030415",
+        datetime="20200310T030415",
         sensor_type="Optical",
         product_type="wonderful_type",
         sun_azimuth=10.0,
         sun_zenith=20.0,
         band_map={BLUE: 1, "GREEN": 2, RED: 3, "NIR": 4, SWIR_1: 5},
     )
     LOGGER.info(prod_some)
-    extent_some = prod_some.extent
-    footprint_some = prod_some.footprint
-    crs_some = prod_some.crs
+    extent_some = prod_some.extent()
+    footprint_some = prod_some.footprint()
+    crs_some = prod_some.crs()
     bands = prod_some.load([HILLSHADE], resolution=200.0)
 
     # Check attributes
@@ -134,16 +136,16 @@ def test_custom():
         custom=True,
         sensor_type=SensorType.SAR,
         name="20210827T162210_ICEYE_SC_GRD",
-        acquisition_datetime="20210827T162210",
+        datetime="20210827T162210",
         platform="ICEYE",
-        default_resolution=6.0,
+        resolution=6.0,
         product_type="GRD",
         band_map={VV: 1, VV_DSPK: 2},
     )
     LOGGER.info(prod_sar)
-    extent_sar = prod_sar.extent
-    footprint_sar = prod_sar.footprint
-    crs_sar = prod_sar.crs
+    extent_sar = prod_sar.extent()
+    footprint_sar = prod_sar.footprint()
+    crs_sar = prod_sar.crs()
     stack_sar = prod_sar.stack([VV, VV_DSPK], prod_sar.resolution * 10)
 
     # Check attributes
@@ -161,12 +163,12 @@ def test_custom():
         custom=True,
         sensor_type=SensorType.SAR,
         band_map={HH: 1, RH: 2},
-        default_resolution=6.0,
+        resolution=6.0,
     )
     LOGGER.info(prod_wtf)
-    extent_wtf = prod_wtf.extent
-    footprint_wtf = prod_wtf.footprint
-    crs_wtf = prod_wtf.crs
+    extent_wtf = prod_wtf.extent()
+    footprint_wtf = prod_wtf.footprint()
+    crs_wtf = prod_wtf.crs()
     stack_wtf = prod_wtf.stack([HH, RH], prod_wtf.resolution * 10)
 
     ci.assert_geom_equal(extent_sar, extent_wtf)
@@ -182,36 +184,38 @@ def test_custom():
         custom=True,
         sensor_type=SensorType.OPTICAL,
         name="SPOT6_WGS84",
-        acquisition_datetime="20181218T0938",
+        datetime="20181218T090308",
         platform="SPOT6",
-        default_resolution=1.5 * 15,
+        resolution=1.5 * 15,
         product_type="ORT",
         band_map={RED: 1, GREEN: 2, BLUE: 3, NIR: 4},
     )
     LOGGER.info(prod_wgs84)
 
     # Check geometries -> assert projected
     with pytest.raises(InvalidProductError):
-        prod_wgs84.extent  # noqa
+        prod_wgs84.extent()  # noqa
 
     with pytest.raises(InvalidProductError):
-        prod_wgs84.footprint  # noqa
+        prod_wgs84.footprint()  # noqa
 
     with pytest.raises(InvalidProductError):
-        prod_wgs84.crs  # noqa
+        prod_wgs84.crs()  # noqa
 
     # Read mtd
     root, nsp = prod_wgs84.read_mtd()
     assert nsp == {}
     assert root.findtext("name") == "SPOT6_WGS84"
-    assert root.findtext("datetime") == "2018-12-18 09:03:08"
-    assert root.findtext("sensor_type") == "SensorType.OPTICAL"
-    assert root.findtext("platform") == "Platform.SPOT6"
+    assert root.findtext("datetime") == "2018-12-18T09:03:08"
+    assert root.findtext("sensor_type") == "Optical"
+    assert root.findtext("platform") == "Spot-6"
     assert root.findtext("resolution") == str(1.5 * 15)
     assert root.findtext("product_type") == "ORT"
-    assert root.findtext("band_names") == "{'BLUE': 3, 'GREEN': 2, 'RED': 1, 'NIR': 4}"
-    assert root.findtext("sun_az") == "None"
-    assert root.findtext("sun_zen") == "None"
+    assert root.findtext("band_map") == "{'BLUE': 3, 'GREEN': 2, 'RED': 1, 'NIR': 4}"
+    assert root.findtext("sun_azimuth") == "None"
+    assert root.findtext("sun_zenith") == "None"
+    assert root.findtext("orbit_direction") == "None"
+    assert root.findtext("cloud_cover") == "None"
 
     # Band paths
     assert prod_wgs84.get_existing_bands() == [BLUE, GREEN, RED, NIR]

CI/SCRIPTS/test_end_to_end.py

@@ -8,7 +8,7 @@
 import pytest
 import xarray as xr
 from cloudpathlib import AnyPath
-from sertit import files
+from sertit import ci, files
 
 from CI.SCRIPTS.scripts_utils import (
     CI_EOREADER_S3,
@@ -18,15 +18,14 @@
     get_db_dir,
     get_db_dir_on_disk,
     opt_path,
-    reduce_verbosity,
 )
 from eoreader.bands import *
 from eoreader.env_vars import DEM_PATH, S3_DB_URL_ROOT, SAR_DEF_RES, TEST_USING_S3_DB
 from eoreader.products.product import Product, SensorType
 from eoreader.reader import CheckMethod
 from eoreader.utils import EOREADER_NAME
 
-reduce_verbosity()
+ci.reduce_verbosity()
 
 LOGGER = logging.getLogger(EOREADER_NAME)
 
@@ -172,8 +171,8 @@ def _test_core(
                 first_band = stack_bands[0]
 
                 # Geometric data
-                footprint = prod.footprint  # noqa
-                extent = prod.extent  # noqa
+                footprint = prod.footprint()  # noqa
+                extent = prod.extent()  # noqa
 
                 # Get stack bands
                 # Stack data

CI/SCRIPTS/test_index.py

@@ -6,7 +6,7 @@
 import numpy as np
 from sertit import ci, rasters
 
-from eoreader.bands.index import get_all_index
+from eoreader.bands.index import get_all_indices
 from eoreader.utils import EOREADER_NAME
 
 from .scripts_utils import READER, dask_env, get_ci_data_dir, opt_path, s3_env
@@ -15,30 +15,26 @@
 
 RES = 2000.0  # 2000 meters
 
+ci.reduce_verbosity()
+
 
 @s3_env
 @dask_env
 def test_index():
-    """Function testing the correct functioning of the index"""
+    """Function testing the correct functioning of the indices"""
     # Load S2 products as it can load every index
     s2_path = opt_path().joinpath(
         r"S2B_MSIL2A_20200114T065229_N0213_R020_T40REQ_20200114T094749.SAFE"
     )
     prod = READER.open(s2_path)
     failed_idx = []
     with tempfile.TemporaryDirectory() as tmp_dir:
-        # tmp_dir = os.path.join("/mnt", "ds2_db3", "CI", "eoreader", "DATA", "OUTPUT")
+        # tmp_dir = os.path.join("/mnt", "ds2_db3", "CI", "eoreader", "DATA", "INDEX")
         prod.output = os.path.join(tmp_dir, prod.condensed_name)
 
         # Load every index
-        LOGGER.info("Load all index")
-        logging.getLogger("boto3").setLevel(
-            logging.WARNING
-        )  # BOTO has way too much verbosity
-        logging.getLogger("botocore").setLevel(
-            logging.WARNING
-        )  # BOTO has way too much verbosity
-        idx_list = [idx for idx in get_all_index() if prod.has_band(idx)]
+        LOGGER.info("Load all indices")
+        idx_list = [idx for idx in get_all_indices() if prod.has_band(idx)]
         idx = prod.load(idx_list, resolution=RES)
 
         for idx_fct, idx_arr in idx.items():
@@ -62,6 +58,6 @@ def test_index():
                 failed_idx.append(idx_name)
 
         # Read the results
-        # Do like that to check all existing index
+        # Do like that to check all existing indices
         if failed_idx:
             raise AssertionError(f"Failed index: {failed_idx}")

CI/SCRIPTS/test_others.py

@@ -6,7 +6,7 @@
 import tempenv
 import xarray as xr
 from cloudpathlib import AnyPath, S3Client
-from sertit import files
+from sertit import ci, files
 
 from eoreader import utils
 from eoreader.bands import *
@@ -27,6 +27,8 @@
     sar_path,
 )
 
+ci.reduce_verbosity()
+
 
 @pytest.mark.xfail
 def test_utils():

CI/SCRIPTS/test_satellites.py

@@ -22,6 +22,7 @@
 )
 from eoreader.keywords import SLSTR_RAD_ADJUST
 from eoreader.products import Product, S2Product, SensorType, SlstrRadAdjust
+from eoreader.products.product import OrbitDirection
 from eoreader.reader import CheckMethod
 from eoreader.utils import EOREADER_NAME
 
@@ -35,12 +36,11 @@
     get_db_dir,
     get_db_dir_on_disk,
     opt_path,
-    reduce_verbosity,
     s3_env,
     sar_path,
 )
 
-reduce_verbosity()
+ci.reduce_verbosity()
 
 LOGGER = logging.getLogger(EOREADER_NAME)
 
@@ -178,7 +178,7 @@ def _test_core(
 
                 # Extent
                 LOGGER.info("Checking extent")
-                extent = prod.extent
+                extent = prod.extent()
                 assert isinstance(extent, gpd.GeoDataFrame)
                 assert extent.crs.is_projected
                 extent_path = get_ci_data_dir().joinpath(
@@ -200,7 +200,7 @@ def _test_core(
 
                 # Footprint
                 LOGGER.info("Checking footprint")
-                footprint = prod.footprint
+                footprint = prod.footprint()
                 assert isinstance(footprint, gpd.GeoDataFrame)
                 assert footprint.crs.is_projected
                 footprint_path = get_ci_data_dir().joinpath(
@@ -273,8 +273,9 @@ def _test_core(
                     raise FileNotFoundError(f"{ci_stack} not found !")
                     # ci_stack = curr_path
 
-                # Test
-                assert_raster_almost_equal(curr_path, ci_stack, decimal=4)
+                else:
+                    # Test
+                    assert_raster_almost_equal(curr_path, ci_stack, decimal=4)
 
                 # Load a band with the size option
                 LOGGER.info("Checking load with size keyword")
@@ -322,19 +323,39 @@ def _test_core(
                 prod.get_existing_band_paths()  # noqa
 
                 # Check if possible to load narrow nir, without checking result
-                if isinstance(prod, S2Product) and not prod._processing_baseline_lt_4_0:
+                if isinstance(prod, S2Product) and not prod._processing_baseline < 4.0:
                     prod.load(NARROW_NIR)
 
                 # CRS
                 LOGGER.info("Checking CRS")
-                assert prod.crs.is_projected
+                assert prod.crs().is_projected
 
                 # MTD
                 LOGGER.info("Checking Mtd")
                 mtd_xml, nmsp = prod.read_mtd()
                 assert isinstance(mtd_xml, etree._Element)
                 assert isinstance(nmsp, dict)
 
+                # Mean sun angle type, cloud cover...
+                if prod.sensor_type == SensorType.OPTICAL:
+                    az, zen = prod.get_mean_sun_angles()
+                    assert isinstance(az, float)
+                    assert isinstance(zen, float)
+
+                    cc = prod.get_cloud_cover()
+                    assert isinstance(cc, float)
+
+                # Quicklook and plot
+                qck_path = prod.get_quicklook_path()
+                if qck_path is not None:
+                    assert isinstance(qck_path, str)
+
+                prod.plot()
+
+                # Orbit direction
+                orbit_dir = prod.get_orbit_direction()
+                assert isinstance(orbit_dir, OrbitDirection)
+
                 # Clean temp
                 if not debug:
                     LOGGER.info("Cleaning tmp")