From c102d837e1bf345a131b36c25be41b13a754b04b Mon Sep 17 00:00:00 2001 From: vdebaecker Date: Tue, 26 Apr 2022 11:46:38 +0200 Subject: [PATCH] Sen2like 4.0.2 --- sen2like/README.md | 1 + sen2like/conf/Sen2Like_GIPP.xsd | 2 + sen2like/conf/config.ini | 2 + sen2like/conf/config.xml | 2 + sen2like/release-notes.md | 10 ++++ .../core/product_archive/product_archive.py | 14 ++++-- sen2like/sen2like/core/readers/landsat.py | 6 ++- sen2like/sen2like/core/readers/sentinel2.py | 48 +++++++++++++++++-- sen2like/sen2like/version.py | 2 +- sen2like/tests/configuration/config.ini | 1 + 10 files changed, 78 insertions(+), 10 deletions(-) diff --git a/sen2like/README.md b/sen2like/README.md index d165da9..3f967f3 100644 --- a/sen2like/README.md +++ b/sen2like/README.md @@ -252,6 +252,7 @@ In addition these parameters are defined in the tool and can be used in brackets * `cloud_cover`: Maximum cloud cover in percent [0, 100] * `url_parameters_pattern_Sentinel2`: Describe storage path for Sentinel 2 products * `url_parameters_pattern_Landsat8`: Describe storage path for Landsat 8 products +* `url_parameters_pattern_Landsat9`: Describe storage path for Landsat 9 products For a Sentinel 2 product on tile 31TFJ: diff --git a/sen2like/conf/Sen2Like_GIPP.xsd b/sen2like/conf/Sen2Like_GIPP.xsd index 594adf0..4f8030c 100644 --- a/sen2like/conf/Sen2Like_GIPP.xsd +++ b/sen2like/conf/Sen2Like_GIPP.xsd @@ -39,9 +39,11 @@ + + diff --git a/sen2like/conf/config.ini b/sen2like/conf/config.ini index ef203b5..63d0271 100644 --- a/sen2like/conf/config.ini +++ b/sen2like/conf/config.ini @@ -29,11 +29,13 @@ base_url = /data/PRODUCTS cloud_cover = 11 url_parameters_pattern_Sentinel2 = {base_url}/{mission}/{tile} url_parameters_pattern_Landsat8 = {base_url}/{mission}/{path}/{row} +url_parameters_pattern_Landsat9 = {base_url}/{mission}/{path}/{row} # Creodias ;base_url = https://finder.creodias.eu/resto/api/collections ;cloud_cover = 11 ;location_Landsat8 = path={path}&row={row} +;location_Landsat9 = path={path}&row={row} ;location_Sentinel2 = processingLevel={s2_processing_level}&productIdentifier=%25{tile}%25 ;url_parameters_pattern = {base_url}/{mission}/search.json?maxRecords=1000&_pretty=true&cloudCover=%5B0%2C{cloud_cover}%5D&startDate={start_date}&completionDate={end_date}&sortParam=startDate&sortOrder=ascending&status=all&{location}&dataset=ESA-DATASET ;thumbnail_property = properties/productIdentifier diff --git a/sen2like/conf/config.xml b/sen2like/conf/config.xml index 54315f4..0cf39f9 100644 --- a/sen2like/conf/config.xml +++ b/sen2like/conf/config.xml @@ -30,11 +30,13 @@ 11 {base_url}/{mission}/{tile} {base_url}/{mission}/{path}/{row} + {base_url}/{mission}/{path}/{row} + diff --git a/sen2like/release-notes.md b/sen2like/release-notes.md index 0735662..15ec715 100644 --- a/sen2like/release-notes.md +++ b/sen2like/release-notes.md @@ -1,5 +1,15 @@ # Sen2Like Release Notes +## v4.0.2 + +### Fix + +* Landsat: collection 2 support: fix BQA extraction (threshold) + +### New features + +* Sentinel-2: support of processing baseline 4.0 (L1 cloud mask as a raster) +* Landsat-9: Add support of local product archive ## v4.0.1 diff --git a/sen2like/sen2like/core/product_archive/product_archive.py b/sen2like/sen2like/core/product_archive/product_archive.py index d451a88..75cdff1 100644 --- a/sen2like/sen2like/core/product_archive/product_archive.py +++ b/sen2like/sen2like/core/product_archive/product_archive.py @@ -341,9 +341,15 @@ def get_products_url_from_tile(self, tile, start_date=None, end_date=None): logger.info("WRS %s_%s does not intersect given ROI. Skip wrs tile." % (path, row)) add_url = False if add_url: - urls.append(( - self.construct_url("Landsat8", tile, start_date=start_date, end_date=end_date, path=path, row=row), - tile_coverage)) + for mission in ['Landsat8', 'Landsat9']: + parameter = self.configuration.get(f'url_parameters_pattern_{mission}') + if parameter is None: + parameter = self.configuration.get(f'location_{mission}') + if parameter is not None: + urls.append(( + self.construct_url(mission, tile, start_date=start_date, end_date=end_date, path=path, row=row), + tile_coverage)) + if not urls: logger.warning( "No product found for tile {} during period {} - {}".format(tile, start_date, end_date)) @@ -373,7 +379,7 @@ def get_products_from_urls(self, urls, start_date=None, end_date=None, product_m [InputProduct(path=os.path.join(url, _dir), tile_coverage=tile_coverage) for _dir in os.listdir(url)]) else: - logger.error("Invalid product path: %s does not exist" % url) + logger.warning("Missing product path: %s does not exist" % url) else: products_urls.extend(self.read_products_from_url(url, tile_coverage=tile_coverage)) diff --git a/sen2like/sen2like/core/readers/landsat.py b/sen2like/sen2like/core/readers/landsat.py index 7d84676..a2676b2 100644 --- a/sen2like/sen2like/core/readers/landsat.py +++ b/sen2like/sen2like/core/readers/landsat.py @@ -458,6 +458,10 @@ def get_valid_pixel_mask(self, mask_filename): else: th = 20480 + #TODO: Check th, 20480 not good for C-2 + if self.collection_number == '02': + th = 21824 + valid_px_mask = np.zeros(bqa_array.shape, np.uint8) valid_px_mask[bqa_array <= th] = 1 valid_px_mask[bqa_array == 1] = 0 # Remove background @@ -467,7 +471,7 @@ def get_valid_pixel_mask(self, mask_filename): mask.write(creation_options=['COMPRESS=LZW'], nodata_value=None) self.mask_filename = mask_filename - # nodata mask (not good when taking it from BQA, getting from B01) + # nodata mask (not good when taking it from BQA, getting from B01): mask_filename = os.path.join(os.path.dirname(mask_filename), 'nodata_pixel_mask.tif') if self.data_type == 'L2A': image_filename = self.surf_image_list[0] diff --git a/sen2like/sen2like/core/readers/sentinel2.py b/sen2like/sen2like/core/readers/sentinel2.py index 6a31fcc..b159465 100644 --- a/sen2like/sen2like/core/readers/sentinel2.py +++ b/sen2like/sen2like/core/readers/sentinel2.py @@ -322,21 +322,32 @@ def __init__(self, product_path, mtd_file=None): if is_compact: # new S2 format maskpath = os.path.join(product_path, maskpath) + log.debug('compact s2 format') else: # old S2 format + log.debug('old s2 format') + maskpath = os.path.join(product_path, 'GRANULE', self.granule_id, 'QI_DATA', maskpath) + log.debug(f'mask path: {maskpath}') + log.debug(f'mask type: {node.getAttribute("type")}') if node.getAttribute('type') == 'MSK_CLOUDS': self.cloudmask = maskpath + elif node.getAttribute('type') == 'MSK_CLASSI': + self.cloudmask = maskpath elif node.getAttribute('type') == 'MSK_NODATA': band = os.path.splitext(maskpath)[0][-3:] self.nodata_mask[band] = maskpath + elif node.getAttribute('type') == 'MSK_QUALIT': + band = os.path.splitext(maskpath)[0][-3:] + self.nodata_mask[band] = maskpath elif node.getAttribute('type') == 'MSK_DETFOO': band = os.path.splitext(maskpath)[0][-3:] self.detfoo_mask[band] = maskpath - log.debug(self.cloudmask) - log.debug(self.nodata_mask) - log.debug(self.detfoo_mask) + + log.debug(f'Cloud Mask: {self.cloudmask}') + log.debug(f'No data mask: {self.nodata_mask}') + log.debug(f'Defective detector: {self.detfoo_mask}') except IndexError: sys.exit(' TILE MTL Parsing Issue ') else: @@ -361,6 +372,7 @@ def get_valid_pixel_mask(self, mask_filename, res=20): :return: """ + log.debug('get valid pixel mask') if self.scene_classif_band: log.info('Generating validity and nodata masks from SCL band') log.debug(f'Read SCL: {self.scene_classif_band}') @@ -399,11 +411,12 @@ def get_valid_pixel_mask(self, mask_filename, res=20): # L1C case for instance -> No SCL, but NODATA and CLD mask else: + log.debug('L1C Case') # Nodata Mask nodata_ref_band = 'B01' band_path = self.bands[nodata_ref_band] log.info(f'Generating nodata mask from band {nodata_ref_band}') - log.debug(f'Read cloud mask: {band_path}') + log.debug(f'Read band file: {band_path}') image = S2L_ImageFile(band_path) array = image.array nodata_mask_filename = os.path.join(os.path.dirname(mask_filename), @@ -455,6 +468,33 @@ def get_valid_pixel_mask(self, mask_filename, res=20): log.info('Written: {}'.format(mask_filename)) self.mask_filename = mask_filename + elif ext =='.jp2': + log.info('Generating validity mask from cloud mask, baseline 4.0') + log.debug(f'no data mask: {self.nodata_mask_filename}') + log.debug(f'mask filename: {mask_filename}') + + log.debug(f'Read cloud mask: {self.cloudmask}') + dataset = gdal.Open(self.cloudmask, gdal.GA_ReadOnly) + clm_1 = dataset.GetRasterBand(1).ReadAsArray() + clm_2 = dataset.GetRasterBand(2).ReadAsArray() + clm_3 = dataset.GetRasterBand(3).ReadAsArray() + tot = clm_1 + clm_2 + clm_3 + valid_px_mask = np.zeros(clm_1.shape, np.uint8) + valid_px_mask[tot == 0] = 1 + # resize valid_px to output res: + shape = (int(valid_px_mask.shape[0] * - image.yRes / res), int(valid_px_mask.shape[1] * image.xRes / res)) + valid_px_mask = skit_resize(valid_px_mask, shape, order=0, preserve_range=True).astype(np.uint8) + #Applied no data mask: + valid_px_mask[nodata == 0] = 0 + + # save to image + mask = image.duplicate(mask_filename, array=valid_px_mask, res=res) + mask.write(creation_options=['COMPRESS=LZW'], nodata_value=None) + log.info('Written: {}'.format(mask_filename)) + self.mask_filename = mask_filename + + dataset = None + return True def get_angle_images(self, DST=None): diff --git a/sen2like/sen2like/version.py b/sen2like/sen2like/version.py index fdbd132..298a38b 100644 --- a/sen2like/sen2like/version.py +++ b/sen2like/sen2like/version.py @@ -1,3 +1,3 @@ """Version of the Application.""" -__version__ = '4.0.1' +__version__ = '4.0.2' diff --git a/sen2like/tests/configuration/config.ini b/sen2like/tests/configuration/config.ini index 9b17dd4..bb8ad2c 100644 --- a/sen2like/tests/configuration/config.ini +++ b/sen2like/tests/configuration/config.ini @@ -18,6 +18,7 @@ coverage = 0.1 ;base_url = /data/PRODUCTS ;url_parameters_pattern_Sentinel2 = {base_url}/{mission}/{tile} ;url_parameters_pattern_Landsat8 = {base_url}/{mission}/{path}/{row} +;url_parameters_pattern_Landsat9 = {base_url}/{mission}/{path}/{row} # Creodias base_url = https://finder.creodias.eu/resto/api/collections