Sen2like 4.0.2

sen2like/README.md

@@ -252,6 +252,7 @@ In addition these parameters are defined in the tool and can be used in brackets
 * `cloud_cover`: Maximum cloud cover in percent [0, 100]
 * `url_parameters_pattern_Sentinel2`: Describe storage path for Sentinel 2 products
 * `url_parameters_pattern_Landsat8`: Describe storage path for Landsat 8 products
+* `url_parameters_pattern_Landsat9`: Describe storage path for Landsat 9 products
 
 For a Sentinel 2 product on tile 31TFJ:
 

sen2like/conf/Sen2Like_GIPP.xsd

@@ -39,9 +39,11 @@
                 <xs:sequence>
                     <xs:element type="xs:string" name="url_parameters_pattern_Sentinel2"/>
                     <xs:element type="xs:string" name="url_parameters_pattern_Landsat8"/>
+                    <xs:element type="xs:string" name="url_parameters_pattern_Landsat9"/>
                 </xs:sequence>
                 <xs:sequence>
                     <xs:element type="xs:string" name="location_Landsat8"/>
+                    <xs:element type="xs:string" name="location_Landsat9"/>
                     <xs:element type="xs:string" name="location_Sentinel2"/>
                     <xs:element type="xs:string" name="url_parameters_pattern"/>
                     <xs:element type="xs:string" name="thumbnail_property"/>

sen2like/conf/config.ini

@@ -29,11 +29,13 @@ base_url = /data/PRODUCTS
 cloud_cover = 11
 url_parameters_pattern_Sentinel2 = {base_url}/{mission}/{tile}
 url_parameters_pattern_Landsat8 = {base_url}/{mission}/{path}/{row}
+url_parameters_pattern_Landsat9 = {base_url}/{mission}/{path}/{row}
 
 # Creodias
 ;base_url = https://finder.creodias.eu/resto/api/collections
 ;cloud_cover = 11
 ;location_Landsat8 = path={path}&row={row}
+;location_Landsat9 = path={path}&row={row}
 ;location_Sentinel2 = processingLevel={s2_processing_level}&productIdentifier=%25{tile}%25
 ;url_parameters_pattern = {base_url}/{mission}/search.json?maxRecords=1000&_pretty=true&cloudCover=%5B0%2C{cloud_cover}%5D&startDate={start_date}&completionDate={end_date}&sortParam=startDate&sortOrder=ascending&status=all&{location}&dataset=ESA-DATASET
 ;thumbnail_property = properties/productIdentifier

sen2like/conf/config.xml

@@ -30,11 +30,13 @@
         <cloud_cover>11</cloud_cover>
         <url_parameters_pattern_Sentinel2>{base_url}/{mission}/{tile}</url_parameters_pattern_Sentinel2>
         <url_parameters_pattern_Landsat8>{base_url}/{mission}/{path}/{row}</url_parameters_pattern_Landsat8>
+        <url_parameters_pattern_Landsat9>{base_url}/{mission}/{path}/{row}</url_parameters_pattern_Landsat9>
 
         <!--  Creodias -->
         <!--        <base_url>https://finder.creodias.eu/resto/api/collections</base_url>-->
         <!--        <cloud_cover>11</cloud_cover>-->
         <!--        <location_Landsat8>path={path}&amp;row={row}</location_Landsat8>-->
+        <!--        <location_Landsat9>path={path}&amp;row={row}</location_Landsat9>-->
         <!--        <location_Sentinel2>processingLevel={s2_processing_level}&amp;productIdentifier=%25{tile}%25</location_Sentinel2>-->
         <!--        <url_parameters_pattern>{base_url}/{mission}/search.json?maxRecords=100&amp;_pretty=true&amp;cloudCover=%5B0%2C{cloud_cover}%5D&amp;startDate={start_date}&amp;completionDate={end_date}&amp;sortParam=startDate&amp;sortOrder=ascending&amp;status=all&amp;{location}&amp;dataset=ESA-DATASET</url_parameters_pattern>-->
         <!--        <thumbnail_property>properties/productIdentifier</thumbnail_property>-->

sen2like/release-notes.md

@@ -1,5 +1,15 @@
 # Sen2Like Release Notes
 
+## v4.0.2
+
+### Fix
+
+* Landsat: collection 2 support: fix BQA extraction (threshold)
+
+### New features
+
+* Sentinel-2: support of processing baseline 4.0 (L1 cloud mask as a raster)
+* Landsat-9: Add support of local product archive
 
 ## v4.0.1
 

sen2like/sen2like/core/product_archive/product_archive.py

@@ -341,9 +341,15 @@ def get_products_url_from_tile(self, tile, start_date=None, end_date=None):
                     logger.info("WRS %s_%s does not intersect given ROI. Skip wrs tile." % (path, row))
                     add_url = False
             if add_url:
-                urls.append((
-                    self.construct_url("Landsat8", tile, start_date=start_date, end_date=end_date, path=path, row=row),
-                    tile_coverage))
+                for mission in ['Landsat8', 'Landsat9']:
+                    parameter = self.configuration.get(f'url_parameters_pattern_{mission}')
+                    if parameter is None:
+                        parameter = self.configuration.get(f'location_{mission}')
+                    if parameter is not None:
+                        urls.append((
+                            self.construct_url(mission, tile, start_date=start_date, end_date=end_date, path=path, row=row),
+                            tile_coverage))
+
         if not urls:
             logger.warning(
                 "No product found for tile {} during period {} - {}".format(tile, start_date, end_date))
@@ -373,7 +379,7 @@ def get_products_from_urls(self, urls, start_date=None, end_date=None, product_m
                             [InputProduct(path=os.path.join(url, _dir), tile_coverage=tile_coverage) for _dir in
                              os.listdir(url)])
                 else:
-                    logger.error("Invalid product path: %s does not exist" % url)
+                    logger.warning("Missing product path: %s does not exist" % url)
             else:
                 products_urls.extend(self.read_products_from_url(url, tile_coverage=tile_coverage))
 

sen2like/sen2like/core/readers/landsat.py

@@ -458,6 +458,10 @@ def get_valid_pixel_mask(self, mask_filename):
             else:
                 th = 20480
 
+            #TODO: Check th, 20480  not good for C-2
+            if self.collection_number == '02':
+                th = 21824
+
             valid_px_mask = np.zeros(bqa_array.shape, np.uint8)
             valid_px_mask[bqa_array <= th] = 1
             valid_px_mask[bqa_array == 1] = 0  # Remove background
@@ -467,7 +471,7 @@ def get_valid_pixel_mask(self, mask_filename):
             mask.write(creation_options=['COMPRESS=LZW'], nodata_value=None)
             self.mask_filename = mask_filename
 
-            # nodata mask (not good when taking it from BQA, getting from B01)
+            # nodata mask (not good when taking it from BQA, getting from B01):
             mask_filename = os.path.join(os.path.dirname(mask_filename), 'nodata_pixel_mask.tif')
             if self.data_type == 'L2A':
                 image_filename = self.surf_image_list[0]

sen2like/sen2like/core/readers/sentinel2.py

@@ -322,21 +322,32 @@ def __init__(self, product_path, mtd_file=None):
                     if is_compact:
                         # new S2 format
                         maskpath = os.path.join(product_path, maskpath)
+                        log.debug('compact s2 format')
                     else:
                         # old S2 format
+                        log.debug('old s2 format')
+
                         maskpath = os.path.join(product_path, 'GRANULE', self.granule_id, 'QI_DATA', maskpath)
 
+                    log.debug(f'mask path: {maskpath}')
+                    log.debug(f'mask type: {node.getAttribute("type")}')
                     if node.getAttribute('type') == 'MSK_CLOUDS':
                         self.cloudmask = maskpath
+                    elif node.getAttribute('type') == 'MSK_CLASSI':
+                        self.cloudmask = maskpath
                     elif node.getAttribute('type') == 'MSK_NODATA':
                         band = os.path.splitext(maskpath)[0][-3:]
                         self.nodata_mask[band] = maskpath
+                    elif node.getAttribute('type') == 'MSK_QUALIT':
+                        band = os.path.splitext(maskpath)[0][-3:]
+                        self.nodata_mask[band] = maskpath
                     elif node.getAttribute('type') == 'MSK_DETFOO':
                         band = os.path.splitext(maskpath)[0][-3:]
                         self.detfoo_mask[band] = maskpath
-                log.debug(self.cloudmask)
-                log.debug(self.nodata_mask)
-                log.debug(self.detfoo_mask)
+
+                log.debug(f'Cloud Mask: {self.cloudmask}')
+                log.debug(f'No data mask: {self.nodata_mask}')
+                log.debug(f'Defective detector: {self.detfoo_mask}')
             except IndexError:
                 sys.exit(' TILE MTL Parsing Issue ')
         else:
@@ -361,6 +372,7 @@ def get_valid_pixel_mask(self, mask_filename, res=20):
         :return:
         """
 
+        log.debug('get valid pixel mask')
         if self.scene_classif_band:
             log.info('Generating validity and nodata masks from SCL band')
             log.debug(f'Read SCL: {self.scene_classif_band}')
@@ -399,11 +411,12 @@ def get_valid_pixel_mask(self, mask_filename, res=20):
 
         # L1C case for instance -> No SCL, but NODATA and CLD mask
         else:
+            log.debug('L1C Case')
             # Nodata Mask
             nodata_ref_band = 'B01'
             band_path = self.bands[nodata_ref_band]
             log.info(f'Generating nodata mask from band {nodata_ref_band}')
-            log.debug(f'Read cloud mask: {band_path}')
+            log.debug(f'Read band file: {band_path}')
             image = S2L_ImageFile(band_path)
             array = image.array
             nodata_mask_filename = os.path.join(os.path.dirname(mask_filename),
@@ -455,6 +468,33 @@ def get_valid_pixel_mask(self, mask_filename, res=20):
                     log.info('Written: {}'.format(mask_filename))
                     self.mask_filename = mask_filename
 
+                elif ext =='.jp2':
+                    log.info('Generating validity mask from cloud mask, baseline 4.0')
+                    log.debug(f'no data mask: {self.nodata_mask_filename}')
+                    log.debug(f'mask filename: {mask_filename}')
+
+                    log.debug(f'Read cloud mask: {self.cloudmask}')
+                    dataset = gdal.Open(self.cloudmask, gdal.GA_ReadOnly)
+                    clm_1 = dataset.GetRasterBand(1).ReadAsArray()
+                    clm_2 = dataset.GetRasterBand(2).ReadAsArray()
+                    clm_3 = dataset.GetRasterBand(3).ReadAsArray()
+                    tot = clm_1 + clm_2 + clm_3
+                    valid_px_mask = np.zeros(clm_1.shape, np.uint8)
+                    valid_px_mask[tot == 0] = 1
+                    # resize valid_px  to output res:
+                    shape = (int(valid_px_mask.shape[0] * - image.yRes / res), int(valid_px_mask.shape[1] * image.xRes / res))
+                    valid_px_mask = skit_resize(valid_px_mask, shape, order=0, preserve_range=True).astype(np.uint8)
+                    #Applied no data mask:
+                    valid_px_mask[nodata == 0] = 0
+
+                    # save to image
+                    mask = image.duplicate(mask_filename, array=valid_px_mask, res=res)
+                    mask.write(creation_options=['COMPRESS=LZW'], nodata_value=None)
+                    log.info('Written: {}'.format(mask_filename))
+                    self.mask_filename = mask_filename
+
+                    dataset = None
+
             return True
 
     def get_angle_images(self, DST=None):

sen2like/sen2like/version.py

@@ -1,3 +1,3 @@
 """Version of the Application."""
 
-__version__ = '4.0.1'
+__version__ = '4.0.2'

sen2like/tests/configuration/config.ini

@@ -18,6 +18,7 @@ coverage = 0.1
 ;base_url = /data/PRODUCTS
 ;url_parameters_pattern_Sentinel2 = {base_url}/{mission}/{tile}
 ;url_parameters_pattern_Landsat8 = {base_url}/{mission}/{path}/{row}
+;url_parameters_pattern_Landsat9 = {base_url}/{mission}/{path}/{row}
 
 # Creodias
 base_url = https://finder.creodias.eu/resto/api/collections