From 57e84e024e28f4688823cc7674c94ed6069fe7da Mon Sep 17 00:00:00 2001 From: romain-bdc Date: Wed, 11 Jan 2023 18:15:53 +0100 Subject: [PATCH 1/6] modification to deactivate irrigation processing when tir ard given in input is empty --- src/ewoc_classif/classif.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/ewoc_classif/classif.py b/src/ewoc_classif/classif.py index add5942..ac2a626 100644 --- a/src/ewoc_classif/classif.py +++ b/src/ewoc_classif/classif.py @@ -5,6 +5,7 @@ import os import shutil import traceback +import csv from json import dump, load from pathlib import Path from tempfile import gettempdir @@ -320,6 +321,14 @@ def run_classif( if tir_csv is None: tir_csv = out_dirpath / f"{uid}_satio_tir.csv" ewoc_ard_bucket.tir_to_satio_csv(tile_id, production_id, filepath=tir_csv) + else: + with open(Path(tir_csv), 'r') as tir_file: + tir_dict = [row for row in csv.DictReader(tir_file)] + no_tir=False + if len(tir_dict) <= 1: + logger.warning(f"TIR ARD is empty for the tile {tile_id}") + no_tir=True + if agera5_csv is None: agera5_csv = out_dirpath / f"{uid}_satio_agera5.csv" ewoc_aux_data_bucket = EWOCAuxDataBucket() @@ -332,6 +341,8 @@ def run_classif( "DEM": "s3://ewoc-aux-data/CopDEM_20m", "METEO": str(agera5_csv), } + + ewoc_config = generate_config_file( ewoc_detector, end_season_year, @@ -343,6 +354,11 @@ def run_classif( csv_dict, feature_blocks_dir= feature_blocks_dir ) + + if no_tir: + ewoc_config["inputs"].pop("TIR") + ewoc_config["parameters"]["irrigation"] = False + ewoc_config_filepath = out_dirpath / f"{uid}_ewoc_config.json" if data_folder is not None: ewoc_config = update_config(ewoc_config, ewoc_detector, data_folder) From dacb6e7025afc28fadf43b4693af2751741c6256 Mon Sep 17 00:00:00 2001 From: msavinaud Date: Thu, 12 Jan 2023 10:09:07 +0100 Subject: [PATCH 2/6] fix pylint warning --- src/ewoc_classif/classif.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ewoc_classif/classif.py b/src/ewoc_classif/classif.py index ac2a626..07c7212 100644 --- a/src/ewoc_classif/classif.py +++ b/src/ewoc_classif/classif.py @@ -322,12 +322,12 @@ def run_classif( tir_csv = out_dirpath / f"{uid}_satio_tir.csv" ewoc_ard_bucket.tir_to_satio_csv(tile_id, production_id, filepath=tir_csv) else: - with open(Path(tir_csv), 'r') as tir_file: + with open(Path(tir_csv), 'r', encoding='utf8') as tir_file: tir_dict = [row for row in csv.DictReader(tir_file)] no_tir=False if len(tir_dict) <= 1: logger.warning(f"TIR ARD is empty for the tile {tile_id}") - no_tir=True + no_tir=True if agera5_csv is None: agera5_csv = out_dirpath / f"{uid}_satio_agera5.csv" From b8699db31a655b5ce55ce7ee4979749ffea40cce Mon Sep 17 00:00:00 2001 From: romain-bdc Date: Thu, 12 Jan 2023 17:07:04 +0100 Subject: [PATCH 3/6] add new test for 40KEC_071, add modifications on no_tir use in utils.py and classif.py --- src/ewoc_classif/classif.py | 38 +++++++++++++++++++++-------------- src/ewoc_classif/utils.py | 40 ++++++++++++++++++++++++------------- tests/test_classif.py | 11 ++++++++++ 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/src/ewoc_classif/classif.py b/src/ewoc_classif/classif.py index ac2a626..f618f61 100644 --- a/src/ewoc_classif/classif.py +++ b/src/ewoc_classif/classif.py @@ -259,7 +259,8 @@ def run_classif( upload_block: bool = True, postprocess: bool = False, out_dirpath: Path = Path(gettempdir()), - clean=True + clean:bool=True, + no_tir:bool=False ) -> None: """ Perform EWoC classification @@ -298,6 +299,8 @@ def run_classif( :type postprocess: bool :param out_dirpath: Output directory path :type out_dirpath: Path + :param no_tir: Boolean specifying if the csv file containing details on ARD TIR is empty or not + :type no_tir: bool :return: None """ uid = uuid4().hex[:6] @@ -322,25 +325,33 @@ def run_classif( tir_csv = out_dirpath / f"{uid}_satio_tir.csv" ewoc_ard_bucket.tir_to_satio_csv(tile_id, production_id, filepath=tir_csv) else: - with open(Path(tir_csv), 'r') as tir_file: + with open(Path(tir_csv), 'r', encoding='utf8') as tir_file: tir_dict = [row for row in csv.DictReader(tir_file)] no_tir=False if len(tir_dict) <= 1: logger.warning(f"TIR ARD is empty for the tile {tile_id}") - no_tir=True + no_tir=True if agera5_csv is None: agera5_csv = out_dirpath / f"{uid}_satio_agera5.csv" ewoc_aux_data_bucket = EWOCAuxDataBucket() ewoc_aux_data_bucket.agera5_to_satio_csv(filepath=agera5_csv) - csv_dict = { - "OPTICAL": str(optical_csv), - "SAR": str(sar_csv), - "TIR": str(tir_csv), - "DEM": "s3://ewoc-aux-data/CopDEM_20m", - "METEO": str(agera5_csv), - } + if not no_tir: + csv_dict = { + "OPTICAL": str(optical_csv), + "SAR": str(sar_csv), + "TIR": str(tir_csv), + "DEM": "s3://ewoc-aux-data/CopDEM_20m", + "METEO": str(agera5_csv), + } + else: + csv_dict = { + "OPTICAL": str(optical_csv), + "SAR": str(sar_csv), + "DEM": "s3://ewoc-aux-data/CopDEM_20m", + "METEO": str(agera5_csv), + } ewoc_config = generate_config_file( @@ -352,13 +363,10 @@ def run_classif( croptype_model_version, irr_model_version, csv_dict, - feature_blocks_dir= feature_blocks_dir + feature_blocks_dir= feature_blocks_dir, + no_tir_data=no_tir ) - if no_tir: - ewoc_config["inputs"].pop("TIR") - ewoc_config["parameters"]["irrigation"] = False - ewoc_config_filepath = out_dirpath / f"{uid}_ewoc_config.json" if data_folder is not None: ewoc_config = update_config(ewoc_config, ewoc_detector, data_folder) diff --git a/src/ewoc_classif/utils.py b/src/ewoc_classif/utils.py index 9870f9b..7ef91ac 100644 --- a/src/ewoc_classif/utils.py +++ b/src/ewoc_classif/utils.py @@ -78,6 +78,7 @@ def generate_config_file( irr_model_version: str, csv_dict: Dict, feature_blocks_dir: Path, + no_tir_data: bool, ) -> Dict: """ Automatic generation of worldcereal config files @@ -103,6 +104,8 @@ def generate_config_file( :type csv_dict: Dict :param feature_blocks_dir: Block features dir :type feature_blocks_dir: Path + :param no_tir: Boolean specifying if the csv file containing details on ARD TIR is empty or not + :type no_tir: bool :return: Dict """ parameters = { @@ -143,20 +146,29 @@ def generate_config_file( parameters["filtersettings"] = {"kernelsize": 7, "conf_threshold": 0.75} parameters["save_features"]= True parameters["features_dir"]=str(feature_blocks_dir) - parameters.update( - { - "active_marker": True, - "cropland_mask": cropland_mask_bucket, - "irrigation": True, - "irrparameters": "irrigation", - "irrmodels": { - "irrigation": f"{ewoc_model_prefix}/models/WorldCerealPixelCatBoost/{irr_model_version}/irrigation_detector_WorldCerealPixelCatBoost_{irr_model_version}/config.json" - }, - } - ) - logger.info( - f"[{featuresettings}] - Using Irrigation model version: {irr_model_version}" - ) + if not no_tir_data: + parameters.update( + { + "active_marker": True, + "cropland_mask": cropland_mask_bucket, + "irrigation": True, + "irrparameters": "irrigation", + "irrmodels": { + "irrigation": f"{ewoc_model_prefix}/models/WorldCerealPixelCatBoost/{irr_model_version}/irrigation_detector_WorldCerealPixelCatBoost_{irr_model_version}/config.json" + }, + } + ) + else: + parameters.update( + { + "active_marker": True, + "cropland_mask": cropland_mask_bucket, + "irrigation" : False + } + ) + logger.info( + f"[{featuresettings}] - Using Irrigation model version: {irr_model_version}" + ) if ewoc_season == "summer1": models = { "maize": f"{ewoc_model_prefix}/models/WorldCerealPixelCatBoost/{croptype_model_version}/maize_detector_WorldCerealPixelCatBoost_{croptype_model_version}/config.json", diff --git a/tests/test_classif.py b/tests/test_classif.py index 416a505..2a8b4a1 100644 --- a/tests/test_classif.py +++ b/tests/test_classif.py @@ -217,3 +217,14 @@ def test_run_classif_cropland_43SCB_50(self): 'c728b264-5c97-4f4c-81fe-1500d4c4dfbd_25147_20220918052128', block_ids=[50], upload_block=False) + + def test_run_classif_winter_40KEC_71(self): + """This test return exit code 2 + """ + run_classif('40KEC', + 'c728b264-5c97-4f4c-81fe-1500d4c4dfbd_9026_20220926141535', + block_ids=[71], + upload_block=False, + tir_csv="./tests/tir_preprocessed_path.csv", + ewoc_detector=EWOC_CROPTYPE_DETECTOR, + ewoc_season=EWOC_SUPPORTED_SEASONS[0]) \ No newline at end of file From cc823b2fd6e776225ca1eead2f75ae69c145f4a3 Mon Sep 17 00:00:00 2001 From: romain-bdc Date: Thu, 12 Jan 2023 17:13:04 +0100 Subject: [PATCH 4/6] add empty csv file to test tir csv is not None but empty --- src/ewoc_classif/classif.py | 1 - tests/tir_preprocessed_path.csv | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/tir_preprocessed_path.csv diff --git a/src/ewoc_classif/classif.py b/src/ewoc_classif/classif.py index f618f61..0c5490c 100644 --- a/src/ewoc_classif/classif.py +++ b/src/ewoc_classif/classif.py @@ -327,7 +327,6 @@ def run_classif( else: with open(Path(tir_csv), 'r', encoding='utf8') as tir_file: tir_dict = [row for row in csv.DictReader(tir_file)] - no_tir=False if len(tir_dict) <= 1: logger.warning(f"TIR ARD is empty for the tile {tile_id}") no_tir=True diff --git a/tests/tir_preprocessed_path.csv b/tests/tir_preprocessed_path.csv new file mode 100644 index 0000000..1f4f13c --- /dev/null +++ b/tests/tir_preprocessed_path.csv @@ -0,0 +1 @@ +,date,tile,level,path From 3c8f0a8202a8059ee73bef07146b44ccf14fcac9 Mon Sep 17 00:00:00 2001 From: msavinaud Date: Tue, 17 Jan 2023 09:29:16 +0100 Subject: [PATCH 5/6] fix some minor issue and add an test --- src/ewoc_classif/classif.py | 2 +- src/ewoc_classif/utils.py | 6 +++--- tests/test_classif.py | 20 ++++++++++++++++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/ewoc_classif/classif.py b/src/ewoc_classif/classif.py index 0c5490c..61ae288 100644 --- a/src/ewoc_classif/classif.py +++ b/src/ewoc_classif/classif.py @@ -328,7 +328,7 @@ def run_classif( with open(Path(tir_csv), 'r', encoding='utf8') as tir_file: tir_dict = [row for row in csv.DictReader(tir_file)] if len(tir_dict) <= 1: - logger.warning(f"TIR ARD is empty for the tile {tile_id}") + logger.warning(f"TIR ARD is empty for the tile {tile_id} => No irrigation computed!") no_tir=True if agera5_csv is None: diff --git a/src/ewoc_classif/utils.py b/src/ewoc_classif/utils.py index 7ef91ac..345e44e 100644 --- a/src/ewoc_classif/utils.py +++ b/src/ewoc_classif/utils.py @@ -158,6 +158,9 @@ def generate_config_file( }, } ) + logger.info( + f"[{featuresettings}] - Using Irrigation model version: {irr_model_version}" + ) else: parameters.update( { @@ -166,9 +169,6 @@ def generate_config_file( "irrigation" : False } ) - logger.info( - f"[{featuresettings}] - Using Irrigation model version: {irr_model_version}" - ) if ewoc_season == "summer1": models = { "maize": f"{ewoc_model_prefix}/models/WorldCerealPixelCatBoost/{croptype_model_version}/maize_detector_WorldCerealPixelCatBoost_{croptype_model_version}/config.json", diff --git a/tests/test_classif.py b/tests/test_classif.py index 2a8b4a1..a460423 100644 --- a/tests/test_classif.py +++ b/tests/test_classif.py @@ -218,13 +218,29 @@ def test_run_classif_cropland_43SCB_50(self): block_ids=[50], upload_block=False) + @unittest.skipIf(os.getenv("EWOC_TEST_VAL_TEST") is None,"env variable not set") def test_run_classif_winter_40KEC_71(self): - """This test return exit code 2 + """ Nominal case with no tir detected + + Island case (Mauritius) """ - run_classif('40KEC', + run_classif('40KEC', 'c728b264-5c97-4f4c-81fe-1500d4c4dfbd_9026_20220926141535', block_ids=[71], upload_block=False, tir_csv="./tests/tir_preprocessed_path.csv", ewoc_detector=EWOC_CROPTYPE_DETECTOR, + ewoc_season=EWOC_SUPPORTED_SEASONS[0]) + + def test_run_classif_winter_40KEC_71_no_csv(self): + """ Nominal case with no tir detected + + Island case (Mauritius) + """ + run_classif('40KEC', + 'c728b264-5c97-4f4c-81fe-1500d4c4dfbd_9026_20220926141535', + block_ids=[71], + upload_block=False, + clean=False, + ewoc_detector=EWOC_CROPTYPE_DETECTOR, ewoc_season=EWOC_SUPPORTED_SEASONS[0]) \ No newline at end of file From 0e4121d8360686b972d739739b63677373a85227 Mon Sep 17 00:00:00 2001 From: msavinaud Date: Tue, 24 Jan 2023 23:22:20 +0100 Subject: [PATCH 6/6] update clean status for a test --- tests/test_classif.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_classif.py b/tests/test_classif.py index 45a7524..c7bb211 100644 --- a/tests/test_classif.py +++ b/tests/test_classif.py @@ -324,6 +324,7 @@ def test_run_classif_winter_40KEC_71(self): 'c728b264-5c97-4f4c-81fe-1500d4c4dfbd_9026_20220926141535', block_ids=[71], upload_block=False, + clean=False, tir_csv="./tests/tir_preprocessed_path.csv", ewoc_detector=EWOC_CROPTYPE_DETECTOR, ewoc_season=EWOC_SUPPORTED_SEASONS[0])