ENH: Use config file for image profiles (theroggy#56)

.env

@@ -1 +1 @@
-PYTHONPATH=.
+PYTHONPATH=.

.github/workflows/tests.yml

@@ -71,6 +71,6 @@ jobs:
         
       - name: Test
         run: |
-          pytest --color=yes --cov=geofileops --cov-append --cov-report term-missing --cov-report xml tests/
+          pytest --color=yes --cov=cropclassification --cov-append --cov-report term-missing --cov-report xml tests/
 
       - uses: codecov/codecov-action@v4

bin_util/calc_periodic_mosaic.py

@@ -15,7 +15,11 @@ def main():
     start_date = datetime(2023, 8, 1)
     end_date = datetime(2023, 10, 1)
     dest_image_data_dir = Path("//dg3.be/alp/Datagis/satellite_periodic/BEFL")
-    sensordata_to_get = [conf.image_profiles["s2-agri"]]
+    sensordata_to_get = [
+        conf._get_image_profiles(
+            Path("x:/monitoring/markers/dev/_config/image_profiles.ini")
+        )["s2-agri"]
+    ]
 
     # As we want a weekly calculation, get nearest monday for start and stop day
     start_date = ts_helper.get_monday(start_date)

cropclassification/general.ini

@@ -129,6 +129,9 @@ parceldata_aggregations_to_use = median
 # Postprocess...
 postprocess_to_groups = 
 
+# Path to the image profiles
+image_profiles_config_filepath = ${dirs:marker_basedir}/image_profiles.ini
+
 [timeseries]
 # The maximum percentage cloudcover an (S2) image can have to be used
 max_cloudcover_pct = 15

cropclassification/helpers/config_helper.py

@@ -117,6 +117,10 @@ def read_config(config_paths: List[Path], default_basedir: Optional[Path] = None
     columns = config["columns"]
     global dirs
     dirs = config["dirs"]
+    global image_profiles
+    image_profiles = _get_image_profiles(
+        marker.getpath("image_profiles_config_filepath")
+    )
 
 
 def parse_sensordata_to_use(input) -> Dict[str, SensorData]:
@@ -154,85 +158,35 @@ def parse_sensordata_to_use(input) -> Dict[str, SensorData]:
     return result
 
 
-def _get_raster_profiles() -> Dict[str, ImageProfile]:
+def _get_image_profiles(image_profiles_path: Path) -> Dict[str, ImageProfile]:
     # Cropclassification gives best results with time_dimension_reducer "mean" for both
     # sentinel 2 and sentinel 1 images.
-    # TODO: this should move to a config file
-
-    job_options_extra_memory = {
-        "executor-memory": "4G",
-        "executor-memoryOverhead": "2G",
-        "executor-cores": "2",
-    }
+    # Init
+    if not image_profiles_path.exists():
+        raise ValueError(f"Config file specified does not exist: {image_profiles_path}")
 
-    profiles = {}
-    profiles["s2-agri"] = ImageProfile(
-        name="s2-agri",
-        satellite="s2",
-        collection="TERRASCOPE_S2_TOC_V2",
-        bands=["B02", "B03", "B04", "B08", "B11", "B12"],
-        # Use the "min" reducer filters out "lightly clouded areas"
-        process_options={
-            "time_dimension_reducer": "mean",
-            "cloud_filter_band_dilated": "SCL",
-        },
-        job_options=None,
-    )
-    profiles["s2-scl"] = ImageProfile(
-        name="s2-scl",
-        satellite="s2",
-        collection="TERRASCOPE_S2_TOC_V2",
-        bands=["SCL"],
-        # Use the "min" reducer filters out "lightly clouded areas"
-        process_options={
-            "time_dimension_reducer": "max",
-            # "cloud_filter_band_dilated": "SCL",
-            "cloud_filter_band": "SCL",
-        },
-        job_options=None,
-    )
-    profiles["s2-ndvi"] = ImageProfile(
-        name="s2-ndvi",
-        satellite="s2",
-        collection="TERRASCOPE_S2_NDVI_V2",
-        bands=["NDVI_10M"],
-        process_options={
-            "time_dimension_reducer": "mean",
-            # "cloud_filter_band_dilated": "SCENECLASSIFICATION_20M",
-            "cloud_filter_band": "SCENECLASSIFICATION_20M",
-        },
-        job_options=job_options_extra_memory,
-    )
-    profiles["s1-grd-sigma0-asc"] = ImageProfile(
-        name="s1-grd-sigma0-asc",
-        satellite="s1",
-        collection="S1_GRD_SIGMA0_ASCENDING",
-        bands=["VV", "VH"],
-        process_options={
-            "time_dimension_reducer": "mean",
-        },
-        job_options=None,
-    )
-    profiles["s1-grd-sigma0-desc"] = ImageProfile(
-        name="s1-grd-sigma0-desc",
-        satellite="s1",
-        collection="S1_GRD_SIGMA0_DESCENDING",
-        bands=["VV", "VH"],
-        process_options={
-            "time_dimension_reducer": "mean",
-        },
-        job_options=None,
-    )
-    profiles["s1-coh"] = ImageProfile(
-        name="s1-coh",
-        satellite="s1",
-        collection="TERRASCOPE_S1_SLC_COHERENCE_V1",
-        bands=["VV", "VH"],
-        process_options={
-            "time_dimension_reducer": "mean",
+    # Read config file...
+    profiles_config = configparser.ConfigParser(
+        interpolation=configparser.ExtendedInterpolation(),
+        converters={
+            "list": lambda x: [i.strip() for i in x.split(",")],
+            "dict": lambda x: None if x == "None" else json.loads(x),
         },
-        job_options=None,
+        allow_no_value=True,
     )
+    profiles_config.read(image_profiles_path)
+
+    # Prepare data
+    profiles = {}
+    for profile in profiles_config.sections():
+        profiles[profile] = ImageProfile(
+            name=profiles_config[profile].get("name"),
+            satellite=profiles_config[profile].get("satellite"),
+            collection=profiles_config[profile].get("collection"),
+            bands=profiles_config[profile].getlist("bands"),
+            process_options=profiles_config[profile].getdict("process_options"),
+            job_options=profiles_config[profile].getdict("job_options"),
+        )
 
     return profiles
 
@@ -264,6 +218,3 @@ def as_dict():
         ].__dict__
 
     return the_dict
-
-
-image_profiles = _get_raster_profiles()

tests/data/image_profiles.ini

@@ -0,0 +1,65 @@
+# In this file, the image profiles are configured
+
+[s2-agri]
+name = s2-agri
+satellite = s2
+collection = TERRASCOPE_S2_TOC_V2
+bands = B02, B03, B04, B08, B11, B12
+# Use the "min" reducer filters out "lightly clouded areas"
+process_options = { "time_dimension_reducer": "mean",
+                    "cloud_filter_band_dilated": "SCL"
+                }
+job_options = None
+
+[s2-scl]
+name = s2-scl
+satellite = s2
+collection = TERRASCOPE_S2_TOC_V2
+bands = SCL
+# Use the "min" reducer filters out "lightly clouded areas"
+process_options = { "time_dimension_reducer": "max",
+                    # "cloud_filter_band_dilated": "SCL",
+                    "cloud_filter_band": "SCL"
+                }
+job_options = None
+
+[s2-ndvi]
+name = s2-ndvi
+satellite = s2
+collection = TERRASCOPE_S2_NDVI_V2
+bands = NDVI_10M
+process_options = { "time_dimension_reducer": "mean",
+                    # "cloud_filter_band_dilated": "SCENECLASSIFICATION_20M",
+                    "cloud_filter_band": "SCENECLASSIFICATION_20M"
+                }
+job_options = { "executor-memory": "4G",
+                "executor-memoryOverhead": "2G",
+                "executor-cores": "2"
+            }
+
+[s1-grd-sigma0-asc]
+name = s1-grd-sigma0-asc
+satellite = s1
+collection = S1_GRD_SIGMA0_ASCENDING
+bands = VV, VH
+process_options = { "time_dimension_reducer": "mean"
+                }
+job_options = None
+
+[s1-grd-sigma0-desc]
+name = s1-grd-sigma0-desc
+satellite = s1
+collection = S1_GRD_SIGMA0_DESCENDING
+bands = VV, VH
+process_options = { "time_dimension_reducer": "mean"
+                }
+job_options = None
+
+[s1-coh]
+name = s1-coh
+satellite = s1
+collection = TERRASCOPE_S1_SLC_COHERENCE_V1
+bands = VV, VH
+process_options = { "time_dimension_reducer": "mean"
+                }
+job_options = None

tests/test_config_helper.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+import pytest
+
+from cropclassification.helpers import config_helper as conf
+
+
+@pytest.mark.parametrize(
+    "sensor",
+    [
+        ("s2-agri"),
+        ("s2-scl"),
+        ("s2-ndvi"),
+        ("s1-grd-sigma0-asc"),
+        ("s1-grd-sigma0-desc"),
+        ("s1-coh"),
+    ],
+)
+def test_get_image_profiles(sensor: str):
+    data_dir = Path(__file__).resolve().parent / "data"
+    config_path = data_dir / "image_profiles.ini"
+    image_profiles = conf._get_image_profiles(config_path)
+
+    profile = image_profiles.get(sensor)
+    assert profile is not None