TST: avoid too long filenames in tests on windows (theroggy#169)

bin_util/calc_periodic_mosaic.py

@@ -41,7 +41,7 @@ def main():
 
     image_profiles_path = (
         Path(__file__).resolve().parent.parent
-        / "sample_marker_basedir/_config/image_profiles.ini"
+        / "markers/_config/image_profiles.ini"
     )
     imageprofiles = conf._get_image_profiles(image_profiles_path)
 

cropclassification/helpers/pandas_helper.py

@@ -1,6 +1,5 @@
 """Module with helper functions to expand on some features of pandas."""
 
-import os
 import sqlite3
 from pathlib import Path
 from typing import Any, Optional, Union
@@ -134,28 +133,28 @@ def to_file(
     """
     if df is None:
         raise ValueError("Dataframe df is None")
+    if not append and path.exists():
+        raise ValueError(f"path already exists and append is False: {path}")
 
     ext_lower = path.suffix.lower()
     if ext_lower == ".csv":
         if append:
-            raise Exception("Append is not supported for csv files")
+            raise ValueError("Append is not supported for csv files")
         df.to_csv(str(path), float_format="%.10f", encoding="utf-8", index=index)
     elif ext_lower == ".tsv":
         if append:
-            raise Exception("Append is not supported for tsv files")
+            raise ValueError("Append is not supported for tsv files")
         df.to_csv(
             str(path), sep="\t", float_format="%.10f", encoding="utf-8", index=index
         )
     elif ext_lower == ".parquet":
         if append:
-            raise Exception("Append is not supported for parquet files")
+            raise ValueError("Append is not supported for parquet files")
         df.to_parquet(str(path), index=index)
     elif ext_lower == ".sqlite":
         if_exists = "fail"
         if append:
             if_exists = "append"
-        elif os.path.exists(path):
-            os.remove(path)
         sql_db = None
         try:
             sql_db = sqlite3.connect(str(path))
@@ -167,9 +166,14 @@ def to_file(
                 chunksize=50000,
             )
         except Exception as ex:
-            raise Exception(f"Error in to_file to file {path!s}") from ex
+            path_length = len(str(path))
+            if path_length > 250:
+                message = f"Error in to_file (note: {path_length=}) with {path=!s}"
+            else:
+                message = f"Error in to_file with {path=!s}"
+            raise RuntimeError(message) from ex
         finally:
             if sql_db is not None:
                 sql_db.close()
     else:
-        raise Exception(f"Not implemented for extension {ext_lower}")
+        raise ValueError(f"Not implemented for extension {ext_lower}")

sample_marker_basedir/_images_periodic/roi_test/s1-grd-sigma0-asc-weekly/s1-grd-sigma0-asc-weekly_2024-03-04_2024-03-10_VV-VH_last.tif.json → markers/_images_periodic/roi_test/s1-grd-sigma0-asc-weekly/s1-grd-sigma0-asc-weekly_2024-03-04_2024-03-10_VV-VH_last.tif.json

@@ -23,6 +23,6 @@
         "VH"
     ],
     "time_reducer": "first",
-    "path": "C:/Users/local_PIEROG/Temp/1/pytest-of-PIEROG/pytest-5/test_calc_periodic_mosaic_loca0/sample_marker_basedir/periodic_mosaic/roi_test/s1-grd-sigma0-asc/s1-grd-sigma0-asc_2024-03-04_2024-03-10_VV-VH_first.tif",
+    "path": "C:/Users/local_PIEROG/Temp/1/pytest-of-PIEROG/pytest-5/test_calc_periodic_mosaic_loca0/markers/periodic_mosaic/roi_test/s1-grd-sigma0-asc/s1-grd-sigma0-asc_2024-03-04_2024-03-10_VV-VH_first.tif",
     "orbit": "asc"
 }

sample_marker_basedir/_images_periodic/roi_test/s1-grd-sigma0-asc-weekly/s1-grd-sigma0-asc-weekly_2024-03-11_2024-03-17_VV-VH_last.tif.json → markers/_images_periodic/roi_test/s1-grd-sigma0-asc-weekly/s1-grd-sigma0-asc-weekly_2024-03-11_2024-03-17_VV-VH_last.tif.json

@@ -25,7 +25,7 @@
         "VH"
     ],
     "time_reducer": "last",
-    "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-36/test_end2end0/sample_marker_basedir/_images_periodic/BEFL/s1-grd-sigma0-asc-weekly/s1-grd-sigma0-asc-weekly_2024-03-11_2024-03-17_VV-VH_last.tif",
+    "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-36/test_end2end0/markers/_images_periodic/BEFL/s1-grd-sigma0-asc-weekly/s1-grd-sigma0-asc-weekly_2024-03-11_2024-03-17_VV-VH_last.tif",
     "job_options": null,
     "process_options": null
 }

sample_marker_basedir/_images_periodic/roi_test/s1-grd-sigma0-desc-weekly/s1-grd-sigma0-desc-weekly_2024-03-04_2024-03-10_VV-VH_last.tif.json → markers/_images_periodic/roi_test/s1-grd-sigma0-desc-weekly/s1-grd-sigma0-desc-weekly_2024-03-04_2024-03-10_VV-VH_last.tif.json

@@ -23,6 +23,6 @@
         "VH"
     ],
     "time_reducer": "first",
-    "path": "C:/Users/local_PIEROG/Temp/1/pytest-of-PIEROG/pytest-5/test_calc_periodic_mosaic_loca0/sample_marker_basedir/periodic_mosaic/roi_test/s1-grd-sigma0-desc/s1-grd-sigma0-desc_2024-03-04_2024-03-10_VV-VH_first.tif",
+    "path": "C:/Users/local_PIEROG/Temp/1/pytest-of-PIEROG/pytest-5/test_calc_periodic_mosaic_loca0/markers/periodic_mosaic/roi_test/s1-grd-sigma0-desc/s1-grd-sigma0-desc_2024-03-04_2024-03-10_VV-VH_first.tif",
     "orbit": "desc"
 }

sample_marker_basedir/_images_periodic/roi_test/s1-grd-sigma0-desc-weekly/s1-grd-sigma0-desc-weekly_2024-03-11_2024-03-17_VV-VH_last.tif.json → markers/_images_periodic/roi_test/s1-grd-sigma0-desc-weekly/s1-grd-sigma0-desc-weekly_2024-03-11_2024-03-17_VV-VH_last.tif.json

@@ -25,7 +25,7 @@
         "VH"
     ],
     "time_reducer": "last",
-    "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-36/test_end2end0/sample_marker_basedir/_images_periodic/BEFL/s1-grd-sigma0-desc-weekly/s1-grd-sigma0-desc-weekly_2024-03-11_2024-03-17_VV-VH_last.tif",
+    "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-36/test_end2end0/markers/_images_periodic/BEFL/s1-grd-sigma0-desc-weekly/s1-grd-sigma0-desc-weekly_2024-03-11_2024-03-17_VV-VH_last.tif",
     "job_options": null,
     "process_options": null
 }

sample_marker_basedir/_images_periodic/roi_test/s2-agri-weekly/s2-agri-weekly_2024-03-11_2024-03-17_B02-B03-B04-B08-B11-B12_mean.tif.json → markers/_images_periodic/roi_test/s2-agri-weekly/s2-agri-weekly_2024-03-11_2024-03-17_B02-B03-B04-B08-B11-B12_mean.tif.json

@@ -29,7 +29,7 @@
         "B12"
     ],
     "time_reducer": "mean",
-    "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-36/test_end2end0/sample_marker_basedir/_images_periodic/BEFL/s2-agri-weekly/s2-agri-weekly_2024-03-11_2024-03-17_B02-B03-B04-B08-B11-B12_mean.tif",
+    "path": "C:/Users/local_KRIWAY/Temp/1/pytest-of-KRIWAY/pytest-36/test_end2end0/markers/_images_periodic/BEFL/s2-agri-weekly/s2-agri-weekly_2024-03-11_2024-03-17_B02-B03-B04-B08-B11-B12_mean.tif",
     "job_options": {
         "driver-memoryOverhead": "5G"
     },

tests/test_config_helper.py

@@ -39,7 +39,7 @@ def test_read_config():
     ]
     conf.read_config(
         config_paths=config_paths,
-        default_basedir=SampleData.marker_basedir,
+        default_basedir=SampleData.markers_dir,
         overrules=[
             "paths.images_periodic_dir=_satellite_periodic/BEFL",
         ],
@@ -80,7 +80,7 @@ def test_read_config_overrule():
     config_paths = SampleData.config_dir / "cropgroup.ini"
     conf.read_config(
         config_paths=config_paths,
-        default_basedir=SampleData.marker_basedir,
+        default_basedir=SampleData.markers_dir,
         overrules=[
             "roi.roi_name=ROI_NAME_TEST",
             "paths.images_periodic_dir=_satellite_periodic/BEFL",

tests/test_end2end.py

@@ -22,13 +22,13 @@
 )
 def test_task_calc_marker(tmp_path, balancing_strategy, cross_pred_models):
     if not HAS_QGIS:
-        pytest.skip("QGIS is not available on this system.")
+        pytest.skip("QGIS is needed for timeseries calculation, but is not available.")
 
-    marker_basedir = tmp_path / test_helper.SampleData.marker_basedir.name
-    shutil.copytree(test_helper.SampleData.marker_basedir, marker_basedir)
+    markers_dir = tmp_path / test_helper.SampleData.markers_dir.name
+    shutil.copytree(test_helper.SampleData.markers_dir, markers_dir)
 
     # Create configparser and read task file!
-    tasks_dir = marker_basedir / "_tasks"
+    tasks_dir = markers_dir / "_tasks"
     ignore_dir = tasks_dir / "ignore"
     task_ini = "task_test_calc_marker.ini"
 
@@ -44,7 +44,7 @@ def test_task_calc_marker(tmp_path, balancing_strategy, cross_pred_models):
     )
 
     today_str = datetime.now().strftime("%Y-%m-%d")
-    run_dir = marker_basedir / f"2024_CROPGROUP/Run_{today_str}_001"
+    run_dir = markers_dir / f"2024_CROPGROUP/Run_{today_str}_001"
     assert run_dir.exists()
     base_stem = "Prc_BEFL_2023_2023-07-24_bufm5_weekly_predict_all"
     assert (run_dir / f"{base_stem}.gpkg").exists()
@@ -71,10 +71,10 @@ def test_task_calc_marker(tmp_path, balancing_strategy, cross_pred_models):
 
 
 def test_task_calc_periodic_mosaic(tmp_path):
-    marker_basedir = tmp_path / test_helper.SampleData.marker_basedir.name
-    shutil.copytree(test_helper.SampleData.marker_basedir, marker_basedir)
+    markers_dir = tmp_path / test_helper.SampleData.markers_dir.name
+    shutil.copytree(test_helper.SampleData.markers_dir, markers_dir)
     # Create configparser and read task file!
-    tasks_dir = marker_basedir / "_tasks"
+    tasks_dir = markers_dir / "_tasks"
     ignore_dir = tasks_dir / "ignore"
     task_ini = "task_test_calc_periodic_mosaic.ini"
 
@@ -83,6 +83,6 @@ def test_task_calc_periodic_mosaic(tmp_path):
     taskrunner.run_tasks(tasksdir=tasks_dir)
 
     # Check if a log file was written
-    log_dir = marker_basedir / "log"
+    log_dir = markers_dir / "log"
     assert log_dir.exists()
     assert len(list(log_dir.glob("*.log"))) == 1

tests/test_helper.py

@@ -7,13 +7,13 @@
 
 
 class SampleData:
-    marker_basedir = Path(__file__).resolve().parent.parent / "sample_marker_basedir"
-    tasks_dir = marker_basedir / "_tasks"
-    config_dir = marker_basedir / "_config"
-    inputdata_dir = marker_basedir / "_inputdata"
-    refe_dir = marker_basedir / "_refe"
-    image_dir = marker_basedir / "_images_periodic"
-    input_dir = marker_basedir / "_inputdata"
+    markers_dir = Path(__file__).resolve().parent.parent / "markers"
+    tasks_dir = markers_dir / "_tasks"
+    config_dir = markers_dir / "_config"
+    inputdata_dir = markers_dir / "_inputdata"
+    refe_dir = markers_dir / "_refe"
+    image_dir = markers_dir / "_images_periodic"
+    input_dir = markers_dir / "_inputdata"
     roi_name = "roi_test"
     image_roi_dir = image_dir / roi_name
 

tests/test_mosaic_util.py

@@ -11,7 +11,7 @@
 
 def test_calc_periodic_mosaic_s2(tmp_path):
     # Prepare test data
-    sample_dir = SampleData.marker_basedir
+    sample_dir = SampleData.markers_dir
     test_dir = tmp_path / sample_dir.name
     shutil.copytree(sample_dir, test_dir)
 
@@ -48,7 +48,7 @@ def test_calc_periodic_mosaic_s2(tmp_path):
 
 def test_calc_periodic_mosaic_s1_local_index_dprvi(tmp_path):
     # Prepare test data
-    sample_dir = SampleData.marker_basedir
+    sample_dir = SampleData.markers_dir
     test_dir = tmp_path / sample_dir.name
     shutil.copytree(sample_dir, test_dir)
 

tests/test_zonal_stats_bulk.py

@@ -15,7 +15,7 @@ def test_zonal_stats_bulk(tmp_path, engine):
         pytest.skip("QGIS is not available on this system.")
 
     # Prepare test data
-    sample_dir = SampleData.marker_basedir
+    sample_dir = SampleData.markers_dir
     test_dir = tmp_path / sample_dir.name
     shutil.copytree(sample_dir, test_dir)