Ml dev add metric (#37)

* power grid benchmark configurations updated with new metric MAPE10 for ML category * external metrics updated with a new MAPE10 metric imported from LeapNet
IRT-SystemX · Apr 24, 2024 · 5a2816f · 5a2816f
1 parent fb4048e
commit 5a2816f
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 15 deletions.
diff --git a/configurations/powergrid/benchmarks/l2rpn_case14_sandbox.ini b/configurations/powergrid/benchmarks/l2rpn_case14_sandbox.ini
@@ -329,17 +329,15 @@ dataset_create_params = {
 		"max_disc": 2} # authorize at most 2 disconnection
 	}
 eval_dict = {
-	"ML": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "TIME_INF"],
+	"ML": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "MAPE_10_avg", "TIME_INF"],
 	"Physics": ["CURRENT_POS", "VOLTAGE_POS", "LOSS_POS", "DISC_LINES", "CHECK_LOSS", "CHECK_GC", "CHECK_LC", "CHECK_JOULE_LAW"],
 	"IndRed": ["TIME_INF"],
-	"OOD": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "TIME_INF"]}
+	"OOD": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "MAPE_10_avg", "TIME_INF"]}
 eval_params = {
 	"inf_batch_size": 14000,
 	"EL_tolerance": 0.04,
 	"GC_tolerance": 1e-3,
 	"LC_tolerance": 1e-2,
-	"KCL": {"tolerance": 1e-2,
-			"ACTIVE_FLOW": True},
-	"JOULE_tolerance": 1e-2
+	"JOULE_tolerance": 1e-2,
 	}
 
diff --git a/configurations/powergrid/benchmarks/l2rpn_idf_2023.ini b/configurations/powergrid/benchmarks/l2rpn_idf_2023.ini
@@ -321,10 +321,10 @@ dataset_create_params = {
 		"action_by_area": True} # authorize at most 2 disconnection
 	}
 eval_dict = {
-	"ML": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "TIME_INF"],
+	"ML": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "MAPE_10_avg", "TIME_INF"],
 	"Physics": ["CURRENT_POS", "VOLTAGE_POS", "LOSS_POS", "DISC_LINES", "CHECK_LOSS", "CHECK_GC", "CHECK_LC", "CHECK_JOULE_LAW"],
 	"IndRed": ["TIME_INF"],
-	"OOD": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "TIME_INF"]}
+	"OOD": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "MAPE_10_avg", "TIME_INF"]}
 eval_params = {
 	"inf_batch_size": 14000,
 	"EL_tolerance": 0.04,

diff --git a/configurations/powergrid/benchmarks/l2rpn_neurips_2020_track1_small.ini b/configurations/powergrid/benchmarks/l2rpn_neurips_2020_track1_small.ini
@@ -336,17 +336,14 @@ dataset_create_params = {
 		"max_disc": 2} # authorize at most 2 disconnection
 	}
 eval_dict = {
-	"ML": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "TIME_INF"],
-	"Physics": ["CURRENT_POS", "VOLTAGE_POS", "LOSS_POS", "DISC_LINES", "CHECK_LOSS", "CHECK_GC", "CHECK_LC", "CHECK_VOLTAGE_EQ"],
+	"ML": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "MAPE_10_avg", "TIME_INF"],
+	"Physics": ["CURRENT_POS", "VOLTAGE_POS", "LOSS_POS", "DISC_LINES", "CHECK_LOSS", "CHECK_GC", "CHECK_LC", "CHECK_JOULE_LAW"],
 	"IndRed": ["TIME_INF"],
-	"OOD": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "TIME_INF"]}
+	"OOD": ["MSE_avg", "MAE_avg", "MAPE_avg", "MAPE_90_avg", "MAPE_10_avg", "TIME_INF"]}
 eval_params = {
-	"inf_batch_size": 59000,
+	"inf_batch_size": 14000,
 	"EL_tolerance": 0.04,
 	"GC_tolerance": 1e-3,
 	"LC_tolerance": 1e-2,
-	"KCL": {"tolerance": 1e-2,
-			"ACTIVE_FLOW": True},
-	"VOLTAGE_EQ": {"tolerance": 1e-4,
-				   "verify_theta": False}
+	"JOULE_tolerance": 1e-2,
 	}
diff --git a/lips/metrics/ml_metrics/external_metrics.py b/lips/metrics/ml_metrics/external_metrics.py
@@ -14,5 +14,12 @@
     metric_factory.register_metric("MAPE_90", lambda y_true, y_pred: mape_quantile(y_true=y_true,
                                                                                    y_pred=y_pred,
                                                                                    multioutput="raw_values"))
+    metric_factory.register_metric("MAPE_10_avg", lambda y_true, y_pred: mape_quantile(y_true=y_true,
+                                                                                       y_pred=y_pred,
+                                                                                       quantile=0.9))
+    metric_factory.register_metric("MAPE_10", lambda y_true, y_pred: mape_quantile(y_true=y_true,
+                                                                                   y_pred=y_pred,
+                                                                                   multioutput="raw_values",
+                                                                                   quantile=0.9))
 except ImportError as exc_:
     pass