[Ant-2223] use area file (#955)

use area file for Benders new outputs.

src/cpp/benders/benders_by_batch/BatchCollection.cpp

@@ -16,7 +16,7 @@ void BatchCollection::BuildBatches() {
         std::string("batch_size(") + std::to_string(batch_size_) +
             ") can not be greater than number of subproblems (" +
             std::to_string(sub_problems_number_) + ")",
-        LogUtils::LOGLEVEL::WARNING);
+        LogUtils::LOGLEVEL::WARNING, "Benders By batch");
     logger_->display_message(
         std::string("Setting batch_size = number of subproblems(") +
         std::to_string(sub_problems_number_) +

src/cpp/benders/benders_core/BendersMathLogger.cpp

@@ -4,7 +4,7 @@
 #include <sstream>
 
 #include "antares-xpansion/xpansion_interfaces/LogUtils.h"
-#include "antares-xpansion/helpers/LoggerUtils.h"
+#include "antares-xpansion/xpansion_interfaces/LoggerUtils.h"
 
 HeadersManager::HeadersManager(HEADERSTYPE type, const BENDERSMETHOD& method)
     : type_(type), method_(method) {}
@@ -113,7 +113,11 @@ MathLogger::MathLogger(std::streamsize width, HEADERSTYPE type)
 void MathLogger::display_message(const std::string& str) {
   LogsDestination() << str << std::endl;
 }
-
+void MathLogger::display_message(const std::string& str,
+                                 LogUtils::LOGLEVEL level,
+                                 const std::string& context) {
+  LogsDestination() << PrefixMessage(level, context) << str << std::endl;
+}
 std::vector<std::string> MathLogger::Headers() const { return headers_; }
 
 LogDestination& MathLogger::LogsDestination() { return log_destination_; }
@@ -260,6 +264,14 @@ void MathLoggerDriver::display_message(const std::string& str) {
   }
 }
 
+void MathLoggerDriver::display_message(const std::string& str,
+                                       LogUtils::LOGLEVEL level,
+                                       const std::string& context) {
+  for (auto logger : math_loggers_) {
+    logger->display_message(str, level, context);
+  }
+}
+
 void MathLoggerDriver::PrintIterationSeparatorBegin() {
   for (auto logger : math_loggers_) {
     logger->PrintIterationSeparatorBegin();
@@ -330,6 +342,11 @@ void MathLoggerImplementation::display_message(const std::string& str) {
   implementation_->display_message(str);
 }
 
+void MathLoggerImplementation::display_message(const std::string& str,
+                                               LogUtils::LOGLEVEL level,
+                                               const std::string& context) {
+  implementation_->display_message(str, level, context);
+}
 void MathLoggerImplementation::Print(const CurrentIterationData& data) {
   implementation_->Print(data);
 }

src/cpp/benders/benders_core/SimulationOptions.cpp

@@ -137,6 +137,7 @@ BaseOptions SimulationOptions::get_base_options() const {
   result.SOLVER_NAME = SOLVER_NAME;
   result.weights = _weights;
   result.RESUME = RESUME;
+  result.AREA_FILE = AREA_FILE;
 
   return result;
 }

src/cpp/benders/benders_core/WorkerMaster.cpp

@@ -301,7 +301,10 @@ void WorkerMaster::_set_alpha_var() {
       solver_addrows(*_solver, rowtype, rowrhs, {}, mstart, mclind, matval);
     }
   } else {
-    logger_->display_message("ERROR a variable named overall_subpb_cost_under_approx is in input", LogUtils::LOGLEVEL::ERR);
+    logger_->display_message(
+        "ERROR There is already a variable called "
+        "overall_subpb_cost_under_approx in the input.",
+        LogUtils::LOGLEVEL::ERR, "Alpha var");
   }
 }
 

src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/BendersMathLogger.h

@@ -78,6 +78,8 @@ struct MathLogger : public MathLoggerBehaviour {
                       HEADERSTYPE type = HEADERSTYPE::SHORT);
 
   void display_message(const std::string& str) override;
+  void display_message(const std::string& str, LogUtils::LOGLEVEL level,
+                       const std::string& context) override;
   virtual void Print(const CurrentIterationData& data) = 0;
   std::vector<std::string> Headers() const override;
   LogDestination& LogsDestination() override;
@@ -137,6 +139,8 @@ struct MathLoggerExternalLoopSpecific : public MathLogger {
   void setHeadersList() override;
   void Print(const CurrentIterationData& data) override;
   void display_message(const std::string& msg) override;
+  void display_message(const std::string& str, LogUtils::LOGLEVEL level,
+                       const std::string& context) override;
   virtual ~MathLoggerExternalLoopSpecific() = default;
 
  private:
@@ -156,6 +160,8 @@ class MathLoggerImplementation : public MathLoggerBehaviour {
   explicit MathLoggerImplementation(std::shared_ptr<MathLogger> implementation);
 
   void display_message(const std::string& str) override;
+  void display_message(const std::string& str, LogUtils::LOGLEVEL level,
+                       const std::string& context) override;
   void Print(const CurrentIterationData& data) override;
   void PrintIterationSeparatorBegin() override;
   void PrintIterationSeparatorEnd() override;
@@ -176,6 +182,8 @@ class MathLoggerDriver : public ILoggerXpansion {
   MathLoggerDriver() = default;
   void write_header();
   void display_message(const std::string& str) override;
+  void display_message(const std::string& str, LogUtils::LOGLEVEL level,
+                       const std::string& context) override;
   void add_logger(std::shared_ptr<MathLoggerImplementation> logger);
   template <class T>
   void add_logger(const std::filesystem::path& file_path,
@@ -218,4 +226,11 @@ template <class T>
 void MathLoggerExternalLoopSpecific<T>::display_message(
     const std::string& msg) {
   // keep empty
+}
+
+template <class T>
+void MathLoggerExternalLoopSpecific<T>::display_message(
+    const std::string& str, LogUtils::LOGLEVEL level,
+    const std::string& context) {
+  // keep empty
 }

src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/OuterLoopInputDataReader.h

@@ -4,7 +4,7 @@
 #include <string>
 #include <vector>
 
-#include "antares-xpansion/helpers/LoggerUtils.h"
+#include "antares-xpansion/xpansion_interfaces/LoggerUtils.h"
 #include "yaml-cpp/yaml.h"
 
 namespace Benders::Criterion {

src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/SimulationOptions.hxx

@@ -79,3 +79,5 @@ BENDERS_OPTIONS_MACRO(DO_OUTER_LOOP, bool, false, asBool())
 // Outer Loop Options file
 BENDERS_OPTIONS_MACRO(OUTER_LOOP_OPTION_FILE, std::string,
                       "adequacy_criterion.yml", asString())
+// area file
+BENDERS_OPTIONS_MACRO(AREA_FILE, std::string, "area.txt", asString())

src/cpp/benders/benders_core/include/antares-xpansion/benders/benders_core/common.h

@@ -57,8 +57,7 @@ enum class BENDERSMETHOD {
   BENDERS_BY_BATCH_OUTERLOOP
 };
 
-constexpr inline std::string_view bendersmethod_to_string(
-    BENDERSMETHOD method) {
+inline std::string bendersmethod_to_string(BENDERSMETHOD method) {
   switch (method) {
     case BENDERSMETHOD::BENDERS:
       return "Benders";
@@ -149,6 +148,7 @@ struct BaseOptions {
   std::string MASTER_NAME;
   std::string SOLVER_NAME;
   std::string SLAVE_WEIGHT;
+  std::string AREA_FILE;
 
   int LOG_LEVEL = 0;
 

src/cpp/benders/benders_mpi/include/antares-xpansion/benders/benders_mpi/BendersMPI.h

@@ -1,15 +1,15 @@
 #pragma once
 
-#include "antares-xpansion/helpers/ArchiveReader.h"
 #include "antares-xpansion/benders/benders_core/BendersBase.h"
 #include "antares-xpansion/benders/benders_core/BendersStructsDatas.h"
-#include "antares-xpansion/xpansion_interfaces/ILogger.h"
-#include "antares-xpansion/helpers/LoggerUtils.h"
 #include "antares-xpansion/benders/benders_core/SubproblemCut.h"
 #include "antares-xpansion/benders/benders_core/SubproblemWorker.h"
-#include "antares-xpansion/helpers/Timer.h"
 #include "antares-xpansion/benders/benders_core/Worker.h"
 #include "antares-xpansion/benders/benders_core/WorkerMaster.h"
+#include "antares-xpansion/helpers/ArchiveReader.h"
+#include "antares-xpansion/helpers/Timer.h"
+#include "antares-xpansion/xpansion_interfaces/ILogger.h"
+#include "antares-xpansion/xpansion_interfaces/LoggerUtils.h"
 #include "common_mpi.h"
 
 /*!

src/cpp/benders/factories/BendersFactory.cpp

@@ -4,16 +4,16 @@
 #include <filesystem>
 
 #include "antares-xpansion/benders/benders_by_batch/BendersByBatch.h"
-#include "antares-xpansion/benders/benders_mpi/BendersMpiOuterLoop.h"
-#include "antares-xpansion/xpansion_interfaces/ILogger.h"
-#include "antares-xpansion/xpansion_interfaces/LogUtils.h"
-#include "antares-xpansion/benders/factories/LoggerFactories.h"
 #include "antares-xpansion/benders/benders_core/MasterUpdate.h"
-#include "antares-xpansion/benders/benders_mpi/OuterLoopBenders.h"
 #include "antares-xpansion/benders/benders_core/StartUp.h"
-#include "antares-xpansion/helpers/Timer.h"
-#include "antares-xpansion/benders/benders_core/Worker.h"
+#include "antares-xpansion/benders/benders_mpi/BendersMpiOuterLoop.h"
+#include "antares-xpansion/benders/benders_mpi/OuterLoopBenders.h"
+#include "antares-xpansion/benders/factories/LoggerFactories.h"
 #include "antares-xpansion/benders/factories/WriterFactories.h"
+#include "antares-xpansion/helpers/AreaParser.h"
+#include "antares-xpansion/helpers/Timer.h"
+#include "antares-xpansion/xpansion_interfaces/ILogger.h"
+#include "antares-xpansion/xpansion_interfaces/LogUtils.h"
 
 BENDERSMETHOD DeduceBendersMethod(size_t coupling_map_size, size_t batch_size,
                                   bool external_loop) {
@@ -44,13 +44,14 @@ pBendersBase BendersMainFactory::PrepareForExecution(bool external_loop) {
 
   method_ = DeduceBendersMethod(coupling_map.size(), options_.BATCH_SIZE,
                                 external_loop);
+  context_ = bendersmethod_to_string(method_);
+
   auto benders_log_console = benders_options.LOG_LEVEL > 0;
   if (pworld_->rank() == 0) {
     auto logger_factory =
         FileAndStdoutLoggerFactory(LogReportsName(), benders_log_console);
     logger_ = logger_factory.get_logger();
-
-
+    math_log_driver = BuildMathLogger(benders_log_console);
     writer_ = build_json_writer(options_.JSON_FILE, options_.RESUME);
     if (Benders::StartUp startup;
         startup.StudyAlreadyAchievedCriterion(options_, writer_, logger_))
@@ -61,16 +62,17 @@ pBendersBase BendersMainFactory::PrepareForExecution(bool external_loop) {
     math_log_driver = MathLoggerFactory::get_void_logger();
   }
 
-  auto outer_loop_input_data = ProcessCriterionInput(coupling_map);
+  benders_loggers_.AddLogger(logger_);
+  benders_loggers_.AddLogger(math_log_driver);
+  auto outer_loop_input_data = ProcessCriterionInput();
   criterion_computation_ =
       std::make_shared<Benders::Criterion::CriterionComputation>(
           outer_loop_input_data);
-  if (pworld_->rank() == 0) {
-    math_log_driver = BuildMathLogger(benders_log_console);
+
+  if (pworld_->rank() == 0 && !outer_loop_input_data.OuterLoopData().empty()) {
+    AddCriterionOutput(math_log_driver);
   }
 
-  benders_loggers_.AddLogger(logger_);
-  benders_loggers_.AddLogger(math_log_driver);
   switch (method_) {
     case BENDERSMETHOD::BENDERS:
       benders = std::make_shared<BendersMpi>(benders_options, logger_, writer_,
@@ -118,17 +120,24 @@ std::shared_ptr<MathLoggerDriver> BendersMainFactory::BuildMathLogger(
 
   auto math_log_driver = math_log_factory.get_logger();
 
+  return math_log_driver;
+}
+
+void BendersMainFactory::AddCriterionOutput(
+    std::shared_ptr<MathLoggerDriver> math_log_driver) {
+  const std::filesystem::path output_root(options_.OUTPUTROOT);
+
   const auto& headers =
       criterion_computation_->getOuterLoopInputData().PatternBodies();
   math_log_driver->add_logger(
-      output_root / "LOLD.txt", headers,
+      output_root / LOLD_FILE, headers,
       &OuterLoopCurrentIterationData::outer_loop_criterion);
+
+  positive_unsupplied_file_ =
+      criterion_computation_->getOuterLoopInputData().PatternsPrefix() + ".txt";
   math_log_driver->add_logger(
-      output_root /
-          (criterion_computation_->getOuterLoopInputData().PatternsPrefix() +
-           ".txt"),
-      headers, &OuterLoopCurrentIterationData::outer_loop_patterns_values);
-  return math_log_driver;
+      output_root / positive_unsupplied_file_, headers,
+      &OuterLoopCurrentIterationData::outer_loop_patterns_values);
 }
 
 int BendersMainFactory::RunBenders() {
@@ -158,17 +167,19 @@ void BendersMainFactory::EndMessage(const double execution_time) {
   std::ostringstream str;
   str << "Optimization results available in : " << options_.JSON_FILE
       << std::endl;
-  benders_loggers_.display_message(str.str());
+  benders_loggers_.display_message(str.str(), LogUtils::LOGLEVEL::INFO,
+                                   context_);
 
   str.str("");
 
   str << bendersmethod_to_string(method_) << " ran in " << execution_time
       << " s" << std::endl;
-  benders_loggers_.display_message(str.str());
+  benders_loggers_.display_message(str.str(), LogUtils::LOGLEVEL::INFO,
+                                   context_);
 }
 
 Benders::Criterion::OuterLoopInputData
-BendersMainFactory::ProcessCriterionInput(const CouplingMap& couplingMap) {
+BendersMainFactory::ProcessCriterionInput() {
   const auto fpath = std::filesystem::path(options_.INPUTROOT) /
                      options_.OUTER_LOOP_OPTION_FILE;
   // if adequacy_criterion.yml is provided read it
@@ -177,36 +188,13 @@ BendersMainFactory::ProcessCriterionInput(const CouplingMap& couplingMap) {
   }
   // else compute criterion for all areas!
   else {
-    return GetInputFromSubProblem(couplingMap);
+    return BuildPatternsUsingAreaFile();
   }
 }
 
 Benders::Criterion::OuterLoopInputData
-BendersMainFactory::GetInputFromSubProblem(const CouplingMap& couplingMap) {
-  auto first_subproblem_pair = std::find_if_not(
-      couplingMap.begin(), couplingMap.end(),
-      [this](const auto& in) { return in.first == options_.MASTER_NAME; });
-  if (first_subproblem_pair == couplingMap.end()) {
-    std::ostringstream stream;
-    auto log_location = LOGLOCATION;
-    stream << "Could not find any Subproblem in structure file "
-           << options_.STRUCTURE_FILE << std::endl;
-    benders_loggers_.display_message(log_location + stream.str());
-    throw InvalidStructureFile(
-        PrefixMessage(LogUtils::LOGLEVEL::FATAL, "Benders"), stream.str(),
-        log_location);
-  } else {
-    const auto first_subproblem_name = first_subproblem_pair->first;
-    return PatternsFromSupbProblem(first_subproblem_name);
-  }
-}
-
-Benders::Criterion::OuterLoopInputData
-BendersMainFactory::PatternsFromSupbProblem(
-    const std::string& first_subproblem_name) const {
-  SolverAbstract::Ptr solver = BuildSolver(first_subproblem_name);
-  const auto all_variables_name = solver->get_col_names();
-  std::set<std::string> unique_areas = UniqueAreas(all_variables_name);
+BendersMainFactory::BuildPatternsUsingAreaFile() {
+  std::set<std::string> unique_areas = ReadAreaFile();
   Benders::Criterion::OuterLoopInputData ret;
   ret.SetCriterionCountThreshold(1);
 
@@ -216,36 +204,28 @@ BendersMainFactory::PatternsFromSupbProblem(
     ret.AddSingleData(singleInputData);
   }
 
-  solver->free();
   return ret;
 }
 
-std::set<std::string> BendersMainFactory::UniqueAreas(
-    const std::vector<std::string>& all_variables_name) const {
+std::set<std::string> BendersMainFactory::ReadAreaFile() {
   std::set<std::string> unique_areas;
-  std::regex area_regex(
-      "area<([^>]+)>");  // Regular expression to match area<...>
-
-  for (const auto& str : all_variables_name) {
-    std::smatch match;
-    if (std::regex_search(str, match, area_regex)) {
-      unique_areas.insert(match[1]);  // Insert the matched area into the set
-    }
+  const auto area_file =
+      std::filesystem::path(options_.INPUTROOT) / options_.AREA_FILE;
+  const auto area_file_data = AreaParser::ReadAreaFile(area_file);
+  if (const auto& msg = area_file_data.error_message; !msg.empty()) {
+    benders_loggers_.display_message(msg, LogUtils::LOGLEVEL::WARNING,
+                                     context_);
+    std::ostringstream ms;
+    ms << " Consequently, " << LOLD_FILE
+       << " and other criterion based files will not be produced!";
+
+    benders_loggers_.display_message(ms.str(), LogUtils::LOGLEVEL::WARNING,
+                                     context_);
+    return {};
   }
-  return unique_areas;
+  return {area_file_data.areas.begin(), area_file_data.areas.end()};
 }
 
-SolverAbstract::Ptr BendersMainFactory::BuildSolver(
-    const std::string& first_subproblem_name) const {
-  SolverFactory factory(logger_);
-  auto solver_log_manager = SolverLogManager(LogReportsName());
-  auto solver = factory.create_solver(
-      options_.SOLVER_NAME, SOLVER_TYPE::CONTINUOUS, solver_log_manager);
-  solver->read_prob_mps(std::filesystem::path(options_.INPUTROOT) /
-                        first_subproblem_name);
-  solver->set_threads(1);
-  return solver;
-}
 
 int BendersMainFactory::RunExternalLoop() {
   try {