Merge remote-tracking branch 'refs/remotes/origin/develop' into ci/an…

…tares_in_memory_ci

# Conflicts:
#	src/cpp/benders/factories/BendersFactory.cpp

CMakeLists.txt

@@ -2,7 +2,7 @@
 # CMake configuration
 # ===========================================================================
 
-CMAKE_MINIMUM_REQUIRED(VERSION 3.0)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.5)
 
 # ===========================================================================
 # Project configuration

cmake/json-cmake-1.1.0/JSONParser.cmake

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.5)
 
 if (DEFINED JSonParserGuard)
     return()

src/cpp/benders/benders_core/BendersBase.cpp

@@ -513,17 +513,34 @@ void BendersBase::post_run_actions() const {
 }
 
 void BendersBase::SaveCurrentIterationInOutputFile() const {
+  if (!_options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) {
+    auto &LastWorkerMasterData = relevantIterationData_.last;
+    if (LastWorkerMasterData._valid) {
+      _writer->write_iteration(iteration(LastWorkerMasterData),
+                               _data.it + iterations_before_resume);
+      _writer->dump();
+    }
+  }
+}
+
+void BendersBase::SaveCurrentOuterLoopIterationInOutputFile() const {
   auto &LastWorkerMasterData = relevantIterationData_.last;
   if (LastWorkerMasterData._valid) {
-    _writer->write_iteration(iteration(LastWorkerMasterData),
-                             _data.it + iterations_before_resume);
+    _writer->write_iteration(
+        iteration(LastWorkerMasterData),
+        _data.outer_loop_current_iteration_data.benders_num_run);
     _writer->dump();
   }
 }
+
 void BendersBase::SaveSolutionInOutputFile() const {
   _writer->write_solution(solution());
   _writer->dump();
 }
+void BendersBase::SaveOuterLoopSolutionInOutputFile() const {
+  _writer->write_solution(GetOuterLoopSolution());
+  _writer->dump();
+}
 
 Output::CandidatesVec candidates_data(const WorkerMasterData &masterDataPtr_l) {
   Output::CandidatesVec candidates_vec;
@@ -562,9 +579,24 @@ Output::Iteration BendersBase::iteration(
 }
 
 Output::SolutionData BendersBase::solution() const {
+  auto solution_data = BendersSolution();
+  solution_data.best_it = _data.best_it + iterations_before_resume;
+
+  return solution_data;
+}
+
+void BendersBase::UpdateOuterLoopSolution() {
+  outer_loop_solution_data_ = BendersSolution();
+  outer_loop_solution_data_.best_it =
+      _data.outer_loop_current_iteration_data.benders_num_run;
+}
+
+Output::SolutionData BendersBase::GetOuterLoopSolution() const {
+  return outer_loop_solution_data_;
+}
+Output::SolutionData BendersBase::BendersSolution() const {
   Output::SolutionData solution_data;
   solution_data.nbWeeks_p = _totalNbProblems;
-  solution_data.best_it = _data.best_it + iterations_before_resume;
   solution_data.problem_status = status_from_criterion();
   const auto optimal_gap(_data.best_ub - _data.lb);
   const auto relative_gap(optimal_gap / _data.best_ub);
@@ -745,6 +777,9 @@ void BendersBase::MatchProblemToId() {
   }
 }
 
+// Search for variables in sub problems that satify patterns
+// var_indices is a vector(for each patterns p) of vector (var indices related
+// to p)
 void BendersBase::SetSubproblemsVariablesIndex() {
   if (!subproblem_map.empty() && _options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) {
     auto subproblem = subproblem_map.begin();
@@ -876,9 +911,13 @@ void BendersBase::ClearCurrentIterationCutTrace() {
 }
 void BendersBase::EndWritingInOutputFile() const {
   _writer->updateEndTime();
+  // TODO duration for outer loop
   _writer->write_duration(_data.benders_time);
-  SaveSolutionInOutputFile();
+  if (!_options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) {
+    SaveSolutionInOutputFile();
+  }
 }
+
 double BendersBase::GetBendersTime() const { return benders_timer.elapsed(); }
 void BendersBase::write_basis() const {
   const auto filename(std::filesystem::path(_options.OUTPUTROOT) /

src/cpp/benders/benders_core/OuterLoopInputDataReader.cpp

@@ -17,7 +17,7 @@ OuterLoopPattern::OuterLoopPattern(std::string prefix,
  * just cat ;)
  */
 std::regex OuterLoopPattern::MakeRegex() const {
-  auto pattern = "(^" + prefix_ + ").*" + body_;
+  auto pattern = "(^" + prefix_ + "area<" + body_ + ">" + ")";
   return std::regex(pattern);
 }
 const std::string &OuterLoopPattern::GetPrefix() const { return prefix_; }

src/cpp/benders/benders_core/include/BendersBase.h

@@ -97,6 +97,9 @@ class BendersBase {
   virtual void ExternalLoopCheckFeasibility() = 0;
   virtual void RunExternalLoopBilevelChecks() = 0;
   double OuterLoopStoppingThreshold() const;
+  Output::SolutionData GetOuterLoopSolution() const;
+  void SaveOuterLoopSolutionInOutputFile() const;
+  void SaveCurrentOuterLoopIterationInOutputFile() const;
 
  protected:
   bool exception_raised_ = false;
@@ -237,6 +240,7 @@ class BendersBase {
       const PlainData::SubProblemData &sub_problem_data);
 
   void UpdateOuterLoopMaxCriterionArea();
+  void UpdateOuterLoopSolution();
 
  private:
   void print_master_and_cut(std::ostream &file, int ite,
@@ -246,6 +250,7 @@ class BendersBase {
   void check_status(const SubProblemDataMap &subproblem_data_map) const;
   [[nodiscard]] LogData build_log_data_from_data() const;
   [[nodiscard]] Output::SolutionData solution() const;
+  [[nodiscard]] Output::SolutionData BendersSolution() const;
   [[nodiscard]] std::string status_from_criterion() const;
   void compute_cut_aggregate(const SubProblemDataMap &subproblem_data_map);
   void compute_cut(const SubProblemDataMap &subproblem_data_map);
@@ -271,6 +276,7 @@ class BendersBase {
   int iterations_before_resume = 0;
   int cumulative_number_of_subproblem_resolved_before_resume = 0;
   Timer benders_timer;
+  Output::SolutionData outer_loop_solution_data_;
 
  public:
   Logger _logger;

src/cpp/benders/benders_mpi/BendersMPI.cpp

@@ -42,10 +42,12 @@ void BendersMpi::InitializeProblems() {
     current_problem_id++;
   }
 
-  // if (_world.rank() == rank_0) {
-  SetSubproblemsVariablesIndex();
-    // }
-    init_problems_ = false;
+  if (_world.rank() == rank_0) {
+    SetSubproblemsVariablesIndex();
+  }
+
+  BroadCast(var_indices_, rank_0);
+  init_problems_ = false;
 }
 void BendersMpi::BuildMasterProblem() {
   if (_world.rank() == rank_0) {
@@ -407,12 +409,16 @@ void BendersMpi::RunExternalLoopBilevelChecks() {
     const WorkerMasterData &workerMasterData = BestIterationWorkerMaster();
     const auto &invest_cost = workerMasterData._invest_cost;
     const auto &overall_cost = invest_cost + workerMasterData._operational_cost;
-    outer_loop_biLevel_.Update_bilevel_data_if_feasible(
-        _data.x_cut,
-        GetOuterLoopCriterionAtBestBenders() /*/!\ must
-                be at best it*/
-        ,
-        overall_cost, invest_cost, _data.outer_loop_current_iteration_data.external_loop_lambda);
-    _data.outer_loop_current_iteration_data.outer_loop_bilevel_best_ub = outer_loop_biLevel_.BilevelBestub();
+    if (outer_loop_biLevel_.Update_bilevel_data_if_feasible(
+            _data.x_cut, GetOuterLoopCriterionAtBestBenders() /*/!\ must
+                                 be at best it*/
+            ,
+            overall_cost, invest_cost,
+            _data.outer_loop_current_iteration_data.external_loop_lambda)) {
+      UpdateOuterLoopSolution();
+    }
+    SaveCurrentOuterLoopIterationInOutputFile();
+    _data.outer_loop_current_iteration_data.outer_loop_bilevel_best_ub =
+        outer_loop_biLevel_.BilevelBestub();
   }
 }

src/cpp/benders/factories/BendersFactory.cpp

@@ -49,10 +49,7 @@ pBendersBase BendersMainFactory::PrepareForExecution(
 
 
   BendersBaseOptions benders_options(options.get_benders_options());
-
-  auto path_to_log =
-      std::filesystem::path(options.OUTPUTROOT) /
-      ("bendersLog-rank" + std::to_string(pworld_->rank()) + ".txt.");
+  benders_options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP = external_loop;
 
   auto log_reports_name =
       std::filesystem::path(options.OUTPUTROOT) / "reportbenders.txt";

src/cpp/benders/outer_loop/CMakeLists.txt

@@ -6,12 +6,15 @@
 # Targets
 # ===========================================================================
 
-find_package(TBB REQUIRED CONFIG)
-if (TBB_VERSION_MAJOR VERSION_LESS "2018")
-	message(FATAL_ERROR "Require tbb 2018 or higher.")
-endif()
-if (TBB_VERSION_MAJOR VERSION_GREATER "2020")
-	message(FATAL_ERROR "Require tbb 2018 to 2020.")
+find_package (MPI REQUIRED COMPONENTS C)
+message ("MPI_C_INCLUDE_PATH ${MPI_C_INCLUDE_PATH}")
+message ("MPI_C_INCLUDE_DIRS ${MPI_C_INCLUDE_DIRS}")
+message ("MPI_C_LIBRARIES ${MPI_C_LIBRARIES}")
+
+find_package(MPI REQUIRED)
+
+if(UNIX)
+set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
 endif()
 
 add_library (outer_loop_lib STATIC
@@ -26,10 +29,14 @@ add_library (outer_loop_lib STATIC
 target_include_directories (outer_loop_lib
 		PUBLIC
 		${CMAKE_CURRENT_SOURCE_DIR}/include
+	    INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}
 		)
 
 target_link_libraries (outer_loop_lib
 	PUBLIC
+		Boost::mpi
+		MPI::MPI_C
+		Boost::serialization
 		helpers
 		benders_core
 		benders_mpi_core

src/cpp/benders/outer_loop/OuterLoop.cpp

@@ -47,7 +47,7 @@ void OuterLoop::Run() {
   // last prints
   PrintLog();
   benders_->mathLoggerDriver_->Print(benders_->GetCurrentIterationData());
-
+  benders_->SaveOuterLoopSolutionInOutputFile();
   // TODO general-case
   //  cuts_manager_->Save(benders_->AllCuts());
   benders_->free();

src/cpp/benders/output/JsonWriter.cpp

@@ -91,8 +91,9 @@ void JsonWriter::write_iteration(const Iteration &iter,
   }
   _output[ITERATIONS_C][strIterCnt_l][CANDIDATES_C] = vectCandidates_l;
 }
+
 void JsonWriter::write_solution(const SolutionData &solution) {
-  // solution
+    // solution
   _output[SOLUTION_C][ITERATION_C] = solution.best_it;
   _output[SOLUTION_C][INVESTMENT_COST_C] = solution.solution.investment_cost;
   _output[SOLUTION_C][OPERATIONAL_COST_C] = solution.solution.operational_cost;

src/cpp/exe/outer_loop/CMakeLists.txt

@@ -20,7 +20,7 @@ set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
 endif()
 
 target_link_libraries(outer_loop
-			${PROJECT_NAME}::benders_mpi_core
+			${PROJECT_NAME}::outer_loop_lib
 			factories
 			)
 

src/cpp/xpansion_interfaces/OutputWriter.h

@@ -18,6 +18,7 @@ const std::string ANTARES_C("antares"), VERSION_C("version"),
     OPERATIONAL_COST_C("operational_cost"), OPTIMALITY_GAP_C("optimality_gap"),
     OVERALL_COST_C("overall_cost"), RELATIVE_GAP_C("relative_gap"), UB_C("ub"),
     NBWEEKS_C("nbWeeks"), OPTIONS_C("options"), SOLUTION_C("solution"),
+    OUTER_LOOP_SOLUTION_C("security criterion solution"),
     ITERATION_C("iteration"), PROBLEM_STATUS_C("problem_status"),
     OPTIMAL_C("OPTIMAL"), LIMIT_REACHED_C("limit reached"), ERROR_C("ERROR"),
     VALUES_C("values"), STOPPING_CRITERION_C("stopping_criterion"),

tests/cpp/TestDoubles/WriterStub.h

@@ -15,6 +15,7 @@ class WriterNOOPStub : public Output::OutputWriter {
   void write_master_name(const std::string& master_name) override {}
   void write_log_level(const int log_level) override {}
   void write_solution(const Output::SolutionData& solution) override {}
+
   void write_iteration(const Output::Iteration& iteration_data,
                        const size_t iteration_num) override {}
   void updateBeginTime() override {}

tests/cpp/outer_loop/outer_loop_test.cpp

@@ -161,12 +161,15 @@ TEST_F(OuterLoopPatternTest, RegexGivenPrefixAndBody) {
 
   auto ret_regex = o.MakeRegex();
 
-  ASSERT_EQ(std::regex_search(prefix + body, ret_regex), true);
+  ASSERT_EQ(std::regex_search(prefix + body, ret_regex), false);
   ASSERT_EQ(std::regex_search(prefix + "::" + body + "::suffix", ret_regex),
-            true);
+            false);
   ASSERT_EQ(std::regex_search(body + prefix, ret_regex), false);
   ASSERT_EQ(std::regex_search(prefix + "::", ret_regex), false);
   ASSERT_EQ(std::regex_search(body, ret_regex), false);
+  ASSERT_EQ(std::regex_search(prefix + "area<" + body + ">", ret_regex), true);
+  ASSERT_EQ(std::regex_search(prefix + "area<" + body + ">::suffix", ret_regex), true);
+  ASSERT_EQ(std::regex_search(prefix + "area<" + body + "_other_area>::suffix", ret_regex), false);
 }
 
 class OuterLoopInputFromYamlTest : public ::testing::Test {};