From 2ed62ff7d1eba92ab4dfc242ae0d2bd3a4dc4e40 Mon Sep 17 00:00:00 2001 From: abdoulbari zaher <32519851+a-zakir@users.noreply.github.com> Date: Thu, 16 May 2024 14:19:35 +0200 Subject: [PATCH] Feature/outer loop migrate criterion computation 2 (#789) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit follows #780 - compute outer loop criterion in Benders inner loop - new mechanism to store vars indices which are linked to patterns (PositiveUnsuppliedEnergy::\***AreaName\***) - Update Python drivers so the outer loop can be launched like other Xpansion steps - Yaml parser (yaml-cpp) ... --------- Co-authored-by: Thomas Bittar Co-authored-by: Jason Maréchal <45510813+JasonMarechal25@users.noreply.github.com> --- .gitignore | 1 + .../external_loop_test/expansion/.gitignore | 4 - .../external_loop_test/expansion/.gitkeep | 0 .../external_loop_test/lp/description.txt | 44 +++ data_test/external_loop_test/lp/options.json | 20 +- .../lp/outer_loop_options.yml | 15 + src/cpp/benders/CMakeLists.txt | 2 +- .../benders_by_batch/BendersByBatch.cpp | 25 +- .../benders_by_batch/include/BendersByBatch.h | 4 +- src/cpp/benders/benders_core/BendersBase.cpp | 125 +++++++- .../benders_core/BendersMathLogger.cpp | 15 +- src/cpp/benders/benders_core/CMakeLists.txt | 7 +- .../benders/benders_core/OuterLoopBiLevel.cpp | 66 +++++ .../benders_core/OuterLoopInputDataReader.cpp | 192 +++++++++++++ .../benders_core/SimulationOptions.cpp | 6 +- .../benders/benders_core/SubproblemWorker.cpp | 10 +- .../benders/benders_core/VariablesGroup.cpp | 28 ++ .../benders_core/include/BendersBase.h | 64 ++++- .../include/BendersStructsDatas.h | 18 +- .../benders_core/include/CustomVector.h | 9 + .../benders_core/include/OuterLoopBiLevel.h | 38 +++ .../include/OuterLoopInputDataReader.h | 115 ++++++++ .../include/SimulationOptions.hxx | 16 +- .../benders_core/include/SubproblemCut.h | 13 +- .../benders_core/include/SubproblemWorker.h | 2 +- .../benders_core/include/VariablesGroup.h | 20 ++ src/cpp/benders/benders_core/include/common.h | 15 +- src/cpp/benders/benders_mpi/BendersMPI.cpp | 112 +++++++- .../benders/benders_mpi/include/BendersMPI.h | 13 +- .../include/BendersSequential.h | 3 + src/cpp/benders/external_loop/OuterLoop.cpp | 90 ------ .../external_loop/OuterloopCriterion.cpp | 49 ---- .../include/OuterLoopCriterion.h | 39 --- src/cpp/benders/factories/BendersFactory.cpp | 77 +++-- src/cpp/benders/factories/CMakeLists.txt | 2 +- .../factories/include/BendersFactory.h | 13 +- .../CMakeLists.txt | 13 +- .../CutsManagement.cpp | 2 + .../MasterUpdateBase.cpp | 89 +++--- src/cpp/benders/outer_loop/OuterLoop.cpp | 71 +++++ .../include/CutsManagement.h | 4 + .../include/MasterUpdate.h | 23 +- .../include/OuterLoop.h | 10 +- src/cpp/exe/CMakeLists.txt | 2 +- src/cpp/exe/benders/main.cpp | 3 +- src/cpp/exe/full_run/main.cpp | 17 +- .../{ExtLoop => outer_loop}/CMakeLists.txt | 12 +- src/cpp/exe/{ExtLoop => outer_loop}/main.cpp | 5 +- src/cpp/full_run/FullRunOptionsParser.cpp | 7 +- .../full_run/include/FullRunOptionsParser.h | 3 + src/cpp/xpansion_interfaces/LogUtils.h | 4 + src/python/antares_xpansion/benders_driver.py | 20 +- .../antares_xpansion/config_file_parser.py | 2 + src/python/antares_xpansion/config_loader.py | 21 +- src/python/antares_xpansion/driver.py | 9 +- .../antares_xpansion/full_run_driver.py | 11 +- src/python/antares_xpansion/input_parser.py | 3 +- .../antares_xpansion/optimisation_keys.py | 16 ++ src/python/antares_xpansion/resume_study.py | 7 +- src/python/antares_xpansion/xpansionConfig.py | 5 + src/python/config.yaml.in | 1 + tests/cpp/CMakeLists.txt | 2 +- tests/cpp/ext_loop/CMakeLists.txt | 21 -- tests/cpp/ext_loop/ext_loop_test.cpp | 251 ---------------- tests/cpp/outer_loop/CMakeLists.txt | 16 ++ tests/cpp/outer_loop/outer_loop_test.cpp | 272 ++++++++++++++++++ tests/python/test_benders_driver.py | 50 +++- tests/python/test_full_run_driver.py | 19 +- vcpkg.json | 3 +- 69 files changed, 1543 insertions(+), 723 deletions(-) delete mode 100644 data_test/external_loop_test/expansion/.gitignore create mode 100644 data_test/external_loop_test/expansion/.gitkeep create mode 100644 data_test/external_loop_test/lp/description.txt create mode 100644 data_test/external_loop_test/lp/outer_loop_options.yml create mode 100644 src/cpp/benders/benders_core/OuterLoopBiLevel.cpp create mode 100644 src/cpp/benders/benders_core/OuterLoopInputDataReader.cpp create mode 100644 src/cpp/benders/benders_core/VariablesGroup.cpp create mode 100644 src/cpp/benders/benders_core/include/CustomVector.h create mode 100644 src/cpp/benders/benders_core/include/OuterLoopBiLevel.h create mode 100644 src/cpp/benders/benders_core/include/OuterLoopInputDataReader.h create mode 100644 src/cpp/benders/benders_core/include/VariablesGroup.h delete mode 100644 src/cpp/benders/external_loop/OuterLoop.cpp delete mode 100644 src/cpp/benders/external_loop/OuterloopCriterion.cpp delete mode 100644 src/cpp/benders/external_loop/include/OuterLoopCriterion.h rename src/cpp/benders/{external_loop => outer_loop}/CMakeLists.txt (75%) rename src/cpp/benders/{external_loop => outer_loop}/CutsManagement.cpp (88%) rename src/cpp/benders/{external_loop => outer_loop}/MasterUpdateBase.cpp (52%) create mode 100644 src/cpp/benders/outer_loop/OuterLoop.cpp rename src/cpp/benders/{external_loop => outer_loop}/include/CutsManagement.h (89%) rename src/cpp/benders/{external_loop => outer_loop}/include/MasterUpdate.h (54%) rename src/cpp/benders/{external_loop => outer_loop}/include/OuterLoop.h (71%) rename src/cpp/exe/{ExtLoop => outer_loop}/CMakeLists.txt (69%) rename src/cpp/exe/{ExtLoop => outer_loop}/main.cpp (54%) delete mode 100644 tests/cpp/ext_loop/CMakeLists.txt delete mode 100644 tests/cpp/ext_loop/ext_loop_test.cpp create mode 100644 tests/cpp/outer_loop/CMakeLists.txt create mode 100644 tests/cpp/outer_loop/outer_loop_test.cpp diff --git a/.gitignore b/.gitignore index 9cf063ffc..3d61cc63d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ _build _install antares-deps include/google +vcpkg_installed # Cmake generated files CMakeCache.txt diff --git a/data_test/external_loop_test/expansion/.gitignore b/data_test/external_loop_test/expansion/.gitignore deleted file mode 100644 index 5e7d2734c..000000000 --- a/data_test/external_loop_test/expansion/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/data_test/external_loop_test/expansion/.gitkeep b/data_test/external_loop_test/expansion/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/data_test/external_loop_test/lp/description.txt b/data_test/external_loop_test/lp/description.txt new file mode 100644 index 000000000..89d38bcf5 --- /dev/null +++ b/data_test/external_loop_test/lp/description.txt @@ -0,0 +1,44 @@ +We describe a system with 4 nodes, on 1 timestep, 2 scenarios : + + + |------- N1 + | + N0 ----- N2 + | + |------- N3 + + On node 0, there is a generator with : + - Initial P_max = 1 + - Prod cost = 1 + - Investment cost = 20 + + On nodes N1, N2, N3, there is a demand that differs between scenarios : + - On N1 : Demand1 = [1, 3] + - On N2 : Demand2 = [3, 3] + - On N3 : Demand3 = [0, 3] + + Over the whole system we have : + - Spillage cost = 1 + - ENS cost = 10 + + There are also transmission costs on lines : + - On L01 : transmission cost = 2 + - On L02 : transmission cost = 1 + - On L03 : transmission cost = 3 + + As the investment cost is higher than the ENS cost, adding 1MW of capacity would cost 20 to reduce ENS cost only by 10, hence an increased overall cost. Therefore the optimal investment is not to add any capacity beyond the existing one. Given the transmission cost, all flow will get to node 2. + + This use case is simple enough so that we can count the number of hours with ENS with respect to the investment. Then we could use this test to check the behavior of any heuristic that aims at reaching a given target of LOLE (not done here). + + We add 1 hour of ENS if there is at least 0.1 MWh of unsupplied energy, the "optimal" situation leads to 5h of ENS hours overall, hence 2.5h in expectation. + + Each invested capacity will first go to node 2, then to node 1 and finally to node 3 given the transmission costs. + + Hence we can deduce the number expected hours of ENS given the total capacity as follows (invested capacity is total capacity - 1 as P_max = 1 initially): + - 0 <= P_max <= 2.9 : 2.5h (1 in area 1, 1 in area 2, 0.5 in area 3) + - 2.9 <= P_max <= 3.9 : 1.5h (1 in area 1, 0 in area 2, 0.5 in area 3) + - 3.9 <= P_max <= 5.9 : 1h (0.5 in area 1, 0 in area 2, 0.5 in area 3) + - 5.9 <= P_max <= 8.9 : 0.5h (0 in area 1, 0 in area 2, 0.5 in area 3) + - 8.9 <= P_max : 0h (0 in area 1, 0 in area 2, 0 in area 3) + + Now, to test the heuristic taking into account the reliability constraint we can easily deduce the expected result depending on the criterion. For the test case, we require that all areas have less than 0.5 hours with ENS, hence, the solution is to have a total capacity of 3.9 (i.e. invest 2.9 as P_max = 1 initially) \ No newline at end of file diff --git a/data_test/external_loop_test/lp/options.json b/data_test/external_loop_test/lp/options.json index 13a960d34..3d3c3828a 100644 --- a/data_test/external_loop_test/lp/options.json +++ b/data_test/external_loop_test/lp/options.json @@ -4,26 +4,26 @@ "RELATIVE_GAP": 1e-06, "RELAXED_GAP": 1e-05, "AGGREGATION": false, - "OUTPUTROOT": "data_test/external_loop_test/lp/", + "OUTPUTROOT": "data_test/external_loop_test/lp", "TRACE": true, "SLAVE_WEIGHT": "CONSTANT", - "SLAVE_WEIGHT_VALUE": 3, + "SLAVE_WEIGHT_VALUE": 0.5, "MASTER_NAME": "master", - "STRUCTURE_FILE": "data_test/external_loop_test/lp/structure.txt", - "INPUTROOT": "data_test/external_loop_test/lp/", + "STRUCTURE_FILE": "structure.txt", + "INPUTROOT": "data_test/external_loop_test/lp", "CSV_NAME": "benders_output_trace", "BOUND_ALPHA": true, "SEPARATION_PARAM": 0.5, "BATCH_SIZE": 0, - "JSON_FILE":"data_test/external_loop_test/expansion/out.json", - "LAST_ITERATION_JSON_FILE":"data_test/external_loop_test/expansion/last_iteration.json", + "JSON_FILE": "data_test/external_loop_test/expansion/out.json", + "LAST_ITERATION_JSON_FILE": "data_test/external_loop_test/expansion/last_iteration.json", "MASTER_FORMULATION": "integer", "SOLVER_NAME": "XPRESS", "TIME_LIMIT": 1000000000000.0, - "LOG_LEVEL": 1, + "LOG_LEVEL": 0, "LAST_MASTER_MPS": "master_last_iteration", "LAST_MASTER_BASIS": "master_last_basis.bss", - "EXT_LOOP_CRITERION_VALUE": 3.0, - "EXT_LOOP_CRITERION_TOLERANCE": 1e-1, - "EXT_LOOP_CRITERION_COUNT_THRESHOLD": 1e-1 + "DO_OUTER_LOOP": true, + "OUTER_LOOP_OPTION_FILE": "data_test/external_loop_test/lp/outer_loop_options.yml", + "OUTER_LOOP_NUMBER_OF_SCENARIOS": 2 } diff --git a/data_test/external_loop_test/lp/outer_loop_options.yml b/data_test/external_loop_test/lp/outer_loop_options.yml new file mode 100644 index 000000000..7fc3b58c1 --- /dev/null +++ b/data_test/external_loop_test/lp/outer_loop_options.yml @@ -0,0 +1,15 @@ +# critère d'arrêt de l'algo +stopping_threshold: 1e-4 +# seuil +criterion_count_threshold: 1e-1 + # tolerance entre seuil et valeur calculée +criterion_tolerance: 1e-5 +patterns: + - area: "N0" + criterion: 1 + - area: "N1" + criterion: 1 + - area: "N2" + criterion: 1 + - area: "N3" + criterion: 1 \ No newline at end of file diff --git a/src/cpp/benders/CMakeLists.txt b/src/cpp/benders/CMakeLists.txt index adea040ed..77979e86b 100644 --- a/src/cpp/benders/CMakeLists.txt +++ b/src/cpp/benders/CMakeLists.txt @@ -4,7 +4,7 @@ add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/benders_core") add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/logger") add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/benders_sequential") add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/benders_by_batch") -add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/external_loop") +add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/outer_loop") add_subdirectory ("${CMAKE_CURRENT_SOURCE_DIR}/benders_mpi") diff --git a/src/cpp/benders/benders_by_batch/BendersByBatch.cpp b/src/cpp/benders/benders_by_batch/BendersByBatch.cpp index 1f5653ed7..b273aa9a0 100644 --- a/src/cpp/benders/benders_by_batch/BendersByBatch.cpp +++ b/src/cpp/benders/benders_by_batch/BendersByBatch.cpp @@ -6,6 +6,7 @@ #include #include "BatchCollection.h" +#include "CustomVector.h" #include "RandomBatchShuffler.h" #include "glog/logging.h" BendersByBatch::BendersByBatch( @@ -46,6 +47,10 @@ void BendersByBatch::InitializeProblems() { problem_count++; } } + + // if (Rank() == rank_0) { + SetSubproblemsVariablesIndex(); + // } init_problems_ = false; } void BendersByBatch::BroadcastSingleSubpbCostsUnderApprox() { @@ -61,7 +66,6 @@ void BendersByBatch::Run() { if (init_data_) { PreRunInitialization(); } else { - // only ? _data.stop = false; } @@ -151,6 +155,9 @@ void BendersByBatch::SeparationLoop() { SolveBatches(); if (Rank() == rank_0) { + outer_loop_criterion_.push_back(_data.outer_loop_current_iteration_data.outer_loop_criterion); + // TODO + // UpdateOuterLoopMaxCriterionArea(); UpdateTrace(); SaveCurrentBendersData(); } @@ -195,8 +202,10 @@ void BendersByBatch::SolveBatches() { const auto &batch_sub_problems = batch.sub_problem_names; double batch_subproblems_costs_contribution_in_gap_per_proc = 0; double batch_subproblems_costs_contribution_in_gap = 0; + std::vector external_loop_criterion_current_batch = {}; BuildCut(batch_sub_problems, - &batch_subproblems_costs_contribution_in_gap_per_proc); + &batch_subproblems_costs_contribution_in_gap_per_proc, + external_loop_criterion_current_batch); Reduce(batch_subproblems_costs_contribution_in_gap_per_proc, batch_subproblems_costs_contribution_in_gap, std::plus(), rank_0); @@ -206,6 +215,9 @@ void BendersByBatch::SolveBatches() { _data.number_of_subproblem_solved += batch_sub_problems.size(); _data.cumulative_number_of_subproblem_solved += batch_sub_problems.size(); remaining_epsilon_ -= batch_subproblems_costs_contribution_in_gap; + // TODO + // AddVectors(_data.outer_loop_current_iteration_data.outer_loop_criterion, + // external_loop_criterion_current_batch); } BroadCast(remaining_epsilon_, rank_0); @@ -222,7 +234,8 @@ void BendersByBatch::SolveBatches() { */ void BendersByBatch::BuildCut( const std::vector &batch_sub_problems, - double *batch_subproblems_costs_contribution_in_gap_per_proc) { + double *batch_subproblems_costs_contribution_in_gap_per_proc, + std::vector &external_loop_criterion_current_batch) { SubProblemDataMap subproblem_data_map; Timer subproblems_timer_per_proc; GetSubproblemCut(subproblem_data_map, batch_sub_problems, @@ -235,7 +248,10 @@ void BendersByBatch::BuildCut( misprice_ = global_misprice; Gather(subproblem_data_map, gathered_subproblem_map, rank_0); SetSubproblemsWalltime(subproblems_timer_per_proc.elapsed()); - + // if (Options().EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) { + // external_loop_criterion_current_batch = + // ComputeSubproblemsContributionToOuterLoopCriterion(subproblem_data_map); + // } for (const auto &subproblem_map : gathered_subproblem_map) { for (auto &&[sub_problem_name, subproblem_data] : subproblem_map) { SetSubproblemCost(GetSubproblemCost() + subproblem_data.subproblem_cost); @@ -270,6 +286,7 @@ void BendersByBatch::GetSubproblemCut( worker->fix_to(_data.x_cut); worker->solve(subproblem_data.lpstatus, Options().OUTPUTROOT, Options().LAST_MASTER_MPS + MPS_SUFFIX, _writer); + worker->get_solution(subproblem_data.solution); worker->get_value(subproblem_data.subproblem_cost); // solution phi(x,s) worker->get_subgradient( subproblem_data.var_name_and_subgradient); // dual pi_s diff --git a/src/cpp/benders/benders_by_batch/include/BendersByBatch.h b/src/cpp/benders/benders_by_batch/include/BendersByBatch.h index 06af08e98..03dde786e 100644 --- a/src/cpp/benders/benders_by_batch/include/BendersByBatch.h +++ b/src/cpp/benders/benders_by_batch/include/BendersByBatch.h @@ -13,8 +13,8 @@ class BendersByBatch : public BendersMpi { std::shared_ptr mathLoggerDriver); ~BendersByBatch() override = default; void Run() override; - void BuildCut(const std::vector &batch_sub_problems, - double *sum); + void BuildCut(const std::vector &batch_sub_problems, double *sum, + std::vector &external_loop_criterion_current_batch); std::string BendersName() const override { return "Benders By Batch mpi"; } protected: diff --git a/src/cpp/benders/benders_core/BendersBase.cpp b/src/cpp/benders/benders_core/BendersBase.cpp index b0c94599c..aad41570e 100644 --- a/src/cpp/benders/benders_core/BendersBase.cpp +++ b/src/cpp/benders/benders_core/BendersBase.cpp @@ -9,18 +9,31 @@ #include "LastIterationReader.h" #include "LastIterationWriter.h" #include "LogUtils.h" +#include "VariablesGroup.h" #include "glog/logging.h" #include "solver_utils.h" -BendersBase::BendersBase(BendersBaseOptions options, Logger logger, +BendersBase::BendersBase(const BendersBaseOptions &options, Logger logger, Writer writer, std::shared_ptr mathLoggerDriver) - : _options(std::move(options)), + : _options(options), _csv_file_path(std::filesystem::path(_options.OUTPUTROOT) / (_options.CSV_NAME + ".csv")), _logger(std::move(logger)), _writer(std::move(writer)), - mathLoggerDriver_(mathLoggerDriver) {} + mathLoggerDriver_(std::move(mathLoggerDriver)) { + if (options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) { + //TODO maybe the input format will change? + outer_loop_input_data_ = + Outerloop::OuterLoopInputFromYaml().Read(OuterloopOptionsFile()); + outer_loop_biLevel_ = OuterLoopBiLevel(outer_loop_input_data_); + } +} + +std::filesystem::path BendersBase::OuterloopOptionsFile() const { + return std::filesystem::path( + _options.EXTERNAL_LOOP_OPTIONS.OUTER_LOOP_OPTION_FILE); +} /*! * \brief Initialize set of data used in the loop @@ -42,6 +55,7 @@ void BendersBase::init_data() { _data.iteration_time = 0; _data.timer_master = 0; _data.subproblems_walltime = 0; + outer_loop_criterion_.clear(); } void BendersBase::OpenCsvFile() { @@ -166,6 +180,10 @@ void BendersBase::update_best_ub() { _data.best_ub = _data.ub; _data.best_it = _data.it; FillWorkerMasterData(relevantIterationData_.best); + _data.outer_loop_current_iteration_data.max_criterion_best_it = + _data.outer_loop_current_iteration_data.max_criterion; + _data.outer_loop_current_iteration_data.max_criterion_area_best_it = + _data.outer_loop_current_iteration_data.max_criterion_area; relevantIterationData_.best._cut_trace = relevantIterationData_.last._cut_trace; best_iteration_data = bendersDataToLogData(_data); } @@ -379,7 +397,7 @@ void BendersBase::GetSubproblemCut(SubProblemDataMap &subproblem_data_map) { worker->solve(subproblem_data.lpstatus, _options.OUTPUTROOT, _options.LAST_MASTER_MPS + MPS_SUFFIX, _writer); worker->get_value(subproblem_data.subproblem_cost); - worker->get_solution(subproblem_data.variables); + worker->get_solution(subproblem_data.solution); worker->get_subgradient(subproblem_data.var_name_and_subgradient); worker->get_splex_num_of_ite_last(subproblem_data.simplex_iter); subproblem_data.subproblem_timer = subproblem_timer.elapsed(); @@ -401,6 +419,7 @@ void BendersBase::GetSubproblemCut(SubProblemDataMap &subproblem_data_map) { * */ void BendersBase::compute_cut(const SubProblemDataMap &subproblem_data_map) { + // current_outer_loop_criterion_ = 0.0; for (auto const &[subproblem_name, subproblem_data] : subproblem_data_map) { _data.ub += subproblem_data.subproblem_cost; @@ -408,7 +427,9 @@ void BendersBase::compute_cut(const SubProblemDataMap &subproblem_data_map) { subproblem_data.var_name_and_subgradient, _data.x_cut, subproblem_data.subproblem_cost); relevantIterationData_.last._cut_trace[subproblem_name] = subproblem_data; + // ComputeOuterLoopCriterion(subproblem_name, subproblem_data); } + // outer_loop_criterion_.push_back(current_outer_loop_criterion_); } void compute_cut_val(const Point &var_name_subgradient, const Point &x_cut, @@ -728,6 +749,17 @@ void BendersBase::MatchProblemToId() { } } +void BendersBase::SetSubproblemsVariablesIndex() { + if (!subproblem_map.empty() && _options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) { + auto subproblem = subproblem_map.begin(); + subproblems_vars_names_.clear(); + subproblems_vars_names_ = subproblem->second->_solver->get_col_names(); + Outerloop::VariablesGroup variablesGroup(subproblems_vars_names_, + outer_loop_input_data_.OuterLoopData()); + var_indices_ = variablesGroup.Indices(); + } +} + void BendersBase::AddSubproblemName(const std::string &name) { subproblems.push_back(name); } @@ -931,16 +963,87 @@ WorkerMasterData BendersBase::BestIterationWorkerMaster() const { return relevantIterationData_.best; } -void BendersBase::ResetData(double criterion) { +void BendersBase::InitExternalValues(bool is_bilevel_check_all, double lambda) { + // _data.outer_loop_current_iteration_data.outer_loop_criterion = 0; + // _data.outer_loop_current_iteration_data.benders_num_run = 1; + is_bilevel_check_all_ = is_bilevel_check_all; + outer_loop_biLevel_.Init(MasterObjectiveFunctionCoeffs(), + BestIterationWorkerMaster().get_max_invest(), + MasterVariables()); + outer_loop_biLevel_.SetLambda(lambda); +} + +CurrentIterationData BendersBase::GetCurrentIterationData() const { + return _data; +} +OuterLoopCurrentIterationData BendersBase::GetOuterLoopData() const { + return _data.outer_loop_current_iteration_data; +} +std::vector BendersBase::GetOuterLoopCriterionAtBestBenders() const { + return ((outer_loop_criterion_.empty()) + ? std::vector() + : outer_loop_criterion_[_data.best_it - 1]); +} + +std::vector BendersBase::ComputeOuterLoopCriterion( + const std::string &subproblem_name, + const PlainData::SubProblemData &sub_problem_data) { + + auto outer_loop_input_size = var_indices_.size(); // num of patterns + std::vector outer_loop_criterion_per_sub_problem(outer_loop_input_size, + {}); + auto subproblem_weight = SubproblemWeight(_data.nsubproblem, subproblem_name); + double criterion_count_threshold = + outer_loop_input_data_.CriterionCountThreshold(); + auto number_of_scenarios = + _options.EXTERNAL_LOOP_OPTIONS.OUTER_LOOP_NUMBER_OF_SCENARIOS; + + for (int pattern_index(0); pattern_index < outer_loop_input_size; + ++pattern_index) { + auto pattern_variables_indices = var_indices_[pattern_index]; + for (auto variables_index : pattern_variables_indices) { + + if (auto solution = sub_problem_data.solution[variables_index]; + solution > criterion_count_threshold) + // 1h of no supplied energy + outer_loop_criterion_per_sub_problem[pattern_index] += + subproblem_weight / number_of_scenarios; + } + } + return outer_loop_criterion_per_sub_problem; +} + +double BendersBase::ExternalLoopLambdaMax() const { + return outer_loop_biLevel_.LambdaMax(); +} +double BendersBase::ExternalLoopLambdaMin() const { + return outer_loop_biLevel_.LambdaMin(); +} + +void BendersBase::init_data(double external_loop_lambda) { + benders_timer.restart(); + auto benders_num_run = _data.outer_loop_current_iteration_data.benders_num_run; + auto outer_loop_bilevel_best_ub = _data.outer_loop_current_iteration_data.outer_loop_bilevel_best_ub; init_data(); - _data.external_loop_criterion = criterion; + _data.outer_loop_current_iteration_data.outer_loop_criterion.clear(); + _data.outer_loop_current_iteration_data.benders_num_run = benders_num_run; + _data.outer_loop_current_iteration_data.outer_loop_bilevel_best_ub = outer_loop_bilevel_best_ub; + _data.outer_loop_current_iteration_data.external_loop_lambda = external_loop_lambda; } -void BendersBase::InitExternalValues() { - _data.external_loop_criterion = 0; - _data.benders_num_run = 0; +bool BendersBase::ExternalLoopFoundFeasible() const { + return outer_loop_biLevel_.FoundFeasible(); } +double BendersBase::OuterLoopStoppingThreshold() const { return outer_loop_input_data_.StoppingThreshold(); } -CurrentIterationData BendersBase::GetCurrentIterationData() const { - return _data; +void BendersBase::UpdateOuterLoopMaxCriterionArea() { + auto criterions_begin = + _data.outer_loop_current_iteration_data.outer_loop_criterion.cbegin(); + auto criterions_end = + _data.outer_loop_current_iteration_data.outer_loop_criterion.cend(); + auto max_criterion_it = std::max_element(criterions_begin, criterions_end); + _data.outer_loop_current_iteration_data.max_criterion = *max_criterion_it; + auto max_criterion_index = std::distance(criterions_begin, max_criterion_it); + _data.outer_loop_current_iteration_data.max_criterion_area = outer_loop_input_data_.OuterLoopData()[max_criterion_index].Pattern().GetBody(); } +bool BendersBase::isExceptionRaised() const { return exception_raised_; } diff --git a/src/cpp/benders/benders_core/BendersMathLogger.cpp b/src/cpp/benders/benders_core/BendersMathLogger.cpp index ccf2ace3d..3022a42cc 100644 --- a/src/cpp/benders/benders_core/BendersMathLogger.cpp +++ b/src/cpp/benders/benders_core/BendersMathLogger.cpp @@ -50,7 +50,9 @@ HeadersManagerExternalLoop::HeadersManagerExternalLoop( std::vector HeadersManagerExternalLoop::HeadersList() { std::vector headers_list; headers_list.push_back("Outer loop"); - headers_list.push_back("Criterion value"); + headers_list.push_back("Max Criterion"); + headers_list.push_back("Area Max Criterion"); + headers_list.push_back("Bilevel best ub"); auto base_headers = HeadersManager::HeadersList(); std::move(base_headers.begin(), base_headers.end(), std::back_inserter(headers_list)); @@ -157,9 +159,16 @@ void PrintExternalLoopData(LogDestination& log_destination, const CurrentIterationData& data, const HEADERSTYPE& type, const BENDERSMETHOD& method) { - log_destination << data.benders_num_run; + log_destination << data.outer_loop_current_iteration_data.benders_num_run; + // TODO + // log_destination << std::scientific << std::setprecision(10) + // << data.outer_loop_criterion; log_destination << std::scientific << std::setprecision(10) - << data.external_loop_criterion; + << data.outer_loop_current_iteration_data.max_criterion; + log_destination << data.outer_loop_current_iteration_data.max_criterion_area; + + log_destination << std::scientific << std::setprecision(10) + << data.outer_loop_current_iteration_data.outer_loop_bilevel_best_ub; PrintBendersData(log_destination, data, type, method); } void MathLoggerBaseExternalLoop::Print(const CurrentIterationData& data) { diff --git a/src/cpp/benders/benders_core/CMakeLists.txt b/src/cpp/benders/benders_core/CMakeLists.txt index a662f8fba..a80756ff6 100644 --- a/src/cpp/benders/benders_core/CMakeLists.txt +++ b/src/cpp/benders/benders_core/CMakeLists.txt @@ -13,7 +13,7 @@ endif() if (TBB_VERSION_MAJOR VERSION_GREATER "2020") message(FATAL_ERROR "Require tbb 2018 to 2020.") endif() - +find_package(yaml-cpp CONFIG REQUIRED) add_library (benders_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/SubproblemWorker.cpp ${CMAKE_CURRENT_SOURCE_DIR}/SimulationOptions.cpp @@ -27,6 +27,10 @@ add_library (benders_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/LastIterationPrinter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/StartUp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/BendersMathLogger.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/VariablesGroup.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/OuterLoopBiLevel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/OuterLoopInputDataReader.cpp + ) @@ -43,6 +47,7 @@ target_link_libraries (benders_core glog::glog TBB::tbb ${JSONCPP_LIB} + yaml-cpp ) add_library (${PROJECT_NAME}::benders_core ALIAS benders_core) diff --git a/src/cpp/benders/benders_core/OuterLoopBiLevel.cpp b/src/cpp/benders/benders_core/OuterLoopBiLevel.cpp new file mode 100644 index 000000000..3a8ef2fee --- /dev/null +++ b/src/cpp/benders/benders_core/OuterLoopBiLevel.cpp @@ -0,0 +1,66 @@ +#include "OuterLoopBiLevel.h" + +OuterLoopBiLevel::OuterLoopBiLevel(const Outerloop::OuterLoopInputData &outer_loop_input_data) + : outer_loop_input_data_(outer_loop_input_data) {} + +bool OuterLoopBiLevel::Update_bilevel_data_if_feasible( + const Point &x, const std::vector &outer_loop_criterion, + double overall_cost, double invest_cost_at_x, double lambda_min) { + if (found_feasible_ = + Check_bilevel_feasibility(outer_loop_criterion, overall_cost); + found_feasible_) { + Update(x, overall_cost, invest_cost_at_x); + } else { + lambda_min_ = lambda_min; + } + + return found_feasible_; +} + +bool OuterLoopBiLevel::Check_bilevel_feasibility( + const std::vector &outer_loop_criterions, double overall_cost) { + return IsCriterionSatisfied(outer_loop_criterions) && + overall_cost < bilevel_best_ub_; + + /** + * else if method == opt ... + */ +} + +bool OuterLoopBiLevel::IsCriterionSatisfied( + const std::vector &outer_loop_criterions) { + const auto& outer_loop_input_data = outer_loop_input_data_.OuterLoopData(); + for (int index(0); index < outer_loop_criterions.size(); ++index) { + if (outer_loop_criterions[index] > + outer_loop_input_data[index].Criterion() + + outer_loop_input_data_.CriterionTolerance()) { + return false; + } + } + return true; +} + +void OuterLoopBiLevel::Update(const Point &x, double overall_cost, + double invest_cost_at_x) { + bilevel_best_x_ = x; + bilevel_best_ub_ = overall_cost; + lambda_max_ = std::min(lambda_max_, invest_cost_at_x); +} + +void OuterLoopBiLevel::Init(const std::vector &obj, + const Point &max_invest, + const VariableMap &master_variable) { + + // TODO log + SetLambdaMaxToMaxInvestmentCosts(obj, max_invest, master_variable); + + found_feasible_ = false; +} +void OuterLoopBiLevel::SetLambdaMaxToMaxInvestmentCosts( + const std::vector &obj, const Point &max_invest, + const VariableMap &master_variable) { + lambda_max_ = 0; + for (const auto &[var_name, var_id] : master_variable) { + lambda_max_ += obj[var_id] * max_invest.at(var_name); + } +} diff --git a/src/cpp/benders/benders_core/OuterLoopInputDataReader.cpp b/src/cpp/benders/benders_core/OuterLoopInputDataReader.cpp new file mode 100644 index 000000000..47dc74ac3 --- /dev/null +++ b/src/cpp/benders/benders_core/OuterLoopInputDataReader.cpp @@ -0,0 +1,192 @@ +#include "OuterLoopInputDataReader.h" + +using namespace Outerloop; + +/** + * prefix could be := PositiveUnsuppliedEnergy:: or something else necessarily + * /!\ body could be := area name or equivalent or nothing + */ +OuterLoopPattern::OuterLoopPattern(const std::string &prefix, + const std::string &body) + : prefix_(prefix), body_(body) {} + +/** + * just do + * just cat ;) + */ +std::regex OuterLoopPattern::MakeRegex() const { + auto pattern = "(^" + prefix_ + ").*" + body_; + return std::regex(pattern); +} +const std::string &OuterLoopPattern::GetPrefix() const { return prefix_; } +void OuterLoopPattern::SetPrefix(const std::string &prefix) { + prefix_ = prefix; +} +const std::string &OuterLoopPattern::GetBody() const { return body_; } + +void OuterLoopPattern::SetBody(const std::string &body) { body_ = body; } + +OuterLoopSingleInputData::OuterLoopSingleInputData(const std::string &prefix, + const std::string &body, + double criterion) + : outer_loop_pattern_(prefix, body), criterion_(criterion) {} + +OuterLoopPattern OuterLoopSingleInputData::Pattern() const { + return outer_loop_pattern_; +} +double OuterLoopSingleInputData::Criterion() const { return criterion_; } + +void OuterLoopSingleInputData::SetCriterion(double criterion) { + criterion_ = criterion; +} +void OuterLoopSingleInputData::ResetPattern(const std::string &prefix, + const std::string &body) { + outer_loop_pattern_.SetPrefix(prefix); + outer_loop_pattern_.SetBody(body); +} + +void OuterLoopInputData::AddSingleData(const OuterLoopSingleInputData &data) { + outer_loop_data_.push_back(data); +} + +std::vector OuterLoopInputData::OuterLoopData() + const { + return outer_loop_data_; +} + +void OuterLoopInputData::SetStoppingThreshold( + double outer_loop_stopping_threshold) { + outer_loop_stopping_threshold_ = outer_loop_stopping_threshold; +} + +double OuterLoopInputData::StoppingThreshold() const { + return outer_loop_stopping_threshold_; +} +void OuterLoopInputData::SetCriterionTolerance(double criterion_tolerance) { + criterion_tolerance_ = criterion_tolerance; +} +double OuterLoopInputData::CriterionTolerance() const { return criterion_tolerance_; } +void OuterLoopInputData::SetCriterionCountThreshold( + double criterion_count_threshold) { + criterion_count_threshold_ = criterion_count_threshold; +} +double OuterLoopInputData::CriterionCountThreshold() const { return criterion_count_threshold_; } + +OuterLoopInputData OuterLoopInputFromYaml::Read( + const std::filesystem::path &input_file) { + YAML::Node yaml_content; + try { + yaml_content = YAML::LoadFile(input_file.string()); + } catch (const std::exception &e) { + std::ostringstream err_msg; + err_msg << "Could not read outer loop input file: " << input_file << "\n" + << e.what(); + throw OuterLoopInputFileError( + PrefixMessage(LogUtils::LOGLEVEL::FATAL, "Outer Loop"), err_msg.str(), + LOGLOCATION); + } + if (yaml_content.IsNull()) { + std::ostringstream err_msg; + err_msg << "outer loop input file is empty: " << input_file << "\n"; + throw OuterLoopInputFileIsEmpty( + PrefixMessage(LogUtils::LOGLEVEL::FATAL, "Outer Loop"), err_msg.str(), + LOGLOCATION); + } + + return yaml_content.as(); +} + +/* +# critère d'arrêt de l'algo +stopping_threshold: 1e-4 +# seuil +criterion_count_threshold: 1e-1 + # tolerance entre seuil et valeur calculée +criterion_tolerance: 1e-5 +patterns: + - area: "N0" + criterion: 1 + - area: "N1" + criterion: 1 + - area: "N2" + criterion: 1 + - area: "N3" + criterion: 1 +*/ + +namespace YAML { + +template <> +struct convert { + static Node encode(const OuterLoopSingleInputData &rhs) { + // + } + + static bool decode(const Node &pattern, OuterLoopSingleInputData &rhs) { + auto body = pattern["area"]; + + // specify line And OR #pattern + if (body.IsNull()) { + std::ostringstream err_msg; + err_msg << PrefixMessage(LogUtils::LOGLEVEL::FATAL, "Outer Loop") + << "Error could not read 'area' field in outer loop input file" + << "\n"; + throw OuterLoopCouldNotReadAreaField(err_msg.str(), LOGLOCATION); + } + auto criterion = pattern["criterion"]; + + if (criterion.IsNull()) { + std::ostringstream err_msg; + err_msg + << PrefixMessage(LogUtils::LOGLEVEL::FATAL, "Outer Loop") + << "Error could not read 'criterion' field in outer loop input file" + << "\n"; + throw OuterLoopCouldNotReadCriterionField(err_msg.str(), LOGLOCATION); + } + + rhs.SetCriterion(criterion.as()); + rhs.ResetPattern("PositiveUnsuppliedEnergy::", body.as()); + return true; + } +}; +template <> +struct convert { + static Node encode(const OuterLoopInputData &rhs) { + // + } + + static void DecodePatterns(const Node &patterns, OuterLoopInputData &rhs) { + if (!patterns.IsSequence()) { + std::ostringstream err_msg; + err_msg << "In outer loop input file 'patterns' should be an array." + << "\n"; + throw OuterLoopInputPatternsShouldBeArray( + PrefixMessage(LogUtils::LOGLEVEL::FATAL, "Outer Loop"), err_msg.str(), + LOGLOCATION); + } + + for (const auto &pattern : patterns) { + rhs.AddSingleData(pattern.as()); + } + } + + static bool decode(const Node &node, OuterLoopInputData &rhs) { + rhs.SetStoppingThreshold(node["stopping_threshold"].as(1e-4)); + rhs.SetCriterionCountThreshold( + node["criterion_count_threshold"].as(1)); + rhs.SetCriterionTolerance(node["criterion_tolerance"].as(1e-1)); + + if (auto patterns = node["patterns"]) { + DecodePatterns(patterns, rhs); + } else { + std::ostringstream err_msg; + err_msg << "outer loop input file must contains at least one pattern." + << "\n"; + throw OuterLoopInputFileNoPatternFound( + PrefixMessage(LogUtils::LOGLEVEL::FATAL, "Outer Loop"), err_msg.str(), + LOGLOCATION); + } + return true; + } +}; +} // namespace YAML \ No newline at end of file diff --git a/src/cpp/benders/benders_core/SimulationOptions.cpp b/src/cpp/benders/benders_core/SimulationOptions.cpp index 90a8ff9ba..cc1901b21 100644 --- a/src/cpp/benders/benders_core/SimulationOptions.cpp +++ b/src/cpp/benders/benders_core/SimulationOptions.cpp @@ -170,7 +170,7 @@ BendersBaseOptions SimulationOptions::get_benders_options() const { result.LAST_MASTER_MPS = LAST_MASTER_MPS; result.LAST_MASTER_BASIS = LAST_MASTER_BASIS; result.BATCH_SIZE = BATCH_SIZE; - + result.EXTERNAL_LOOP_OPTIONS = GetExternalLoopOptions(); return result; } SimulationOptions::InvalidOptionFileException::InvalidOptionFileException( @@ -179,6 +179,6 @@ SimulationOptions::InvalidOptionFileException::InvalidOptionFileException( ExternalLoopOptions SimulationOptions::GetExternalLoopOptions() const { - return {EXT_LOOP_CRITERION_VALUE, EXT_LOOP_CRITERION_TOLERANCE, - EXT_LOOP_CRITERION_COUNT_THRESHOLD}; + return {DO_OUTER_LOOP, OUTER_LOOP_OPTION_FILE, + OUTER_LOOP_NUMBER_OF_SCENARIOS}; } \ No newline at end of file diff --git a/src/cpp/benders/benders_core/SubproblemWorker.cpp b/src/cpp/benders/benders_core/SubproblemWorker.cpp index e88cea2a5..5778710da 100644 --- a/src/cpp/benders/benders_core/SubproblemWorker.cpp +++ b/src/cpp/benders/benders_core/SubproblemWorker.cpp @@ -73,14 +73,12 @@ void SubproblemWorker::get_subgradient(Point &s) const { * * \param lb : reference to a map */ -void SubproblemWorker::get_solution(PlainData::Variables &vars) const { - vars.values = std::vector(_solver->get_ncols()); +void SubproblemWorker::get_solution(std::vector &solution) const { + solution = std::vector(_solver->get_ncols()); if (_solver->get_n_integer_vars() > 0) { - _solver->get_mip_sol(vars.values.data()); + _solver->get_mip_sol(solution.data()); } else { - _solver->get_lp_sol(vars.values.data(), NULL, NULL); + _solver->get_lp_sol(solution.data(), NULL, NULL); } - - vars.names = _solver->get_col_names(); } \ No newline at end of file diff --git a/src/cpp/benders/benders_core/VariablesGroup.cpp b/src/cpp/benders/benders_core/VariablesGroup.cpp new file mode 100644 index 000000000..4b42bef77 --- /dev/null +++ b/src/cpp/benders/benders_core/VariablesGroup.cpp @@ -0,0 +1,28 @@ +#include "VariablesGroup.h" +using namespace Outerloop; + +VariablesGroup::VariablesGroup( + const std::vector& all_variables, + const std::vector& outer_loop_single_input_data) + : all_variables_(all_variables), outer_loop_single_input_data_(outer_loop_single_input_data) { + Search(); +} + +std::vector> VariablesGroup::Indices() const { + return indices_; +} + +void VariablesGroup::Search() { + indices_.assign(outer_loop_single_input_data_.size(), {}); + int var_index(0); + for (const auto& variable : all_variables_) { + int pattern_index(0); + for (const auto& single_input_data : outer_loop_single_input_data_) { + if (std::regex_search(variable, single_input_data.Pattern().MakeRegex())) { + indices_[pattern_index].push_back(var_index); + } + ++pattern_index; + } + ++var_index; + } +} diff --git a/src/cpp/benders/benders_core/include/BendersBase.h b/src/cpp/benders/benders_core/include/BendersBase.h index 3054d3712..705a622c7 100644 --- a/src/cpp/benders/benders_core/include/BendersBase.h +++ b/src/cpp/benders/benders_core/include/BendersBase.h @@ -2,10 +2,13 @@ #include #include +#include #include "BendersMathLogger.h" #include "BendersStructsDatas.h" #include "ILogger.h" +#include "OuterLoopBiLevel.h" +#include "OuterLoopInputDataReader.h" #include "OutputWriter.h" #include "SimulationOptions.h" #include "SubproblemCut.h" @@ -16,9 +19,9 @@ #include "common.h" /** - * std execution policies don't share a base type so we can't just select them - *in place in the foreach This function allow the selection of policy via - *template deduction + * std execution policies don't share a base type so we can't just select + *them in place in the foreach This function allow the selection of policy + *via template deduction **/ template auto selectPolicy(lambda f, bool shouldParallelize) { @@ -30,7 +33,7 @@ auto selectPolicy(lambda f, bool shouldParallelize) { class BendersBase { public: virtual ~BendersBase() = default; - BendersBase(BendersBaseOptions options, Logger logger, Writer writer, + BendersBase(const BendersBaseOptions &options, Logger logger, Writer writer, std::shared_ptr mathLoggerDriver); virtual void launch() = 0; void set_solver_log_file(const std::filesystem::path &log_file); @@ -79,11 +82,27 @@ class BendersBase { _options.MAX_ITERATIONS = max_iteration; } BendersBaseOptions Options() const { return _options; } - void ResetData(double criterion); virtual void free() = 0; - void InitExternalValues(); - int GetBendersRunNumber() const { return _data.benders_num_run; } + void InitExternalValues(bool is_bilevel_check_all, double lambda); + int GetBendersRunNumber() const { return _data.outer_loop_current_iteration_data.benders_num_run; } CurrentIterationData GetCurrentIterationData() const; + OuterLoopCurrentIterationData GetOuterLoopData() const; + std::vector GetOuterLoopCriterionAtBestBenders() const; + virtual void init_data(); + void init_data(double external_loop_lambda); + + double ExternalLoopLambdaMax() const; + double ExternalLoopLambdaMin() const; + bool ExternalLoopFoundFeasible() const; + virtual void ExternalLoopCheckFeasibility() = 0; + virtual void RunExternalLoopBilevelChecks() = 0; + double OuterLoopStoppingThreshold() const; + + protected: + bool exception_raised_ = false; + + public: + bool isExceptionRaised() const; protected: CurrentIterationData _data; @@ -92,14 +111,20 @@ class BendersBase { // BendersCuts current_iteration_cuts_; VariableMap master_variable_map_; CouplingMap coupling_map_; - // for warmstart initialize all data, master, subproblem etc... + BendersRelevantIterationsData relevantIterationData_ = {WorkerMasterData(), + WorkerMasterData()}; bool init_data_ = true; bool init_problems_ = true; bool free_problems_ = true; - protected: + std::vector> outer_loop_criterion_; + std::vector subproblems_vars_names_ = {}; + std::vector> var_indices_; + OuterLoopBiLevel outer_loop_biLevel_; + bool is_bilevel_check_all_ = false; + Outerloop::OuterLoopInputData outer_loop_input_data_; + virtual void Run() = 0; - virtual void init_data(); void update_best_ub(); bool ShouldBendersStop(); bool is_initial_relaxation_requested() const; @@ -122,6 +147,7 @@ class BendersBase { std::string const &name) const; [[nodiscard]] std::filesystem::path get_master_path() const; [[nodiscard]] std::filesystem::path get_structure_path() const; + [[nodiscard]] std::filesystem::path OuterloopOptionsFile() const; [[nodiscard]] LogData bendersDataToLogData( const CurrentIterationData &data) const; virtual void reset_master(WorkerMaster *worker_master); @@ -130,6 +156,15 @@ class BendersBase { void AddSubproblem(const std::pair &kvp); [[nodiscard]] WorkerMasterPtr get_master() const; void MatchProblemToId(); + /** + * for the nth variable name, Subproblems shares the same prefix , only the + suffix is different + * ex variable at index = 0 is named in: + + * subproblems-1-1 --> NTCDirect::link::hour<0> + * subproblems-3-5 --> NTCDirect::link::hour<672> + */ + void SetSubproblemsVariablesIndex(); void AddSubproblemName(const std::string &name); [[nodiscard]] std::string get_master_name() const; [[nodiscard]] std::string get_solver_name() const; @@ -192,6 +227,13 @@ class BendersBase { SolverLogManager solver_log_manager_; + // outer loop criterion per pattern + std::vector ComputeOuterLoopCriterion( + const std::string &subproblem_name, + const PlainData::SubProblemData &sub_problem_data); + + void UpdateOuterLoopMaxCriterionArea(); + private: void print_master_and_cut(std::ostream &file, int ite, WorkerMasterData &trace, Point const &xopt); @@ -215,8 +257,6 @@ class BendersBase { BendersBaseOptions _options; unsigned int _totalNbProblems = 0; std::filesystem::path solver_log_file_ = ""; - BendersRelevantIterationsData relevantIterationData_ = {WorkerMasterData(), - WorkerMasterData()}; WorkerMasterPtr _master; VariableMap _problem_to_id; SubproblemsMapPtr subproblem_map; diff --git a/src/cpp/benders/benders_core/include/BendersStructsDatas.h b/src/cpp/benders/benders_core/include/BendersStructsDatas.h index 30318ab6b..9329a19ca 100644 --- a/src/cpp/benders/benders_core/include/BendersStructsDatas.h +++ b/src/cpp/benders/benders_core/include/BendersStructsDatas.h @@ -5,8 +5,19 @@ #include "Worker.h" #include "common.h" -/*! \struct struct that hold current benders iteration - */ +struct OuterLoopCurrentIterationData{ + int benders_num_run = 0; + std::vector outer_loop_criterion = {}; + double max_criterion = 0.; + double max_criterion_best_it = 0.; + double outer_loop_bilevel_best_ub = +1e20; + double external_loop_lambda = 0.; + std::string max_criterion_area; + std::string max_criterion_area_best_it; +}; +/*! \struct + * struct that hold current Benders iteration + */ struct CurrentIterationData { double subproblems_walltime; double subproblems_cputime; @@ -39,8 +50,7 @@ struct CurrentIterationData { int min_simplexiter; int max_simplexiter; // ugly - int benders_num_run; - double external_loop_criterion; + OuterLoopCurrentIterationData outer_loop_current_iteration_data; }; // /*! \struct to store benders cuts data diff --git a/src/cpp/benders/benders_core/include/CustomVector.h b/src/cpp/benders/benders_core/include/CustomVector.h new file mode 100644 index 000000000..60844a510 --- /dev/null +++ b/src/cpp/benders/benders_core/include/CustomVector.h @@ -0,0 +1,9 @@ +#pragma once +#include + +template +void AddVectors(std::vector& a, const std::vector& b) { + if (a.size() == b.size()) { + std::transform(a.begin(), a.end(), b.begin(), a.begin(), std::plus()); + } +} diff --git a/src/cpp/benders/benders_core/include/OuterLoopBiLevel.h b/src/cpp/benders/benders_core/include/OuterLoopBiLevel.h new file mode 100644 index 000000000..76802f70c --- /dev/null +++ b/src/cpp/benders/benders_core/include/OuterLoopBiLevel.h @@ -0,0 +1,38 @@ +#pragma once +#include "OuterLoopInputDataReader.h" +#include "SubproblemCut.h" + +class OuterLoopBiLevel { + public: + explicit OuterLoopBiLevel(const Outerloop::OuterLoopInputData &outer_loop_input_data); + OuterLoopBiLevel() = default; + bool Update_bilevel_data_if_feasible( + const Point &x, + const std::vector &outer_loop_criterion, double overall_cost, + double invest_cost_at_x, double lambda_min); + bool Check_bilevel_feasibility( + const std::vector &outer_loop_criterion, double overall_cost); + // double LambdaMax() const { return lambda_max_; } + void Init(const std::vector &obj, const Point &max_invest, + const VariableMap &master_variable); + double LambdaMax() const { return lambda_max_; } + double LambdaMin() const { return lambda_min_; } + void SetLambda(double lambda) { lambda_ = lambda; } + double BilevelBestub() const { return bilevel_best_ub_; } + bool FoundFeasible() const { return found_feasible_; } + + private: + void SetLambdaMaxToMaxInvestmentCosts(const std::vector &obj, + const Point &max_invest, + const VariableMap &master_variable); + void Update(const Point &x, double overall_cost, + double invest_cost_at_x); + bool IsCriterionSatisfied(const std::vector &outer_loop_criterions); + bool found_feasible_ = false; + double bilevel_best_ub_ = +1e20; + Point bilevel_best_x_; + double lambda_max_ = 0.0; + double lambda_min_ = 0.0; + double lambda_ = 0.0; + Outerloop::OuterLoopInputData outer_loop_input_data_; +}; \ No newline at end of file diff --git a/src/cpp/benders/benders_core/include/OuterLoopInputDataReader.h b/src/cpp/benders/benders_core/include/OuterLoopInputDataReader.h new file mode 100644 index 000000000..3ce2dd650 --- /dev/null +++ b/src/cpp/benders/benders_core/include/OuterLoopInputDataReader.h @@ -0,0 +1,115 @@ +#pragma once +#include +#include +#include +#include + +#include "LoggerUtils.h" +#include "yaml-cpp/yaml.h" + +namespace Outerloop { + +class OuterLoopInputFileError + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; +}; +class OuterLoopInputFileIsEmpty + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; +}; + +class OuterLoopInputFileNoPatternFound + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; +}; + +class OuterLoopInputPatternsShouldBeArray + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; +}; + +class OuterLoopCouldNotReadAreaField + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; +}; + +class OuterLoopCouldNotReadCriterionField + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; +}; + +/// @brief lovely class +class OuterLoopPattern { + public: + explicit OuterLoopPattern(const std::string &prefix, const std::string &body); + OuterLoopPattern() = default; + [[nodiscard]] std::regex MakeRegex() const; + [[nodiscard]] const std::string &GetPrefix() const; + void SetPrefix(const std::string &prefix); + [[nodiscard]] const std::string &GetBody() const; + void SetBody(const std::string &body); + + private: + std::string prefix_; + std::string body_; + +}; + +/// @brief holds the pattern and the criterion of the outer loop +class OuterLoopSingleInputData { + public: + OuterLoopSingleInputData() = default; + /// @brief constructor + /// @param prefix the prefix in the variable's name + /// @param body any string that could be in the variable's name + /// @param criterion the criterion that should be satisfied + OuterLoopSingleInputData(const std::string &prefix, const std::string &body, + double criterion); + + [[nodiscard]] OuterLoopPattern Pattern() const; + [[nodiscard]] double Criterion() const; + void SetCriterion(double criterion); + void ResetPattern(const std::string &prefix, const std::string &body); + private: + OuterLoopPattern outer_loop_pattern_; + double criterion_ = 0; +}; + +/// @brief this class contains all data read from user input file +class OuterLoopInputData { + public: + OuterLoopInputData() = default; + + [[nodiscard]] std::vector OuterLoopData() const; + + void SetStoppingThreshold(double outer_loop_stopping_threshold); + [[nodiscard]] double StoppingThreshold() const; + void SetCriterionTolerance(double criterion_tolerance); + [[nodiscard]] double CriterionTolerance() const; + void SetCriterionCountThreshold(double criterion_count_threshold); + [[nodiscard]] double CriterionCountThreshold() const; + void AddSingleData(const OuterLoopSingleInputData &data); + + private: + double outer_loop_stopping_threshold_ = 1e-4; + std::vector outer_loop_data_; + double criterion_tolerance_ = 1e-1; + double criterion_count_threshold_ = 1; +}; + +/// @brief Abstract /*** +class IOuterLoopInputDataReader { + public: + virtual OuterLoopInputData Read(const std::filesystem::path &input_file) = 0; +}; + +class OuterLoopInputFromYaml : public IOuterLoopInputDataReader { + public: + OuterLoopInputFromYaml() = default; + OuterLoopInputData Read(const std::filesystem::path &input_file) override; + + private: + OuterLoopInputData outerLoopInputData_; +}; + +} // namespace Outerloop \ No newline at end of file diff --git a/src/cpp/benders/benders_core/include/SimulationOptions.hxx b/src/cpp/benders/benders_core/include/SimulationOptions.hxx index cd695af88..2c0c65210 100644 --- a/src/cpp/benders/benders_core/include/SimulationOptions.hxx +++ b/src/cpp/benders/benders_core/include/SimulationOptions.hxx @@ -33,7 +33,7 @@ BENDERS_OPTIONS_MACRO(TRACE, bool, true, asBool()) BENDERS_OPTIONS_MACRO(SLAVE_WEIGHT, std::string, "CONSTANT", asString()) // If SLAVE_WEIGHT is CONSTANT, set here the divisor required -BENDERS_OPTIONS_MACRO(SLAVE_WEIGHT_VALUE, double, 1, asInt()) +BENDERS_OPTIONS_MACRO(SLAVE_WEIGHT_VALUE, double, 1, asDouble()) // Name of the master problem file, if different from 'master' BENDERS_OPTIONS_MACRO(MASTER_NAME, std::string, "master", asString()) @@ -73,12 +73,12 @@ BENDERS_OPTIONS_MACRO(LAST_MASTER_BASIS, std::string, "master_last_basis", // BATCH SIZE (Benders by batch) BENDERS_OPTIONS_MACRO(BATCH_SIZE, size_t, 0, asUInt()) -// EXTERNAL Loop Loss of Load thresold -BENDERS_OPTIONS_MACRO(EXT_LOOP_CRITERION_VALUE, double, 1.0, asDouble()) +// is this an outer Loop +BENDERS_OPTIONS_MACRO(DO_OUTER_LOOP, bool, false, asBool()) -// EXTERNAL Loop epsilon -BENDERS_OPTIONS_MACRO(EXT_LOOP_CRITERION_TOLERANCE, double, 1e-1, asDouble()) +// Outer Loop Options file +BENDERS_OPTIONS_MACRO(OUTER_LOOP_OPTION_FILE, std::string, + "outer_loop_options.json", asString()) -// EXTERNAL Loop Max unsupplied energy per timestep -BENDERS_OPTIONS_MACRO(EXT_LOOP_CRITERION_COUNT_THRESHOLD, double, 1e-1, - asDouble()) +// Outer Loop number of scenarios (mc years) +BENDERS_OPTIONS_MACRO(OUTER_LOOP_NUMBER_OF_SCENARIOS, unsigned int, 1, asUInt()) diff --git a/src/cpp/benders/benders_core/include/SubproblemCut.h b/src/cpp/benders/benders_core/include/SubproblemCut.h index ac85fbeba..69c0f392c 100644 --- a/src/cpp/benders/benders_core/include/SubproblemCut.h +++ b/src/cpp/benders/benders_core/include/SubproblemCut.h @@ -5,20 +5,11 @@ #include "Worker.h" #include "common.h" namespace PlainData { -struct Variables { - std::vector names; - std::vector values; - template - void serialize(Archive &ar, const unsigned int version) { - ar & names; - ar & values; - } -}; struct SubProblemData { double subproblem_cost; Point var_name_and_subgradient; - Variables variables; + std::vector solution; double single_subpb_costs_under_approx; double subproblem_timer; int simplex_iter; @@ -28,7 +19,7 @@ struct SubProblemData { void serialize(Archive &ar, const unsigned int version) { ar & subproblem_cost; ar & var_name_and_subgradient; - ar & variables; + ar & solution; ar & single_subpb_costs_under_approx; ar & subproblem_timer; ar & simplex_iter; diff --git a/src/cpp/benders/benders_core/include/SubproblemWorker.h b/src/cpp/benders/benders_core/include/SubproblemWorker.h index f84c80d93..1369b2a8d 100644 --- a/src/cpp/benders/benders_core/include/SubproblemWorker.h +++ b/src/cpp/benders/benders_core/include/SubproblemWorker.h @@ -22,7 +22,7 @@ class SubproblemWorker : public Worker { SolverLogManager&solver_log_manager, Logger logger); virtual ~SubproblemWorker() = default; - void get_solution(PlainData::Variables &vars) const; + void get_solution(std::vector &solution) const; public: void fix_to(Point const &x0) const; diff --git a/src/cpp/benders/benders_core/include/VariablesGroup.h b/src/cpp/benders/benders_core/include/VariablesGroup.h new file mode 100644 index 000000000..d68efdfc3 --- /dev/null +++ b/src/cpp/benders/benders_core/include/VariablesGroup.h @@ -0,0 +1,20 @@ +#pragma once +#include +#include +#include + +#include "OuterLoopInputDataReader.h" +namespace Outerloop { +class VariablesGroup { + public: + explicit VariablesGroup(const std::vector& all_variables, + const std::vector& outer_loop_single_input_data); + [[nodiscard]] std::vector> Indices() const; + + private: + void Search(); + const std::vector& all_variables_; + const std::vector& outer_loop_single_input_data_; + std::vector> indices_; +}; +} // namespace Outerloop \ No newline at end of file diff --git a/src/cpp/benders/benders_core/include/common.h b/src/cpp/benders/benders_core/include/common.h index 3cfb2d115..1eff42d1e 100644 --- a/src/cpp/benders/benders_core/include/common.h +++ b/src/cpp/benders/benders_core/include/common.h @@ -24,6 +24,7 @@ #include enum class MasterFormulation { INTEGER, RELAXED }; +enum class SOLVER { BENDERS, OUTER_LOOP, MERGE_MPS }; struct Predicate; typedef std::map Point; @@ -141,6 +142,13 @@ struct BaseOptions { Str2Dbl weights; }; typedef BaseOptions MergeMPSOptions; + +struct ExternalLoopOptions { + bool DO_OUTER_LOOP = false; + std::string OUTER_LOOP_OPTION_FILE; + unsigned int OUTER_LOOP_NUMBER_OF_SCENARIOS = 1; +}; + struct BendersBaseOptions : public BaseOptions { explicit BendersBaseOptions(const BaseOptions &base_to_copy) : BaseOptions(base_to_copy) {} @@ -164,12 +172,7 @@ struct BendersBaseOptions : public BaseOptions { std::string LAST_MASTER_BASIS; size_t BATCH_SIZE; -}; - -struct ExternalLoopOptions { - double EXT_LOOP_CRITERION_VALUE = 1.0; - double EXT_LOOP_CRITERION_TOLERANCE = 1e-1; - double EXT_LOOP_CRITERION_COUNT_THRESHOLD = 1e-1; + ExternalLoopOptions EXTERNAL_LOOP_OPTIONS; }; void usage(int argc); diff --git a/src/cpp/benders/benders_mpi/BendersMPI.cpp b/src/cpp/benders/benders_mpi/BendersMPI.cpp index a4088a98a..faed57825 100644 --- a/src/cpp/benders/benders_mpi/BendersMPI.cpp +++ b/src/cpp/benders/benders_mpi/BendersMPI.cpp @@ -4,6 +4,7 @@ #include #include +#include "CustomVector.h" #include "Timer.h" #include "glog/logging.h" @@ -40,7 +41,11 @@ void BendersMpi::InitializeProblems() { } current_problem_id++; } - init_problems_ = false; + + // if (_world.rank() == rank_0) { + SetSubproblemsVariablesIndex(); + // } + init_problems_ = false; } void BendersMpi::BuildMasterProblem() { if (_world.rank() == rank_0) { @@ -81,7 +86,7 @@ void BendersMpi::do_solve_master_create_trace_and_update_cuts() { } void BendersMpi::BroadcastXCut() { - if (!_exceptionRaised) { + if (!exception_raised_) { Point x_cut = get_x_cut(); mpi::broadcast(_world, x_cut, rank_0); set_x_cut(x_cut); @@ -131,7 +136,7 @@ void BendersMpi::step_2_solve_subproblems_and_build_cuts() { void BendersMpi::gather_subproblems_cut_package_and_build_cuts( const SubProblemDataMap &subproblem_data_map, const Timer &walltime) { - if (!_exceptionRaised) { + if (!exception_raised_) { std::vector gathered_subproblem_map; mpi::gather(_world, subproblem_data_map, gathered_subproblem_map, rank_0); SetSubproblemsWalltime(walltime.elapsed()); @@ -139,11 +144,41 @@ void BendersMpi::gather_subproblems_cut_package_and_build_cuts( Reduce(GetSubproblemsCpuTime(), cumulative_subproblems_timer_per_iter, std::plus(), rank_0); SetSubproblemsCumulativeCpuTime(cumulative_subproblems_timer_per_iter); + if (Options().EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP) { + _data.outer_loop_current_iteration_data.outer_loop_criterion = + ComputeSubproblemsContributionToOuterLoopCriterion( + subproblem_data_map); + if (_world.rank() == rank_0) { + outer_loop_criterion_.push_back( + _data.outer_loop_current_iteration_data.outer_loop_criterion); + UpdateOuterLoopMaxCriterionArea(); + } + } // only rank_0 receive non-emtpy gathered_subproblem_map master_build_cuts(gathered_subproblem_map); } } +std::vector +BendersMpi::ComputeSubproblemsContributionToOuterLoopCriterion( + const SubProblemDataMap &subproblem_data_map) { + std::vector outer_loop_criterion_per_sub_problem_per_pattern( + var_indices_.size(), {}); + std::vector outer_loop_criterion_sub_problems_map_result( + var_indices_.size(), {}); + for (const auto &[subproblem_name, subproblem_data] : subproblem_data_map) { + AddVectors( + outer_loop_criterion_per_sub_problem_per_pattern, + ComputeOuterLoopCriterion(subproblem_name, subproblem_data)); + } + Reduce(outer_loop_criterion_per_sub_problem_per_pattern, + outer_loop_criterion_sub_problems_map_result, std::plus(), + rank_0); + // outer_loop_criterion_sub_problems_map_result/=nbyears; + + return outer_loop_criterion_sub_problems_map_result; +} + SubProblemDataMap BendersMpi::get_subproblem_cut_package() { SubProblemDataMap subproblem_data_map; GetSubproblemCut(subproblem_data_map); @@ -199,7 +234,7 @@ void BendersMpi::check_if_some_proc_had_a_failure(int success) { int global_success; mpi::all_reduce(_world, success, global_success, mpi::bitwise_and()); if (global_success == 0) { - _exceptionRaised = true; + exception_raised_ = true; } } @@ -259,14 +294,14 @@ void BendersMpi::Run() { /*Gather cut from each subproblem in master thread and add them to Master * problem*/ - if (!_exceptionRaised) { + if (!exception_raised_) { step_2_solve_subproblems_and_build_cuts(); } - if (!_exceptionRaised) { + if (!exception_raised_) { step_4_update_best_solution(_world.rank()); } - _data.stop |= _exceptionRaised; + _data.stop |= exception_raised_; broadcast(_world, _data.is_in_initial_relaxation, rank_0); broadcast(_world, _data.stop, rank_0); @@ -308,7 +343,7 @@ void BendersMpi::PreRunInitialization() { } void BendersMpi::launch() { - ++_data.benders_num_run; + ++_data.outer_loop_current_iteration_data.benders_num_run; if (init_problems_) { InitializeProblems(); } @@ -318,8 +353,67 @@ void BendersMpi::launch() { _world.barrier(); post_run_actions(); + if (free_problems_) { free(); } _world.barrier(); -} \ No newline at end of file +} + +void BendersMpi::ExternalLoopCheckFeasibility() { + std::vector obj_coeff; + if (_world.rank() == 0) { + obj_coeff = MasterObjectiveFunctionCoeffs(); + + // /!\ partially + SetMasterObjectiveFunctionCoeffsToZeros(); + + // PrintLog(); + } + + launch(); + if (_world.rank() == 0) { + SetMasterObjectiveFunction(obj_coeff.data(), 0, obj_coeff.size() - 1); + UpdateOverallCosts(); + RunExternalLoopBilevelChecks(); + // de-comment for general case + // cuts_manager_->Save(benders_->AllCuts()); + // auto cuts = cuts_manager_->Load(); + // High + if (!ExternalLoopFoundFeasible()) { + std::ostringstream err_msg; + err_msg << PrefixMessage(LogUtils::LOGLEVEL::FATAL, "External Loop") + << "Criterion cannot be satisfied for your study:\n"; + throw CriterionCouldNotBeSatisfied(err_msg.str(), LOGLOCATION); + } + // lambda_max + // benders_->InitExternalValues(false, master_updater_->Rhs()); + InitExternalValues(false, 0.0); + } +} + +void BendersMpi::UpdateOverallCosts() { + auto obj = MasterObjectiveFunctionCoeffs(); + _data.invest_cost = 0; + for (const auto &[var_name, var_id] : MasterVariables()) { + _data.invest_cost += obj[var_id] * _data.x_cut.at(var_name); + } + + relevantIterationData_.best._invest_cost = _data.invest_cost; +} + +void BendersMpi::RunExternalLoopBilevelChecks() { + if (_world.rank() == rank_0 && Options().EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP && + !is_bilevel_check_all_) { + const WorkerMasterData &workerMasterData = BestIterationWorkerMaster(); + const auto &invest_cost = workerMasterData._invest_cost; + const auto &overall_cost = invest_cost + workerMasterData._operational_cost; + outer_loop_biLevel_.Update_bilevel_data_if_feasible( + _data.x_cut, + GetOuterLoopCriterionAtBestBenders() /*/!\ must + be at best it*/ + , + overall_cost, invest_cost, _data.outer_loop_current_iteration_data.external_loop_lambda); + _data.outer_loop_current_iteration_data.outer_loop_bilevel_best_ub = outer_loop_biLevel_.BilevelBestub(); + } +} diff --git a/src/cpp/benders/benders_mpi/include/BendersMPI.h b/src/cpp/benders/benders_mpi/include/BendersMPI.h index fd3bedaa5..179af7da7 100644 --- a/src/cpp/benders/benders_mpi/include/BendersMPI.h +++ b/src/cpp/benders/benders_mpi/include/BendersMPI.h @@ -4,6 +4,7 @@ #include "BendersBase.h" #include "BendersStructsDatas.h" #include "ILogger.h" +#include "LoggerUtils.h" #include "SubproblemCut.h" #include "SubproblemWorker.h" #include "Timer.h" @@ -11,6 +12,10 @@ #include "WorkerMaster.h" #include "common_mpi.h" +class CriterionCouldNotBeSatisfied + : public LogUtils::XpansionError { + using LogUtils::XpansionError::XpansionError; +}; /*! * \class BendersMpi * \brief Class use run the benders algorithm in parallel @@ -25,6 +30,7 @@ class BendersMpi : public BendersBase { void launch() override; virtual std::string BendersName() const { return "Benders mpi"; } const unsigned int rank_0 = 0; + virtual void ExternalLoopCheckFeasibility() override; protected: void free() override; @@ -43,7 +49,6 @@ class BendersMpi : public BendersBase { void solve_master_and_create_trace(); - bool _exceptionRaised = false; void do_solve_master_create_trace_and_update_cuts(); @@ -53,6 +58,9 @@ class BendersMpi : public BendersBase { void write_exception_message(const std::exception &ex) const; void check_if_some_proc_had_a_failure(int success); + + void UpdateOverallCosts(); + void RunExternalLoopBilevelChecks() override; mpi::environment &_env; mpi::communicator &_world; @@ -85,4 +93,7 @@ class BendersMpi : public BendersBase { void AllReduce(const T &in_value, T &out_value, Op op) const { mpi::all_reduce(_world, in_value, out_value, op); } + virtual std::vector + ComputeSubproblemsContributionToOuterLoopCriterion( + const SubProblemDataMap &subproblem_data_map); }; diff --git a/src/cpp/benders/benders_sequential/include/BendersSequential.h b/src/cpp/benders/benders_sequential/include/BendersSequential.h index 141226e1e..2e54632e7 100644 --- a/src/cpp/benders/benders_sequential/include/BendersSequential.h +++ b/src/cpp/benders/benders_sequential/include/BendersSequential.h @@ -20,6 +20,9 @@ class BendersSequential : public BendersBase { virtual void InitializeProblems(); std::string BendersName() const { return "Sequential"; } + void ExternalLoopCheckFeasibility() override {} + void RunExternalLoopBilevelChecks() override {} + protected: virtual void free(); virtual void Run(); diff --git a/src/cpp/benders/external_loop/OuterLoop.cpp b/src/cpp/benders/external_loop/OuterLoop.cpp deleted file mode 100644 index f86ca3d92..000000000 --- a/src/cpp/benders/external_loop/OuterLoop.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include "OuterLoop.h" - -#include "LoggerUtils.h" - -OuterLoop::OuterLoop(std::shared_ptr criterion, - std::shared_ptr master_updater, - std::shared_ptr cuts_manager, - pBendersBase benders, mpi::environment& env, - mpi::communicator& world) - : criterion_(std::move(criterion)), - master_updater_(std::move(master_updater)), - cuts_manager_(std::move(cuts_manager)), - benders_(std::move(benders)), - env_(env), - world_(world) { - loggers_.AddLogger(benders_->_logger); - loggers_.AddLogger(benders_->mathLoggerDriver_); -} - -void OuterLoop::Run() { - benders_->DoFreeProblems(false); - benders_->InitializeProblems(); - benders_->InitExternalValues(); - CRITERION criterion = CRITERION::IS_MET; - std::vector obj_coeff; - if (world_.rank() == 0) { - obj_coeff = benders_->MasterObjectiveFunctionCoeffs(); - - // /!\ partially - benders_->SetMasterObjectiveFunctionCoeffsToZeros(); - - PrintLog(); - } - benders_->launch(); - if (world_.rank() == 0) { - benders_->SetMasterObjectiveFunction(obj_coeff.data(), 0, - obj_coeff.size() - 1); - // de-comment for general case - // cuts_manager_->Save(benders_->AllCuts()); - // auto cuts = cuts_manager_->Load(); - criterion = - criterion_->IsCriterionSatisfied(benders_->BestIterationWorkerMaster()); - if (criterion == CRITERION::HIGH) { - std::ostringstream err_msg; - err_msg << PrefixMessage(LogUtils::LOGLEVEL::FATAL, "External Loop") - << "Criterion cannot be satisfied for your study:\n" - << criterion_->StateAsString(); - throw CriterionCouldNotBeSatisfied(err_msg.str(), LOGLOCATION); - } - // lambda_max - master_updater_->Init(); - } - - mpi::broadcast(world_, criterion, 0); - - while (criterion != CRITERION::IS_MET) { - benders_->ResetData(criterion_->CriterionValue()); - PrintLog(); - benders_->launch(); - if (world_.rank() == 0) { - criterion = criterion_->IsCriterionSatisfied( - benders_->BestIterationWorkerMaster()); - master_updater_->Update(criterion); - } - - mpi::broadcast(world_, criterion, 0); - } - // last prints - PrintLog(); - auto benders_data = benders_->GetCurrentIterationData(); - benders_data.external_loop_criterion = criterion_->CriterionValue(); - benders_->mathLoggerDriver_->Print(benders_data); - - // TODO general-case - // cuts_manager_->Save(benders_->AllCuts()); - benders_->free(); -} - -void OuterLoop::PrintLog() { - std::ostringstream msg; - auto logger = benders_->_logger; - logger->PrintIterationSeparatorBegin(); - msg << "*** Outer loop: " << benders_->GetBendersRunNumber(); - logger->display_message(msg.str()); - msg.str(""); - msg << "*** Criterion value: " << std::scientific << std::setprecision(10) - << criterion_->CriterionValue(); - logger->display_message(msg.str()); - logger->PrintIterationSeparatorEnd(); -} \ No newline at end of file diff --git a/src/cpp/benders/external_loop/OuterloopCriterion.cpp b/src/cpp/benders/external_loop/OuterloopCriterion.cpp deleted file mode 100644 index 842546b7a..000000000 --- a/src/cpp/benders/external_loop/OuterloopCriterion.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "OuterLoopCriterion.h" - -#include "LoggerUtils.h" - -OuterloopCriterionLossOfLoad::OuterloopCriterionLossOfLoad( - const ExternalLoopOptions& options) - : options_(options) {} - -CRITERION OuterloopCriterionLossOfLoad::IsCriterionSatisfied( - const WorkerMasterData& worker_master_data) { - ProcessSum(worker_master_data); - - if (sum_loss_ <= options_.EXT_LOOP_CRITERION_VALUE + - options_.EXT_LOOP_CRITERION_TOLERANCE) { - if (sum_loss_ >= options_.EXT_LOOP_CRITERION_VALUE - - options_.EXT_LOOP_CRITERION_TOLERANCE) { - return CRITERION::IS_MET; - } - return CRITERION::LOW; - } else { - return CRITERION::HIGH; - } -} - -void OuterloopCriterionLossOfLoad::ProcessSum( - const WorkerMasterData& worker_master_data) { - sum_loss_ = 0; - for (const auto& [sub_problem_name, sub_problem_data] : - worker_master_data._cut_trace) { - for (auto i(0); i < sub_problem_data.variables.names.size(); ++i) { - auto var_name = sub_problem_data.variables.names[i]; - auto solution = sub_problem_data.variables.values[i]; - if (std::regex_search(var_name, rgx_) && - solution > options_.EXT_LOOP_CRITERION_COUNT_THRESHOLD) { - // 1h of unsupplied energy - sum_loss_ += 1; - } - } - } -} - -std::string OuterloopCriterionLossOfLoad::StateAsString() const { - std::ostringstream msg; - msg << "Sum loss = " << sum_loss_ << "\n" - << "threshold: " << options_.EXT_LOOP_CRITERION_VALUE << "\n" - << "epsilon: " << options_.EXT_LOOP_CRITERION_TOLERANCE << "\n"; - - return msg.str(); -} diff --git a/src/cpp/benders/external_loop/include/OuterLoopCriterion.h b/src/cpp/benders/external_loop/include/OuterLoopCriterion.h deleted file mode 100644 index 128380873..000000000 --- a/src/cpp/benders/external_loop/include/OuterLoopCriterion.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once -#include -#include - -#include "BendersBase.h" -#include "LogUtils.h" -#include "common.h" - -class CriterionCouldNotBeSatisfied - : public LogUtils::XpansionError { - using LogUtils::XpansionError::XpansionError; -}; - -enum class CRITERION { LOW, IS_MET, HIGH }; -class IOuterLoopCriterion { - public: - virtual CRITERION IsCriterionSatisfied( - const WorkerMasterData& worker_master_data) = 0; - virtual std::string StateAsString() const = 0; - virtual double CriterionValue() const = 0; -}; - -class OuterloopCriterionLossOfLoad : public IOuterLoopCriterion { - public: - explicit OuterloopCriterionLossOfLoad(const ExternalLoopOptions& options); - CRITERION IsCriterionSatisfied( - const WorkerMasterData& milp_solution) override; - std::string StateAsString() const override; - double CriterionValue() const override { return sum_loss_; } - - private: - void ProcessSum(const WorkerMasterData& worker_master_data); - const std::string positive_unsupplied_vars_prefix_ = - "^PositiveUnsuppliedEnergy::"; - const std::regex rgx_ = std::regex(positive_unsupplied_vars_prefix_); - - ExternalLoopOptions options_; - double sum_loss_ = 0.0; -}; diff --git a/src/cpp/benders/factories/BendersFactory.cpp b/src/cpp/benders/factories/BendersFactory.cpp index 45f808ee6..12c745306 100644 --- a/src/cpp/benders/factories/BendersFactory.cpp +++ b/src/cpp/benders/factories/BendersFactory.cpp @@ -42,21 +42,20 @@ BENDERSMETHOD DeduceBendersMethod(size_t coupling_map_size, size_t batch_size, } } -pBendersBase PrepareForExecution(BendersLoggerBase& benders_loggers, - const SimulationOptions& options, - const char* argv0, bool external_loop, - mpi::environment& env, - mpi::communicator& world) { +pBendersBase BendersMainFactory::PrepareForExecution( + BendersLoggerBase& benders_loggers, const SimulationOptions& options, + bool external_loop) const { pBendersBase benders; Logger logger; std::shared_ptr math_log_driver; + BendersBaseOptions benders_options(options.get_benders_options()); - google::InitGoogleLogging(argv0); + google::InitGoogleLogging(argv_[0]); auto path_to_log = std::filesystem::path(options.OUTPUTROOT) / - ("bendersLog-rank" + std::to_string(world.rank()) + ".txt."); + ("bendersLog-rank" + std::to_string(pworld_->rank()) + ".txt."); google::SetLogDestination(google::GLOG_INFO, path_to_log.string().c_str()); auto log_reports_name = @@ -70,7 +69,7 @@ pBendersBase PrepareForExecution(BendersLoggerBase& benders_loggers, const auto method = DeduceBendersMethod(coupling_map.size(), options.BATCH_SIZE, external_loop); - if (world.rank() == 0) { + if (pworld_->rank() == 0) { auto benders_log_console = benders_options.LOG_LEVEL > 0; auto logger_factory = FileAndStdoutLoggerFactory(log_reports_name, benders_log_console); @@ -95,12 +94,12 @@ pBendersBase PrepareForExecution(BendersLoggerBase& benders_loggers, case BENDERSMETHOD::BENDERS: case BENDERSMETHOD::BENDERS_EXTERNAL_LOOP: benders = std::make_shared(benders_options, logger, writer, - env, world, math_log_driver); + *penv_, *pworld_, math_log_driver); break; case BENDERSMETHOD::BENDERS_BY_BATCH: case BENDERSMETHOD::BENDERS_BY_BATCH_EXTERNAL_LOOP: benders = std::make_shared( - benders_options, logger, writer, env, world, math_log_driver); + benders_options, logger, writer, *penv_, *pworld_, math_log_driver); break; } benders->set_input_map(coupling_map); @@ -111,7 +110,7 @@ pBendersBase PrepareForExecution(BendersLoggerBase& benders_loggers, if (benders_options.LOG_LEVEL > 1) { auto solver_log = std::filesystem::path(options.OUTPUTROOT) / (std::string("solver_log_proc_") + - std::to_string(world.rank()) + ".txt"); + std::to_string(pworld_->rank()) + ".txt"); benders->set_solver_log_file(solver_log); } @@ -121,15 +120,13 @@ pBendersBase PrepareForExecution(BendersLoggerBase& benders_loggers, return benders; } -int RunBenders(char** argv, const std::filesystem::path& options_file, - mpi::environment& env, mpi::communicator& world) { +int BendersMainFactory::RunBenders() const { // Read options, needed to have options.OUTPUTROOT BendersLoggerBase benders_loggers; try { - SimulationOptions options(options_file); - auto benders = PrepareForExecution(benders_loggers, options, argv[0], false, - env, world); + SimulationOptions options(options_file_); + auto benders = PrepareForExecution(benders_loggers, options, false); if (benders) { benders->launch(); @@ -159,25 +156,22 @@ int RunBenders(char** argv, const std::filesystem::path& options_file, } return 0; } -int RunExternalLoop_(char** argv, const std::filesystem::path& options_file, - mpi::environment& env, mpi::communicator& world) { +int BendersMainFactory::RunExternalLoop() const { BendersLoggerBase benders_loggers; try { - SimulationOptions options(options_file); - auto benders = PrepareForExecution(benders_loggers, options, argv[0], true, - env, world); + SimulationOptions options(options_file_); + auto benders = PrepareForExecution(benders_loggers, options, true); double tau = 0.5; - std::shared_ptr criterion = - std::make_shared( - options.GetExternalLoopOptions()); - std::shared_ptr master_updater = - std::make_shared(benders, tau); - std::shared_ptr cuts_manager = - std::make_shared(); - - OuterLoop ext_loop(criterion, master_updater, cuts_manager, benders, env, - world); + double epsilon_lambda = 0.1; + std::shared_ptr master_updater = + std::make_shared( + benders, tau); + std::shared_ptr cuts_manager = + std::make_shared(); + + Outerloop::OuterLoop ext_loop(master_updater, cuts_manager, benders, *penv_, + *pworld_); ext_loop.Run(); } catch (std::exception& e) { @@ -198,8 +192,9 @@ int RunExternalLoop_(char** argv, const std::filesystem::path& options_file, BendersMainFactory::BendersMainFactory(int argc, char** argv, mpi::environment& env, - mpi::communicator& world) - : argv_(argv), penv_(&env), pworld_(&world) { + mpi::communicator& world, + const SOLVER& solver) + : argv_(argv), penv_(&env), pworld_(&world), solver_(solver) { // First check usage (options are given) if (world.rank() == 0) { usage(argc); @@ -210,20 +205,20 @@ BendersMainFactory::BendersMainFactory(int argc, char** argv, BendersMainFactory::BendersMainFactory( int argc, char** argv, const std::filesystem::path& options_file, - mpi::environment& env, mpi::communicator& world) + mpi::environment& env, mpi::communicator& world, const SOLVER& solver) : argv_(argv), options_file_(options_file), penv_(&env), - pworld_(&world) { + pworld_(&world), + solver_(solver) { // First check usage (options are given) if (world.rank() == 0) { usage(argc); } } - int BendersMainFactory::Run() const { - return RunBenders(argv_, options_file_, *penv_, *pworld_); -} - -int BendersMainFactory::RunExternalLoop() const { - return RunExternalLoop_(argv_, options_file_, *penv_, *pworld_); + if (solver_ == SOLVER::BENDERS) { + return RunBenders(); + } else { + return RunExternalLoop(); + } } diff --git a/src/cpp/benders/factories/CMakeLists.txt b/src/cpp/benders/factories/CMakeLists.txt index 188101411..14486d5d2 100644 --- a/src/cpp/benders/factories/CMakeLists.txt +++ b/src/cpp/benders/factories/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries (factories output_core logger_lib ${PROJECT_NAME}::benders_mpi_core - external_loop + outer_loop_lib ) target_include_directories (factories diff --git a/src/cpp/benders/factories/include/BendersFactory.h b/src/cpp/benders/factories/include/BendersFactory.h index 4ec0dd4b4..76e24ae14 100644 --- a/src/cpp/benders/factories/include/BendersFactory.h +++ b/src/cpp/benders/factories/include/BendersFactory.h @@ -9,16 +9,23 @@ class BendersMainFactory { std::filesystem::path options_file_; boost::mpi::environment* penv_ = nullptr; boost::mpi::communicator* pworld_ = nullptr; + SOLVER solver_ = SOLVER::BENDERS; + [[nodiscard]] int RunExternalLoop() const; + [[nodiscard]] int RunBenders() const; + pBendersBase PrepareForExecution(BendersLoggerBase& benders_loggers, + const SimulationOptions& options, + bool external_loop) const; public: explicit BendersMainFactory(int argc, char** argv, boost::mpi::environment& env, - boost::mpi::communicator& world); + boost::mpi::communicator& world, + const SOLVER& solver); explicit BendersMainFactory(int argc, char** argv, const std::filesystem::path& options_file, boost::mpi::environment& env, - boost::mpi::communicator& world); + boost::mpi::communicator& world, + const SOLVER& solver); int Run() const; - int RunExternalLoop() const; }; #endif // ANTARES_XPANSION_SRC_CPP_BENDERS_FACTORIES_INCLUDE_BENDERSFACTORY_H \ No newline at end of file diff --git a/src/cpp/benders/external_loop/CMakeLists.txt b/src/cpp/benders/outer_loop/CMakeLists.txt similarity index 75% rename from src/cpp/benders/external_loop/CMakeLists.txt rename to src/cpp/benders/outer_loop/CMakeLists.txt index 68c3c1293..cdd7a5d06 100644 --- a/src/cpp/benders/external_loop/CMakeLists.txt +++ b/src/cpp/benders/outer_loop/CMakeLists.txt @@ -14,9 +14,8 @@ if (TBB_VERSION_MAJOR VERSION_GREATER "2020") message(FATAL_ERROR "Require tbb 2018 to 2020.") endif() -add_library (external_loop STATIC +add_library (outer_loop_lib STATIC ${CMAKE_CURRENT_SOURCE_DIR}/OuterLoop.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/OuterloopCriterion.cpp ${CMAKE_CURRENT_SOURCE_DIR}/MasterUpdateBase.cpp ${CMAKE_CURRENT_SOURCE_DIR}/CutsManagement.cpp ) @@ -24,12 +23,12 @@ add_library (external_loop STATIC -target_include_directories (external_loop +target_include_directories (outer_loop_lib PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include -) + ${CMAKE_CURRENT_SOURCE_DIR}/include + ) -target_link_libraries (external_loop +target_link_libraries (outer_loop_lib PUBLIC helpers benders_core @@ -37,4 +36,4 @@ target_link_libraries (external_loop benders_by_batch_core ) -add_library (${PROJECT_NAME}::external_loop ALIAS external_loop) \ No newline at end of file +add_library (${PROJECT_NAME}::outer_loop_lib ALIAS outer_loop_lib) diff --git a/src/cpp/benders/external_loop/CutsManagement.cpp b/src/cpp/benders/outer_loop/CutsManagement.cpp similarity index 88% rename from src/cpp/benders/external_loop/CutsManagement.cpp rename to src/cpp/benders/outer_loop/CutsManagement.cpp index a0fd00625..17ca06db8 100644 --- a/src/cpp/benders/external_loop/CutsManagement.cpp +++ b/src/cpp/benders/outer_loop/CutsManagement.cpp @@ -1,5 +1,7 @@ #include "CutsManagement.h" +using namespace Outerloop; + void CutsManagerRunTime::Save(const WorkerMasterDataVect& benders_cuts) { benders_cuts_ = benders_cuts; } diff --git a/src/cpp/benders/external_loop/MasterUpdateBase.cpp b/src/cpp/benders/outer_loop/MasterUpdateBase.cpp similarity index 52% rename from src/cpp/benders/external_loop/MasterUpdateBase.cpp rename to src/cpp/benders/outer_loop/MasterUpdateBase.cpp index 2217dc88a..f39e4653e 100644 --- a/src/cpp/benders/external_loop/MasterUpdateBase.cpp +++ b/src/cpp/benders/outer_loop/MasterUpdateBase.cpp @@ -1,29 +1,19 @@ +#include + #include "MasterUpdate.h" +using namespace Outerloop; + MasterUpdateBase::MasterUpdateBase(pBendersBase benders, double tau) - : benders_(std::move(benders)), lambda_(0), lambda_min_(0) { + : benders_(std::move(benders)), + lambda_(0), + outer_loop_stopping_threshold_(benders_->OuterLoopStoppingThreshold()) { CheckTau(tau); } MasterUpdateBase::MasterUpdateBase(pBendersBase benders, double tau, const std::string &name) - : MasterUpdateBase(benders, tau) { - min_invest_constraint_name_ = name; -} -MasterUpdateBase::MasterUpdateBase(pBendersBase benders, double lambda, - double lambda_min, double lambda_max, - double tau) - : benders_(std::move(benders)), - lambda_(lambda), - lambda_min_(lambda_min), - lambda_max_(lambda_max) { - CheckTau(tau); -} - -MasterUpdateBase::MasterUpdateBase(pBendersBase benders, double lambda, - double lambda_min, double lambda_max, - double tau, const std::string &name) - : MasterUpdateBase(benders, lambda, lambda_min, lambda_max, tau) { + : MasterUpdateBase(std::move(benders), tau) { min_invest_constraint_name_ = name; } @@ -35,37 +25,42 @@ void MasterUpdateBase::CheckTau(double tau) { } void MasterUpdateBase::Init() { - // check lambda_max_ - if (lambda_max_ <= 0 || lambda_max_ < lambda_min_) { - // TODO log - SetLambdaMaxToMaxInvestmentCosts(); - } + // check lambda_max + // if (lambda_max <= 0 || lambda_max < lambda_min) { + // // TODO log + // SetLambdaMaxToMaxInvestmentCosts(); + // } } -void MasterUpdateBase::SetLambdaMaxToMaxInvestmentCosts() { - const auto &obj = benders_->MasterObjectiveFunctionCoeffs(); - const auto max_invest = - benders_->BestIterationWorkerMaster().get_max_invest(); - lambda_max_ = 0; - for (const auto &[var_name, var_id] : benders_->MasterVariables()) { - lambda_max_ += obj[var_id] * max_invest.at(var_name); - } -} -void MasterUpdateBase::Update(const CRITERION &criterion) { - switch (criterion) { - case CRITERION::LOW: - lambda_max_ = - std::min(lambda_max_, benders_->GetBestIterationData().invest_cost); - break; - case CRITERION::HIGH: - lambda_min_ = lambda_; - break; - default: - return; +bool MasterUpdateBase::Update(double lambda_min, double lambda_max) { + // if (is_criterion_high) { + // lambda_min = lambda_; + // } else { + // lambda_max = + // std::min(lambda_max, benders_->GetBestIterationData().invest_cost); + // } + // WorkerMasterData workerMasterData = benders_->BestIterationWorkerMaster(); + // double invest_cost = workerMasterData._invest_cost; + // double overall_cost = invest_cost + workerMasterData._operational_cost; + // if (outerLoopBiLevel_.Update_bilevel_data_if_feasible( + // workerMasterData._cut_trace, + // benders_ + // ->GetOuterLoopCriterionAtBestBenders() /*/!\ must be at best + // it*/, + // overall_cost)) { + // lambda_max = std::min(lambda_max, invest_cost); + // } else { + // lambda_min = lambda_; + // } + + stop_update_ = + std::abs(lambda_max - lambda_min) < outer_loop_stopping_threshold_; + if (!stop_update_) { + lambda_ = dichotomy_weight_coeff_ * lambda_max + + (1 - dichotomy_weight_coeff_) * lambda_min; + UpdateConstraints(); } - lambda_ = dichotomy_weight_coeff_ * lambda_max_ + - (1 - dichotomy_weight_coeff_) * lambda_min_; - UpdateConstraints(); + return stop_update_; } void MasterUpdateBase::UpdateConstraints() { @@ -110,3 +105,5 @@ void MasterUpdateBase::AddMinInvestConstraint() { } additional_constraint_index_ = benders_->MasterGetnrows() - 1; } + +double MasterUpdateBase::Rhs() const { return lambda_; } \ No newline at end of file diff --git a/src/cpp/benders/outer_loop/OuterLoop.cpp b/src/cpp/benders/outer_loop/OuterLoop.cpp new file mode 100644 index 000000000..49642fee2 --- /dev/null +++ b/src/cpp/benders/outer_loop/OuterLoop.cpp @@ -0,0 +1,71 @@ +#include "OuterLoop.h" + +#include "LoggerUtils.h" +using namespace Outerloop; + +OuterLoop::OuterLoop(std::shared_ptr master_updater, + std::shared_ptr cuts_manager, + pBendersBase benders, mpi::environment& env, + mpi::communicator& world) + : master_updater_(std::move(master_updater)), + cuts_manager_(std::move(cuts_manager)), + benders_(std::move(benders)), + env_(env), + world_(world) { + loggers_.AddLogger(benders_->_logger); + loggers_.AddLogger(benders_->mathLoggerDriver_); +} + +void OuterLoop::Run() { + benders_->DoFreeProblems(false); + benders_->InitializeProblems(); + + benders_->ExternalLoopCheckFeasibility(); + + bool stop_update_master = false; + while (!stop_update_master) { + PrintLog(); + benders_->init_data(master_updater_->Rhs()); + benders_->launch(); + if(!benders_->isExceptionRaised()) { + benders_->RunExternalLoopBilevelChecks(); + if (world_.rank() == 0) { + stop_update_master = + master_updater_->Update(benders_->ExternalLoopLambdaMin(), + benders_->ExternalLoopLambdaMax()); + } + + mpi::broadcast(world_, stop_update_master, 0); + } + else { + stop_update_master = true; + } + } + + // last prints + PrintLog(); + benders_->mathLoggerDriver_->Print(benders_->GetCurrentIterationData()); + + // TODO general-case + // cuts_manager_->Save(benders_->AllCuts()); + benders_->free(); +} + +void OuterLoop::PrintLog() { + std::ostringstream msg; + auto logger = benders_->_logger; + logger->PrintIterationSeparatorBegin(); + msg << "*** Outer loop: " << benders_->GetBendersRunNumber(); + logger->display_message(msg.str()); + msg.str(""); + // TODO criterion per pattern (aka prefix+area) at best Benders ? + const auto outer_loop_data = benders_->GetOuterLoopData(); + msg << "*** Max Criterion: " << std::scientific << std::setprecision(10) + << outer_loop_data.max_criterion_best_it; + logger->display_message(msg.str()); + msg.str(""); + msg << "*** Max Criterion Area: " + << outer_loop_data.max_criterion_area_best_it; + logger->display_message(msg.str()); + logger->PrintIterationSeparatorEnd(); +} diff --git a/src/cpp/benders/external_loop/include/CutsManagement.h b/src/cpp/benders/outer_loop/include/CutsManagement.h similarity index 89% rename from src/cpp/benders/external_loop/include/CutsManagement.h rename to src/cpp/benders/outer_loop/include/CutsManagement.h index 5a1addfcd..bb3c7bdca 100644 --- a/src/cpp/benders/external_loop/include/CutsManagement.h +++ b/src/cpp/benders/outer_loop/include/CutsManagement.h @@ -1,6 +1,8 @@ #pragma once #include "BendersStructsDatas.h" +namespace Outerloop { + class ICutsManager { public: ICutsManager() = default; @@ -16,3 +18,5 @@ class CutsManagerRunTime : public ICutsManager { private: WorkerMasterDataVect benders_cuts_; }; + +} // namespace Outerloop diff --git a/src/cpp/benders/external_loop/include/MasterUpdate.h b/src/cpp/benders/outer_loop/include/MasterUpdate.h similarity index 54% rename from src/cpp/benders/external_loop/include/MasterUpdate.h rename to src/cpp/benders/outer_loop/include/MasterUpdate.h index 51f3a746b..1b1f5c059 100644 --- a/src/cpp/benders/external_loop/include/MasterUpdate.h +++ b/src/cpp/benders/outer_loop/include/MasterUpdate.h @@ -1,28 +1,26 @@ #pragma once -#include "OuterLoopCriterion.h" +#include "BendersBase.h" +#include "common.h" +namespace Outerloop { class IMasterUpdate { public: - virtual void Update(const CRITERION &criterion) = 0; + virtual bool Update(double lambda_min, double lambda_max) = 0; virtual void Init() = 0; + [[nodiscard]] virtual double Rhs() const = 0; }; class MasterUpdateBase : public IMasterUpdate { public: - explicit MasterUpdateBase(pBendersBase benders, double lambda, - double lambda_min, double lambda_max, double tau); - explicit MasterUpdateBase(pBendersBase benders, double lambda, - double lambda_min, double lambda_max, double tau, - const std::string &name); + explicit MasterUpdateBase(pBendersBase benders, double tau); explicit MasterUpdateBase(pBendersBase benders, double tau, const std::string &name); - explicit MasterUpdateBase(pBendersBase benders, double tau); - void Update(const CRITERION &criterion) override; + bool Update(double lambda_min, double lambda_max) override; void Init() override; + [[nodiscard]] double Rhs() const override; private: void CheckTau(double tau); - void SetLambdaMaxToMaxInvestmentCosts(); void UpdateConstraints(); void AddMinInvestConstraint(); // rename min invest constraint @@ -30,8 +28,9 @@ class MasterUpdateBase : public IMasterUpdate { int additional_constraint_index_ = -1; pBendersBase benders_; double lambda_ = 0; - double lambda_min_ = 0; - double lambda_max_ = -1; // tau double dichotomy_weight_coeff_ = 0.5; + double outer_loop_stopping_threshold_ = 1e-1; + bool stop_update_ = true; }; +} // namespace Outerloop \ No newline at end of file diff --git a/src/cpp/benders/external_loop/include/OuterLoop.h b/src/cpp/benders/outer_loop/include/OuterLoop.h similarity index 71% rename from src/cpp/benders/external_loop/include/OuterLoop.h rename to src/cpp/benders/outer_loop/include/OuterLoop.h index 080c1f65f..0fe642de0 100644 --- a/src/cpp/benders/external_loop/include/OuterLoop.h +++ b/src/cpp/benders/outer_loop/include/OuterLoop.h @@ -1,13 +1,12 @@ #pragma once #include "CutsManagement.h" #include "MasterUpdate.h" -#include "OuterLoopCriterion.h" #include "common_mpi.h" +namespace Outerloop { class OuterLoop { public: - explicit OuterLoop(std::shared_ptr criterion, - std::shared_ptr master_updater, + explicit OuterLoop(std::shared_ptr master_updater, std::shared_ptr cuts_manager, pBendersBase benders, mpi::environment& env, mpi::communicator& world); @@ -15,11 +14,12 @@ class OuterLoop { private: void PrintLog(); - std::shared_ptr criterion_; std::shared_ptr master_updater_; std::shared_ptr cuts_manager_; pBendersBase benders_; BendersLoggerBase loggers_; mpi::environment& env_; mpi::communicator& world_; -}; \ No newline at end of file +}; + +} // namespace Outerloop \ No newline at end of file diff --git a/src/cpp/exe/CMakeLists.txt b/src/cpp/exe/CMakeLists.txt index 85a7d6427..90ef014aa 100644 --- a/src/cpp/exe/CMakeLists.txt +++ b/src/cpp/exe/CMakeLists.txt @@ -14,5 +14,5 @@ add_subdirectory ("${CMAKE_CURRENT_SOURCE_DIR}/full_run") add_subdirectory ("${CMAKE_CURRENT_SOURCE_DIR}/antares_archive_updater") add_subdirectory ("${CMAKE_CURRENT_SOURCE_DIR}/benders") -add_subdirectory ("${CMAKE_CURRENT_SOURCE_DIR}/ExtLoop") +add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/outer_loop") diff --git a/src/cpp/exe/benders/main.cpp b/src/cpp/exe/benders/main.cpp index 5e27445f6..3db774816 100644 --- a/src/cpp/exe/benders/main.cpp +++ b/src/cpp/exe/benders/main.cpp @@ -4,6 +4,7 @@ int main(int argc, char **argv) { mpi::environment env(argc, argv); mpi::communicator world; - auto benders_factory = BendersMainFactory(argc, argv, env, world); + auto benders_factory = + BendersMainFactory(argc, argv, env, world, SOLVER::BENDERS); return benders_factory.Run(); } diff --git a/src/cpp/exe/full_run/main.cpp b/src/cpp/exe/full_run/main.cpp index 59203a50a..f24351cad 100644 --- a/src/cpp/exe/full_run/main.cpp +++ b/src/cpp/exe/full_run/main.cpp @@ -31,10 +31,19 @@ int main(int argc, char** argv) { int argc_ = 2; const auto options_file = options_parser.BendersOptionsFile(); - auto benders_factory = - BendersMainFactory(argc_, argv, options_file, env, world); - benders_factory.Run(); - + auto solver = options_parser.Solver(); + if (solver == "benders") { + auto benders_factory = BendersMainFactory(argc_, argv, options_file, env, + world, SOLVER::BENDERS); + benders_factory.Run(); + } + if (solver == "outer_loop") { + auto benders_factory = BendersMainFactory(argc_, argv, options_file, env, + world, SOLVER::OUTER_LOOP); + benders_factory.Run(); + } else { + // TODO merge mps? + } if (world.rank() == 0) { auto log_file_path = xpansion_output_dir / "lp" / "StudyUpdateLog.txt"; auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout, diff --git a/src/cpp/exe/ExtLoop/CMakeLists.txt b/src/cpp/exe/outer_loop/CMakeLists.txt similarity index 69% rename from src/cpp/exe/ExtLoop/CMakeLists.txt rename to src/cpp/exe/outer_loop/CMakeLists.txt index 7a2dc6170..478eb76e7 100644 --- a/src/cpp/exe/ExtLoop/CMakeLists.txt +++ b/src/cpp/exe/outer_loop/CMakeLists.txt @@ -10,7 +10,7 @@ # MPI Benders Exe # --------------------------------------------------------------------------- -add_executable (ext_loop +add_executable(outer_loop ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp ) @@ -19,15 +19,9 @@ if(UNIX) set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER}) endif() -#IF (WIN32) -# target_link_libraries (benders -# ${PROJECT_NAME}::benders_sequential_core ${PROJECT_NAME}::benders_mpi_core -# msmpi libboost_serialization-vc141-mt-x64-1_67) -#ELSE (WIN32) - target_link_libraries (ext_loop +target_link_libraries(outer_loop ${PROJECT_NAME}::benders_mpi_core factories ) -#ENDIF (WIN32) -install(TARGETS ext_loop DESTINATION bin) +install(TARGETS outer_loop DESTINATION bin) diff --git a/src/cpp/exe/ExtLoop/main.cpp b/src/cpp/exe/outer_loop/main.cpp similarity index 54% rename from src/cpp/exe/ExtLoop/main.cpp rename to src/cpp/exe/outer_loop/main.cpp index 08b804404..7556135eb 100644 --- a/src/cpp/exe/ExtLoop/main.cpp +++ b/src/cpp/exe/outer_loop/main.cpp @@ -4,6 +4,7 @@ int main(int argc, char **argv) { mpi::environment env(argc, argv); mpi::communicator world; - auto benders_factory = BendersMainFactory(argc, argv, env, world); - return benders_factory.RunExternalLoop(); + auto benders_factory = + BendersMainFactory(argc, argv, env, world, SOLVER::OUTER_LOOP); + return benders_factory.Run(); } diff --git a/src/cpp/full_run/FullRunOptionsParser.cpp b/src/cpp/full_run/FullRunOptionsParser.cpp index aa0efe03c..a75bd648f 100644 --- a/src/cpp/full_run/FullRunOptionsParser.cpp +++ b/src/cpp/full_run/FullRunOptionsParser.cpp @@ -10,8 +10,11 @@ FullRunOptionsParser::FullRunOptionsParser() : ProblemGenerationExeOptions() { "benders options file")( "solution,s", po::value(&solutionFile_)->required(), - "path to json solution file"); + "path to json solution file")( + "solver", po::value(&solver_)->default_value("benders"), + "solver (benders, outer_loop, "); // Add mergeMps? } void FullRunOptionsParser::Parse(unsigned int argc, const char* const* argv) { ProblemGenerationExeOptions::Parse(argc, argv); -} \ No newline at end of file +} +std::string FullRunOptionsParser::Solver() const { return solver_; } diff --git a/src/cpp/full_run/include/FullRunOptionsParser.h b/src/cpp/full_run/include/FullRunOptionsParser.h index 02b49a1a8..31a441bdc 100644 --- a/src/cpp/full_run/include/FullRunOptionsParser.h +++ b/src/cpp/full_run/include/FullRunOptionsParser.h @@ -16,9 +16,12 @@ class FullRunOptionsParser : public ProblemGenerationExeOptions { } [[nodiscard]] std::filesystem::path SolutionFile() const { return solutionFile_; } + std::string Solver() const; + private: std::filesystem::path benders_options_file_; std::filesystem::path solutionFile_; + std::string solver_; }; #endif // ANTARES_XPANSION_SRC_CPP_FULL_RUN_FULLRUNOPTIONSPARSER_H diff --git a/src/cpp/xpansion_interfaces/LogUtils.h b/src/cpp/xpansion_interfaces/LogUtils.h index 3c97b681d..b893d0633 100644 --- a/src/cpp/xpansion_interfaces/LogUtils.h +++ b/src/cpp/xpansion_interfaces/LogUtils.h @@ -14,6 +14,10 @@ class XpansionError : public T { explicit XpansionError(const std::string& err_message, const std::string& log_location) : T(log_location + err_message), err_message_(err_message) {} + explicit XpansionError(const std::string& prefix, + const std::string& err_message, + const std::string& log_location) + : T(log_location + prefix + err_message), err_message_(err_message) {} public: std::string ErrorMessage() const { return err_message_; } diff --git a/src/python/antares_xpansion/benders_driver.py b/src/python/antares_xpansion/benders_driver.py index 6f1763cc5..3027cf874 100644 --- a/src/python/antares_xpansion/benders_driver.py +++ b/src/python/antares_xpansion/benders_driver.py @@ -10,20 +10,30 @@ from antares_xpansion.logger import step_logger from antares_xpansion.study_output_cleaner import StudyOutputCleaner +from dataclasses import dataclass +@dataclass +class SolversExe: + benders: Path + merge_mps: Path + outer_loop: Path + class BendersDriver: - def __init__(self, benders, merge_mps, options_file, mpiexec=None) -> None: + def __init__(self, solvers_exe: SolversExe, options_file, mpiexec=None) -> None: self.oversubscribe = False self.allow_run_as_root = False - self.benders = benders - self.merge_mps = merge_mps + self.benders = solvers_exe.benders + self.merge_mps = solvers_exe.merge_mps + self.outer_loop = solvers_exe.outer_loop self.mpiexec = mpiexec + self.method = "benders" + self.n_mpi = 1 self.logger = step_logger(__name__, __class__.__name__) - if (options_file != ""): + if options_file != "": self.options_file = options_file else: raise BendersDriver.BendersOptionsFileError( @@ -91,6 +101,8 @@ def get_lp_path(self): def set_solver(self): if self.method == "benders": self.solver = self.benders + elif self.method == "outer_loop": + self.solver = self.outer_loop elif self.method == "mergeMPS": self.solver = self.merge_mps else: diff --git a/src/python/antares_xpansion/config_file_parser.py b/src/python/antares_xpansion/config_file_parser.py index e8d3eceda..0739af7dd 100644 --- a/src/python/antares_xpansion/config_file_parser.py +++ b/src/python/antares_xpansion/config_file_parser.py @@ -14,6 +14,7 @@ def __init__(self, config_file) -> None: self.LP_NAMER_DEFAULT = "lp_namer" self.STUDY_UPDATER_DEFAULT = "study_updater" self.FULL_RUN_DEFAULT = "full_run" + self.OUTER_LOOP_DEFAULT = "outer_loop" self.ANTARES_ARCHIVE_UPDATER_DEFAULT = "antares_archive_updater" self.SENSITIVITY_DEFAULT = "sensitivity" self.MPIEXEC_DEFAULT = "mpiexec" @@ -35,6 +36,7 @@ def get_config_parameters(self) -> ConfigParameters: LP_NAMER=content.get("LP_NAMER", self.LP_NAMER_DEFAULT), STUDY_UPDATER=content.get("STUDY_UPDATER", self.STUDY_UPDATER_DEFAULT), FULL_RUN=content.get("FULL_RUN", self.FULL_RUN_DEFAULT), + OUTER_LOOP=content.get("OUTER_LOOP", self.FULL_RUN_DEFAULT), ANTARES_ARCHIVE_UPDATER=content.get( "ANTARES_ARCHIVE_UPDATER", self.ANTARES_ARCHIVE_UPDATER_DEFAULT ), diff --git a/src/python/antares_xpansion/config_loader.py b/src/python/antares_xpansion/config_loader.py index a3b2591d8..356e86ae8 100644 --- a/src/python/antares_xpansion/config_loader.py +++ b/src/python/antares_xpansion/config_loader.py @@ -482,9 +482,11 @@ def _set_options_for_benders_solver(self): options_values[ OptimisationKeys.last_mps_master_name_key() ] = self._config.LAST_MASTER_MPS - options_values["LAST_MASTER_BASIS"] = self._config.LAST_MASTER_BASIS - options_values[OptimisationKeys.batch_size_key() - ] = self.get_batch_size() + options_values[OptimisationKeys.last_master_basis_key()] = self._config.LAST_MASTER_BASIS + options_values[OptimisationKeys.batch_size_key()] = self.get_batch_size() + options_values[OptimisationKeys.do_outer_loop_key()] = self._config.method == "outer_loop" + options_values[OptimisationKeys.outer_loop_option_file_key()] = self.outer_loop_options_path() + options_values[OptimisationKeys.outer_loop_number_of_scenarios_key()] = len(self.active_years) # generate options file for the solver with open(self.options_file_path(), "w") as options_file: json.dump(options_values, options_file, indent=4) @@ -626,6 +628,9 @@ def full_run_exe(self): def antares_archive_updater_exe(self): return self.exe_path(self._config.ANTARES_ARCHIVE_UPDATER) + def outer_loop_exe(self): + return self.exe_path(self._config.OUTER_LOOP) + def method(self): return self._config.method @@ -724,3 +729,13 @@ def check_NTC_column_constraints(self, antares_version): def mpi_exe(self): return self.exe_path(Path(self._config.MPIEXEC).name) + + def outer_loop_options_path(self): + return os.path.join(self.outer_loop_dir(), self._config.OUTER_LOOP_FILE) + + def outer_loop_dir(self): + return os.path.normpath(os.path.join( + self.data_dir(), + self._config.USER, + self._config.EXPANSION, + self._config.OUTER_LOOP_DIR)) diff --git a/src/python/antares_xpansion/driver.py b/src/python/antares_xpansion/driver.py index fe8c0167c..cfde50ea2 100644 --- a/src/python/antares_xpansion/driver.py +++ b/src/python/antares_xpansion/driver.py @@ -8,7 +8,7 @@ from pathlib import Path from antares_xpansion.antares_driver import AntaresDriver -from antares_xpansion.benders_driver import BendersDriver +from antares_xpansion.benders_driver import BendersDriver, SolversExe from antares_xpansion.config_loader import ConfigLoader from antares_xpansion.logger import step_logger from antares_xpansion.general_data_processor import GeneralDataProcessor @@ -46,8 +46,10 @@ def __init__(self, config_loader: ConfigLoader): )) self.benders_driver = BendersDriver( - self.config_loader.benders_exe(), - self.config_loader.merge_mps_exe(), + SolversExe( + self.config_loader.benders_exe(), + self.config_loader.merge_mps_exe(), + self.config_loader.outer_loop_exe()), self.config_loader.options_file_name(), self.config_loader.mpi_exe(), ) @@ -76,6 +78,7 @@ def launch(self): self.config_loader.benders_pre_actions() self.full_run_driver.launch(self.config_loader.simulation_output_path(), self.config_loader.is_relaxed(), + self.config_loader.method(), self.config_loader.json_file_path(), self.config_loader.keep_mps(), self.config_loader.n_mpi(), diff --git a/src/python/antares_xpansion/full_run_driver.py b/src/python/antares_xpansion/full_run_driver.py index 77a4487ef..49b8d96b9 100644 --- a/src/python/antares_xpansion/full_run_driver.py +++ b/src/python/antares_xpansion/full_run_driver.py @@ -20,6 +20,7 @@ def __init__(self, full_exe, problem_generation_driver: ProblemGeneratorDriver, def prepare_drivers(self, output_path: Path, problem_generation_is_relaxed: bool, + method: str, json_file_path, benders_keep_mps=False, benders_n_mpi=1, @@ -36,7 +37,7 @@ def prepare_drivers(self, output_path: Path, self.keep_mps = benders_keep_mps # Benders pre-step - self.benders_driver.method = "benders" + self.benders_driver.method = method self.benders_driver.n_mpi = benders_n_mpi self.benders_driver.oversubscribe = benders_oversubscribe self.benders_driver.allow_run_as_root = benders_allow_run_as_root @@ -47,13 +48,14 @@ def prepare_drivers(self, output_path: Path, def launch(self, output_path: Path, problem_generation_is_relaxed: bool, + method: str, json_file_path, benders_keep_mps=False, benders_n_mpi=1, benders_oversubscribe=False, benders_allow_run_as_root=False): self.prepare_drivers( - output_path, problem_generation_is_relaxed, + output_path, problem_generation_is_relaxed, method, json_file_path, benders_keep_mps, benders_n_mpi, benders_oversubscribe, benders_allow_run_as_root) self.run() @@ -79,11 +81,12 @@ def run(self): def full_command(self) -> List: bare_solver_command = [ self.full_exe, "--benders_options", self.benders_driver.options_file, "-s", - str(self.json_file_path)] + str(self.json_file_path), "--solver", self.benders_driver.method] bare_solver_command.extend( self.problem_generation_driver.lp_namer_options()) - if self.benders_driver.solver == self.benders_driver.benders and self.benders_driver.n_mpi > 1: + if self.benders_driver.solver in [self.benders_driver.benders, + self.benders_driver.outer_loop] and self.benders_driver.n_mpi > 1: mpi_command = self.benders_driver.get_mpi_run_command_root() mpi_command.extend(bare_solver_command) return mpi_command diff --git a/src/python/antares_xpansion/input_parser.py b/src/python/antares_xpansion/input_parser.py index f2145f565..c496c1a1a 100644 --- a/src/python/antares_xpansion/input_parser.py +++ b/src/python/antares_xpansion/input_parser.py @@ -38,7 +38,8 @@ def _initialize_parser(self): dest=LauncherOptionsKeys.method_key(), type=str, choices=["benders", - "mergeMPS"], + "mergeMPS", + "outer_loop"], help="Choose the optimization method", default=LauncherOptionsDefaultValues.DEFAULT_VALUE()) self.parser.add_argument("-n", "--np", diff --git a/src/python/antares_xpansion/optimisation_keys.py b/src/python/antares_xpansion/optimisation_keys.py index fcc6d3a6a..c6d25e89f 100644 --- a/src/python/antares_xpansion/optimisation_keys.py +++ b/src/python/antares_xpansion/optimisation_keys.py @@ -63,6 +63,10 @@ def master_name_key(): def last_mps_master_name_key(): return "LAST_MASTER_MPS" + @staticmethod + def last_master_basis_key(): + return "LAST_MASTER_BASIS" + @staticmethod def json_file_key(): return "JSON_FILE" @@ -94,3 +98,15 @@ def separation_key(): @staticmethod def batch_size_key(): return "BATCH_SIZE" + + @staticmethod + def do_outer_loop_key(): + return "DO_OUTER_LOOP" + + @staticmethod + def outer_loop_option_file_key(): + return "OUTER_LOOP_OPTION_FILE" + + @staticmethod + def outer_loop_number_of_scenarios_key(): + return "OUTER_LOOP_NUMBER_OF_SCENARIOS" diff --git a/src/python/antares_xpansion/resume_study.py b/src/python/antares_xpansion/resume_study.py index 3e12c6e5a..d84b900a1 100644 --- a/src/python/antares_xpansion/resume_study.py +++ b/src/python/antares_xpansion/resume_study.py @@ -4,7 +4,7 @@ import json from pathlib import Path import shutil -from antares_xpansion.benders_driver import BendersDriver +from antares_xpansion.benders_driver import BendersDriver, SolversExe from antares_xpansion.launcher_options_default_value import LauncherOptionsDefaultValues from antares_xpansion.launcher_options_keys import LauncherOptionsKeys from antares_xpansion.optimisation_keys import OptimisationKeys @@ -74,7 +74,7 @@ def _update_options_file(self): master_keyword = OptimisationKeys.master_name_key() if not options_file_path.exists(): raise ResumeStudy.OptionsFileNotFound( - f"last master file: {options_file_path} not found") + f"Options file: {options_file_path} not found") options_file_path = self._simulation_output_path / self.benders_options_file with open(options_file_path, "r") as json_file: @@ -108,8 +108,7 @@ def _update_options_file(self): json.dump(options, options_json, indent=4) benders_driver = BendersDriver( - self.benders_exe, - self.merge_mps_exe, + SolversExe(self.benders_exe, self.merge_mps_exe, ""), self.benders_options_file ) benders_driver.launch(self._simulation_output_path.parent, self.method, diff --git a/src/python/antares_xpansion/xpansionConfig.py b/src/python/antares_xpansion/xpansionConfig.py index 81b5b5bf0..5344e8b66 100644 --- a/src/python/antares_xpansion/xpansionConfig.py +++ b/src/python/antares_xpansion/xpansionConfig.py @@ -20,6 +20,7 @@ class ConfigParameters: STUDY_UPDATER: str SENSITIVITY_EXE: str FULL_RUN: str + OUTER_LOOP: str ANTARES_ARCHIVE_UPDATER: str MPIEXEC: str AVAILABLE_SOLVERS: List[str] @@ -60,6 +61,7 @@ def __init__( self.STUDY_UPDATER: str = "" self.SENSITIVITY_EXE: str = "" self.FULL_RUN: str = "" + self.OUTER_LOOP: str = "" self.ANTARES_ARCHIVE_UPDATER: str = "" self.MPI_LAUNCHER: str = "" self.MPI_N: str = "" @@ -144,6 +146,8 @@ def _set_constants(self): self.LAST_MASTER_BASIS = "master_last_basis.bss" self.WEIGHTS = "weights" self.CONSTRAINTS = "constraints" + self.OUTER_LOOP_FILE = "outer_loop.yml" + self.OUTER_LOOP_DIR = "outer_loop" def _set_default_settings(self): self.settings_default = { @@ -242,6 +246,7 @@ def _get_config_values(self): self.LP_NAMER = self.config_parameters.LP_NAMER self.STUDY_UPDATER = self.config_parameters.STUDY_UPDATER self.FULL_RUN = self.config_parameters.FULL_RUN + self.OUTER_LOOP = self.config_parameters.OUTER_LOOP self.ANTARES_ARCHIVE_UPDATER = self.config_parameters.ANTARES_ARCHIVE_UPDATER self.SENSITIVITY_EXE = self.config_parameters.SENSITIVITY_EXE self.MPIEXEC = self.config_parameters.MPIEXEC diff --git a/src/python/config.yaml.in b/src/python/config.yaml.in index d37c9a549..3660c00a1 100644 --- a/src/python/config.yaml.in +++ b/src/python/config.yaml.in @@ -5,6 +5,7 @@ BENDERS : @BENDERS@ LP_NAMER : $ STUDY_UPDATER : $ FULL_RUN : $ +OUTER_LOOP : $ SENSITIVITY : $ ANTARES_ARCHIVE_UPDATER : $ mpiexec : @MPIEXEC_EXECUTABLE@ diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 579066af5..9f3f7f706 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -31,4 +31,4 @@ add_subdirectory(restart_benders) add_subdirectory(zip_mps) add_subdirectory(benders) add_subdirectory(full_run) -add_subdirectory(ext_loop) +add_subdirectory(outer_loop) diff --git a/tests/cpp/ext_loop/CMakeLists.txt b/tests/cpp/ext_loop/CMakeLists.txt deleted file mode 100644 index 4121ae77e..000000000 --- a/tests/cpp/ext_loop/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -add_executable(ext_loop_test - ext_loop_test.cpp) - -target_link_libraries(ext_loop_test - PRIVATE - solvers - external_loop - benders_by_batch_core - benders_core - output_core - logger_lib - GTest::Main - tests_utils) - -# target_include_directories(ext_loop_test -# PRIVATE -# ${CMAKE_CURRENT_SOURCE_DIR}/../TestDoubles/ -# ) - -add_test(NAME unit_ext_loop COMMAND ext_loop_test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) -set_property(TEST unit_ext_loop PROPERTY LABELS unit) \ No newline at end of file diff --git a/tests/cpp/ext_loop/ext_loop_test.cpp b/tests/cpp/ext_loop/ext_loop_test.cpp deleted file mode 100644 index 050ac0c27..000000000 --- a/tests/cpp/ext_loop/ext_loop_test.cpp +++ /dev/null @@ -1,251 +0,0 @@ - -// #include "MasterUpdate.h" -// #include "ext_loop_test.h" - -#include "LoggerFactories.h" -#include "MasterUpdate.h" -#include "OuterLoopCriterion.h" -#include "WriterFactories.h" -#include "gtest/gtest.h" -#include "multisolver_interface/environment.h" - -int my_argc; -char** my_argv; - -boost::mpi::environment* penv = nullptr; -boost::mpi::communicator* pworld = nullptr; -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - - mpi::environment env(my_argc, my_argv); - mpi::communicator world; - penv = &env; - pworld = &world; - my_argc = argc; - my_argv = argv; - return RUN_ALL_TESTS(); -} - -//-------------------- OuterLoopCriterionTest ------------------------- - -class OuterLoopCriterionTest : public ::testing::Test {}; - -TEST_F(OuterLoopCriterionTest, IsCriterionHigh) { - double threshold = 1.4; - double epsilon = 1e-1; - double max_unsup_energy = 0.1; - const ExternalLoopOptions options = {threshold, epsilon, max_unsup_energy}; - PlainData::Variables variables = { - {"PositiveUnsuppliedEnergy::1", "PositiveUnsuppliedEnergy::2", "var3"}, - {0.2, 0.3, 68}}; - double criterion_value = 2.0; // two vars named ^PositiveUnsuppliedEnergy - // with value > max_unsup_energy - - PlainData::SubProblemData subProblemData; - subProblemData.variables = variables; - SubProblemDataMap cut_trace = { - std::make_pair(std::string("P1"), subProblemData)}; - - WorkerMasterData worker_master_data; - worker_master_data._cut_trace = cut_trace; - - OuterloopCriterionLossOfLoad criterion(options); - - EXPECT_EQ(criterion.IsCriterionSatisfied(worker_master_data), - CRITERION::HIGH); - EXPECT_EQ(criterion.CriterionValue(), criterion_value); -} - -TEST_F(OuterLoopCriterionTest, IsCriterionLow) { - double threshold = 4; - double epsilon = 1e-1; - double max_unsup_energy = 0.1; - const ExternalLoopOptions options = {threshold, epsilon, max_unsup_energy}; - PlainData::Variables variables = { - {"PositiveUnsuppliedEnergy::1", "PositiveUnsuppliedEnergy::2", "var3"}, - {0.2, 0.3, 68}}; - double criterion_value = 2.0; // two vars named PositiveUnsuppliedEnergy with - // value > max_unsup_energy - - PlainData::SubProblemData subProblemData; - subProblemData.variables = variables; - SubProblemDataMap cut_trace = { - std::make_pair(std::string("P1"), subProblemData)}; - - WorkerMasterData worker_master_data; - worker_master_data._cut_trace = cut_trace; - - OuterloopCriterionLossOfLoad criterion(options); - - EXPECT_EQ(criterion.IsCriterionSatisfied(worker_master_data), CRITERION::LOW); - EXPECT_EQ(criterion.CriterionValue(), criterion_value); -} - -TEST_F(OuterLoopCriterionTest, IsMet) { - double threshold = 2.0; - double epsilon = 1e-1; - double max_unsup_energy = 0.1; - const ExternalLoopOptions options = {threshold, epsilon, max_unsup_energy}; - PlainData::Variables variables = { - {"PositiveUnsuppliedEnergy::1", "PositiveUnsuppliedEnergy::2", "var3"}, - {0.2, 0.3, 68}}; - double criterion_value = 2.0; // two vars named PositiveUnsuppliedEnergy with - // value > max_unsup_energy - - PlainData::SubProblemData subProblemData; - subProblemData.variables = variables; - SubProblemDataMap cut_trace = { - std::make_pair(std::string("P1"), subProblemData)}; - - WorkerMasterData worker_master_data; - worker_master_data._cut_trace = cut_trace; - - OuterloopCriterionLossOfLoad criterion(options); - - EXPECT_EQ(criterion.IsCriterionSatisfied(worker_master_data), - CRITERION::IS_MET); - EXPECT_EQ(criterion.CriterionValue(), criterion_value); -} - -//-------------------- MasterUpdateBaseTest ------------------------- -const auto STUDY_PATH = - std::filesystem::path("data_test") / "external_loop_test"; -const auto OPTIONS_FILE = STUDY_PATH / "lp" / "options.json"; - -class MasterUpdateBaseTest : public ::testing::TestWithParam { - public: - pBendersBase benders; - std::shared_ptr math_log_driver; - Logger logger; - Writer writer; - - void SetUp() override { - math_log_driver = MathLoggerFactory::get_void_logger(); - logger = build_void_logger(); - writer = build_void_writer(); - } - BendersBaseOptions BuildBendersOptions() { - SimulationOptions options(OPTIONS_FILE); - return options.get_benders_options(); - } -}; - -auto solvers() { - std::vector solvers_name; - solvers_name.push_back("COIN"); - LoadXpress::XpressLoader xpress_loader; - if (xpress_loader.XpressIsCorrectlyInstalled()) { - solvers_name.push_back("XPRESS"); - } - return solvers_name; -} - -INSTANTIATE_TEST_SUITE_P(availsolvers, MasterUpdateBaseTest, - ::testing::ValuesIn(solvers())); -double LambdaMax(pBendersBase benders) { - const auto& obj = benders->MasterObjectiveFunctionCoeffs(); - const auto max_invest = benders->BestIterationWorkerMaster().get_max_invest(); - double lambda_max = 0; - for (const auto& [var_name, var_id] : benders->MasterVariables()) { - lambda_max += obj[var_id] * max_invest.at(var_name); - } - return lambda_max; -} - -void CheckMinInvestmentConstraint(const VariableMap& master_variables, - const std::vector& expected_coeffs, - const double expected_rhs, char expected_sign, - const std::vector& coeffs, - const double rhs, char sign) { - for (auto const& [name, var_id] : master_variables) { - ASSERT_EQ(expected_coeffs[var_id], coeffs[var_id]); - } - - ASSERT_EQ(expected_rhs, rhs); - ASSERT_EQ(expected_sign, sign); -} - -TEST_P(MasterUpdateBaseTest, ConstraintIsAddedBendersMPI) { - BendersBaseOptions benders_options = BuildBendersOptions(); - CouplingMap coupling_map = build_input(benders_options.STRUCTURE_FILE); - // override solver - benders_options.SOLVER_NAME = GetParam(); - benders = std::make_shared(benders_options, logger, writer, *penv, - *pworld, math_log_driver); - benders->set_input_map(coupling_map); - benders->DoFreeProblems(false); - benders->InitializeProblems(); - benders->launch(); - - MasterUpdateBase master_updater(benders, 0.5); - // update lambda_max - master_updater.Init(); - benders->ResetData(3.0); - benders->launch(); - auto num_constraints_master_before = benders->MasterGetnrows(); - master_updater.Update(CRITERION::LOW); - auto num_constraints_master_after = benders->MasterGetnrows(); - - auto master_variables = benders->MasterVariables(); - auto expected_coeffs = benders->MasterObjectiveFunctionCoeffs(); - - // criterion is low <=> lambda_max = min(lambda_max, invest_cost) - auto lambda_max = (std::min)(LambdaMax(benders), - benders->GetBestIterationData().invest_cost); - auto expected_rhs = 0.5 * lambda_max; - - // - ASSERT_EQ(num_constraints_master_after, num_constraints_master_before + 1); - - std::vector mstart(1 + 1); - auto n_elems = benders->MasterGetNElems(); - auto nnz = master_variables.size(); - std::vector mclind(n_elems); - std::vector matval(n_elems); - std::vector p_nels(1, 0); - - auto added_row_index = num_constraints_master_after - 1; - benders->MasterRowsCoeffs(mstart, mclind, matval, n_elems, p_nels, - added_row_index, added_row_index); - std::vector coeffs(benders->MasterGetncols()); - - for (auto ind = mstart[0]; ind < mstart[1]; ++ind) { - coeffs[mclind[ind]] = matval[ind]; - } - double rhs; - benders->MasterGetRhs(rhs, added_row_index); - std::vector qrtype(1); - benders->MasterGetRowType(qrtype, added_row_index, added_row_index); - CheckMinInvestmentConstraint(master_variables, expected_coeffs, expected_rhs, - 'G', coeffs, rhs, qrtype[0]); - benders->free(); -} - -TEST_P(MasterUpdateBaseTest, InitialRhs) { - BendersBaseOptions benders_options = BuildBendersOptions(); - benders_options.SOLVER_NAME = GetParam(); - CouplingMap coupling_map = build_input(benders_options.STRUCTURE_FILE); - - benders = std::make_shared(benders_options, logger, writer, *penv, - *pworld, math_log_driver); - benders->set_input_map(coupling_map); - benders->DoFreeProblems(false); - benders->InitializeProblems(); - - benders->launch(); - - MasterUpdateBase master_updater(benders, 0.5); - // update lambda_max - master_updater.Init(); - auto lambda_max = LambdaMax(benders); - benders->ResetData(3.0); - benders->launch(); - master_updater.Update(CRITERION::HIGH); - auto expected_initial_rhs = lambda_max * 0.5; - - auto added_row_index = benders->MasterGetnrows() - 1; - double rhs; - benders->MasterGetRhs(rhs, added_row_index); - EXPECT_EQ(expected_initial_rhs, rhs); - benders->free(); -} \ No newline at end of file diff --git a/tests/cpp/outer_loop/CMakeLists.txt b/tests/cpp/outer_loop/CMakeLists.txt new file mode 100644 index 000000000..b4da36560 --- /dev/null +++ b/tests/cpp/outer_loop/CMakeLists.txt @@ -0,0 +1,16 @@ +add_executable(outer_loop_test + outer_loop_test.cpp) + +target_link_libraries(outer_loop_test + PRIVATE + solvers + outer_loop_lib + benders_by_batch_core + benders_core + output_core + logger_lib + GTest::Main + tests_utils) + +add_test(NAME unit_outer_loop COMMAND outer_loop_test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) +set_property(TEST unit_outer_loop PROPERTY LABELS unit) \ No newline at end of file diff --git a/tests/cpp/outer_loop/outer_loop_test.cpp b/tests/cpp/outer_loop/outer_loop_test.cpp new file mode 100644 index 000000000..258c4b9a8 --- /dev/null +++ b/tests/cpp/outer_loop/outer_loop_test.cpp @@ -0,0 +1,272 @@ + +// #include "MasterUpdate.h" + +#include "LoggerFactories.h" +#include "MasterUpdate.h" +#include "OuterLoopInputDataReader.h" +#include "WriterFactories.h" +#include "gtest/gtest.h" +#include "multisolver_interface/environment.h" + +int my_argc; +char** my_argv; + +boost::mpi::environment* penv = nullptr; +boost::mpi::communicator* pworld = nullptr; + +using namespace Outerloop; + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + + mpi::environment env(my_argc, my_argv); + mpi::communicator world; + penv = &env; + pworld = &world; + my_argc = argc; + my_argv = argv; + return RUN_ALL_TESTS(); +} + +//-------------------- MasterUpdateBaseTest ------------------------- +const auto STUDY_PATH = + std::filesystem::path("data_test") / "external_loop_test"; +const auto LP_DIR = STUDY_PATH / "lp"; +const auto OPTIONS_FILE = LP_DIR / "options.json"; +const auto OUTER_OPTIONS_FILE = LP_DIR / "outer_loop_options.yml"; + +class MasterUpdateBaseTest : public ::testing::TestWithParam { + public: + pBendersBase benders; + std::shared_ptr math_log_driver; + Logger logger; + Writer writer; + + void SetUp() override { + math_log_driver = MathLoggerFactory::get_void_logger(); + logger = build_void_logger(); + writer = build_void_writer(); + } + BendersBaseOptions BuildBendersOptions() { + SimulationOptions options(OPTIONS_FILE); + return options.get_benders_options(); + } +}; + +auto solvers() { + std::vector solvers_name; + solvers_name.push_back("COIN"); + if (LoadXpress::XpressLoader xpressLoader; + xpressLoader.XpressIsCorrectlyInstalled()) { + solvers_name.push_back("XPRESS"); + } + return solvers_name; +} + +INSTANTIATE_TEST_SUITE_P(availsolvers, MasterUpdateBaseTest, + ::testing::ValuesIn(solvers())); +double LambdaMax(pBendersBase benders) { + const auto& obj = benders->MasterObjectiveFunctionCoeffs(); + const auto max_invest = benders->BestIterationWorkerMaster().get_max_invest(); + double lambda_max = 0; + for (const auto& [var_name, var_id] : benders->MasterVariables()) { + lambda_max += obj[var_id] * max_invest.at(var_name); + } + return lambda_max; +} + +void CheckMinInvestmentConstraint(const VariableMap& master_variables, + const std::vector& expected_coeffs, + const double expected_rhs, char expected_sign, + const std::vector& coeffs, + const double rhs, char sign) { + for (auto const& [name, var_id] : master_variables) { + ASSERT_EQ(expected_coeffs[var_id], coeffs[var_id]); + } + + ASSERT_EQ(expected_rhs, rhs); + ASSERT_EQ(expected_sign, sign); +} + +TEST_P(MasterUpdateBaseTest, ConstraintIsAddedBendersMPI) { + BendersBaseOptions benders_options = BuildBendersOptions(); + CouplingMap coupling_map = + build_input(std::filesystem::path(benders_options.INPUTROOT) / + benders_options.STRUCTURE_FILE); + // override solver + benders_options.SOLVER_NAME = GetParam(); + benders_options.EXTERNAL_LOOP_OPTIONS.DO_OUTER_LOOP = true; + benders_options.EXTERNAL_LOOP_OPTIONS.OUTER_LOOP_OPTION_FILE = + OUTER_OPTIONS_FILE.string(); + benders = std::make_shared(benders_options, logger, writer, *penv, + *pworld, math_log_driver); + benders->set_input_map(coupling_map); + benders->DoFreeProblems(false); + benders->InitializeProblems(); + benders->ExternalLoopCheckFeasibility(); + + auto num_constraints_master_before = benders->MasterGetnrows(); + auto lambda_min = benders->ExternalLoopLambdaMin(); + auto lambda_max = benders->ExternalLoopLambdaMax(); + auto expected_lambda_max = LambdaMax(benders); + //-------- + ASSERT_EQ(lambda_max, expected_lambda_max); + //-------- + MasterUpdateBase master_updater(benders, 0.5); + + master_updater.Update(lambda_min, lambda_max); + auto num_constraints_master_after = benders->MasterGetnrows(); + + //------ + ASSERT_EQ(num_constraints_master_after, num_constraints_master_before + 1); + //------ + + auto master_variables = benders->MasterVariables(); + auto expected_coeffs = benders->MasterObjectiveFunctionCoeffs(); + + // criterion is low <=> lambda_max = min(lambda_max, invest_cost) + auto expected_rhs = 0.5 * lambda_max; + + // get added constraint infos (coeff, sign & rhs) + std::vector mstart(1 + 1); + auto n_elems = benders->MasterGetNElems(); + auto nnz = master_variables.size(); + std::vector mclind(n_elems); + std::vector matval(n_elems); + std::vector p_nels(1, 0); + + auto added_row_index = num_constraints_master_after - 1; + benders->MasterRowsCoeffs(mstart, mclind, matval, n_elems, p_nels, + added_row_index, added_row_index); + std::vector coeffs(benders->MasterGetncols()); + + for (auto ind = mstart[0]; ind < mstart[1]; ++ind) { + coeffs[mclind[ind]] = matval[ind]; + } + double rhs; + benders->MasterGetRhs(rhs, added_row_index); + std::vector qrtype(1); + benders->MasterGetRowType(qrtype, added_row_index, added_row_index); + CheckMinInvestmentConstraint(master_variables, expected_coeffs, expected_rhs, + 'G', coeffs, rhs, qrtype[0]); + benders->free(); +} + +class OuterLoopPatternTest : public ::testing::Test {}; + +TEST_F(OuterLoopPatternTest, RegexGivenPrefixAndBody) { + const std::string prefix = "prefix"; + const std::string body = "body"; + OuterLoopPattern o(prefix, body); + + auto ret_regex = o.MakeRegex(); + + ASSERT_EQ(std::regex_search(prefix + body, ret_regex), true); + ASSERT_EQ(std::regex_search(prefix + "::" + body + "::suffix", ret_regex), + true); + ASSERT_EQ(std::regex_search(body + prefix, ret_regex), false); + ASSERT_EQ(std::regex_search(prefix + "::", ret_regex), false); + ASSERT_EQ(std::regex_search(body, ret_regex), false); +} + +class OuterLoopInputFromYamlTest : public ::testing::Test {}; + +TEST_F(OuterLoopInputFromYamlTest, YamlFileDoesNotExist) { + std::filesystem::path empty(""); + std::ostringstream expected_msg; + expected_msg << "Could not read outer loop input file: " << empty << "\n" + << "bad file: " << empty.string(); + try { + OuterLoopInputFromYaml parser; + parser.Read(empty); + } catch (const OuterLoopInputFileError& e) { + ASSERT_EQ(expected_msg.str(), e.ErrorMessage()); + } +} + +TEST_F(OuterLoopInputFromYamlTest, YamlFileIsEmpty) { + std::filesystem::path empty(std::filesystem::temp_directory_path() / + "empty.yml"); + std::ofstream of(empty); + of.close(); + + std::ostringstream expected_msg; + expected_msg << "outer loop input file is empty: " << empty << "\n"; + try { + OuterLoopInputFromYaml parser; + parser.Read(empty); + } catch (const OuterLoopInputFileIsEmpty& e) { + ASSERT_EQ(expected_msg.str(), e.ErrorMessage()); + } +} + +TEST_F(OuterLoopInputFromYamlTest, YamlFileShouldContainsAtLeast1Pattern) { + std::filesystem::path empty_patterns(std::filesystem::temp_directory_path() / + "empty_patterns.yml"); + std::ofstream of(empty_patterns); + of << "stopping_threshold: " << 17.07; + of.close(); + + std::ostringstream expected_msg; + expected_msg << "outer loop input file must contains at least one pattern." + << "\n"; + try { + OuterLoopInputFromYaml parser; + parser.Read(empty_patterns); + } catch (const OuterLoopInputFileNoPatternFound& e) { + ASSERT_EQ(expected_msg.str(), e.ErrorMessage()); + } +} + +TEST_F(OuterLoopInputFromYamlTest, YamlFilePatternsShouldBeAnArray) { + std::filesystem::path patterns_not_array( + std::filesystem::temp_directory_path() / "patterns_not_array.yml"); + std::ofstream of(patterns_not_array); + of << "stopping_threshold: " << 17.07 << "\n" + << "patterns: " << 786; + of.close(); + + std::ostringstream expected_msg; + expected_msg << "In outer loop input file 'patterns' should be an array." + << "\n"; + try { + OuterLoopInputFromYaml parser; + parser.Read(patterns_not_array); + } catch (const OuterLoopInputPatternsShouldBeArray& e) { + ASSERT_EQ(expected_msg.str(), e.ErrorMessage()); + } +} + +TEST_F(OuterLoopInputFromYamlTest, ReadValidFile) { + std::filesystem::path valid_file(std::filesystem::temp_directory_path() / + "valid_file.yml"); + + std::ofstream of(valid_file); + auto my_yaml = R"(stopping_threshold: 1e-4 +# seuil +criterion_count_threshold: 1e-1 + # tolerance entre seuil et valeur calculée +criterion_tolerance: 1e-5 +patterns: + - area: "N0" + criterion: 185 + - area: "N1" + criterion: 1 + - area: "N2" + criterion: 1 + - area: "N3" + criterion: 1)"; + of << my_yaml; + of.close(); + auto data = OuterLoopInputFromYaml().Read(valid_file); + + ASSERT_EQ(data.StoppingThreshold(), 1e-4); + ASSERT_EQ(data.CriterionTolerance(), 1e-5); + ASSERT_EQ(data.CriterionCountThreshold(), 1e-1); + + auto patterns = data.OuterLoopData(); + ASSERT_EQ(patterns.size(), 4); + auto pattern1 = patterns[0]; + ASSERT_EQ(pattern1.Criterion(), 185.0); + ASSERT_EQ(pattern1.Pattern().GetBody(), "N0"); +} diff --git a/tests/python/test_benders_driver.py b/tests/python/test_benders_driver.py index 3e88d0776..8d863502d 100644 --- a/tests/python/test_benders_driver.py +++ b/tests/python/test_benders_driver.py @@ -4,7 +4,7 @@ from unittest.mock import patch import pytest -from antares_xpansion.benders_driver import BendersDriver +from antares_xpansion.benders_driver import BendersDriver, SolversExe MOCK_SUBPROCESS_RUN = "antares_xpansion.benders_driver.subprocess.run" MOCK_SUBPROCESS_POPEN = "antares_xpansion.benders_driver.subprocess.Popen" @@ -25,24 +25,24 @@ def setup_method(self): def test_lp_path(self, tmp_path): lp_path = tmp_path / "lp" os.mkdir(lp_path) - benders_driver = BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER) + benders_driver = BendersDriver(SolversExe("", "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER) benders_driver.set_simulation_output_path(tmp_path) assert benders_driver.get_lp_path() == lp_path def test_non_existing_output_path(self, tmp_path): - benders_driver = BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER) + benders_driver = BendersDriver(SolversExe("", "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER) with pytest.raises(BendersDriver.BendersOutputPathError): benders_driver.launch(tmp_path / "i_dont_exist", "test", False, 13) def test_empty_output_path(self, tmp_path): - benders_driver = BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER) + benders_driver = BendersDriver(SolversExe("", "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER) with pytest.raises(BendersDriver.BendersLpPathError): benders_driver.launch(tmp_path, "") def test_illegal_method(self, tmp_path): lp_path = tmp_path / "lp" os.mkdir(lp_path) - benders_driver = BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER) + benders_driver = BendersDriver(SolversExe("", "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER) with pytest.raises(BendersDriver.BendersSolverError): benders_driver.launch(tmp_path, "test") @@ -54,7 +54,7 @@ def test_benders_cmd_mpibenders(self, tmp_path): os.path.join(my_install_dir, my_benders_mpi)) benders_driver = BendersDriver( - exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER + SolversExe(exe_path, "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER ) simulation_output_path = tmp_path @@ -80,7 +80,7 @@ def test_benders_cmd_mpibenders_with_oversubscribe_linux_only(self, tmp_path): os.path.join(my_install_dir, my_benders_mpi)) benders_driver = BendersDriver( - exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER + SolversExe(exe_path, "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER ) simulation_output_path = tmp_path @@ -104,7 +104,7 @@ def test_benders_cmd_sequential(self, tmp_path): os.path.join(my_install_dir, my_sequential)) benders_driver = BendersDriver( - exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER + SolversExe(exe_path, "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER ) simulation_output_path = tmp_path @@ -126,7 +126,7 @@ def test_benders_cmd_merge_mps(self, tmp_path): os.path.join(my_install_dir, my_merges_mps)) benders_driver = BendersDriver( - "", exe_path, self.OPTIONS_JSON, self.MPI_LAUNCHER + SolversExe("", exe_path, ""), self.OPTIONS_JSON, self.MPI_LAUNCHER ) simulation_output_path = tmp_path @@ -141,6 +141,28 @@ def test_benders_cmd_merge_mps(self, tmp_path): args, _ = run_function.call_args_list[0] assert args[0] == expected_cmd + def test_benders_cmd_outer_loop(self, tmp_path): + my_outer_loop = "something" + my_install_dir = Path("Path/to/") + exe_path = os.path.normpath( + os.path.join(my_install_dir, my_outer_loop)) + + benders_driver = BendersDriver( + SolversExe("", "", exe_path), self.OPTIONS_JSON, self.MPI_LAUNCHER + ) + + simulation_output_path = tmp_path + lp_path = Path(os.path.normpath( + os.path.join(simulation_output_path, 'lp'))) + lp_path.mkdir() + os.chdir(lp_path) + with patch(MOCK_SUBPROCESS_RUN, autospec=True) as run_function: + expected_cmd = [exe_path, self.OPTIONS_JSON] + run_function.return_value.returncode = 0 + benders_driver.launch(simulation_output_path, "outer_loop", True) + args, _ = run_function.call_args_list[0] + assert args[0] == expected_cmd + def test_raise_execution_error(self, tmp_path): my_benders_mpi = "something" @@ -150,7 +172,7 @@ def test_raise_execution_error(self, tmp_path): os.path.join(my_install_dir, my_benders_mpi)) benders_driver = BendersDriver( - exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER + SolversExe(exe_path, "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER ) simulation_output_path = tmp_path @@ -171,7 +193,7 @@ def test_clean_solver_log_file(self, tmp_path): exe_path = os.path.normpath( os.path.join(my_install_dir, my_benders_mpi)) benders_driver = BendersDriver( - exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER + SolversExe(exe_path, "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER ) simulation_output_path = tmp_path @@ -202,7 +224,7 @@ def test_unsupported_platform(self, tmp_path): with patch(MOCK_SYS, autospec=True) as sys_: sys_.platform = "exotic_platform" with pytest.raises(BendersDriver.BendersUnsupportedPlatform): - BendersDriver("", "", self.OPTIONS_JSON, self.MPI_LAUNCHER) + BendersDriver(SolversExe("", "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER) def test_clean_benders_step_if_not_keep_mps(self, tmp_path): my_benders_mpi = "something" @@ -212,7 +234,7 @@ def test_clean_benders_step_if_not_keep_mps(self, tmp_path): keep_mps = False benders_driver = BendersDriver( - exe_path, "", self.OPTIONS_JSON, self.MPI_LAUNCHER + SolversExe(exe_path, "", ""), self.OPTIONS_JSON, self.MPI_LAUNCHER ) simulation_output_path = tmp_path @@ -246,7 +268,7 @@ def test_clean_benders_step_if_not_keep_mps(self, tmp_path): def test_invalid_options_file(self): with pytest.raises(BendersDriver.BendersOptionsFileError): - BendersDriver("", "", "") + BendersDriver(SolversExe("", "", ""), "") def _create_empty_file(self, tmp_path: Path, fname: str): file = tmp_path / fname diff --git a/tests/python/test_full_run_driver.py b/tests/python/test_full_run_driver.py index 30c742f37..498a8b3ab 100644 --- a/tests/python/test_full_run_driver.py +++ b/tests/python/test_full_run_driver.py @@ -1,6 +1,6 @@ from pathlib import Path -from antares_xpansion.benders_driver import BendersDriver +from antares_xpansion.benders_driver import BendersDriver, SolversExe from antares_xpansion.full_run_driver import FullRunDriver from antares_xpansion.problem_generator_driver import ProblemGeneratorDriver, ProblemGeneratorData @@ -38,15 +38,17 @@ def test_sequential_command(self, tmp_path): problem_generation.set_output_path(output_path) problem_generation.create_lp_dir() benders_driver = BendersDriver( - "benders.exe", "", self.benders_driver_options_file) + SolversExe("benders.exe", "", ""), self.benders_driver_options_file) full_run_driver = FullRunDriver(self.full_run_exe, problem_generation, benders_driver) - full_run_driver.prepare_drivers(output_path, is_relaxed, json_file_path, + benders_method = "benders" + full_run_driver.prepare_drivers(output_path, is_relaxed, benders_method, json_file_path, benders_keep_mps=benders_keep_mps, benders_oversubscribe=benders_oversubscribe, benders_allow_run_as_root=benders_allow_run_as_root) xpansion_output_dir = output_path.parent / \ (output_path.stem+"-Xpansion") expected_command = [self.full_run_exe, "--benders_options", self.benders_driver_options_file, - "-s", str(json_file_path), "-a", str(output_path), "-f", "integer", "-e", self.pb_gen_data.additional_constraints] + "-s", str(json_file_path), "-a", str(output_path), "-f", "integer", "-e", + self.pb_gen_data.additional_constraints, "--solver", benders_method] command = full_run_driver.full_command() assert len(expected_command) == len(command) @@ -72,15 +74,18 @@ def test_mpi_command(self, tmp_path): problem_generation.create_lp_dir() benders_driver = BendersDriver( - "benders.exe", "", self.benders_driver_options_file) + SolversExe("benders.exe", "", ""), self.benders_driver_options_file) full_run_driver = FullRunDriver(self.full_run_exe, problem_generation, benders_driver) - full_run_driver.prepare_drivers(output_path, is_relaxed, json_file_path, + benders_method = "benders" + full_run_driver.prepare_drivers(output_path, is_relaxed, benders_method, + json_file_path, benders_keep_mps, benders_n_mpi, benders_oversubscribe, benders_allow_run_as_root) xpansion_output_dir = output_path.parent / \ (output_path.stem+"-Xpansion") expected_command = [benders_driver.MPI_LAUNCHER, "-n", str(benders_n_mpi), self.full_run_exe, "--benders_options", self.benders_driver_options_file, - "-s", str(json_file_path), "-a", str(output_path), "-f", "integer", "-e", self.pb_gen_data.additional_constraints] + "-s", str(json_file_path), "-a", str(output_path), "-f", "integer", "-e", + self.pb_gen_data.additional_constraints, "--solver", benders_method] command = full_run_driver.full_command() diff --git a/vcpkg.json b/vcpkg.json index 35e3b00cf..89eb11e18 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -7,7 +7,8 @@ "gtest", "boost-mpi", "boost-program-options", - "zlib" + "zlib", + "yaml-cpp" ], "overrides": [ { "name": "boost-mpi", "version": "1.81.0" },