diff --git a/.github/workflows/download-extract-precompiled-libraries-zip/action.yml b/.github/workflows/download-extract-precompiled-libraries-zip/action.yml index 9244ad0a8..6b1c62c2c 100644 --- a/.github/workflows/download-extract-precompiled-libraries-zip/action.yml +++ b/.github/workflows/download-extract-precompiled-libraries-zip/action.yml @@ -31,4 +31,4 @@ runs: wget https://github.com/AntaresSimulatorTeam/Antares_Simulator/releases/download/v${{inputs.antares-version}}/rte-antares-${{inputs.antares-version}}-installer-64bits.zip unzip rte-antares-${{inputs.antares-version}}-installer-64bits.zip - rm -rf rte-antares-${{inputs.antares-version}}-installer-64bits.zip \ No newline at end of file + rm -rf rte-antares-${{inputs.antares-version}}-installer-64bits.zip diff --git a/CMakeLists.txt b/CMakeLists.txt index ea83d64fa..c452a6013 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,6 +164,7 @@ sbeParseJson(antares_version jsonContent) #need to know antares solver version because antares-solver targets still refers to antares version SET(ANTARES_VERSION ${antares_version.antares_version_executable}) SET(ANTARES_VERSION_TAG ${antares_version.antares_version}) +SET(MINIZIP_TAG ${antares_version.minizip_ng_version}) sbeClearJson(antares_version) @@ -183,7 +184,7 @@ endif() add_subdirectory(antares-deps) add_subdirectory(cmake/dependencies ${CMAKE_CURRENT_BINARY_DIR}/build_deps) - +find_package(minizip-ng REQUIRED) # --------------------------------------------------------------------------- # Boost # --------------------------------------------------------------------------- diff --git a/antares-version.json b/antares-version.json index 0eb50e9a8..0b7b7589b 100644 --- a/antares-version.json +++ b/antares-version.json @@ -2,5 +2,6 @@ "antares_version": "8.4.0-rc2", "antares_version_executable": "8.4", "antares_xpansion_version": "0.8.0", - "antares_deps_version": "2.0.5" + "antares_deps_version": "2.0.5", + "minizip_ng_version": "3.0.6" } diff --git a/cmake/dependencies/CMakeLists.txt b/cmake/dependencies/CMakeLists.txt index f8058f4d6..e65c3c3b1 100644 --- a/cmake/dependencies/CMakeLists.txt +++ b/cmake/dependencies/CMakeLists.txt @@ -34,3 +34,4 @@ if (NOT antares-solver_FOUND) endif() endif() + diff --git a/data_test/mini_instance_LP/MPS_ZIP_FILE.zip b/data_test/mini_instance_LP/MPS_ZIP_FILE.zip new file mode 100644 index 000000000..f516f5ba1 Binary files /dev/null and b/data_test/mini_instance_LP/MPS_ZIP_FILE.zip differ diff --git a/data_test/mini_instance_LP/SP1.mps b/data_test/mini_instance_LP/SP1.mps deleted file mode 100644 index 32cba9a2d..000000000 --- a/data_test/mini_instance_LP/SP1.mps +++ /dev/null @@ -1,13 +0,0 @@ -NAME SP1 -ROWS - N OBJ - G C1 -COLUMNS - x C1 1 - perte OBJ 1 - perte C1 1 -RHS - RHS C1 1.5 -BOUNDS - LO BND perte 0.0 -ENDATA \ No newline at end of file diff --git a/data_test/mini_instance_LP/SP2.mps b/data_test/mini_instance_LP/SP2.mps deleted file mode 100644 index 4360a354a..000000000 --- a/data_test/mini_instance_LP/SP2.mps +++ /dev/null @@ -1,13 +0,0 @@ -NAME SP2 -ROWS - N OBJ - G C2 -COLUMNS - x C2 1.0 - perte_2 OBJ 1.0 - perte_2 C2 1.0 -RHS - RHS C2 2.5 -BOUNDS - LO BND perte_2 0.0 -ENDATA \ No newline at end of file diff --git a/data_test/mini_instance_MIP/MPS_ZIP_FILE.zip b/data_test/mini_instance_MIP/MPS_ZIP_FILE.zip new file mode 100644 index 000000000..978131411 Binary files /dev/null and b/data_test/mini_instance_MIP/MPS_ZIP_FILE.zip differ diff --git a/data_test/mini_instance_MIP/SP1.mps b/data_test/mini_instance_MIP/SP1.mps deleted file mode 100644 index 89244bdbf..000000000 --- a/data_test/mini_instance_MIP/SP1.mps +++ /dev/null @@ -1,12 +0,0 @@ -* ENCODING=ISO-8859-1 -NAME SP1.mps -ROWS - N OBJ - G C1 -COLUMNS - x C1 1 - perte OBJ 2 - perte C1 1 -RHS - rhs C1 1.5 -ENDATA diff --git a/data_test/mini_instance_MIP/SP2.mps b/data_test/mini_instance_MIP/SP2.mps deleted file mode 100644 index 8ba48694f..000000000 --- a/data_test/mini_instance_MIP/SP2.mps +++ /dev/null @@ -1,12 +0,0 @@ -* ENCODING=ISO-8859-1 -NAME SP2.mps -ROWS - N OBJ - G C1 -COLUMNS - x C1 1 - perte_2 OBJ 1 - perte_2 C1 1 -RHS - rhs C1 2.5 -ENDATA diff --git a/data_test/mini_instance_MIP/options_default.json b/data_test/mini_instance_MIP/options_default.json index 8946b2887..b494b3854 100644 --- a/data_test/mini_instance_MIP/options_default.json +++ b/data_test/mini_instance_MIP/options_default.json @@ -12,5 +12,6 @@ "CSV_NAME": "benders_output_trace", "BOUND_ALPHA": true, "JSON_FILE": "./expansion/out.json", - "LAST_ITERATION_JSON_FILE": "./expansion/last_iteration.json" -} \ No newline at end of file + "LAST_ITERATION_JSON_FILE": "./expansion/last_iteration.json", + "MPS_ZIP_FILE": "MPS_ZIP_FILE.zip" +} diff --git a/data_test/mini_network/MPS_ZIP_FILE.zip b/data_test/mini_network/MPS_ZIP_FILE.zip new file mode 100644 index 000000000..26a78a608 Binary files /dev/null and b/data_test/mini_network/MPS_ZIP_FILE.zip differ diff --git a/data_test/mini_network/SP1.mps b/data_test/mini_network/SubProblem1.mps similarity index 100% rename from data_test/mini_network/SP1.mps rename to data_test/mini_network/SubProblem1.mps diff --git a/data_test/mini_network/SP2.mps b/data_test/mini_network/SubProblem2.mps similarity index 100% rename from data_test/mini_network/SP2.mps rename to data_test/mini_network/SubProblem2.mps diff --git a/data_test/mini_network/options_default.json b/data_test/mini_network/options_default.json index ede703d90..9fa8a6105 100644 --- a/data_test/mini_network/options_default.json +++ b/data_test/mini_network/options_default.json @@ -15,5 +15,6 @@ "BOUND_ALPHA": true, "SOLVER_NAME": "COIN", "JSON_FILE": "./expansion/out.json", - "LAST_ITERATION_JSON_FILE": "./expansion/last_iteration.json" -} \ No newline at end of file + "LAST_ITERATION_JSON_FILE": "./expansion/last_iteration.json", + "MPS_ZIP_FILE": "MPS_ZIP_FILE.zip" +} diff --git a/data_test/mps_zip/archive1.zip b/data_test/mps_zip/archive1.zip new file mode 100644 index 000000000..6d43dffa5 Binary files /dev/null and b/data_test/mps_zip/archive1.zip differ diff --git a/data_test/mps_zip/archive1/file1 b/data_test/mps_zip/archive1/file1 new file mode 100644 index 000000000..4dcd42526 --- /dev/null +++ b/data_test/mps_zip/archive1/file1 @@ -0,0 +1 @@ +DATA in file 1 diff --git a/data_test/mps_zip/archive1/file2 b/data_test/mps_zip/archive1/file2 new file mode 100644 index 000000000..e69de29bb diff --git a/data_test/mps_zip/archive1/file3 b/data_test/mps_zip/archive1/file3 new file mode 100644 index 000000000..3cde61338 --- /dev/null +++ b/data_test/mps_zip/archive1/file3 @@ -0,0 +1,7 @@ +File 3 content +1 +2 +4 +2 +2 +3 diff --git a/src/cpp/benders/benders_core/BendersBase.cpp b/src/cpp/benders/benders_core/BendersBase.cpp index 6bb68d0a1..e35e55cbc 100644 --- a/src/cpp/benders/benders_core/BendersBase.cpp +++ b/src/cpp/benders/benders_core/BendersBase.cpp @@ -710,6 +710,13 @@ std::filesystem::path BendersBase::get_structure_path() const { return std::filesystem::path(_options.INPUTROOT) / _options.STRUCTURE_FILE; } +/*! + * \brief Get path to mps zip file from options + */ +std::filesystem::path BendersBase::GetMpsZipPath() const { + return std::filesystem::path(_options.INPUTROOT) / _options.MPS_ZIP_FILE; +} + LogData BendersBase::bendersDataToLogData(const BendersData &data) const { auto optimal_gap(data.best_ub - data.lb); return {data.lb, diff --git a/src/cpp/benders/benders_core/SimulationOptions.cpp b/src/cpp/benders/benders_core/SimulationOptions.cpp index 15f42adf2..73cdccce2 100644 --- a/src/cpp/benders/benders_core/SimulationOptions.cpp +++ b/src/cpp/benders/benders_core/SimulationOptions.cpp @@ -130,6 +130,7 @@ BaseOptions SimulationOptions::get_base_options() const { result.SOLVER_NAME = SOLVER_NAME; result.weights = _weights; result.RESUME = RESUME; + result.MPS_ZIP_FILE = MPS_ZIP_FILE; return result; } diff --git a/src/cpp/benders/benders_core/include/BendersBase.h b/src/cpp/benders/benders_core/include/BendersBase.h index 5b6f9d6fa..d7bdc0f0e 100644 --- a/src/cpp/benders/benders_core/include/BendersBase.h +++ b/src/cpp/benders/benders_core/include/BendersBase.h @@ -54,6 +54,7 @@ class BendersBase { std::string const &name) const; [[nodiscard]] std::filesystem::path get_master_path() const; [[nodiscard]] std::filesystem::path get_structure_path() const; + [[nodiscard]] std::filesystem::path GetMpsZipPath() const; [[nodiscard]] LogData bendersDataToLogData(const BendersData &data) const; virtual void build_input_map(); void push_in_trace(const WorkerMasterDataPtr &worker_master_data); diff --git a/src/cpp/benders/benders_core/include/SimulationOptions.hxx b/src/cpp/benders/benders_core/include/SimulationOptions.hxx index 314c39cf5..4e5f54372 100644 --- a/src/cpp/benders/benders_core/include/SimulationOptions.hxx +++ b/src/cpp/benders/benders_core/include/SimulationOptions.hxx @@ -69,3 +69,6 @@ BENDERS_OPTIONS_MACRO(RESUME, bool, false, asBool()) // Name of the last master basis file BENDERS_OPTIONS_MACRO(LAST_MASTER_BASIS, std::string, "master_last_basis", asString()) + +// Name of the MPS ZIP file +BENDERS_OPTIONS_MACRO(MPS_ZIP_FILE, std::string, "MPS_ZIP_FILE.zip", asString()) diff --git a/src/cpp/benders/benders_core/include/common.h b/src/cpp/benders/benders_core/include/common.h index 44c8e751e..f50319078 100644 --- a/src/cpp/benders/benders_core/include/common.h +++ b/src/cpp/benders/benders_core/include/common.h @@ -132,6 +132,7 @@ struct BaseOptions { bool RESUME = false; Str2Dbl weights; + std::string MPS_ZIP_FILE; }; typedef BaseOptions MergeMPSOptions; struct BendersBaseOptions : public BaseOptions { diff --git a/src/cpp/benders/benders_mpi/BendersMPI.cpp b/src/cpp/benders/benders_mpi/BendersMPI.cpp index a83dced38..d23f2c3ef 100644 --- a/src/cpp/benders/benders_mpi/BendersMPI.cpp +++ b/src/cpp/benders/benders_mpi/BendersMPI.cpp @@ -12,7 +12,10 @@ BendersMpi::BendersMpi(BendersBaseOptions const &options, Logger &logger, mpi::communicator &world) : BendersBase(options, logger, std::move(writer)), _env(env), - _world(world) {} + _world(world) { + reader_.SetArchivePath(GetMpsZipPath()); + reader_.Open(); +} /*! * \brief Method to load each problem in a thread @@ -41,8 +44,12 @@ void BendersMpi::initialize_problems() { process_to_feed == _world .rank()) { // Assign [problemNumber % processCount] to processID + + const auto subProblemFilePath = GetSubproblemPath(problem.first); + reader_.ExtractFile(subProblemFilePath.filename()); addSubproblem(problem); AddSubproblemName(problem.first); + std::filesystem::remove(subProblemFilePath); } current_problem_id++; } @@ -277,6 +284,10 @@ void BendersMpi::launch() { initialize_problems(); _world.barrier(); + if (_world.rank() == rank_0) { + reader_.Close(); + reader_.Delete(); + } run(); _world.barrier(); diff --git a/src/cpp/benders/benders_mpi/include/BendersMPI.h b/src/cpp/benders/benders_mpi/include/BendersMPI.h index 524dd0061..f6ad02345 100644 --- a/src/cpp/benders/benders_mpi/include/BendersMPI.h +++ b/src/cpp/benders/benders_mpi/include/BendersMPI.h @@ -1,5 +1,6 @@ #pragma once +#include "ArchiveReader.h" #include "BendersBase.h" #include "BendersStructsDatas.h" #include "SubproblemCut.h" @@ -52,6 +53,7 @@ class BendersMpi : public BendersBase { void check_if_some_proc_had_a_failure(int success); [[nodiscard]] bool shouldParallelize() const final { return false; } + ArchiveReader reader_; mpi::environment &_env; mpi::communicator &_world; const unsigned int rank_0 = 0; diff --git a/src/cpp/benders/benders_sequential/BendersSequential.cpp b/src/cpp/benders/benders_sequential/BendersSequential.cpp index 668ecb87b..b05ee3840 100644 --- a/src/cpp/benders/benders_sequential/BendersSequential.cpp +++ b/src/cpp/benders/benders_sequential/BendersSequential.cpp @@ -4,6 +4,7 @@ #include #include +#include "ArchiveReader.h" #include "Timer.h" #include "glog/logging.h" #include "solver_utils.h" @@ -27,10 +28,18 @@ void BendersSequential::initialize_problems() { reset_master(new WorkerMaster(master_variable_map, get_master_path(), get_solver_name(), get_log_level(), _data.nsubproblem, log_name(), IsResumeMode())); + auto reader = ArchiveReader(GetMpsZipPath()); + reader.Open(); for (const auto &problem : coupling_map) { + const auto subProblemFilePath = GetSubproblemPath(problem.first); + reader.ExtractFile(subProblemFilePath.filename()); addSubproblem(problem); AddSubproblemName(problem.first); + std::filesystem::remove(subProblemFilePath); } + + reader.Close(); + reader.Delete(); } /*! diff --git a/src/cpp/benders/merge_mps/MergeMPS.cpp b/src/cpp/benders/merge_mps/MergeMPS.cpp index 7280f4f3a..198a71699 100644 --- a/src/cpp/benders/merge_mps/MergeMPS.cpp +++ b/src/cpp/benders/merge_mps/MergeMPS.cpp @@ -2,6 +2,7 @@ #include +#include "ArchiveReader.h" #include "Timer.h" #include "glog/logging.h" MergeMPS::MergeMPS(const MergeMPSOptions &options, Logger &logger, @@ -9,8 +10,8 @@ MergeMPS::MergeMPS(const MergeMPSOptions &options, Logger &logger, : _options(options), _logger(logger), _writer(writer) {} void MergeMPS::launch() { - auto structure_path(std::filesystem::path(_options.INPUTROOT) / - _options.STRUCTURE_FILE); + const auto inputRootDir = std::filesystem::path(_options.INPUTROOT); + auto structure_path(inputRootDir / _options.STRUCTURE_FILE); CouplingMap input = build_input(structure_path); SolverFactory factory; @@ -25,16 +26,18 @@ void MergeMPS::launch() { int cntProblems_l(0); LOG(INFO) << "Merging problems..." << std::endl; + auto reader = ArchiveReader(inputRootDir / _options.MPS_ZIP_FILE); + reader.Open(); for (auto const &kvp : input) { - auto problem_name(std::filesystem::path(_options.INPUTROOT) / - (kvp.first + MPS_SUFFIX)); - + auto problem_name(inputRootDir / (kvp.first + MPS_SUFFIX)); SolverAbstract::Ptr solver_l = factory.create_solver(solver_to_use); solver_l->init(); solver_l->set_output_log_level(_options.LOG_LEVEL); - solver_l->read_prob_mps(problem_name); if (kvp.first != _options.MASTER_NAME) { + reader.ExtractFile(problem_name.filename()); + solver_l->read_prob_mps(problem_name); + std::filesystem::remove(problem_name); int mps_ncols(solver_l->get_ncols()); DblVector o(mps_ncols); @@ -48,6 +51,8 @@ void MergeMPS::launch() { c *= weigth; } solver_l->chg_obj(sequence, o); + } else { + solver_l->read_prob_mps(problem_name); } StandardLp lpData(solver_l); std::string varPrefix_l = "prob" + std::to_string(cntProblems_l) + "_"; @@ -73,6 +78,9 @@ void MergeMPS::launch() { ++cntProblems_l; } + reader.Close(); + reader.Delete(); + IntVector mstart; IntVector cindex; DblVector values; diff --git a/src/cpp/exe/lpnamer/main.cpp b/src/cpp/exe/lpnamer/main.cpp index 7f825786d..bafa83350 100644 --- a/src/cpp/exe/lpnamer/main.cpp +++ b/src/cpp/exe/lpnamer/main.cpp @@ -39,6 +39,7 @@ namespace po = boost::program_options; int main(int argc, char **argv) { try { std::filesystem::path root; + std::filesystem::path archivePath; std::string master_formulation; std::string additionalConstraintFilename_l; @@ -47,6 +48,8 @@ int main(int argc, char **argv) { desc.add_options()("help,h", "produce help message")( "output,o", po::value(&root)->required(), "antares-xpansion study output")( + "archive,a", po::value(&archivePath)->required(), + "antares-xpansion study zip")( "formulation,f", po::value(&master_formulation)->default_value("relaxed"), "master formulation (relaxed or integer)")( @@ -98,7 +101,8 @@ int main(int argc, char **argv) { std::vector links = linkBuilder.getLinks(); LinkProblemsGenerator linkProblemsGenerator(links, solver_name, logger, log_file_path); - linkProblemsGenerator.treatloop(root, couplings); + linkProblemsGenerator.treatloop(root, archivePath, couplings); + MasterGeneration master_generation(root, links, additionalConstraints, couplings, master_formulation, diff --git a/src/cpp/helpers/ArchiveIO.h b/src/cpp/helpers/ArchiveIO.h new file mode 100644 index 000000000..00805d9ac --- /dev/null +++ b/src/cpp/helpers/ArchiveIO.h @@ -0,0 +1,55 @@ +#ifndef _ARCHIVEIO_H +#define _ARCHIVEIO_H + +extern "C" { +#include +#include +#include +#include +} +#include +#include + +class ArchiveIOGeneralException : public std::runtime_error { + public: + ArchiveIOGeneralException(int32_t status, const std::string& action, + int32_t expectedStatus = MZ_OK) + : std::runtime_error("Failed to " + action + + " invalid status: " + std::to_string(status) + " (" + + std::to_string(expectedStatus) + " expected)") {} +}; +class ArchiveIOSpecificException : public std::runtime_error { + public: + ArchiveIOSpecificException(int32_t status, const std::string& errMessage, + int32_t expectedStatus = MZ_OK) + : std::runtime_error( + errMessage + "\ninvalid status: " + std::to_string(status) + " (" + + std::to_string(expectedStatus) + " expected)") {} +}; +#include +#include +class ArchiveIO { + private: + std::filesystem::path archivePath_; + + protected: + mutable std::shared_mutex mutex_; + + virtual void Create() = 0; + + public: + explicit ArchiveIO(const std::filesystem::path& archivePath) + : archivePath_(archivePath) {} + ArchiveIO() = default; + std::filesystem::path ArchivePath() const { return archivePath_; } + void SetArchivePath(const std::filesystem::path& archivePath) { + archivePath_ = archivePath; + }; + + virtual int32_t Close() = 0; + virtual void Delete() = 0; + + virtual int Open() = 0; +}; + +#endif // _ARCHIVEIO_H \ No newline at end of file diff --git a/src/cpp/helpers/ArchiveReader.cpp b/src/cpp/helpers/ArchiveReader.cpp new file mode 100644 index 000000000..cdfc1515e --- /dev/null +++ b/src/cpp/helpers/ArchiveReader.cpp @@ -0,0 +1,94 @@ +#include "ArchiveReader.h" + +#include +#include +ArchiveReader::ArchiveReader(const std::filesystem::path& archivePath) + : ArchiveIO(archivePath) { + Create(); +} +// void* ArchiveReader::InternalPointer() const { return mz_zip_reader; } +ArchiveReader::ArchiveReader() : ArchiveIO() { Create(); } +void ArchiveReader::Create() { mz_zip_reader_create(&internalPointer_); } + +int32_t ArchiveReader::Open() { + const auto err = + mz_zip_reader_open_file(internalPointer_, ArchivePath().string().c_str()); + if (err != MZ_OK) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "Open Archive: " << ArchivePath().string() << std::endl; + throw ArchiveIOGeneralException(err, errMsg.str()); + } + return err; +} +int32_t ArchiveReader::Close() { return mz_zip_reader_close(internalPointer_); } +void ArchiveReader::Delete() { mz_zip_reader_delete(&internalPointer_); } + +int32_t ArchiveReader::ExtractFile( + const std::filesystem::path& fileToExtractPath) { + return ExtractFile(fileToExtractPath, ArchivePath().parent_path() / + fileToExtractPath.filename()); +} + +int32_t ArchiveReader::ExtractFile( + const std::filesystem::path& fileToExtractPath, + const std::filesystem::path& destination) { + std::unique_lock lock(mutex_); + int32_t err = MZ_OK; + LocateEntry(fileToExtractPath); + OpenEntry(fileToExtractPath); + + std::filesystem::path targetFile(destination); + if (std::filesystem::is_directory(destination)) { + targetFile = destination / fileToExtractPath.filename(); + } + err = mz_zip_reader_entry_save_file(internalPointer_, + targetFile.string().c_str()); + mz_zip_reader_entry_close(internalPointer_); + return err; +} +void ArchiveReader::LocateEntry( + const std::filesystem::path& fileToExtractPath) { + auto err = mz_zip_reader_locate_entry(internalPointer_, + fileToExtractPath.string().c_str(), 1); + if (err != MZ_OK) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "File : " << fileToExtractPath.string().c_str() + << " is not found in archive :" << ArchivePath().c_str() + << std::endl; + throw ArchiveIOSpecificException(err, errMsg.str()); + } +} +void ArchiveReader::OpenEntry(const std::filesystem::path& fileToExtractPath) { + auto err = mz_zip_reader_entry_open(internalPointer_); + if (err != MZ_OK) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "open " << fileToExtractPath.string() + << " in archive :" << ArchivePath().string() << std::endl; + throw ArchiveIOGeneralException(err, errMsg.str()); + } +} +std::istringstream ArchiveReader::ExtractFileInStringStream( + const std::filesystem::path& FileToExtractPath) { + std::unique_lock lock(mutex_); + LocateEntry(FileToExtractPath); + OpenEntry(FileToExtractPath); + int32_t len = mz_zip_reader_entry_save_buffer_length(internalPointer_); + std::vector buf(len); + auto err = mz_zip_reader_entry_save_buffer(internalPointer_, buf.data(), len); + if (err != MZ_OK) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "Extract file " << FileToExtractPath.string() + << "in archive: " << ArchivePath().string() << std::endl; + throw ArchiveIOGeneralException(err, errMsg.str()); + } + mz_zip_reader_entry_close(internalPointer_); + return std::istringstream(std::string(buf.begin(), buf.end())); +} \ No newline at end of file diff --git a/src/cpp/helpers/ArchiveReader.h b/src/cpp/helpers/ArchiveReader.h new file mode 100644 index 000000000..d9470e256 --- /dev/null +++ b/src/cpp/helpers/ArchiveReader.h @@ -0,0 +1,31 @@ +#ifndef _ARCHIVEREADER_H +#define _ARCHIVEREADER_H +#include +#include + +#include "ArchiveIO.h" +class ArchiveReader : public ArchiveIO { + private: + void* internalPointer_ = NULL; + void* handle_ = NULL; + void Create() override; + + public: + explicit ArchiveReader(const std::filesystem::path& archivePath); + ArchiveReader(); + + int32_t Close() override; + void Delete() override; + // void* InternalPointer() const override; + // void* handle() const override; + + int Open() override; + int32_t ExtractFile(const std::filesystem::path& FileToExtractPath); + int32_t ExtractFile(const std::filesystem::path& FileToExtractPath, + const std::filesystem::path& destination); + void LocateEntry(const std::filesystem::path& fileToExtractPath); + void OpenEntry(const std::filesystem::path& fileToExtractPath); + std::istringstream ExtractFileInStringStream( + const std::filesystem::path& FileToExtractPath); +}; +#endif // _ARCHIVEREADER_H diff --git a/src/cpp/helpers/ArchiveWriter.cpp b/src/cpp/helpers/ArchiveWriter.cpp new file mode 100644 index 000000000..712f9b5bf --- /dev/null +++ b/src/cpp/helpers/ArchiveWriter.cpp @@ -0,0 +1,93 @@ +#include "ArchiveWriter.h" + +#include + +#include +#include + +ArchiveWriter::ArchiveWriter(const std::filesystem::path& archivePath) + : ArchiveIO(archivePath) { + Create(); + InitFileInfo(); +} +// void* ArchiveWriter::InternalPointer() const { return mz_zip_reader; } +ArchiveWriter::ArchiveWriter() : ArchiveIO() { + Create(); + InitFileInfo(); +} + +void ArchiveWriter::Create() { mz_zip_writer_create(&internalPointer_); } +void ArchiveWriter::InitFileInfo() { + fileInfo_.compression_method = MZ_COMPRESS_METHOD_DEFLATE; + fileInfo_.flag = MZ_ZIP_FLAG_UTF8; +} +int32_t ArchiveWriter::Open() { + // disk-spanning is disabled, meaning that only one file is created + const auto err = + mz_zip_writer_open_file(internalPointer_, ArchivePath().string().c_str(), + 0 /* disk-spanning disabled */, 1 /* append */); + if (err != MZ_OK) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "Open Archive: " << ArchivePath().string() << std::endl; + throw ArchiveIOGeneralException(err, errMsg.str()); + } + return err; +} +int32_t ArchiveWriter::Close() { return mz_zip_writer_close(internalPointer_); } +void ArchiveWriter::Delete() { mz_zip_writer_delete(&internalPointer_); } + +int32_t ArchiveWriter::AddFileInArchive(const FileBuffer& FileBufferToAdd) { + std::unique_lock lock(mutex_); + fileInfo_.filename = FileBufferToAdd.fname.c_str(); + fileInfo_.creation_date = std::time(0); + int32_t err = mz_zip_writer_entry_open(internalPointer_, &fileInfo_); + if (err != MZ_OK) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "Open entry: " << FileBufferToAdd.fname + << " in archive: " << ArchivePath().string() << std::endl; + throw ArchiveIOGeneralException(err, errMsg.str()); + } + const auto len = FileBufferToAdd.buffer.size(); + auto bw = mz_zip_writer_entry_write(internalPointer_, + FileBufferToAdd.buffer.c_str(), len); + if (bw != len) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "[KO] mz_zip_writer_entry_write, expected " << len + << "data to be read got" << bw << std::endl; + throw ArchiveIOSpecificException(err, errMsg.str()); + } + err = mz_zip_writer_entry_close(internalPointer_); + if (MZ_OK != err) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "[KO] mz_zip_writer_entry_close error: could not close entry: " + << FileBufferToAdd.fname << std::endl; + throw ArchiveIOSpecificException(err, errMsg.str()); + } + + return MZ_OK; +} +int32_t ArchiveWriter::AddFileInArchive( + const std::filesystem::path& FileToAdd) { + std::unique_lock lock(mutex_); + auto err = + mz_zip_writer_add_file(internalPointer_, FileToAdd.string().c_str(), + FileToAdd.filename().string().c_str()); + if (err != MZ_OK) { + Close(); + Delete(); + std::stringstream errMsg; + errMsg << "[KO] mz_zip_writer_add_file: Failed to add file: " << FileToAdd + << " in archive: " << ArchivePath().string() << std::endl; + throw ArchiveIOSpecificException(err, errMsg.str()); + } + + return MZ_OK; +} diff --git a/src/cpp/helpers/ArchiveWriter.h b/src/cpp/helpers/ArchiveWriter.h new file mode 100644 index 000000000..d37048dbe --- /dev/null +++ b/src/cpp/helpers/ArchiveWriter.h @@ -0,0 +1,30 @@ +#ifndef _ARCHIVEWRITER_H +#define _ARCHIVEWRITER_H + +#include + +#include "ArchiveIO.h" +#include "FileInBuffer.h" + +class ArchiveWriter : public ArchiveIO { + private: + void* internalPointer_ = NULL; + void* handle_ = NULL; + void Create() override; + mz_zip_file fileInfo_ = {0}; + + public: + explicit ArchiveWriter(const std::filesystem::path& archivePath); + ArchiveWriter(); + + int32_t Close() override; + void Delete() override; + // void* InternalPointer() const override; + // void* handle() const override; + + int Open() override; + void InitFileInfo(); + int32_t AddFileInArchive(const FileBuffer& FileBufferToAdd); + int32_t AddFileInArchive(const std::filesystem::path& FileToAdd); +}; +#endif // _ARCHIVEWRITER_H \ No newline at end of file diff --git a/src/cpp/helpers/CMakeLists.txt b/src/cpp/helpers/CMakeLists.txt index 88f6605b3..6b876784c 100644 --- a/src/cpp/helpers/CMakeLists.txt +++ b/src/cpp/helpers/CMakeLists.txt @@ -19,6 +19,13 @@ add_library (helpers STATIC ${CMAKE_CURRENT_SOURCE_DIR}/solver_utils.cc ${CMAKE_CURRENT_SOURCE_DIR}/JsonXpansionReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/StringUtils.cpp AntaresVersionProvider.cpp AntaresVersionProvider.h + ${CMAKE_CURRENT_SOURCE_DIR}/ArchiveIO.h + ${CMAKE_CURRENT_SOURCE_DIR}/ArchiveReader.h + ${CMAKE_CURRENT_SOURCE_DIR}/ArchiveReader.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ArchiveWriter.h + ${CMAKE_CURRENT_SOURCE_DIR}/ArchiveWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/FileInBuffer.h + ${CMAKE_CURRENT_SOURCE_DIR}/FileInBuffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Clock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Clock.h) @@ -37,4 +44,5 @@ target_link_libraries (helpers solvers glog::glog gflags::gflags + MINIZIP::minizip-ng ) diff --git a/src/cpp/helpers/FileInBuffer.cpp b/src/cpp/helpers/FileInBuffer.cpp new file mode 100644 index 000000000..3c903ceff --- /dev/null +++ b/src/cpp/helpers/FileInBuffer.cpp @@ -0,0 +1,20 @@ +#include "FileInBuffer.h" + +#include +#include +#include + +FileBuffer FileInBuffer::run(const std::filesystem::path& filePath) { + std::ifstream file(filePath); + std::stringstream buffer; + if (file.bad()) { + std::cerr << "Error while reading file : " << filePath.c_str() << std::endl; + return {}; + } + buffer << file.rdbuf(); + file.close(); + // if (!keepFile_) { + std::filesystem::remove(filePath); + // } + return {filePath.filename().string(), buffer.str()}; +} diff --git a/src/cpp/helpers/FileInBuffer.h b/src/cpp/helpers/FileInBuffer.h new file mode 100644 index 000000000..04831027a --- /dev/null +++ b/src/cpp/helpers/FileInBuffer.h @@ -0,0 +1,23 @@ +#ifndef _FILEINBUFFER_H +#define _FILEINBUFFER_H +#include +#include + +struct FileBuffer { + std::string fname; + std::string buffer; +}; +using FileBufferVector = std::vector; + +class FileInBuffer { + private: + // std::filesystem::path filePath_; + // bool keepFile_ = false; + + public: + FileInBuffer() = default; + // explicit FileInBuffer(bool keepFile) : keepFile_(keepFile) {} + + FileBuffer run(const std::filesystem::path& filePath); +}; +#endif //_FILEINBUFFER_H \ No newline at end of file diff --git a/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp b/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp index b28bffdb2..b535410b0 100644 --- a/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp +++ b/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp @@ -25,9 +25,7 @@ CandidatesINIReader::CandidatesINIReader( } std::vector CandidatesINIReader::ReadAntaresIntercoFile( - const std::filesystem::path &antaresIntercoFile)const { - std::vector result; - + const std::filesystem::path &antaresIntercoFile) const { std::ifstream interco_filestream(antaresIntercoFile); if (!interco_filestream.good()) { (*logger_)(ProblemGenerationLog::LOGLEVEL::FATAL) @@ -35,8 +33,20 @@ std::vector CandidatesINIReader::ReadAntaresIntercoFile( std::exit(1); } + return ReadLineByLineInterco(interco_filestream); +} + +std::vector CandidatesINIReader::ReadAntaresIntercoFile( + std::istringstream &antaresIntercoFileInStringStream) const { + return ReadLineByLineInterco(antaresIntercoFileInStringStream); +} + +std::vector CandidatesINIReader::ReadLineByLineInterco( + std::istream &stream) const { + std::vector result; + std::string line; - while (std::getline(interco_filestream, line)) { + while (std::getline(stream, line)) { std::stringstream buffer(line); if (!line.empty() && line.front() != '#') { IntercoFileData intercoData; @@ -49,10 +59,9 @@ std::vector CandidatesINIReader::ReadAntaresIntercoFile( } return result; } + std::vector CandidatesINIReader::ReadAreaFile( const std::filesystem::path &areaFile) const { - std::vector result; - std::ifstream area_filestream(areaFile); if (!area_filestream.good()) { (*logger_)(ProblemGenerationLog::LOGLEVEL::FATAL) @@ -60,8 +69,20 @@ std::vector CandidatesINIReader::ReadAreaFile( std::exit(1); } + return ReadLineByLineArea(area_filestream); +} + +std::vector CandidatesINIReader::ReadAreaFile( + std::istringstream &areaFileInStringStream) const { + return ReadLineByLineArea(areaFileInStringStream); +} + +std::vector CandidatesINIReader::ReadLineByLineArea( + std::istream &stream) const { + std::vector result; + std::string line; - while (std::getline(area_filestream, line)) { + while (std::getline(stream, line)) { if (!line.empty() && line.front() != '#') { result.push_back(StringUtils::ToLowercase(line)); } @@ -105,7 +126,7 @@ bool CandidatesINIReader::checkArea(std::string const &areaName_p) const { } std::vector CandidatesINIReader::readCandidateData( - const std::filesystem::path &candidateFile) { + const std::filesystem::path &candidateFile) const { std::vector result; INIReader reader(candidateFile.string()); @@ -122,7 +143,7 @@ std::vector CandidatesINIReader::readCandidateData( CandidateData CandidatesINIReader::readCandidateSection( const std::filesystem::path &candidateFile, const INIReader &reader, - const std::string §ionName) { + const std::string §ionName) const { CandidateData candidateData; candidateData.name = StringUtils::ToLowercase(getStrVal(reader, sectionName, "name")); diff --git a/src/cpp/lpnamer/input_reader/CandidatesINIReader.h b/src/cpp/lpnamer/input_reader/CandidatesINIReader.h index 2cc3a1d77..8967a8d2d 100644 --- a/src/cpp/lpnamer/input_reader/CandidatesINIReader.h +++ b/src/cpp/lpnamer/input_reader/CandidatesINIReader.h @@ -5,6 +5,7 @@ #include #include +#include #include #include "Candidate.h" @@ -23,22 +24,28 @@ class CandidatesINIReader { const std::filesystem::path& antaresIntercoFile, const std::filesystem::path& areaFile, ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger); - CandidatesINIReader( + explicit CandidatesINIReader( ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger) : logger_(logger) {} - std::vector ReadAntaresIntercoFile( - const std::filesystem::path& antaresIntercoFile)const; - std::vector ReadAreaFile(const std::filesystem::path& areaFile)const; - + const std::filesystem::path& antaresIntercoFile) const; + std::vector ReadAntaresIntercoFile( + std::istringstream& antaresIntercoFileInStringStream) const; + std::vector ReadAreaFile( + const std::filesystem::path& areaFile) const; + std::vector ReadAreaFile( + std::istringstream& areaFileInStringStream) const; + std::vector ReadLineByLineInterco( + std::istream& stream) const; + std::vector ReadLineByLineArea(std::istream& stream) const; std::vector readCandidateData( - const std::filesystem::path& candidateFile); + const std::filesystem::path& candidateFile) const; private: bool checkArea(std::string const& areaName_p) const; CandidateData readCandidateSection(const std::filesystem::path& candidateFile, const INIReader& reader, - const std::string& sectionName); + const std::string& sectionName) const; std::map _intercoIndexMap; std::vector _intercoFileData; diff --git a/src/cpp/lpnamer/input_reader/VariableFileReader.cpp b/src/cpp/lpnamer/input_reader/VariableFileReader.cpp index 81a4175d9..99526b571 100644 --- a/src/cpp/lpnamer/input_reader/VariableFileReader.cpp +++ b/src/cpp/lpnamer/input_reader/VariableFileReader.cpp @@ -29,7 +29,23 @@ VariableFileReader::VariableFileReader( (*logger_)(ProblemGenerationLog::LOGLEVEL::FATAL) << errMsg; throw std::runtime_error(errMsg); } - while (std::getline(file, line)) { + ReadVarsFromStream(file, links, variable_name_config); + file.close(); +} + +VariableFileReader::VariableFileReader( + std::istringstream& fileInIStringStream, + const std::vector& links, + const VariableFileReadNameConfiguration& variable_name_config, + ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger) + : logger_(logger) { + ReadVarsFromStream(fileInIStringStream, links, variable_name_config); +} +void VariableFileReader::ReadVarsFromStream( + std::istream& stream, const std::vector& links, + const VariableFileReadNameConfiguration& variable_name_config) { + std::string line; + while (std::getline(stream, line)) { std::string name = getVarNameFromLine(line); _variables.push_back(name); @@ -63,9 +79,7 @@ VariableFileReader::VariableFileReader( _direct_cost_p_var_columns); } } - file.close(); } - std::string VariableFileReader::getVarNameFromLine( const std::string& line) const { std::ostringstream name; diff --git a/src/cpp/lpnamer/input_reader/VariableFileReader.h b/src/cpp/lpnamer/input_reader/VariableFileReader.h index fdad0f28f..17f162e93 100644 --- a/src/cpp/lpnamer/input_reader/VariableFileReader.h +++ b/src/cpp/lpnamer/input_reader/VariableFileReader.h @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include @@ -21,7 +23,14 @@ class VariableFileReader { const std::string& fileName, const std::vector& links, const VariableFileReadNameConfiguration& variable_name_config, ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger); - + VariableFileReader( + std::istringstream& fileInIStringStream, + const std::vector& links, + const VariableFileReadNameConfiguration& variable_name_config, + ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger); + void ReadVarsFromStream( + std::istream& stream, const std::vector& links, + const VariableFileReadNameConfiguration& variable_name_config); const std::vector& getVariables() const; const std::map& getNtcVarColumns() const; const std::map& getDirectCostVarColumns() const; diff --git a/src/cpp/lpnamer/problem_modifier/LauncherHelpers.cpp b/src/cpp/lpnamer/problem_modifier/LauncherHelpers.cpp index 7ee36bc7a..b5af5d516 100644 --- a/src/cpp/lpnamer/problem_modifier/LauncherHelpers.cpp +++ b/src/cpp/lpnamer/problem_modifier/LauncherHelpers.cpp @@ -1,5 +1,7 @@ #include "LauncherHelpers.h" +#include + #include #include "Candidate.h" @@ -128,6 +130,7 @@ ActiveLinksBuilder get_link_builders( const std::filesystem::path &root, ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger) { const auto area_file_name = root / "area.txt"; + const auto interco_file_name = root / "interco.txt"; const auto ts_root = root / "ts-numbers/ntc"; diff --git a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp index b3727332d..90d748d08 100644 --- a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp +++ b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.cpp @@ -1,6 +1,7 @@ #include "LinkProblemsGenerator.h" #include +#include #include #include "VariableFileReader.h" @@ -52,15 +53,10 @@ std::vector LinkProblemsGenerator::readMPSList( */ void LinkProblemsGenerator::treat(const std::filesystem::path &root, ProblemData const &problemData, - Couplings &couplings) const { + Couplings &couplings, ArchiveReader &reader, + ArchiveWriter &writer) const { // get path of file problem***.mps, variable***.txt and constraints***.txt - auto const mps_name = root / problemData._problem_mps; - auto const var_name = root / problemData._variables_txt; - - // new mps file in the new lp directory - std::string const lp_name = - problemData._problem_mps.substr(0, problemData._problem_mps.size() - 4); - auto const lp_mps_name = root / "lp" / (lp_name + ".mps"); + auto const lp_mps_name = lpDir_ / problemData._problem_mps; // List of variables VariableFileReadNameConfiguration variable_name_config; @@ -69,8 +65,9 @@ void LinkProblemsGenerator::treat(const std::filesystem::path &root, "CoutOrigineVersExtremiteDeLInterconnexion"; variable_name_config.cost_extremite_variable_name = "CoutExtremiteVersOrigineDeLInterconnexion"; - - auto variableReader = VariableFileReader(var_name.string(), _links, + auto variableFileContent = + reader.ExtractFileInStringStream(problemData._variables_txt); + auto variableReader = VariableFileReader(variableFileContent, _links, variable_name_config, logger_); std::vector var_names = variableReader.getVariables(); @@ -84,7 +81,8 @@ void LinkProblemsGenerator::treat(const std::filesystem::path &root, SolverFactory factory; auto in_prblm = std::make_shared( factory.create_solver(_solver_name, log_file_path_)); - in_prblm->read_prob_mps(mps_name); + reader.ExtractFile(problemData._problem_mps, lpDir_); + in_prblm->read_prob_mps(lp_mps_name); solver_rename_vars(in_prblm, var_names); @@ -98,13 +96,15 @@ void LinkProblemsGenerator::treat(const std::filesystem::path &root, for (const Candidate &candidate : link.getCandidates()) { if (problem_modifier.has_candidate_col_id(candidate.get_name())) { std::lock_guard guard(coupling_mutex_); - couplings[{candidate.get_name(), mps_name}] = + couplings[{candidate.get_name(), lp_mps_name}] = problem_modifier.get_candidate_col_id(candidate.get_name()); } } } in_prblm->write_prob_mps(lp_mps_name); + writer.AddFileInArchive(lp_mps_name); + std::filesystem::remove(lp_mps_name); } /** @@ -116,9 +116,24 @@ void LinkProblemsGenerator::treat(const std::filesystem::path &root, * \return void */ void LinkProblemsGenerator::treatloop(const std::filesystem::path &root, - Couplings &couplings) const { + const std::filesystem::path &archivePath, + Couplings &couplings) { auto const mps_file_name = root / MPS_TXT; + lpDir_ = root / "lp"; + auto reader = ArchiveReader(archivePath); + reader.Open(); + const auto tmpArchiveName = MPS_ZIP_FILE + "-tmp" + ZIP_EXT; + const auto tmpArchivePath = lpDir_ / tmpArchiveName; + auto writer = ArchiveWriter(tmpArchivePath); + writer.Open(); auto mpsList = readMPSList(mps_file_name); - std::for_each(std::execution::par, mpsList.begin(), mpsList.end(), - [&](const auto &mps) { treat(root, mps, couplings); }); + std::for_each( + std::execution::par, mpsList.begin(), mpsList.end(), + [&](const auto &mps) { treat(root, mps, couplings, reader, writer); }); + reader.Close(); + reader.Delete(); + writer.Close(); + writer.Delete(); + std::filesystem::remove(archivePath); + std::filesystem::rename(tmpArchivePath, lpDir_ / (MPS_ZIP_FILE + ZIP_EXT)); } diff --git a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h index 9506bd653..a520cb947 100644 --- a/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h +++ b/src/cpp/lpnamer/problem_modifier/LinkProblemsGenerator.h @@ -7,7 +7,10 @@ #include #include "ActiveLinks.h" +#include "ArchiveReader.h" +#include "ArchiveWriter.h" #include "Candidate.h" +#include "FileInBuffer.h" #include "ProblemGenerationLogger.h" #include "ProblemModifier.h" #include "common_lpnamer.h" @@ -15,6 +18,8 @@ const std::string CANDIDATES_INI{"candidates.ini"}; const std::string STRUCTURE_FILE{"structure.txt"}; const std::string MPS_TXT{"mps.txt"}; +const std::string MPS_ZIP_FILE{"MPS_ZIP_FILE"}; +const std::string ZIP_EXT{".zip"}; const std::string STUDY_FILE{"study.antares"}; typedef std::pair CandidateNameAndMpsFilePath; @@ -38,17 +43,21 @@ class LinkProblemsGenerator { logger_(logger), log_file_path_(log_file_path) {} - void treatloop(const std::filesystem::path& root, Couplings& couplings) const; + void treatloop(const std::filesystem::path& root, + const std::filesystem::path& archivePath, + Couplings& couplings); private: std::vector readMPSList( const std::filesystem::path& mps_filePath_p) const; void treat(const std::filesystem::path& root, ProblemData const&, - Couplings& couplings) const; + Couplings& couplings, ArchiveReader& reader, + ArchiveWriter& writer) const; const std::vector& _links; std::string _solver_name; + std::filesystem::path lpDir_ = ""; ProblemGenerationLog::ProblemGenerationLoggerSharedPointer logger_; mutable std::mutex coupling_mutex_; std::filesystem::path log_file_path_; diff --git a/src/cpp/multisolver_interface/SolverClp.cpp b/src/cpp/multisolver_interface/SolverClp.cpp index 4067ec714..e66f10e04 100644 --- a/src/cpp/multisolver_interface/SolverClp.cpp +++ b/src/cpp/multisolver_interface/SolverClp.cpp @@ -98,7 +98,8 @@ void SolverClp::write_basis(const std::filesystem::path &filename) { } void SolverClp::read_prob_mps(const std::filesystem::path &filename) { - _clp.readMps(filename.string().c_str(), true, false); + int status = _clp.readMps(filename.string().c_str(), true, false); + zero_status_check(status, "Clp readMps"); } void SolverClp::read_prob_lp(const std::filesystem::path &filename) { diff --git a/src/python/antares_xpansion/antares_driver.py b/src/python/antares_xpansion/antares_driver.py index f40899326..8c3981215 100644 --- a/src/python/antares_xpansion/antares_driver.py +++ b/src/python/antares_xpansion/antares_driver.py @@ -25,6 +25,7 @@ def __init__(self, antares_exe_path: Path) -> None: self.output = 'output' self.ANTARES_N_CPU_OPTION = "--force-parallel" self.antares_n_cpu = 1 # default + self.zip_option = "-z" def launch(self, antares_study_path, antares_n_cpu: int) -> bool: self._set_antares_n_cpu(antares_n_cpu) @@ -87,7 +88,8 @@ def _set_simulation_name(self): self.simulation_name = list_of_dirs[-1] def _get_antares_cmd(self): - return [str(self.antares_exe_path), self.data_dir, self.ANTARES_N_CPU_OPTION, str(self.antares_n_cpu)] + return [str(self.antares_exe_path), self.data_dir, self.ANTARES_N_CPU_OPTION, str(self.antares_n_cpu), + self.zip_option] class Error(Exception): pass diff --git a/src/python/antares_xpansion/benders_driver.py b/src/python/antares_xpansion/benders_driver.py index efa2b405f..c87e97de7 100644 --- a/src/python/antares_xpansion/benders_driver.py +++ b/src/python/antares_xpansion/benders_driver.py @@ -4,13 +4,12 @@ import glob import os -from pathlib import Path import subprocess import sys +from pathlib import Path - -from antares_xpansion.study_output_cleaner import StudyOutputCleaner from antares_xpansion.flushed_print import flushed_print +from antares_xpansion.study_output_cleaner import StudyOutputCleaner class BendersDriver: diff --git a/src/python/antares_xpansion/config_loader.py b/src/python/antares_xpansion/config_loader.py index 171e6877b..f605cba2f 100644 --- a/src/python/antares_xpansion/config_loader.py +++ b/src/python/antares_xpansion/config_loader.py @@ -2,24 +2,22 @@ Class to work on config """ +import json import os import re import shutil import sys -import json - from pathlib import Path +from antares_xpansion.chronicles_checker import ChronicleChecker +from antares_xpansion.flushed_print import flushed_print from antares_xpansion.general_data_reader import GeneralDataIniReader from antares_xpansion.input_checker import check_candidates_file, check_options from antares_xpansion.launcher_options_default_value import LauncherOptionsDefaultValues +from antares_xpansion.launcher_options_keys import LauncherOptionsKeys from antares_xpansion.optimisation_keys import OptimisationKeys from antares_xpansion.xpansionConfig import XpansionConfig from antares_xpansion.xpansion_study_reader import XpansionStudyReader -from antares_xpansion.flushed_print import flushed_print -from antares_xpansion.launcher_options_keys import LauncherOptionsKeys - -from antares_xpansion.chronicles_checker import ChronicleChecker class NTCColumnConstraintError(Exception): @@ -51,6 +49,7 @@ def __init__(self, config: XpansionConfig): self.platform = sys.platform self._INFO_MSG = "<< INFO >>" self._config = config + self._last_zip = None if self._config.step == "resume": self._config.simulation_name = ( LauncherOptionsDefaultValues.DEFAULT_SIMULATION_NAME() @@ -76,7 +75,8 @@ def __init__(self, config: XpansionConfig): def _set_simulation_name(self): if not self._config.simulation_name: - raise ConfigLoader.MissingSimulationName("Missing argument simulationName") + raise ConfigLoader.MissingSimulationName( + "Missing argument simulationName") else: self._simulation_name = self._config.simulation_name @@ -86,9 +86,11 @@ def _restore_launcher_options(self): self._config.method = options[LauncherOptionsKeys.method_key()] self._config.n_mpi = options[LauncherOptionsKeys.n_mpi_key()] - self._config.antares_n_cpu = options[LauncherOptionsKeys.antares_n_cpu_key()] + self._config.antares_n_cpu = options[LauncherOptionsKeys.antares_n_cpu_key( + )] self._config.keep_mps = options[LauncherOptionsKeys.keep_mps_key()] - self._config.oversubscribe = options[LauncherOptionsKeys.oversubscribe_key()] + self._config.oversubscribe = options[LauncherOptionsKeys.oversubscribe_key( + )] self._config.allow_run_as_root = options[ LauncherOptionsKeys.allow_run_as_root_key() ] @@ -330,8 +332,12 @@ def simulation_lp_path(self): return self._simulation_lp_path() def _simulation_lp_path(self): - lp_path = os.path.normpath(os.path.join(self.simulation_output_path(), "lp")) - return lp_path + return self.xpansion_simulation_output() / "lp" + + def xpansion_simulation_output(self): + if self._simulation_name == "last": + self._set_last_simulation_name() + return self._simulation_name def _verify_additional_constraints_file(self): if self.options.get("additional-constraints", "") != "": @@ -363,9 +369,8 @@ def _verify_solver(self): def simulation_output_path(self) -> Path: if self._simulation_name == "last": self._set_last_simulation_name() - return Path( - os.path.normpath(os.path.join(self.antares_output(), self._simulation_name)) - ) + + return self._last_zip def benders_pre_actions(self): self.save_launcher_options() @@ -381,14 +386,16 @@ def save_launcher_options(self): options[LauncherOptionsKeys.keep_mps_key()] = self.keep_mps() options[LauncherOptionsKeys.oversubscribe_key()] = self.oversubscribe() options[LauncherOptionsKeys.oversubscribe_key()] = self.oversubscribe() - options[LauncherOptionsKeys.allow_run_as_root_key()] = self.allow_run_as_root() + options[LauncherOptionsKeys.allow_run_as_root_key() + ] = self.allow_run_as_root() with open(self.launcher_options_file_path(), "w") as launcher_options: json.dump(options, launcher_options, indent=4) def launcher_options_file_path(self): return os.path.normpath( - os.path.join(self._simulation_lp_path(), self._config.LAUNCHER_OPTIONS_JSON) + os.path.join(self._simulation_lp_path(), + self._config.LAUNCHER_OPTIONS_JSON) ) def create_expansion_dir(self): @@ -405,12 +412,12 @@ def _create_sensitivity_dir(self): def _expansion_dir(self): return os.path.normpath( - os.path.join(self.simulation_output_path(), "expansion") + os.path.join(self.xpansion_simulation_output(), "expansion") ) def _sensitivity_dir(self): return os.path.normpath( - os.path.join(self.simulation_output_path(), "sensitivity") + os.path.join(self.xpansion_simulation_output(), "sensitivity") ) def _set_options_for_benders_solver(self): @@ -423,7 +430,8 @@ def _set_options_for_benders_solver(self): options_values[OptimisationKeys.slave_weight_value_key()] = len( self.active_years ) - options_values[OptimisationKeys.json_file_key()] = self.json_file_path() + options_values[OptimisationKeys.json_file_key() + ] = self.json_file_path() options_values[ OptimisationKeys.last_iteration_json_file_key() ] = self.last_iteration_json_file_path() @@ -475,17 +483,55 @@ def _set_last_simulation_name(self): """ return last simulation name """ + last_dir_path = self.get_last_modified_dir(self.antares_output()) + + if self.step() == "resume": + self._simulation_name = last_dir_path + else: + # TODO temp function to zip study output -- for now antares does not provide archive output + # self.zip_last_study(last_dir_path) + # Get list of all dirs only in the given directory + list_of_zip_filter = Path(self.antares_output()).glob("*.zip") + + # Sort list of files based on last modification time in ascending order + list_of_zip = sorted( + list_of_zip_filter, + key=lambda x: os.path.getmtime( + os.path.join(self.antares_output(), x)), + ) + self._last_zip = list_of_zip[-1] + self._simulation_name = self._last_zip.parent / \ + (self._last_zip.stem+"-Xpansion") + + def zip_last_study(self, last_dir_path): + """ + zip last simulation and delete it + """ # Get list of all dirs only in the given directory - list_of_dirs_filter = filter( - lambda x: os.path.isdir(os.path.join(self.antares_output(), x)), - os.listdir(self.antares_output()), + shutil.make_archive(str(last_dir_path), "zip", last_dir_path) + + shutil.rmtree(last_dir_path, ignore_errors=True) + + def is_zip(self, file): + filename, ext = os.path.splitext(file) + return ext == ".zip" + + def get_last_modified_dir(self, root_dir): + list_dir = os.listdir(root_dir) + list_of_zip = filter( + lambda x: self.is_zip(x), list_dir ) # Sort list of files based on last modification time in ascending order - list_of_dirs = sorted( - list_of_dirs_filter, - key=lambda x: os.path.getmtime(os.path.join(self.antares_output(), x)), + zip_sorted = sorted( + list_of_zip, + key=lambda x: os.path.getmtime( + os.path.join(root_dir, x)), ) - self._simulation_name = list_of_dirs[-1] + + last_zip = Path(root_dir) / zip_sorted[-1] + filename, ext = os.path.splitext(last_zip) + output_dir = os.path.basename(filename) + return output_dir def is_accurate(self): """ @@ -498,7 +544,8 @@ def is_accurate(self): uc_type = self.options.get( self._config.UC_TYPE, self._config.settings_default[self._config.UC_TYPE] ) - assert uc_type in [self._config.EXPANSION_ACCURATE, self._config.EXPANSION_FAST] + assert uc_type in [self._config.EXPANSION_ACCURATE, + self._config.EXPANSION_FAST] return uc_type == self._config.EXPANSION_ACCURATE class MissingFile(Exception): @@ -573,14 +620,16 @@ def json_sensitivity_out_path(self): def structure_file_path(self): # Assumes that structure file is always the default, ok for now as the user cannot set it, but could it be dangerous if later we wish for some reasons modify its name to a non-default one return os.path.join( - self.simulation_lp_path(), self._config.options_default["STRUCTURE_FILE"] + self.simulation_lp_path( + ), self._config.options_default["STRUCTURE_FILE"] ) def last_master_file_path(self): # The 'last_iteration' literal is only hard-coded in Worker.cpp, should we introduce a new variable in _config.options_default ? return os.path.join( self.simulation_lp_path(), - self._config.options_default["MASTER_NAME"] + "_last_iteration.mps", + self._config.options_default["MASTER_NAME"] + + "_last_iteration.mps", ) def last_master_basis_path(self): @@ -617,7 +666,8 @@ def log_level(self): def sensitivity_log_file(self) -> Path: return Path( - os.path.join(self._sensitivity_dir(), self._config.SENSITIVITY_LOG_FILE) + os.path.join(self._sensitivity_dir(), + self._config.SENSITIVITY_LOG_FILE) ) class MissingSimulationName(Exception): diff --git a/src/python/antares_xpansion/driver.py b/src/python/antares_xpansion/driver.py index 12d1bfc82..677058ef2 100644 --- a/src/python/antares_xpansion/driver.py +++ b/src/python/antares_xpansion/driver.py @@ -67,7 +67,7 @@ def launch(self): self.launch_problem_generation_step() self.launch_benders_step() self.study_update_driver.launch( - self.config_loader.simulation_output_path(), self.config_loader.json_file_path(), self.config_loader.keep_mps()) + self.config_loader.xpansion_simulation_output(), self.config_loader.json_file_path(), self.config_loader.keep_mps()) elif self.config_loader.step() == "antares": self.launch_antares_step() @@ -77,7 +77,7 @@ def launch(self): elif self.config_loader.step() == "study_update": self.study_update_driver.launch( - self.config_loader.simulation_output_path(), self.config_loader.json_file_path(), self.config_loader.keep_mps()) + self.config_loader.xpansion_simulation_output(), self.config_loader.json_file_path(), self.config_loader.keep_mps()) elif self.config_loader.step() == "benders": self.launch_benders_step() @@ -120,7 +120,7 @@ def launch_problem_generation_step(self): def launch_benders_step(self): self.config_loader.benders_pre_actions() self.benders_driver.launch( - self.config_loader.simulation_output_path(), + self.config_loader.xpansion_simulation_output(), self.config_loader.method(), self.config_loader.keep_mps(), self.config_loader.n_mpi(), diff --git a/src/python/antares_xpansion/optimisation_keys.py b/src/python/antares_xpansion/optimisation_keys.py index fa08a11a0..a460e198d 100644 --- a/src/python/antares_xpansion/optimisation_keys.py +++ b/src/python/antares_xpansion/optimisation_keys.py @@ -87,6 +87,10 @@ def log_level_key(): def resume_key(): return "RESUME" + @staticmethod + def mps_zip_file_key(): + return "MPS_ZIP_FILE" + @staticmethod def separation_key(): return "SEPARATION_PARAM" diff --git a/src/python/antares_xpansion/problem_generator_driver.py b/src/python/antares_xpansion/problem_generator_driver.py index c77bfe998..a163af891 100644 --- a/src/python/antares_xpansion/problem_generator_driver.py +++ b/src/python/antares_xpansion/problem_generator_driver.py @@ -2,21 +2,20 @@ Class to control the Problem Generation """ -import shutil import os +import shutil import subprocess -from typing import List - import sys -from datetime import datetime +import zipfile from dataclasses import dataclass +from datetime import datetime from pathlib import Path +from typing import List +from antares_xpansion.flushed_print import flushed_print +from antares_xpansion.xpansion_study_reader import XpansionStudyReader from antares_xpansion.xpansion_utils import read_and_write_mps -from antares_xpansion.study_output_cleaner import StudyOutputCleaner from antares_xpansion.yearly_weight_writer import YearlyWeightWriter -from antares_xpansion.xpansion_study_reader import XpansionStudyReader -from antares_xpansion.flushed_print import flushed_print @dataclass @@ -33,6 +32,9 @@ class ProblemGeneratorDriver: class BasicException(Exception): pass + class MpsZipFileException(BasicException): + pass + class AreaFileException(BasicException): pass @@ -63,6 +65,9 @@ def __init__(self, problem_generator_data: ProblemGeneratorData) -> None: self.MPS_TXT = "mps.txt" self.is_relaxed = False self._lp_path = None + self.mps_zip_filename = "MPS_ZIP" + self.zip_ext = ".zip" + self.mps_zip_file = self.mps_zip_filename+self.zip_ext def launch(self, output_path: Path, is_relaxed: bool): """ @@ -81,8 +86,13 @@ def set_output_path(self, output_path): if output_path.exists(): self._output_path = output_path + self.xpansion_output_dir = output_path.parent / \ + (output_path.stem+"-Xpansion") + if self.xpansion_output_dir.exists(): + shutil.rmtree(self.xpansion_output_dir) + os.makedirs(self.xpansion_output_dir) self._lp_path = os.path.normpath( - os.path.join(self._output_path, 'lp')) + os.path.join(self.xpansion_output_dir, 'lp')) else: raise ProblemGeneratorDriver.OutputPathError( f"{output_path} not found") @@ -104,29 +114,46 @@ def _get_names(self): produces a file named with xpansionConfig.MPS_TXT """ + self._create_lp_dir() + + self._process_weights_file() mps_txt = read_and_write_mps(self.output_path) - with open(os.path.normpath(os.path.join(self.output_path, self.MPS_TXT)), 'w') as file_l: + with open(os.path.normpath(os.path.join(self.xpansion_output_dir, self.MPS_TXT)), 'w') as file_l: for line in mps_txt.items(): - file_l.write(line[1][0] + ' ' + line[1] + mps_sub_problem_file = line[1][0] + file_l.write(mps_sub_problem_file + ' ' + line[1] [1] + ' ' + line[1][2] + '\n') - self._check_and_copy_area_file() - self._check_and_copy_interco_file() - - def _check_and_copy_area_file(self): - self._check_and_copy_txt_file( + with zipfile.ZipFile(self.output_path, 'r') as study_archive: + for e in study_archive.namelist(): + if '.txt' in e and '/' not in e: + if 'area' in e: + study_archive.extract( + e, self.xpansion_output_dir) + if 'interco' in e: + study_archive.extract( + e, self.xpansion_output_dir) + if 'ts-numbers' in e: + study_archive.extract( + e, self.xpansion_output_dir) + + self._check_and_rename_area_file() + self._check_and_rename_interco_file() + + def _check_and_rename_area_file(self): + self._check_and_rename_txt_file( "area", ProblemGeneratorDriver.AreaFileException) - def _check_and_copy_interco_file(self): - self._check_and_copy_txt_file( + def _check_and_rename_interco_file(self): + self._check_and_rename_txt_file( "interco", ProblemGeneratorDriver.IntercoFilesException) - def _check_and_copy_txt_file(self, prefix, exception_to_raise: BasicException): - self._check_and_copy_file(prefix, "txt", exception_to_raise) + def _check_and_rename_txt_file(self, prefix, exception_to_raise: BasicException): + self._check_and_rename_file(prefix, "txt", exception_to_raise) - def _check_and_copy_file(self, prefix, extension, exception_to_raise: BasicException): - glob_path = Path(self.output_path) + def _check_and_rename_file(self, prefix, extension, exception_to_raise: BasicException): + glob_path = Path(self.xpansion_output_dir) files = [str(pp) for pp in glob_path.glob(prefix + "*" + extension)] if len(files) == 0: raise exception_to_raise("No %s*.txt file found" % prefix) @@ -136,7 +163,7 @@ def _check_and_copy_file(self, prefix, extension, exception_to_raise: BasicExcep "More than one %s*.txt file found" % prefix) shutil.copy(files[0], os.path.normpath( - os.path.join(self.output_path, prefix + '.' + extension))) + os.path.join(self.xpansion_output_dir, prefix + '.' + extension))) def _lp_step(self): """ @@ -145,22 +172,15 @@ def _lp_step(self): produces a file named with xpansionConfig.MPS_TXT """ - if os.path.isdir(self._lp_path): - shutil.rmtree(self._lp_path) - os.makedirs(self._lp_path) - - if self.weight_file_name_for_lp: - XpansionStudyReader.check_weights_file( - self.user_weights_file_path, len(self.active_years)) - weight_list = XpansionStudyReader.get_years_weight_from_file( - self.user_weights_file_path) - YearlyWeightWriter(Path(self.output_path)).create_weight_file(weight_list, self.weight_file_name_for_lp, - self.active_years) + start_time = datetime.now() + flushed_print(f"LPNamer command {self._get_lp_namer_command()}") returned_l = subprocess.run(self._get_lp_namer_command(), shell=False, stdout=sys.stdout, stderr=sys.stderr) + + end_time = datetime.now() flushed_print('Post antares step duration: {}'.format( end_time - start_time)) @@ -168,8 +188,24 @@ def _lp_step(self): if returned_l.returncode != 0: raise ProblemGeneratorDriver.LPNamerExecutionError( "ERROR: exited lpnamer with status %d" % returned_l.returncode) - elif not self.keep_mps: - StudyOutputCleaner.clean_lpnamer_step(Path(self.output_path)) + # TODO will not be needed + # elif not self.keep_mps: + # StudyOutputCleaner.clean_lpnamer_step(Path(self.output_path)) + + def _create_lp_dir(self): + if os.path.isdir(self._lp_path): + shutil.rmtree(self._lp_path) + os.makedirs(self._lp_path) + + def _process_weights_file(self): + if self.weight_file_name_for_lp: + XpansionStudyReader.check_weights_file( + self.user_weights_file_path, len(self.active_years)) + weight_list = XpansionStudyReader.get_years_weight_from_file( + self.user_weights_file_path) + YearlyWeightWriter(Path(self.xpansion_output_dir), self.output_path).create_weight_file(weight_list, self.weight_file_name_for_lp, + self.active_years) + def _get_lp_namer_command(self): @@ -178,7 +214,7 @@ def _get_lp_namer_command(self): raise ProblemGeneratorDriver.LPNamerExeError( f"LP namer exe: {self.lp_namer_exe_path} not found") - return [self.lp_namer_exe_path, "-o", str(self.output_path), "-f", is_relaxed, "-e", + return [self.lp_namer_exe_path, "-o", str(self.xpansion_output_dir), "-a", self.output_path, "-f", is_relaxed, "-e", self.additional_constraints] output_path = property(get_output_path, set_output_path) diff --git a/src/python/antares_xpansion/study_output_cleaner.py b/src/python/antares_xpansion/study_output_cleaner.py index 67edf8616..c73cd7fef 100644 --- a/src/python/antares_xpansion/study_output_cleaner.py +++ b/src/python/antares_xpansion/study_output_cleaner.py @@ -17,7 +17,7 @@ def rename_master(old_master_path: Path, new_master_path: Path): def get_last_master_path(dirpath: Path) -> Path: - return Path(dirpath / 'lp/master_last_iteration.mps') + return Path(dirpath / "lp"/'master_last_iteration.mps') def get_tmp_master_path(master_path: Path) -> Path: @@ -42,11 +42,13 @@ def clean_benders_step(study_output: Path): master_path = get_last_master_path(study_output) tmp_master_path = get_tmp_master_path(master_path) + lp_dir = study_output / 'lp' rename_master(master_path, tmp_master_path) - remove_files_containing_str_from_dir('.mps', study_output / 'lp') + remove_files_containing_str_from_dir('.mps', lp_dir) rename_master(tmp_master_path, master_path) - remove_files_containing_str_from_dir('.lp', study_output / 'lp') + remove_files_containing_str_from_dir('.lp', lp_dir) + remove_files_containing_str_from_dir(".zip", lp_dir) @staticmethod def clean_study_update_step(study_output: Path): diff --git a/src/python/antares_xpansion/xpansionConfig.py b/src/python/antares_xpansion/xpansionConfig.py index 15e0fb7a5..00ddbcc9d 100644 --- a/src/python/antares_xpansion/xpansionConfig.py +++ b/src/python/antares_xpansion/xpansionConfig.py @@ -172,6 +172,7 @@ def _set_default_options(self): OptimisationKeys.input_root_key(): self.input_root_default_value(), OptimisationKeys.csv_name_key(): self.csv_name_default_value(), OptimisationKeys.bound_alpha_key(): self.bound_alpha_default_value(), + OptimisationKeys.mps_zip_file_key(): self.mps_zip_file_default_value(), OptimisationKeys.separation_key(): self.separation_default_value(), } @@ -217,6 +218,12 @@ def relaxed_gap_default_value(self): def max_iterations_default_value(self): return "-1" + def initial_master_relaxation_default_value(self): + return False + + def mps_zip_file_default_value(self): + return "MPS_ZIP_FILE.zip" + def separation_default_value(self): return "0.5" diff --git a/src/python/antares_xpansion/xpansion_utils.py b/src/python/antares_xpansion/xpansion_utils.py index ac4ae0ed0..71f06f270 100644 --- a/src/python/antares_xpansion/xpansion_utils.py +++ b/src/python/antares_xpansion/xpansion_utils.py @@ -3,15 +3,17 @@ """ import os +from zipfile import ZipFile -def read_and_write_mps(root_path): +def read_and_write_mps(study_archive): """ :return: a dictionary giving instance file, variables file and constraints file per (year, week) """ result = {} - sorted_root_dir = sorted(os.listdir(root_path)) + with ZipFile(study_archive, 'r') as zip_object: + sorted_root_dir = sorted(zip_object.namelist()) mps_ext = '.mps' txt_ext = ".txt" diff --git a/src/python/antares_xpansion/yearly_weight_writer.py b/src/python/antares_xpansion/yearly_weight_writer.py index 4a2aae76b..f8c87c04c 100644 --- a/src/python/antares_xpansion/yearly_weight_writer.py +++ b/src/python/antares_xpansion/yearly_weight_writer.py @@ -1,20 +1,22 @@ import os from pathlib import Path from typing import List +from zipfile import ZipFile class YearlyWeightWriter: - def __init__(self, simulation_path: Path): + def __init__(self, xpansion_output_dir: Path, zipped_output_path: Path): self.MPS_DIR = "lp" - self.simulation_path = simulation_path + self.zipped_output_path = zipped_output_path + self.xpansion_output_dir = xpansion_output_dir self.file_content = [] if not os.path.isdir(self.output_dir): os.mkdir(self.output_dir) @property def output_dir(self): - return self.simulation_path / self.MPS_DIR + return self.xpansion_output_dir / self.MPS_DIR def create_weight_file(self, weight_list: List[float], file_name: str, active_years): self.file_content = [] @@ -23,16 +25,19 @@ def create_weight_file(self, weight_list: List[float], file_name: str, active_ye self._write_content_to_file(file_name) def _add_mps_lines_with_weights_to_content(self, weight_list, active_years): - sorted_dir = sorted(os.listdir(self.simulation_path)) + with ZipFile(self.zipped_output_path, 'r') as zip_object: + sorted_dir = sorted(zip_object.namelist()) for mps_file in sorted_dir: if self._file_should_be_added(mps_file): - self._add_mps_file_to_output_file_content(mps_file, weight_list, active_years) + self._add_mps_file_to_output_file_content( + mps_file, weight_list, active_years) def _add_mps_file_to_output_file_content(self, file_name, weight_list, active_years): year = self._get_year_index_from_name(file_name) year_index = active_years.index(int(year)) mps_file_name = Path(file_name).with_suffix('').name - self.file_content.append(mps_file_name + " " + str(weight_list[year_index]) + "\n") + self.file_content.append( + mps_file_name + " " + str(weight_list[year_index]) + "\n") def _add_last_line_to_content(self, weight_list): self.file_content.append("WEIGHT_SUM " + str(sum(weight_list))) diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 46675d816..585c98ef0 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -16,6 +16,7 @@ message ("Gtest found here: " ${GTEST_INCLUDE_DIRS}) # -------------------------------------------------------------------------- # unit tests # -------------------------------------------------------------------------- +add_subdirectory(tests_utils) add_subdirectory(solvers_interface) add_subdirectory(logger) add_subdirectory(json_output_writer) @@ -23,4 +24,5 @@ add_subdirectory(helpers) add_subdirectory(lp_namer) add_subdirectory(sensitivity) add_subdirectory(restart_benders) +add_subdirectory(zip_mps) add_subdirectory(benders) diff --git a/tests/cpp/benders/CMakeLists.txt b/tests/cpp/benders/CMakeLists.txt index 0b2fba765..deef43a1a 100644 --- a/tests/cpp/benders/CMakeLists.txt +++ b/tests/cpp/benders/CMakeLists.txt @@ -7,7 +7,8 @@ target_link_libraries(benders_sequential_test benders_core output_core logger_lib - GTest::Main) + GTest::Main + tests_utils) target_include_directories(benders_sequential_test PRIVATE diff --git a/tests/cpp/benders/benders_sequential_test.cpp b/tests/cpp/benders/benders_sequential_test.cpp index 557d3fe28..ce637b375 100644 --- a/tests/cpp/benders/benders_sequential_test.cpp +++ b/tests/cpp/benders/benders_sequential_test.cpp @@ -1,7 +1,9 @@ #include +#include "ArchiveWriter.h" #include "BendersSequential.h" #include "JsonWriter.h" +#include "RandomDirGenerator.h" #include "LoggerStub.h" #include "gtest/gtest.h" @@ -72,7 +74,19 @@ class BendersSequentialDouble : public BendersSequential { BendersBase::reset_master(new FakeWorkerMaster(var)); }; void free() override{}; - + void initialize_problems() override { + match_problem_to_id(); + + reset_master(new WorkerMaster( + master_variable_map, get_master_path(), get_solver_name(), + get_log_level(), _data.nsubproblem, log_name(), IsResumeMode())); + for (const auto &problem : coupling_map) { + const auto subProblemFilePath = GetSubproblemPath(problem.first); + addSubproblem(problem); + AddSubproblemName(problem.first); + std::filesystem::remove(subProblemFilePath); + } + } // No override as the base class function is const void DeactivateIntegrityConstraints() const override { _deactivateIntConstraintCall = true; @@ -116,6 +130,9 @@ class BendersSequentialTest : public ::testing::Test { Logger logger; Writer writer; const std::filesystem::path data_test_dir = "data_test"; + const std::filesystem::path mps_dir = data_test_dir / "mps"; + std::filesystem::path tmpDir; + const std::string MPS_ZIP_FILE = "MPS_ZIP_FILE.zip"; protected: void SetUp() override { @@ -123,7 +140,12 @@ class BendersSequentialTest : public ::testing::Test { writer = std::make_shared(std::make_shared(), std::tmpnam(nullptr)); } + void copyMasterMps() { + tmpDir = CreateRandomSubDir(std::filesystem::temp_directory_path()); + std::filesystem::copy(mps_dir / "mip_toy_prob.mps", tmpDir, + std::filesystem::copy_options::update_existing); + } BaseOptions init_base_options() const { BaseOptions base_options; @@ -133,10 +155,11 @@ class BendersSequentialTest : public ::testing::Test { base_options.SLAVE_WEIGHT = "CONSTANT"; base_options.MASTER_NAME = "mip_toy_prob"; base_options.STRUCTURE_FILE = "my_structure.txt"; - base_options.INPUTROOT = (data_test_dir / "mps").string(); + base_options.INPUTROOT = tmpDir.string(); base_options.SOLVER_NAME = "COIN"; base_options.weights = {}; base_options.RESUME = false; + base_options.MPS_ZIP_FILE = MPS_ZIP_FILE; return base_options; } @@ -188,6 +211,7 @@ class BendersSequentialTest : public ::testing::Test { }; TEST_F(BendersSequentialTest, MasterNotRelaxedWhenSepSetToOne) { + copyMasterMps(); MasterFormulation master_formulation = MasterFormulation::INTEGER; int max_iter = 1; double relaxed_gap = 1e-2; @@ -196,8 +220,8 @@ TEST_F(BendersSequentialTest, MasterNotRelaxedWhenSepSetToOne) { master_formulation, max_iter, relaxed_gap, sep_param); benders.set_data(true, 0); - benders.launch(); + benders.launch(); std::vector nb_units_col_types = get_nb_units_col_types(benders); EXPECT_EQ(benders._deactivateIntConstraintCall, false); @@ -207,6 +231,7 @@ TEST_F(BendersSequentialTest, MasterNotRelaxedWhenSepSetToOne) { } TEST_F(BendersSequentialTest, MasterRelaxedWhenSepLowerThanOne) { + copyMasterMps(); MasterFormulation master_formulation = MasterFormulation::INTEGER; int max_iter = 1; double relaxed_gap = 1e-2; @@ -226,6 +251,7 @@ TEST_F(BendersSequentialTest, MasterRelaxedWhenSepLowerThanOne) { } TEST_F(BendersSequentialTest, ReactivateIntConstraintAfterRelaxedGapReached) { + copyMasterMps(); MasterFormulation master_formulation = MasterFormulation::INTEGER; int max_iter = 1; double relaxed_gap = 1e-2; @@ -247,6 +273,7 @@ TEST_F(BendersSequentialTest, ReactivateIntConstraintAfterRelaxedGapReached) { TEST_F(BendersSequentialTest, MaxIterReachedBeforeRelaxedGapShouldEndRunWithAnIntegerMasterIteration) { + copyMasterMps(); MasterFormulation master_formulation = MasterFormulation::INTEGER; int max_iter = 1; double relaxed_gap = 1e-5; @@ -271,6 +298,7 @@ TEST_F(BendersSequentialTest, } TEST_F(BendersSequentialTest, CheckDataPostRelaxation) { + copyMasterMps(); MasterFormulation master_formulation = MasterFormulation::INTEGER; int max_iter = 1; double relaxed_gap = 1e-2; @@ -292,6 +320,7 @@ TEST_F(BendersSequentialTest, CheckDataPostRelaxation) { } TEST_F(BendersSequentialTest, CheckInOutDataWhithoutImprovement) { + copyMasterMps(); MasterFormulation master_formulation = MasterFormulation::RELAXED; double sep_param = 0.8; double relaxed_gap = 1e-2; @@ -335,6 +364,7 @@ TEST_F(BendersSequentialTest, CheckInOutDataWhithoutImprovement) { } TEST_F(BendersSequentialTest, CheckInOutDataWhenImprovement) { + copyMasterMps(); MasterFormulation master_formulation = MasterFormulation::RELAXED; double relaxed_gap = 1e-2; double sep_param = 0.8; diff --git a/tests/cpp/helpers/CMakeLists.txt b/tests/cpp/helpers/CMakeLists.txt index a953c646f..0edb83aca 100644 --- a/tests/cpp/helpers/CMakeLists.txt +++ b/tests/cpp/helpers/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable (helpers_test JsonXpansionReaderTest.cc - AntaresVersionProviderTest.cpp) + AntaresVersionProviderTest.cpp + ) target_include_directories (helpers_test SYSTEM PRIVATE diff --git a/tests/cpp/solvers_interface/define_datas.cpp b/tests/cpp/solvers_interface/define_datas.cpp index fa2e12379..782e45798 100644 --- a/tests/cpp/solvers_interface/define_datas.cpp +++ b/tests/cpp/solvers_interface/define_datas.cpp @@ -1,14 +1,15 @@ #include "define_datas.hpp" +#include void fill_datas(AllDatas& datas) { datas.clear(); - std::string data_test_dir = "data_test"; + std::filesystem::path data_test_dir = "data_test"; //================================================================== // 1. mip toy - InstanceData miptoy = InstanceData(); - miptoy._path = data_test_dir + "/mps/mip_toy_prob.mps"; + auto miptoy = InstanceData(); + miptoy._path = data_test_dir / "mps" / "mip_toy_prob.mps"; miptoy._ncols = 2; miptoy._nintegervars = 2; miptoy._nrows = 2; @@ -44,8 +45,8 @@ void fill_datas(AllDatas& datas) { //================================================================== // LP toy - InstanceData lptoy = InstanceData(); - lptoy._path = data_test_dir + "/mps/lp_toy_prob.mps"; + auto lptoy = InstanceData(); + lptoy._path = data_test_dir / "mps" / "lp_toy_prob.mps"; lptoy._ncols = 2; lptoy._nintegervars = 2; lptoy._nrows = 2; @@ -81,8 +82,8 @@ void fill_datas(AllDatas& datas) { //================================================================== // 2. multi knapsack - InstanceData multikp = InstanceData(); - multikp._path = data_test_dir + "/mps/lp1.mps"; + auto multikp = InstanceData(); + multikp._path = data_test_dir / "mps" / "lp1.mps"; multikp._ncols = 9; multikp._nintegervars = 0; multikp._nrows = 5; @@ -121,8 +122,8 @@ void fill_datas(AllDatas& datas) { //================================================================== // 3. unbounded - InstanceData unbd = InstanceData(); - unbd._path = data_test_dir + "/mps/unbounded.mps"; + auto unbd = InstanceData(); + unbd._path = data_test_dir / "mps" / "unbounded.mps"; unbd._ncols = 2; unbd._nintegervars = 0; unbd._nrows = 2; @@ -158,8 +159,8 @@ void fill_datas(AllDatas& datas) { //================================================================== // 3. unbounded - InstanceData infeas = InstanceData(); - infeas._path = data_test_dir + "/mps/infeas.mps"; + auto infeas = InstanceData(); + infeas._path = data_test_dir / "mps" / "infeas.mps"; infeas._ncols = 2; infeas._nintegervars = 0; infeas._nrows = 2; @@ -195,8 +196,8 @@ void fill_datas(AllDatas& datas) { //================================================================== // 5. NETWORK instance -- master - InstanceData net_master = InstanceData(); - net_master._path = data_test_dir + "/mini_network/master.mps"; + auto net_master = InstanceData(); + net_master._path = data_test_dir / "mini_network" / "master.mps"; net_master._ncols = 2; net_master._nintegervars = 0; net_master._nrows = 1; @@ -224,9 +225,9 @@ void fill_datas(AllDatas& datas) { datas.push_back(net_master); //================================================================== - // 6. NETWORK instance -- SP1 - InstanceData net_sp1 = InstanceData(); - net_sp1._path = data_test_dir + "/mini_network/SP1.mps"; + // 6. NETWORK instance -- SubProblem1 + auto net_sp1 = InstanceData(); + net_sp1._path = data_test_dir / "mini_network" / "SubProblem1.mps"; net_sp1._ncols = 4; net_sp1._nintegervars = 0; net_sp1._nrows = 3; @@ -254,9 +255,9 @@ void fill_datas(AllDatas& datas) { datas.push_back(net_sp1); //================================================================== - // 7. NETWORK instance -- SP2 - InstanceData net_sp2 = InstanceData(); - net_sp2._path = data_test_dir + "/mini_network/SP2.mps"; + // 7. NETWORK instance -- SubProblem2 + auto net_sp2 = InstanceData(); + net_sp2._path = data_test_dir / "mini_network" / "SubProblem2.mps"; net_sp2._ncols = 4; net_sp2._nintegervars = 0; net_sp2._nrows = 3; @@ -285,8 +286,8 @@ void fill_datas(AllDatas& datas) { //================================================================== // test slacks computation - InstanceData test_slacks = InstanceData(); - test_slacks._path = data_test_dir + "/mps/test_slacks.mps"; + auto test_slacks = InstanceData(); + test_slacks._path = data_test_dir / "mps" / "test_slacks.mps"; test_slacks._ncols = 1; test_slacks._nintegervars = 0; test_slacks._nrows = 3; @@ -315,8 +316,8 @@ void fill_datas(AllDatas& datas) { //================================================================== // test slacks computation - InstanceData test_reduced = InstanceData(); - test_reduced._path = data_test_dir + "/mps/test_reduced_costs.mps"; + auto test_reduced = InstanceData(); + test_reduced._path = data_test_dir / "mps" / "test_reduced_costs.mps"; test_reduced._ncols = 3; test_reduced._nintegervars = 0; test_reduced._nrows = 1; diff --git a/tests/cpp/solvers_interface/define_datas.hpp b/tests/cpp/solvers_interface/define_datas.hpp index 64e26a1d3..605186f72 100644 --- a/tests/cpp/solvers_interface/define_datas.hpp +++ b/tests/cpp/solvers_interface/define_datas.hpp @@ -3,13 +3,14 @@ #include #include #include +#include #include "multisolver_interface/SolverAbstract.h" /* Contains all the data to check the results of the tests on an instance*/ class InstanceData { public: - std::string _path; + std::filesystem::path _path; int _ncols; int _nintegervars; int _nrows; diff --git a/tests/cpp/solvers_interface/test_basis.cpp b/tests/cpp/solvers_interface/test_basis.cpp index b6227ed65..2f47357d1 100644 --- a/tests/cpp/solvers_interface/test_basis.cpp +++ b/tests/cpp/solvers_interface/test_basis.cpp @@ -27,7 +27,7 @@ TEST_CASE("Write and read basis", "[basis]") { SolverFactory factory; auto inst = GENERATE(MIP_TOY, LP_TOY, MULTIKP, NET_MASTER, SLACKS, REDUCED); - std::string instance = datas[inst]._path; + auto instance = datas[inst]._path; SECTION("Loop on instances and solvers") { for (auto const& solver_name : factory.get_solvers_list()) { @@ -38,7 +38,7 @@ TEST_CASE("Write and read basis", "[basis]") { expec_solver->read_prob_mps(instance); expec_solver->solve_mip(); - std::string basis_file = std::tmpnam(nullptr); + std::filesystem::path basis_file = std::tmpnam(nullptr); expec_solver->write_basis(basis_file); SolverAbstract::Ptr current_solver = factory.create_solver(solver_name); diff --git a/tests/cpp/solvers_interface/test_modifying_problem.cpp b/tests/cpp/solvers_interface/test_modifying_problem.cpp index 2cc5e04c1..7edc4daaf 100644 --- a/tests/cpp/solvers_interface/test_modifying_problem.cpp +++ b/tests/cpp/solvers_interface/test_modifying_problem.cpp @@ -13,7 +13,7 @@ TEST_CASE("Modification: deleting rows", "[modif][del-rows]") { auto inst = GENERATE(MIP_TOY, MULTIKP, UNBD_PRB, INFEAS_PRB); SECTION("Loop on instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // solver declaration @@ -54,7 +54,7 @@ TEST_CASE("Modification: add rows", "[modif][add-rows]") { for (auto const& solver_name : factory.get_solvers_list()) { // trying each type of row for (auto const& constr_type : {'L', 'G', 'E'}) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // solver declaration @@ -140,7 +140,7 @@ TEST_CASE("Modification: change obj", "[modif][chg-obj]") { auto inst = GENERATE(MIP_TOY, MULTIKP, UNBD_PRB, INFEAS_PRB); SECTION("Loop on instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // solver declaration @@ -182,7 +182,7 @@ TEST_CASE("Modification: change right-hand side", "[modif][chg-rhs]") { auto inst = GENERATE(MIP_TOY, MULTIKP, UNBD_PRB, INFEAS_PRB); SECTION("Loop on instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // solver declaration @@ -222,7 +222,7 @@ TEST_CASE("Modification: change matrix coefficient", "[modif][chg-coef]") { auto inst = GENERATE(MIP_TOY, MULTIKP, UNBD_PRB, INFEAS_PRB); SECTION("Loop on instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // solver declaration @@ -279,7 +279,7 @@ TEST_CASE("Modification: add columns", "[modif][add-cols]") { for (auto const& solver_name : factory.get_solvers_list()) { // testing add 1 and 2 columns for (int newcol = 1; newcol < 3; newcol++) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // solver declaration @@ -411,7 +411,7 @@ TEST_CASE("Modification: change name of row and column", "[modif][chg-names]") { auto inst = GENERATE(MIP_TOY, MULTIKP); SECTION("Loop on instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; SolverAbstract::Ptr solver = factory.create_solver(solver_name); solver->init(); solver->read_prob_mps(instance); @@ -450,7 +450,7 @@ TEST_CASE("Modification: add cols and a row associated to those columns", auto inst = GENERATE(NET_MASTER); SECTION("Loop on instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; SolverAbstract::Ptr solver = factory.create_solver(solver_name); solver->init(); solver->read_prob_mps(instance); diff --git a/tests/cpp/solvers_interface/test_reading_problem.cpp b/tests/cpp/solvers_interface/test_reading_problem.cpp index aa42875fa..7b3ee6313 100644 --- a/tests/cpp/solvers_interface/test_reading_problem.cpp +++ b/tests/cpp/solvers_interface/test_reading_problem.cpp @@ -13,7 +13,7 @@ TEST_CASE("Un objet solveur peut etre cree et detruit", "[read][init]") { auto inst = GENERATE(MIP_TOY, MULTIKP); SECTION("Construction and destruction") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // solver declaration @@ -40,7 +40,7 @@ TEST_CASE("MPS file can be read and we can get number of columns", SECTION("Reading instance") { namespace fs = std::filesystem; for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; std::cout << "Current dir " << fs::current_path() << std::endl; std::cout << instance << std::endl; //======================================================================================== @@ -72,7 +72,7 @@ TEST_CASE("MPS file can be read and we can get number of rows", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -99,7 +99,7 @@ TEST_CASE("MPS file can be read and we can get number of integer variables", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -128,7 +128,7 @@ TEST_CASE( NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -155,7 +155,7 @@ TEST_CASE("MPS file can be read and we can get objective function coefficients", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -188,7 +188,7 @@ TEST_CASE("MPS file can be read and we can get matrix coefficients", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -235,7 +235,7 @@ TEST_CASE("MPS file can be read and we can get right hand side", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -270,7 +270,7 @@ TEST_CASE("MPS file can be read and we can get row types", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver Declaration SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -303,7 +303,7 @@ TEST_CASE("MPS file can be read and we can get types of columns", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver Declaration SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -335,7 +335,7 @@ TEST_CASE("MPS file can be read and we can get lower bounds on variables", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -367,7 +367,7 @@ TEST_CASE("MPS file can be read and we can get upper bounds on variables", NET_SP1, NET_SP2); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -407,7 +407,7 @@ TEST_CASE( NET_SP1, NET_SP2, SLACKS, REDUCED); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); REQUIRE(solver->get_number_of_instances() == 1); @@ -512,7 +512,7 @@ TEST_CASE( auto inst = GENERATE(MIP_TOY, MULTIKP); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -561,7 +561,7 @@ TEST_CASE("We can get the indices of rows and columns by their names", auto inst = GENERATE(MIP_TOY, MULTIKP); SECTION("Reading instance") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -605,7 +605,7 @@ TEST_CASE("Testing copy constructor", "[init][copy-constructor]") { auto inst = GENERATE(MIP_TOY, MULTIKP); SECTION("Construction and destruction") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance= datas[inst]._path; //======================================================================================== // Intial solver declaration and read problem diff --git a/tests/cpp/solvers_interface/test_solving_problem.cpp b/tests/cpp/solvers_interface/test_solving_problem.cpp index b9dd24370..30aa1dd17 100644 --- a/tests/cpp/solvers_interface/test_solving_problem.cpp +++ b/tests/cpp/solvers_interface/test_solving_problem.cpp @@ -13,7 +13,7 @@ TEST_CASE("A LP problem is solved", "[solve-lp]") { auto inst = GENERATE(MULTIKP); SECTION("Loop on the instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // Solver declaration and read problem @@ -64,7 +64,7 @@ TEST_CASE("A LP problem is solved and we can get the LP value", auto inst = GENERATE(MULTIKP, NET_MASTER, NET_SP1, NET_SP2); SECTION("Loop on the instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + auto instance = datas[inst]._path; //======================================================================================== // Solver declaration SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -88,7 +88,7 @@ TEST_CASE("A LP problem is solved and we can get the LP value", } success = false; - for (auto stat : datas[inst]._status) { + for (const auto &stat : datas[inst]._status) { if (stat == solver->SOLVER_STRING_STATUS[slv_status]) { SUCCEED(); success = true; @@ -122,7 +122,7 @@ TEST_CASE("A LP problem is solved and we can get the LP solution", auto inst = GENERATE(LP_TOY, SLACKS, REDUCED); SECTION("Loop on the instances") { for (auto const& solver_name : factory.get_solvers_list()) { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // Solver declaration and read problem SolverAbstract::Ptr solver = factory.create_solver(solver_name); @@ -146,7 +146,7 @@ TEST_CASE("A LP problem is solved and we can get the LP solution", } success = false; - for (auto stat : datas[inst]._status) { + for (const auto &stat : datas[inst]._status) { if (stat == solver->SOLVER_STRING_STATUS[slv_status]) { SUCCEED(); success = true; @@ -207,7 +207,7 @@ TEST_CASE("A problem is solved and we can get the optimal solution", for (auto const& solver_name : factory.get_solvers_list()) { // As CLP is a pure LP solver, it cannot pass this test if (solver_name != "CLP") { - std::string instance = datas[inst]._path; + std::filesystem::path instance = datas[inst]._path; //======================================================================================== // Solver declaration SolverAbstract::Ptr solver = factory.create_solver(solver_name); diff --git a/tests/cpp/tests_utils/CMakeLists.txt b/tests/cpp/tests_utils/CMakeLists.txt new file mode 100644 index 000000000..d6754296a --- /dev/null +++ b/tests/cpp/tests_utils/CMakeLists.txt @@ -0,0 +1,7 @@ + +add_library(tests_utils STATIC RandomDirGenerator.h RandomDirGenerator.cpp) +target_include_directories (helpers + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} +) +add_library (${PROJECT_NAME}::tests_utils ALIAS tests_utils) diff --git a/tests/cpp/tests_utils/RandomDirGenerator.cpp b/tests/cpp/tests_utils/RandomDirGenerator.cpp new file mode 100644 index 000000000..cd315fc60 --- /dev/null +++ b/tests/cpp/tests_utils/RandomDirGenerator.cpp @@ -0,0 +1,37 @@ +#include "RandomDirGenerator.h" +std::string timeToStr(const std::time_t& time_p) { + struct tm local_time; +#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__) + localtime_s(&local_time, &time_p); +#else // defined(__unix__) || (__APPLE__) + localtime_r(&time_p, &local_time); +#endif + const char* FORMAT = "%d-%m-%Y-%H-%M-%S"; + char buffer_l[100]; + strftime(buffer_l, sizeof(buffer_l), FORMAT, &local_time); + + return buffer_l; +} + +std::string GenerateRandomString(size_t len) { + srand(time(NULL)); + const auto abc = "abcdefghijklmnopqrstuvwxyz"; + + std::string ret = "XXXXXX"; + + for (int i = 0; i < len; ++i) { + ret[i] = abc[rand() % (sizeof(abc) - 1)]; + } + return ret; +} +std::filesystem::path GetRandomSubDirPath( + const std::filesystem::path& parentDir) { + return parentDir / + (timeToStr(std::time(nullptr)) + "-" + GenerateRandomString(6)); +} +std::filesystem::path CreateRandomSubDir( + const std::filesystem::path& parentDir) { + const auto subDirPath = GetRandomSubDirPath(parentDir); + std::filesystem::create_directory(subDirPath); + return subDirPath; +} diff --git a/tests/cpp/tests_utils/RandomDirGenerator.h b/tests/cpp/tests_utils/RandomDirGenerator.h new file mode 100644 index 000000000..e17fdf89e --- /dev/null +++ b/tests/cpp/tests_utils/RandomDirGenerator.h @@ -0,0 +1,15 @@ +#ifndef __RANDOMDIRGENERATOR_H_ +#define __RANDOMDIRGENERATOR_H_ +#include + +#include +#include + +std::string timeToStr(const std::time_t& time_p); + +std::string GenerateRandomString(size_t len); +std::filesystem::path CreateRandomSubDir( + const std::filesystem::path& parentDir); +std::filesystem::path GetRandomSubDirPath( + const std::filesystem::path& parentDir); +#endif //__RANDOMDIRGENERATOR_H_ \ No newline at end of file diff --git a/tests/cpp/zip_mps/CMakeLists.txt b/tests/cpp/zip_mps/CMakeLists.txt new file mode 100644 index 000000000..5925207e4 --- /dev/null +++ b/tests/cpp/zip_mps/CMakeLists.txt @@ -0,0 +1,9 @@ +add_executable(zip_mps_lib_tests zip_mps_lib_tests.cpp) + +target_link_libraries(zip_mps_lib_tests + PRIVATE + helpers + tests_utils + GTest::Main) +add_test(NAME zip_mps_lib_tests COMMAND zip_mps_lib_tests WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) +set_property(TEST zip_mps_lib_tests PROPERTY LABELS unit) diff --git a/tests/cpp/zip_mps/zip_mps_lib_tests.cpp b/tests/cpp/zip_mps/zip_mps_lib_tests.cpp new file mode 100644 index 000000000..b34ccb274 --- /dev/null +++ b/tests/cpp/zip_mps/zip_mps_lib_tests.cpp @@ -0,0 +1,139 @@ + + +#include +#include +#include +#include + +#include +#include + +#include "ArchiveReader.h" +#include "ArchiveWriter.h" +#include "FileInBuffer.h" +#include "RandomDirGenerator.h" +#include "gtest/gtest.h" + +/* +ARCHIVES COMPOSITION: + +data/mps_zip/archive1.zip +data/mps_zip/archive1 + |__ file1 + |__ file2 + |__ file3 + +*/ + +const auto mpsZipDir = std::filesystem::path ("data_test") / "mps_zip"; +const auto archive1 = mpsZipDir / "archive1.zip"; +const auto archive1Dir = mpsZipDir / "archive1"; +const auto archive1File1 = archive1Dir / "file1"; +const auto archive1File2 = archive1Dir / "file2"; +const auto archive1File3 = archive1Dir / "file3"; + +class ArchiveReaderTest : public ::testing::Test { + public: + ArchiveReaderTest() = default; + + const std::filesystem::path invalid_file_path = ""; + const std::string invalid_file_name = ""; +}; + +TEST_F(ArchiveReaderTest, ShouldFailIfInvalidFileIsGiven) { + std::stringstream expectedErrorString; + expectedErrorString << "Failed to Open Archive: " + << invalid_file_path.string() << std::endl + << " invalid status: " + std::to_string(MZ_OPEN_ERROR) + + " (" + std::to_string(MZ_OK) + " expected)"; + + try { + auto fileExt = ArchiveReader(invalid_file_path); + } catch (const ArchiveIOGeneralException& e) { + EXPECT_EQ(e.what(), expectedErrorString.str()); + } +} +bool equal_files(const std::filesystem::path& a, + const std::filesystem::path& b) { + std::ifstream stream{a}; + std::string file1{std::istreambuf_iterator(stream), + std::istreambuf_iterator()}; + + stream = std::ifstream{b}; + std::string file2{std::istreambuf_iterator(stream), + std::istreambuf_iterator()}; + + return file1 == file2; +} +TEST_F(ArchiveReaderTest, ShouldExtractFile1FromArchive1) { + auto fileExt = ArchiveReader(archive1); + ASSERT_EQ(fileExt.Open(), MZ_OK); + const auto tmpDir = std::filesystem::temp_directory_path(); + const auto expectedFilePath = tmpDir / archive1File1.filename(); + ASSERT_EQ(fileExt.ExtractFile(archive1File1.filename(), tmpDir), MZ_OK); + ASSERT_TRUE(std::filesystem::exists(expectedFilePath)); + ASSERT_TRUE(equal_files(expectedFilePath, archive1File1)); + ASSERT_EQ(fileExt.Close(), MZ_OK); + fileExt.Delete(); +} +class ArchiveWriterTest : public ::testing::Test { + public: + ArchiveWriterTest() = default; +}; + +FileBufferVector GetBufferVectorOfFilesInDir(const std::filesystem::path& dir) { + FileBufferVector result; + for (const auto& file : std::filesystem::directory_iterator(dir)) { + const auto& pathToFile = file.path(); + std::ifstream fileStream(pathToFile); + std::stringstream buffer; + buffer << fileStream.rdbuf(); + result.push_back({pathToFile.filename().string(), buffer.str()}); + } + return result; +} +void compareArchiveAndDir(const std::filesystem::path& archivePath, + const std::filesystem::path& dirPath, + const std::filesystem::path& tmpDir) { + void* reader = NULL; + + mz_zip_reader_create(&reader); + const auto& archive_path_str = archivePath.string(); + auto archive_path_c_str = archive_path_str.c_str(); + assert(mz_zip_reader_open_file(reader, archive_path_c_str) == MZ_OK); + //assert(mz_zip_reader_entry_open(reader) == MZ_OK); + + for (const auto &file : std::filesystem::directory_iterator(dirPath)) { + const auto &filename_path = file.path().filename(); + const auto &filename_str = filename_path.string(); + const auto searchFilename = filename_str.c_str(); + assert(mz_zip_reader_locate_entry(reader, searchFilename, 1) == MZ_OK); + assert(mz_zip_reader_entry_open(reader) == MZ_OK); + const auto extractedFilePath = tmpDir / searchFilename; + mz_zip_reader_entry_save_file(reader, extractedFilePath.string().c_str()); + assert(equal_files(extractedFilePath, file.path())); + } + mz_zip_reader_close(reader); + mz_zip_reader_delete(&reader); +} +TEST_F(ArchiveWriterTest, ShouldCreateArchiveWithVecBuffer) { + const auto tmpArchiveRoot = std::filesystem::temp_directory_path(); + const auto tmpDir = + CreateRandomSubDir(std::filesystem::temp_directory_path()); + std::string archiveName = GenerateRandomString(6); + archiveName += ".zip"; + const auto archivePath = tmpDir / archiveName; + ArchiveWriter writer(archivePath); + ASSERT_EQ(writer.Open(), MZ_OK); + const auto mpsBufferVec = GetBufferVectorOfFilesInDir(archive1Dir); + for (const auto& mpsBuf : mpsBufferVec) { + ASSERT_EQ(writer.AddFileInArchive(mpsBuf), MZ_OK); + } + ASSERT_EQ(writer.Close(), MZ_OK); + writer.Delete(); + compareArchiveAndDir(archivePath, archive1Dir, tmpDir); +} +class FileInBufferTest : public ::testing::Test { + public: + FileInBufferTest() = default; +}; diff --git a/tests/end_to_end/benders/test_bendersEndToEnd.py b/tests/end_to_end/benders/test_bendersEndToEnd.py index 687e10b87..a23091a0b 100644 --- a/tests/end_to_end/benders/test_bendersEndToEnd.py +++ b/tests/end_to_end/benders/test_bendersEndToEnd.py @@ -6,6 +6,7 @@ import sys import yaml import numpy as np +from zipfile import ZipFile, ZIP_DEFLATED # File RESULT_FILE_PATH # Json file containing @@ -72,15 +73,14 @@ def launch_optimization(data_path, commands, status=None): # arguments : # - expected_results_dict : Dict of expected values to compare with ones present in # in file out.json +# - output_path : Path to the output file out.json -def check_optimization_json_output(expected_results_dict): +def check_optimization_json_output(expected_results_dict, output_path: Path): # Loading output from optimization process curr_instance_json = {} - output_path = expected_results_dict['path'] + \ - "/expansion/out.json" with open(output_path, 'r') as jsonFile: curr_instance_json = json.load(jsonFile) @@ -109,7 +109,7 @@ def check_optimization_json_output(expected_results_dict): rtol=1e-6, atol=0) -def run_solver(install_dir, solver, allow_run_as_root=False): +def run_solver(install_dir, solver, tmp_path, allow_run_as_root=False): # Loading expected results from json RESULT_FILE_PATH with open(RESULT_FILE_PATH, 'r') as jsonFile: expected_results_dict = json.load(jsonFile) @@ -128,13 +128,18 @@ def run_solver(install_dir, solver, allow_run_as_root=False): instance_path = expected_results_dict[instance]['path'] command = [e for e in pre_command] command.append(executable_path) + options_file = expected_results_dict[instance]['option_file'] command.append( - expected_results_dict[instance]['option_file'] + options_file ) status = expected_results_dict[instance]["status"] if "status" in expected_results_dict[instance] else None - launch_optimization(instance_path, command, status) + tmp_study = tmp_path / \ + (Path(instance_path).name+"-"+Path(options_file).stem) + shutil.copytree(instance_path, tmp_study) + + launch_optimization(tmp_study, command, status) check_optimization_json_output( - expected_results_dict[instance]) + expected_results_dict[instance], tmp_study/"expansion/out.json") def get_solver_exe(solver: str): diff --git a/tests/end_to_end/benders/test_bendersMergeMpsEndToEnd.py b/tests/end_to_end/benders/test_bendersMergeMpsEndToEnd.py index f141674ef..34d3db759 100644 --- a/tests/end_to_end/benders/test_bendersMergeMpsEndToEnd.py +++ b/tests/end_to_end/benders/test_bendersMergeMpsEndToEnd.py @@ -2,7 +2,8 @@ import pytest from test_bendersEndToEnd import run_solver + @pytest.mark.optim @pytest.mark.mergemps -def test_001_mergemps(install_dir): - run_solver(install_dir, 'MERGE_MPS') \ No newline at end of file +def test_001_mergemps(install_dir, tmp_path): + run_solver(install_dir, 'MERGE_MPS', tmp_path) diff --git a/tests/end_to_end/benders/test_bendersSequentialEndToEnd.py b/tests/end_to_end/benders/test_bendersSequentialEndToEnd.py index 014205f32..088c2b1d4 100644 --- a/tests/end_to_end/benders/test_bendersSequentialEndToEnd.py +++ b/tests/end_to_end/benders/test_bendersSequentialEndToEnd.py @@ -2,8 +2,9 @@ from test_bendersEndToEnd import run_solver ## TESTS ## + + @pytest.mark.optim @pytest.mark.benderssequential -def test_001_sequential(install_dir): - run_solver(install_dir, 'BENDERS_SEQUENTIAL') - +def test_001_sequential(install_dir, tmp_path): + run_solver(install_dir, 'BENDERS_SEQUENTIAL', tmp_path) diff --git a/tests/end_to_end/benders/test_bendersmpibendersEndToEnd.py b/tests/end_to_end/benders/test_bendersmpibendersEndToEnd.py index 4d94439ed..8496b16ca 100644 --- a/tests/end_to_end/benders/test_bendersmpibendersEndToEnd.py +++ b/tests/end_to_end/benders/test_bendersmpibendersEndToEnd.py @@ -4,5 +4,5 @@ @pytest.mark.optim @pytest.mark.bendersmpi -def test_001_mpibenders(install_dir, allow_run_as_root): - run_solver(install_dir, 'BENDERS_MPI', allow_run_as_root) +def test_001_mpibenders(install_dir, allow_run_as_root, tmp_path): + run_solver(install_dir, 'BENDERS_MPI', tmp_path, allow_run_as_root) diff --git a/tests/end_to_end/lpnamer/test_lpnamerEndToEnd.py b/tests/end_to_end/lpnamer/test_lpnamerEndToEnd.py index bc7b67b36..84d5766d7 100644 --- a/tests/end_to_end/lpnamer/test_lpnamerEndToEnd.py +++ b/tests/end_to_end/lpnamer/test_lpnamerEndToEnd.py @@ -1,27 +1,34 @@ import filecmp +import glob import os import shutil import subprocess import pytest +import zipfile from pathlib import Path +MPS_ZIP = "MPS_ZIP_FILE.zip" DATA_TEST = Path("../../../data_test/") DATA_TEST_INTEGER = DATA_TEST / "tests_lpnamer" / "tests_integer" DATA_TEST_RELAXED = DATA_TEST / "tests_lpnamer" / "tests_relaxed" -TEST_LP_INTEGER_01 = DATA_TEST_INTEGER / "test_lpnamer_01" / "output" / "economy" -TEST_LP_INTEGER_02 = DATA_TEST_INTEGER / "test_one_link_one_candidate_1week" / "output" / "economy/" +TEST_LP_INTEGER_01 = DATA_TEST_INTEGER / \ + "test_lpnamer_01" / "output" / "economy" +TEST_LP_INTEGER_02 = DATA_TEST_INTEGER / \ + "test_one_link_one_candidate_1week" / "output" / "economy/" TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB = DATA_TEST_INTEGER / "test_one_link_two_candidates_simple_prob" \ - / "output" / "economy" -TEST_LP_INTEGER_MULTIPLE_CANDIDATES = DATA_TEST_INTEGER / "test_one_link_two_candidates_1week" / "output" / "economy" + / "output" / "economy" +TEST_LP_INTEGER_MULTIPLE_CANDIDATES = DATA_TEST_INTEGER / \ + "test_one_link_two_candidates_1week" / "output" / "economy" TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB_HURDLES = DATA_TEST_INTEGER / "test_one_link_two_candidates_simple_prob_hurdle_cost" \ - / "output" / "economy" + / "output" / "economy" TEST_LP_INTEGER_MULTIPLE_CANDIDATES_SIMPLE_PROB_NULL_PROFILE = DATA_TEST_INTEGER / "test_one_link_two_candidates_simple_prob_null_profile" \ - / "output" / "economy" + / "output" / "economy" -TEST_LP_RELAXED_01 = DATA_TEST_RELAXED / "test_one_link_one_candidate-relaxed" / "output" / "economy/" +TEST_LP_RELAXED_01 = DATA_TEST_RELAXED / \ + "test_one_link_one_candidate-relaxed" / "output" / "economy/" TEST_LP_RELAXED_02 = DATA_TEST_RELAXED / "SmallTestSixCandidatesWithAlreadyInstalledCapacity-relaxed" / "output" \ - / "economy" + / "economy" test_data = [ (TEST_LP_INTEGER_01, "integer"), (TEST_LP_INTEGER_02, "integer"), @@ -44,17 +51,23 @@ def setup_and_teardown_lp_directory(request): if Path(lp_dir).is_dir(): shutil.rmtree(lp_dir) Path(lp_dir).mkdir(exist_ok=True) + + list_files = list(Path(test_dir).glob("*.mps")) + list_files.extend(list(Path(test_dir).glob("variables*.txt"))) + with zipfile.ZipFile(lp_dir/MPS_ZIP, "w") as write_mps_zip: + for file in list_files: + write_mps_zip.write( + file, file.name, compress_type=zipfile.ZIP_DEFLATED) yield - #shutil.rmtree(lp_dir) -@pytest.mark.parametrize("test_dir,master_mode", test_data) +@ pytest.mark.parametrize("test_dir,master_mode", test_data) def test_lp_directory_files(install_dir, test_dir, master_mode, setup_and_teardown_lp_directory): # given launch_and_compare_lp_with_reference(install_dir, master_mode, test_dir) -@pytest.mark.parametrize("test_dir,master_mode", test_data_multiple_candidates) +@ pytest.mark.parametrize("test_dir,master_mode", test_data_multiple_candidates) def test_lp_multiple_candidates(install_dir, test_dir, master_mode, setup_and_teardown_lp_directory): launch_and_compare_lp_with_reference(install_dir, master_mode, test_dir) @@ -64,14 +77,23 @@ def launch_and_compare_lp_with_reference(install_dir, master_mode, test_dir): reference_lp_dir = test_dir / "reference_lp" lp_dir = test_dir / "lp" lp_namer_exe = Path(install_dir) / "lp_namer" + zip_path = (lp_dir/MPS_ZIP).resolve() + study_dir = test_dir.resolve() os.chdir(test_dir.parent) - launch_command = [str(lp_namer_exe), "-o", str(test_dir.name), "-e", "contraintes.txt", "-f", master_mode] + launch_command = [str(lp_namer_exe), "-o", str(study_dir), "-a", str(zip_path), + "-e", "contraintes.txt", "-f", master_mode] # when returned_l = subprocess.run(launch_command, shell=False) # then os.chdir(old_path) + + with zipfile.ZipFile(zip_path, "r") as mps_zip_file: + mps_zip_file.extractall(zip_path.parent) + + os.remove(zip_path) files_to_compare = os.listdir(reference_lp_dir) - match, mismatch, errors = filecmp.cmpfiles(reference_lp_dir, lp_dir, files_to_compare) + match, mismatch, errors = filecmp.cmpfiles( + reference_lp_dir, lp_dir, files_to_compare) assert len(match) == len(files_to_compare) assert len(mismatch) == 0 assert returned_l.returncode == 0 diff --git a/tests/end_to_end/restart/studies.py b/tests/end_to_end/restart/studies.py new file mode 100644 index 000000000..23900c7da --- /dev/null +++ b/tests/end_to_end/restart/studies.py @@ -0,0 +1,47 @@ +study_parameters = "study" +study_values = [ + { + "path": "../../../data_test/mini_network/", + "option_file": "options_default_restart.json", + "status": "OPTIMAL", + "optimal_value": 233.0, + "optimal_variables": { + "t": 4.0, + "p": 6.0 + }, + "output_file": "out_default_test_restart.json", + "last_iteration_file": "last_iteration_default_test_restart.json" + }, { + "path": "../../../data_test/mini_network/", + "option_file": "options_weights_restart.json", + "status": "OPTIMAL", + "optimal_value": 19.0, + "optimal_variables": { + "t": 2.0, + "p": 3.0 + }, + "output_file": "out_weights_test_restart.json", + "last_iteration_file": "last_iteration_weights_test_restart.json" + }, + { + "path": "../../../data_test/mini_instance_LP/", + "option_file": "options_default_restart.json", + "status": "OPTIMAL", + "optimal_value": 3.25, + "optimal_variables": { + "x": 1.5 + }, + "output_file": "out_default_test_restart.json", + "last_iteration_file": "last_iteration_default_test_restart.json" + }, { + "path": "../../../data_test/mini_instance_MIP/", + "option_file": "options_default_restart.json", + "status": "OPTIMAL", + "optimal_value": 3.50, + "optimal_variables": { + "x": 2.0 + }, + "output_file": "out_default_test_restart.json", + "last_iteration_file": "last_iteration_default_test_restart.json" + } +] diff --git a/tests/end_to_end/restart/test_restartBendersEndToEnd.py b/tests/end_to_end/restart/test_restartBendersEndToEnd.py index 54587e237..ae79e8f93 100644 --- a/tests/end_to_end/restart/test_restartBendersEndToEnd.py +++ b/tests/end_to_end/restart/test_restartBendersEndToEnd.py @@ -4,6 +4,7 @@ import subprocess from pathlib import Path import sys +from zipfile import ZIP_DEFLATED, ZipFile import yaml import numpy as np @@ -18,7 +19,6 @@ # - optimal_variables : # * one entry by variable with variable name as key : optimal # value of the variable -RESULT_FILE_PATH = Path('resultTest.json') # File CONFIG_FILE_PATH # yaml file containing executable name @@ -54,14 +54,11 @@ def launch_optimization(data_path, commands, status=None): data_path_ = Path(data_path) expansion_dir = data_path_ / "expansion" assert expansion_dir.is_dir() - shutil.copyfile(data_path_ / "out_default_test_restart.json", - expansion_dir / "out.json") - shutil.copyfile(data_path_ / "last_iteration_default_test_restart.json", - expansion_dir / "last_iteration.json") os.chdir(data_path) # Launching optimization from instance folder - process = subprocess.run(commands, stdout=subprocess.PIPE, stderr=None) + process = subprocess.run( + commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Back to working directory os.chdir(owd) @@ -75,15 +72,15 @@ def launch_optimization(data_path, commands, status=None): # arguments : # - expected_results_dict : Dict of expected values to compare with ones present in # in file out.json +# - output_path : Path to the output file out.json -def check_optimization_json_output(expected_results_dict): +def check_optimization_json_output(expected_results_dict, output_path: Path): # Loading output from optimization process curr_instance_json = {} - output_path = expected_results_dict['path'] + \ - "/expansion/out.json" + # output_path = expected_results_dict['path'] / "expansion/out.json" with open(output_path, 'r') as jsonFile: curr_instance_json = json.load(jsonFile) @@ -112,10 +109,7 @@ def check_optimization_json_output(expected_results_dict): rtol=1e-6, atol=0) -def run_solver(install_dir, solver, allow_run_as_root=False): - # Loading expected results from json RESULT_FILE_PATH - with open(RESULT_FILE_PATH, 'r') as jsonFile: - expected_results_dict = json.load(jsonFile) +def run_solver(install_dir, solver, tmp_study, instance, allow_run_as_root=False): solver_executable = get_solver_exe(solver) @@ -127,29 +121,26 @@ def run_solver(install_dir, solver, allow_run_as_root=False): executable_path = str( (Path(install_dir) / Path(solver_executable)).resolve()) - for instance in expected_results_dict: - instance_path = expected_results_dict[instance]['path'] - command = [e for e in pre_command] - command.append(executable_path) - command.append( - expected_results_dict[instance]['option_file'] - ) - status = expected_results_dict[instance]["status"] if "status" in expected_results_dict[instance] else None - - data_path_ = Path(instance_path) - expansion_dir = data_path_ / "expansion" - if expansion_dir.is_dir(): - shutil.rmtree(expansion_dir) - - expansion_dir.mkdir() - shutil.copyfile(data_path_ / expected_results_dict[instance]["output_file"], - expansion_dir / "out.json") - shutil.copyfile(data_path_ / expected_results_dict[instance]["last_iteration_file"], - expansion_dir / "last_iteration.json") - - launch_optimization(instance_path, command, status) + command = [e for e in pre_command] + command.append(executable_path) + command.append( + instance['option_file'] + ) + status = instance["status"] if "status" in instance else None + + expansion_dir = tmp_study / "expansion" + if expansion_dir.is_dir(): + shutil.rmtree(expansion_dir) + + expansion_dir.mkdir() + shutil.copyfile(tmp_study / instance["output_file"], + expansion_dir / "out.json") + shutil.copyfile(tmp_study / instance["last_iteration_file"], + expansion_dir / "last_iteration.json") + + launch_optimization(tmp_study, command, status) check_optimization_json_output( - expected_results_dict[instance]) + instance, tmp_study/"expansion/out.json") def get_solver_exe(solver: str): diff --git a/tests/end_to_end/restart/test_restartBendersSequentialEndToEnd.py b/tests/end_to_end/restart/test_restartBendersSequentialEndToEnd.py index 69626bc61..1d107f0b8 100644 --- a/tests/end_to_end/restart/test_restartBendersSequentialEndToEnd.py +++ b/tests/end_to_end/restart/test_restartBendersSequentialEndToEnd.py @@ -1,10 +1,31 @@ +from pathlib import Path +import shutil import pytest -from test_restartBendersEndToEnd import run_solver +from .test_restartBendersEndToEnd import run_solver ## TESTS ## +from .studies import study_parameters, study_values + + +@pytest.mark.parametrize( + study_parameters, + study_values, +) @pytest.mark.optim @pytest.mark.benderssequential -def test_001_sequential(install_dir): - run_solver(install_dir, 'BENDERS_SEQUENTIAL') +def test_001_sequential(install_dir, tmp_path, study): + + instance_path = Path(study['path']) + + expansion_dir = instance_path / "expansion" + if expansion_dir.is_dir(): + shutil.rmtree(expansion_dir) + + expansion_dir.mkdir() + shutil.copyfile(instance_path / study["output_file"], + expansion_dir / "out.json") + shutil.copyfile(instance_path / study["last_iteration_file"], + expansion_dir / "last_iteration.json") + run_solver(install_dir, 'BENDERS_SEQUENTIAL', instance_path, study) diff --git a/tests/end_to_end/restart/test_restartBendersmpibendersEndToEnd.py b/tests/end_to_end/restart/test_restartBendersmpibendersEndToEnd.py index b24e6ef4a..6a451fd2b 100644 --- a/tests/end_to_end/restart/test_restartBendersmpibendersEndToEnd.py +++ b/tests/end_to_end/restart/test_restartBendersmpibendersEndToEnd.py @@ -1,8 +1,28 @@ +from pathlib import Path +import shutil import pytest -from test_restartBendersEndToEnd import run_solver +from .test_restartBendersEndToEnd import run_solver +from .studies import study_parameters, study_values +@pytest.mark.parametrize( + study_parameters, + study_values, +) @pytest.mark.optim @pytest.mark.bendersmpi -def test_001_mpibenders(install_dir, allow_run_as_root): - run_solver(install_dir, 'BENDERS_MPI', allow_run_as_root) +def test_001_mpibenders(install_dir, tmp_path, study, allow_run_as_root): + + instance_path = Path(study['path']) + + expansion_dir = instance_path / "expansion" + if expansion_dir.is_dir(): + shutil.rmtree(expansion_dir) + + expansion_dir.mkdir() + shutil.copyfile(instance_path / study["output_file"], + expansion_dir / "out.json") + shutil.copyfile(instance_path / study["last_iteration_file"], + expansion_dir / "last_iteration.json") + run_solver(install_dir, 'BENDERS_MPI', + instance_path, study, allow_run_as_root) diff --git a/tests/python/test_antares_driver.py b/tests/python/test_antares_driver.py index 692267801..93c6def17 100644 --- a/tests/python/test_antares_driver.py +++ b/tests/python/test_antares_driver.py @@ -231,7 +231,7 @@ def test_antares_cmd(self, tmp_path): # mock subprocess.run with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, 1) - expected_cmd = [exe_path, study_dir, "--force-parallel", "1"] + expected_cmd = [exe_path, study_dir, "--force-parallel", "1", "-z"] run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 ) @@ -243,7 +243,7 @@ def test_antares_cmd_force_parallel_option(self, tmp_path): antares_driver = AntaresDriver(exe_path) with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, n_cpu) - expected_cmd = [exe_path, study_dir, "--force-parallel", str(n_cpu)] + expected_cmd = [exe_path, study_dir, "--force-parallel", str(n_cpu), "-z"] run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 ) @@ -261,6 +261,7 @@ def test_invalid_n_cpu(self, tmp_path): study_dir, "--force-parallel", str(expected_n_cpu), + "-z" ] run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 @@ -275,7 +276,7 @@ def test_remove_log_file(self, tmp_path): antares_driver = AntaresDriver(exe_path) with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, n_cpu) - expected_cmd = [str(exe_path), study_dir, "--force-parallel", str(n_cpu)] + expected_cmd = [str(exe_path), study_dir, "--force-parallel", str(n_cpu), "-z"] run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 ) @@ -336,7 +337,7 @@ def test_preserve_adequacy_option_after_run(self, tmp_path): antares_driver = AntaresDriver(exe_path) with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, n_cpu) - expected_cmd = [str(exe_path), study_dir, "--force-parallel", str(n_cpu)] + expected_cmd = [str(exe_path), study_dir, "--force-parallel", str(n_cpu), "-z"] run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 ) @@ -358,7 +359,7 @@ def test_preserve_general_file_section_missing(self, tmp_path): antares_driver = AntaresDriver(exe_path) with patch(SUBPROCESS_RUN, autospec=True) as run_function: antares_driver.launch(study_dir, n_cpu) - expected_cmd = [str(exe_path), study_dir, "--force-parallel", str(n_cpu)] + expected_cmd = [str(exe_path), study_dir, "--force-parallel", str(n_cpu), "-z"] run_function.assert_called_once_with( expected_cmd, shell=False, stdout=-3, stderr=-3 ) diff --git a/tests/python/test_problem_generator_driver.py b/tests/python/test_problem_generator_driver.py index 268bbcd0c..4a705d0ab 100644 --- a/tests/python/test_problem_generator_driver.py +++ b/tests/python/test_problem_generator_driver.py @@ -4,6 +4,7 @@ from pathlib import Path from datetime import date, datetime from unittest.mock import ANY, mock_open, patch +import shutil from .file_creation import _create_weight_file from antares_xpansion.problem_generator_driver import ProblemGeneratorData, ProblemGeneratorDriver @@ -12,6 +13,7 @@ from antares_xpansion.general_data_reader import GeneralDataIniReader SUBPROCESS_RUN = "antares_xpansion.problem_generator_driver.subprocess.run" +zipfile_ZipFile = "antares_xpansion.problem_generator_driver.zipfile.ZipFile" class TestProblemGeneratorDriver: @@ -48,73 +50,87 @@ def test_no_area_file(self, tmp_path): problem_generator_driver = ProblemGeneratorDriver( self.empty_pblm_gen_data) + zipped_output = self.get_zipped_output(tmp_path) + with pytest.raises(ProblemGeneratorDriver.AreaFileException): - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(zipped_output, False) + + def get_zipped_output(self, tmp_path): + zipped_output = tmp_path.parent / (tmp_path.name+".zip") + shutil.make_archive(tmp_path, "zip", tmp_path) + return zipped_output def test_more_than_1_area_file(self, tmp_path): self._create_empty_area_file(tmp_path) self._create_empty_area_file(tmp_path) - + zipped_output = self.get_zipped_output(tmp_path) problem_generator_driver = ProblemGeneratorDriver( self.empty_pblm_gen_data) with pytest.raises(ProblemGeneratorDriver.AreaFileException): - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(zipped_output, False) def test_no_interco_file(self, tmp_path): self._create_empty_area_file(tmp_path) + zipped_output = self.get_zipped_output(tmp_path) problem_generator_driver = ProblemGeneratorDriver( self.empty_pblm_gen_data) with pytest.raises(ProblemGeneratorDriver.IntercoFilesException): - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(zipped_output, False) def test_more_than_1_interco_file(self, tmp_path): self._create_empty_area_file(tmp_path) self._create_empty_interco_file(tmp_path) self._create_empty_interco_file(tmp_path) - + zipped_output = self.get_zipped_output(tmp_path) problem_generator_driver = ProblemGeneratorDriver( self.empty_pblm_gen_data) with pytest.raises(ProblemGeneratorDriver.IntercoFilesException): - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(zipped_output, False) def test_lp_namer_exe_does_not_exit(self, tmp_path): self._create_empty_area_file(tmp_path) self._create_empty_interco_file(tmp_path) - + zipped_output = self.get_zipped_output(tmp_path) problem_generator_driver = ProblemGeneratorDriver( self.empty_pblm_gen_data) with pytest.raises(ProblemGeneratorDriver.LPNamerExeError): - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(zipped_output, False) def test_mps_txt_creation(self, tmp_path): self._create_empty_area_file(tmp_path) self._create_empty_interco_file(tmp_path) + zipped_output = self.get_zipped_output(tmp_path) problem_generator_driver = ProblemGeneratorDriver( self.empty_pblm_gen_data) with pytest.raises(ProblemGeneratorDriver.LPNamerExeError): - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(zipped_output, False) - mps_txt_file = tmp_path / "mps.txt" + xpansion_output = self.xpansion_output(tmp_path) + mps_txt_file = xpansion_output / "mps.txt" assert mps_txt_file.exists() + def xpansion_output(self, tmp_path): + return tmp_path.parent / (tmp_path.name + '-Xpansion') + def test_mps_txt_content(self, tmp_path): expected_results = self._get_expected_mps_txt(tmp_path) self._create_empty_area_file(tmp_path) self._create_empty_interco_file(tmp_path) - + zipped_output = self.get_zipped_output(tmp_path) problem_generator_driver = ProblemGeneratorDriver( self.empty_pblm_gen_data) with pytest.raises(ProblemGeneratorDriver.LPNamerExeError): - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(zipped_output, False) - mps_txt_file = tmp_path / problem_generator_driver.MPS_TXT + mps_txt_file = self.xpansion_output( + tmp_path) / problem_generator_driver.MPS_TXT assert mps_txt_file.exists() with open(mps_txt_file) as mps_txt: @@ -134,16 +150,18 @@ def test_clear_old_log(self, tmp_path): active_years=[]) self._create_empty_area_file(tmp_path) self._create_empty_interco_file(tmp_path) - + output_zipped = self.get_zipped_output(tmp_path) log_file_name = self.lp_exe + ".log" - log_file = tmp_path / log_file_name + xpansion_dir = self.xpansion_output(tmp_path) + log_file = xpansion_dir / log_file_name + xpansion_dir.mkdir() log_file.write_text("bla bla") assert log_file.exists() pblm_gen = ProblemGeneratorDriver(pblm_gen_data) with patch(SUBPROCESS_RUN, autospec=True): with pytest.raises(ProblemGeneratorDriver.LPNamerExecutionError): - pblm_gen.launch(tmp_path, False) + pblm_gen.launch(output_zipped, False) assert not log_file.exists() @@ -159,12 +177,14 @@ def test_clean_lp_dir_before_run(self, tmp_path): active_years=[]) self._create_empty_area_file(tmp_path) self._create_empty_interco_file(tmp_path) - + output_zipped = self.get_zipped_output(tmp_path) log_file_name = self.lp_exe + ".log" - log_file = tmp_path / log_file_name + xpansion_dir = self.xpansion_output(tmp_path) + xpansion_dir.mkdir() + log_file = xpansion_dir / log_file_name log_file.write_text("bla bla") - lp_dir = tmp_path / "lp" + lp_dir = xpansion_dir / "lp" lp_dir.mkdir() lp_dir_sub_file_1 = lp_dir / "file1" lp_dir_sub_file_1.write_text("") @@ -174,7 +194,7 @@ def test_clean_lp_dir_before_run(self, tmp_path): problem_generator_driver = ProblemGeneratorDriver(pblm_gen_data) with patch(SUBPROCESS_RUN, autospec=True) as run_function: run_function.return_value.returncode = 0 - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(output_zipped, False) assert lp_dir.exists() assert not lp_dir_sub_file_1.exists() @@ -192,12 +212,12 @@ def test_weight_file_name_fails_if_file_does_not_exist(self, tmp_path): active_years=[]) self._create_empty_area_file(tmp_path) self._create_empty_interco_file(tmp_path) - + output_zipped = self.get_zipped_output(tmp_path) expected_message = f'Illegal value : {str(file_path)} is not an existent yearly-weights file' problem_generator_driver = ProblemGeneratorDriver(pblm_gen_data) with pytest.raises(FileNotFoundError) as expect: - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(output_zipped, False) assert str(expect.value) == expected_message def test_weight_file_name_fails_if_there_is_one_negative_value(self, tmp_path): @@ -296,15 +316,16 @@ def test_weight_file_is_created(self, tmp_path): "WEIGHT_SUM " + str(float(sum(weight_list)))) self._create_empty_area_file(tmp_path) self._create_empty_interco_file(tmp_path) + zipped_output = self.get_zipped_output(tmp_path) problem_generator_driver = ProblemGeneratorDriver(pblm_gen_data) with patch(SUBPROCESS_RUN, autospec=True) as run_function: run_function.return_value.returncode = 0 - problem_generator_driver.launch(tmp_path, False) + problem_generator_driver.launch(zipped_output, False) assert file_path.exists() - with open(tmp_path / "lp" / weight_file_name, "r") as file: + with open(self.xpansion_output(tmp_path) / "lp" / weight_file_name, "r") as file: lines = file.readlines() assert lines == expected_weight_file_content diff --git a/tests/python/test_yearly_weigth_writer.py b/tests/python/test_yearly_weigth_writer.py index b31b66244..4cf1d42f3 100644 --- a/tests/python/test_yearly_weigth_writer.py +++ b/tests/python/test_yearly_weigth_writer.py @@ -1,4 +1,5 @@ from pathlib import Path +import shutil from antares_xpansion.yearly_weight_writer import YearlyWeightWriter from file_creation import _create_weight_file, _create_empty_file_from_list @@ -17,7 +18,10 @@ def test_weight_file_write(tmp_path): f"problem-{active_years[2]}-1-AAAAMMDD-hhmmss.mps", f"problem-{active_years[2]}-24-AAAAMMDD-hhmmss.mps") _create_empty_file_from_list(tmp_path, mps_files) - YearlyWeightWriter(tmp_path).create_weight_file(weight_list, input_weight_file_name, active_years) + shutil.make_archive(tmp_path, "zip", tmp_path) + zipped_out = tmp_path.parent/(tmp_path.name+".zip") + YearlyWeightWriter(tmp_path, zipped_out).create_weight_file( + weight_list, input_weight_file_name, active_years) expected_content = f"problem-{active_years[0]}-1-AAAAMMDD-hhmmss {weight_list[0]}\n" \ f"problem-{active_years[0]}-24-AAAAMMDD-hhmmss {weight_list[0]}\n" \ @@ -27,6 +31,6 @@ def test_weight_file_write(tmp_path): f"problem-{active_years[2]}-24-AAAAMMDD-hhmmss {weight_list[2]}\n" \ f"WEIGHT_SUM {sum(weight_list)}" - with open(YearlyWeightWriter(tmp_path).output_dir / input_weight_file_name, 'r') as write_file: + with open(YearlyWeightWriter(tmp_path, zipped_out).output_dir / input_weight_file_name, 'r') as write_file: content = write_file.read() assert content == expected_content