Merge pull request #492 from AntaresSimulatorTeam/feature/zip_mps

Feature/zip mps

.github/workflows/download-extract-precompiled-libraries-zip/action.yml

@@ -31,4 +31,4 @@ runs:
            
            wget https://github.com/AntaresSimulatorTeam/Antares_Simulator/releases/download/v${{inputs.antares-version}}/rte-antares-${{inputs.antares-version}}-installer-64bits.zip
            unzip rte-antares-${{inputs.antares-version}}-installer-64bits.zip
-           rm -rf rte-antares-${{inputs.antares-version}}-installer-64bits.zip
+           rm -rf rte-antares-${{inputs.antares-version}}-installer-64bits.zip

CMakeLists.txt

@@ -164,6 +164,7 @@ sbeParseJson(antares_version jsonContent)
 #need to know antares solver version because antares-solver targets still refers to antares version
 SET(ANTARES_VERSION ${antares_version.antares_version_executable})
 SET(ANTARES_VERSION_TAG ${antares_version.antares_version})
+SET(MINIZIP_TAG ${antares_version.minizip_ng_version})
 
 sbeClearJson(antares_version)
 
@@ -183,7 +184,7 @@ endif()
 add_subdirectory(antares-deps)
 
 add_subdirectory(cmake/dependencies ${CMAKE_CURRENT_BINARY_DIR}/build_deps)
-
+find_package(minizip-ng REQUIRED)
 # ---------------------------------------------------------------------------
 # Boost
 # ---------------------------------------------------------------------------

antares-version.json

@@ -2,5 +2,6 @@
   "antares_version": "8.4.0-rc2",
   "antares_version_executable": "8.4",
   "antares_xpansion_version": "0.8.0",
-  "antares_deps_version": "2.0.5"
+  "antares_deps_version": "2.0.5",
+  "minizip_ng_version": "3.0.6"
 }

cmake/dependencies/CMakeLists.txt

@@ -34,3 +34,4 @@ if (NOT antares-solver_FOUND)
 endif()
 
 endif()
+

data_test/mini_instance_MIP/options_default.json

@@ -12,5 +12,6 @@
      "CSV_NAME": "benders_output_trace",
      "BOUND_ALPHA": true,
      "JSON_FILE": "./expansion/out.json",
-     "LAST_ITERATION_JSON_FILE": "./expansion/last_iteration.json"
-}
+     "LAST_ITERATION_JSON_FILE": "./expansion/last_iteration.json",
+     "MPS_ZIP_FILE": "MPS_ZIP_FILE.zip"
+}

data_test/mini_network/options_default.json

@@ -15,5 +15,6 @@
      "BOUND_ALPHA": true,
      "SOLVER_NAME": "COIN",
      "JSON_FILE": "./expansion/out.json",
-     "LAST_ITERATION_JSON_FILE": "./expansion/last_iteration.json"
-}
+     "LAST_ITERATION_JSON_FILE": "./expansion/last_iteration.json",
+     "MPS_ZIP_FILE": "MPS_ZIP_FILE.zip"
+}

data_test/mps_zip/archive1/file1

@@ -0,0 +1 @@
+DATA in file 1

data_test/mps_zip/archive1/file3

@@ -0,0 +1,7 @@
+File 3 content
+1 
+2
+4
+2
+2
+3

src/cpp/benders/benders_core/BendersBase.cpp

@@ -710,6 +710,13 @@ std::filesystem::path BendersBase::get_structure_path() const {
   return std::filesystem::path(_options.INPUTROOT) / _options.STRUCTURE_FILE;
 }
 
+/*!
+ *  \brief Get path to mps zip file from options
+ */
+std::filesystem::path BendersBase::GetMpsZipPath() const {
+  return std::filesystem::path(_options.INPUTROOT) / _options.MPS_ZIP_FILE;
+}
+
 LogData BendersBase::bendersDataToLogData(const BendersData &data) const {
   auto optimal_gap(data.best_ub - data.lb);
   return {data.lb,

src/cpp/benders/benders_core/SimulationOptions.cpp

@@ -130,6 +130,7 @@ BaseOptions SimulationOptions::get_base_options() const {
   result.SOLVER_NAME = SOLVER_NAME;
   result.weights = _weights;
   result.RESUME = RESUME;
+  result.MPS_ZIP_FILE = MPS_ZIP_FILE;
 
   return result;
 }

src/cpp/benders/benders_core/include/BendersBase.h

@@ -54,6 +54,7 @@ class BendersBase {
                                         std::string const &name) const;
   [[nodiscard]] std::filesystem::path get_master_path() const;
   [[nodiscard]] std::filesystem::path get_structure_path() const;
+  [[nodiscard]] std::filesystem::path GetMpsZipPath() const;
   [[nodiscard]] LogData bendersDataToLogData(const BendersData &data) const;
   virtual void build_input_map();
   void push_in_trace(const WorkerMasterDataPtr &worker_master_data);

src/cpp/benders/benders_core/include/SimulationOptions.hxx

@@ -69,3 +69,6 @@ BENDERS_OPTIONS_MACRO(RESUME, bool, false, asBool())
 // Name of the last master basis file
 BENDERS_OPTIONS_MACRO(LAST_MASTER_BASIS, std::string, "master_last_basis",
                       asString())
+
+// Name of the MPS ZIP file
+BENDERS_OPTIONS_MACRO(MPS_ZIP_FILE, std::string, "MPS_ZIP_FILE.zip", asString())

src/cpp/benders/benders_core/include/common.h

@@ -132,6 +132,7 @@ struct BaseOptions {
   bool RESUME = false;
 
   Str2Dbl weights;
+  std::string MPS_ZIP_FILE;
 };
 typedef BaseOptions MergeMPSOptions;
 struct BendersBaseOptions : public BaseOptions {

src/cpp/benders/benders_mpi/BendersMPI.cpp

@@ -12,7 +12,10 @@ BendersMpi::BendersMpi(BendersBaseOptions const &options, Logger &logger,
                        mpi::communicator &world)
     : BendersBase(options, logger, std::move(writer)),
       _env(env),
-      _world(world) {}
+      _world(world) {
+  reader_.SetArchivePath(GetMpsZipPath());
+  reader_.Open();
+}
 
 /*!
  *  \brief Method to load each problem in a thread
@@ -41,8 +44,12 @@ void BendersMpi::initialize_problems() {
           process_to_feed ==
           _world
               .rank()) {  // Assign  [problemNumber % processCount] to processID
+
+        const auto subProblemFilePath = GetSubproblemPath(problem.first);
+        reader_.ExtractFile(subProblemFilePath.filename());
         addSubproblem(problem);
         AddSubproblemName(problem.first);
+        std::filesystem::remove(subProblemFilePath);
       }
       current_problem_id++;
     }
@@ -277,6 +284,10 @@ void BendersMpi::launch() {
 
   initialize_problems();
   _world.barrier();
+  if (_world.rank() == rank_0) {
+    reader_.Close();
+    reader_.Delete();
+  }
 
   run();
   _world.barrier();

src/cpp/benders/benders_mpi/include/BendersMPI.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "ArchiveReader.h"
 #include "BendersBase.h"
 #include "BendersStructsDatas.h"
 #include "SubproblemCut.h"
@@ -52,6 +53,7 @@ class BendersMpi : public BendersBase {
   void check_if_some_proc_had_a_failure(int success);
   [[nodiscard]] bool shouldParallelize() const final { return false; }
 
+  ArchiveReader reader_;
   mpi::environment &_env;
   mpi::communicator &_world;
   const unsigned int rank_0 = 0;

src/cpp/benders/benders_sequential/BendersSequential.cpp

@@ -4,6 +4,7 @@
 #include <iomanip>
 #include <utility>
 
+#include "ArchiveReader.h"
 #include "Timer.h"
 #include "glog/logging.h"
 #include "solver_utils.h"
@@ -27,10 +28,18 @@ void BendersSequential::initialize_problems() {
   reset_master(new WorkerMaster(master_variable_map, get_master_path(),
                                 get_solver_name(), get_log_level(),
                                 _data.nsubproblem, log_name(), IsResumeMode()));
+  auto reader = ArchiveReader(GetMpsZipPath());
+  reader.Open();
   for (const auto &problem : coupling_map) {
+    const auto subProblemFilePath = GetSubproblemPath(problem.first);
+    reader.ExtractFile(subProblemFilePath.filename());
     addSubproblem(problem);
     AddSubproblemName(problem.first);
+    std::filesystem::remove(subProblemFilePath);
   }
+
+  reader.Close();
+  reader.Delete();
 }
 
 /*!

src/cpp/benders/merge_mps/MergeMPS.cpp

@@ -2,15 +2,16 @@
 
 #include <filesystem>
 
+#include "ArchiveReader.h"
 #include "Timer.h"
 #include "glog/logging.h"
 MergeMPS::MergeMPS(const MergeMPSOptions &options, Logger &logger,
                    Writer writer)
     : _options(options), _logger(logger), _writer(writer) {}
 
 void MergeMPS::launch() {
-  auto structure_path(std::filesystem::path(_options.INPUTROOT) /
-                      _options.STRUCTURE_FILE);
+  const auto inputRootDir = std::filesystem::path(_options.INPUTROOT);
+  auto structure_path(inputRootDir / _options.STRUCTURE_FILE);
   CouplingMap input = build_input(structure_path);
 
   SolverFactory factory;
@@ -25,16 +26,18 @@ void MergeMPS::launch() {
   int cntProblems_l(0);
 
   LOG(INFO) << "Merging problems..." << std::endl;
+  auto reader = ArchiveReader(inputRootDir / _options.MPS_ZIP_FILE);
+  reader.Open();
   for (auto const &kvp : input) {
-    auto problem_name(std::filesystem::path(_options.INPUTROOT) /
-                      (kvp.first + MPS_SUFFIX));
-
+    auto problem_name(inputRootDir / (kvp.first + MPS_SUFFIX));
     SolverAbstract::Ptr solver_l = factory.create_solver(solver_to_use);
     solver_l->init();
     solver_l->set_output_log_level(_options.LOG_LEVEL);
-    solver_l->read_prob_mps(problem_name);
 
     if (kvp.first != _options.MASTER_NAME) {
+      reader.ExtractFile(problem_name.filename());
+      solver_l->read_prob_mps(problem_name);
+      std::filesystem::remove(problem_name);
       int mps_ncols(solver_l->get_ncols());
 
       DblVector o(mps_ncols);
@@ -48,6 +51,8 @@ void MergeMPS::launch() {
         c *= weigth;
       }
       solver_l->chg_obj(sequence, o);
+    } else {
+      solver_l->read_prob_mps(problem_name);
     }
     StandardLp lpData(solver_l);
     std::string varPrefix_l = "prob" + std::to_string(cntProblems_l) + "_";
@@ -73,6 +78,9 @@ void MergeMPS::launch() {
     ++cntProblems_l;
   }
 
+  reader.Close();
+  reader.Delete();
+
   IntVector mstart;
   IntVector cindex;
   DblVector values;

src/cpp/exe/lpnamer/main.cpp

@@ -39,6 +39,7 @@ namespace po = boost::program_options;
 int main(int argc, char **argv) {
   try {
     std::filesystem::path root;
+    std::filesystem::path archivePath;
     std::string master_formulation;
     std::string additionalConstraintFilename_l;
 
@@ -47,6 +48,8 @@ int main(int argc, char **argv) {
     desc.add_options()("help,h", "produce help message")(
         "output,o", po::value<std::filesystem::path>(&root)->required(),
         "antares-xpansion study output")(
+        "archive,a", po::value<std::filesystem::path>(&archivePath)->required(),
+        "antares-xpansion study zip")(
         "formulation,f",
         po::value<std::string>(&master_formulation)->default_value("relaxed"),
         "master formulation (relaxed or integer)")(
@@ -98,7 +101,8 @@ int main(int argc, char **argv) {
     std::vector<ActiveLink> links = linkBuilder.getLinks();
     LinkProblemsGenerator linkProblemsGenerator(links, solver_name, logger,
                                                 log_file_path);
-    linkProblemsGenerator.treatloop(root, couplings);
+    linkProblemsGenerator.treatloop(root, archivePath, couplings);
+
 
     MasterGeneration master_generation(root, links, additionalConstraints,
                                        couplings, master_formulation,

src/cpp/helpers/ArchiveIO.h

@@ -0,0 +1,55 @@
+#ifndef _ARCHIVEIO_H
+#define _ARCHIVEIO_H
+
+extern "C" {
+#include <mz.h>
+#include <mz_strm.h>
+#include <mz_zip.h>
+#include <mz_zip_rw.h>
+}
+#include <stdexcept>
+#include <string>
+
+class ArchiveIOGeneralException : public std::runtime_error {
+ public:
+  ArchiveIOGeneralException(int32_t status, const std::string& action,
+                            int32_t expectedStatus = MZ_OK)
+      : std::runtime_error("Failed to " + action +
+                           " invalid status: " + std::to_string(status) + " (" +
+                           std::to_string(expectedStatus) + " expected)") {}
+};
+class ArchiveIOSpecificException : public std::runtime_error {
+ public:
+  ArchiveIOSpecificException(int32_t status, const std::string& errMessage,
+                             int32_t expectedStatus = MZ_OK)
+      : std::runtime_error(
+            errMessage + "\ninvalid status: " + std::to_string(status) + " (" +
+            std::to_string(expectedStatus) + " expected)") {}
+};
+#include <filesystem>
+#include <shared_mutex>
+class ArchiveIO {
+ private:
+  std::filesystem::path archivePath_;
+
+ protected:
+  mutable std::shared_mutex mutex_;
+
+  virtual void Create() = 0;
+
+ public:
+  explicit ArchiveIO(const std::filesystem::path& archivePath)
+      : archivePath_(archivePath) {}
+  ArchiveIO() = default;
+  std::filesystem::path ArchivePath() const { return archivePath_; }
+  void SetArchivePath(const std::filesystem::path& archivePath) {
+    archivePath_ = archivePath;
+  };
+
+  virtual int32_t Close() = 0;
+  virtual void Delete() = 0;
+
+  virtual int Open() = 0;
+};
+
+#endif  // _ARCHIVEIO_H