Simplify API of Problem generation (#745)

Change how -o and -a options work. -a options doesn't need -o option anymore. Given the options _-a simulation.zip_ it is now automatically deduced and output dir _simulation-Xpansion.zip_ -o options works by reading a directory (by opposition to reading an archive) Move some code in RunProblemGeneration to classes - Version is in its own file - RunProblemGeneration is part of a ProblemGeneration object (simpler testing)
AntaresSimulatorTeam · Feb 2, 2024 · 56828a2 · 56828a2
1 parent 9b7a3c3
commit 56828a2
Show file tree

Hide file tree

Showing 39 changed files with 774 additions and 338 deletions.
diff --git a/.github/workflows/build_ubuntu.yml b/.github/workflows/build_ubuntu.yml
@@ -65,6 +65,16 @@ jobs:
           sudo apt-get install -y ccache cmake libgtest-dev libjsoncpp-dev libtbb-dev libopenmpi-dev
           sudo apt-get install -y g++-10 gcc-10
 
+      - name: Update alternatives
+        #mpicxx  uses "g++" so we need g++ to be symbolic link to g++-10
+        run: |
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10
+          sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30
+          sudo update-alternatives --set cc /usr/bin/gcc
+          sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30
+          sudo update-alternatives --set c++ /usr/bin/g++
+
       - name: Read antares-solver version
         id: antares-version
         uses: ./.github/actions/read-json-value

diff --git a/.github/workflows/compile-boost/action.yml b/.github/workflows/compile-boost/action.yml
@@ -4,9 +4,6 @@ inputs:
   prefix:
     description: 'extra qualifiers'
     required: true
-  load-toolset:
-    description: 'load devtoolset 10'
-    default: 'true'
 runs:
   using: "composite"
   steps: 

diff --git a/.github/workflows/compile-tbb/action.yml b/.github/workflows/compile-tbb/action.yml
@@ -4,9 +4,7 @@ inputs:
   cmake:
     description: 'cmake version'
     default: cmake
-  load-toolset:
-    description: 'load devtoolset 10'
-    default: 'true'
+
 runs:
   using: "composite"
   steps: 

diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
@@ -31,6 +31,11 @@ jobs:
       - name: Install sonar-scanner and build-wrapper
         uses: SonarSource/sonarcloud-github-c-cpp@v2
 
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: sonarcloud-${{ env.SONAR_SCANNER_VERSION }}
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -48,10 +53,21 @@ jobs:
         run: |
           sudo apt-get update --fix-missing
           sudo apt-get install libjsoncpp-dev libgtest-dev libboost-mpi-dev libboost-program-options-dev libtbb-dev
+          sudo apt-get install g++-10 gcc-10
           cd /usr/src/googletest/
           sudo cmake .
           sudo cmake --build . --target install
 
+      - name: Update alternatives
+        #mpicxx  uses "g++" so we need g++ to be symbolic link to g++-10
+        run: |
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 10
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 10
+          sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30
+          sudo update-alternatives --set cc /usr/bin/gcc
+          sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30
+          sudo update-alternatives --set c++ /usr/bin/g++
+
       - name: Read antares-solver version
         id: antares-version
         uses: ./.github/actions/read-json-value
@@ -80,7 +96,6 @@ jobs:
           antares-version: ${{steps.antares-version.outputs.result}}
           os: ${{matrix.os}}
           os-full-name: Ubuntu-20.04
-          buildtype: Debug
 
       - name: Compile Boost
         uses: ./.github/workflows/compile-boost
@@ -94,11 +109,22 @@ jobs:
           git submodule update --init --recursive .
 
       - name: Configure
+        shell: bash
         run: |
-          cmake -B _build -S . -DDEPS_INSTALL_DIR=rte-antares-deps-Debug -DCODE_COVERAGE=ON -DBUILD_TESTING=ON -DBUILD_antares_solver=OFF -DCMAKE_BUILD_TYPE=Debug  -DCMAKE_INSTALL_PREFIX=_install
+          cmake -B _build -S . \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_C_COMPILER=/usr/bin/gcc-10 \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_CXX_COMPILER=/usr/bin/g++-10 \
+          -DDEPS_INSTALL_DIR=rte-antares-deps-Release \
+          -DCODE_COVERAGE=ON \
+          -DBUILD_TESTING=ON \
+          -DBUILD_antares_solver=OFF \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=_install
 
       - name: Build
-        run: build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/_build/output cmake --build _build --config Debug -j2
+        run: build-wrapper-linux-x86-64 --out-dir $GITHUB_WORKSPACE/_build/output cmake --build _build --config Release -j2
 
       - name: Test and generate coverage
         continue-on-error: true

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -98,7 +98,6 @@ message ("CMAKE_SOURCE_DIR : ${CMAKE_SOURCE_DIR}")
 message ("CMAKE_C_COMPILER_VERSION : ${CMAKE_C_COMPILER_VERSION}")
 message ("CMAKE_CXX_COMPILER_VERSION : ${CMAKE_CXX_COMPILER_VERSION}")
 
-set(CODE_COVERAGE FALSE CACHE BOOL "Enable code coverage")
 if (CODE_COVERAGE)
 	# if code-coverage is ON, force tests build
 	set(BUILD_TESTING ON)

diff --git a/src/cpp/exe/full_run/main.cpp b/src/cpp/exe/full_run/main.cpp
@@ -1,11 +1,10 @@
 #include <boost/program_options.hpp>
 #include <exception>
 #include <iostream>
-#include <string>
 
 #include "FullRunOptionsParser.h"
+#include "ProblemGeneration.h"
 #include "ProblemGenerationLogger.h"
-#include "RunProblemGeneration.h"
 #include "StudyUpdateRunner.h"
 #include "common_mpi.h"
 namespace po = boost::program_options;
@@ -15,26 +14,11 @@ int main(int argc, char** argv) {
   mpi::communicator world;
   auto options_parser = FullRunOptionsParser();
   std::filesystem::path xpansion_output_dir;
-  std::filesystem::path archive_path;
   options_parser.Parse(argc, argv);
   if (world.rank() == 0) {
     try {
-      xpansion_output_dir = options_parser.XpansionOutputDir();
-      archive_path = options_parser.ArchivePath();
-
-      const auto log_file_path =
-          xpansion_output_dir / "lp" / "ProblemGenerationLog.txt";
-      auto logger = ProblemGenerationLog::BuildLogger(
-          log_file_path, std::cout, "Full Run - Problem Generation");
-
-      auto master_formulation = options_parser.MasterFormulation();
-      auto additionalConstraintFilename_l =
-          options_parser.AdditionalConstraintsFilename();
-      auto weights_file = options_parser.WeightsFile();
-      const auto unnamed_problems = options_parser.UnnamedProblems();
-      RunProblemGeneration(xpansion_output_dir, master_formulation,
-                           additionalConstraintFilename_l, archive_path, logger,
-                           log_file_path, weights_file, unnamed_problems);
+      ProblemGeneration pbg(options_parser);
+      xpansion_output_dir = pbg.updateProblems();
 
     } catch (std::exception& e) {
       std::cerr << "error: " << e.what() << std::endl;

diff --git a/src/cpp/exe/lpnamer/main.cpp b/src/cpp/exe/lpnamer/main.cpp
@@ -1,45 +1,16 @@
 
 #include <iostream>
 
+#include "ProblemGeneration.h"
 #include "ProblemGenerationExeOptions.h"
-#include "ProblemGenerationLogger.h"
-#include "RunProblemGeneration.h"
-
-static const std::string LP_DIRNAME = "lp";
-
-void CreateDirectories(const std::filesystem::path& output_path) {
-  if (!std::filesystem::exists(output_path)) {
-    std::filesystem::create_directories(output_path);
-  }
-  auto lp_path = output_path / LP_DIRNAME;
-  if (!std::filesystem::exists(lp_path)) {
-    std::filesystem::create_directories(lp_path);
-  }
-}
 
 int main(int argc, char** argv) {
   try {
     auto options_parser = ProblemGenerationExeOptions();
     options_parser.Parse(argc, argv);
 
-    auto xpansion_output_dir = options_parser.XpansionOutputDir();
-    auto archive_path = options_parser.ArchivePath();
-    using namespace ProblemGenerationLog;
-    const auto log_file_path =
-        xpansion_output_dir / "lp" / "ProblemGenerationLog.txt";
-
-    CreateDirectories(xpansion_output_dir);
-    auto logger = ProblemGenerationLog::BuildLogger(log_file_path, std::cout,
-                                                    "Problem Generation");
-
-    auto master_formulation = options_parser.MasterFormulation();
-    auto additionalConstraintFilename_l =
-        options_parser.AdditionalConstraintsFilename();
-    auto weights_file = options_parser.WeightsFile();
-    auto unnamed_problems = options_parser.UnnamedProblems();
-    RunProblemGeneration(xpansion_output_dir, master_formulation,
-                         additionalConstraintFilename_l, archive_path, logger,
-                         log_file_path, weights_file, unnamed_problems);
+    ProblemGeneration pbg(options_parser);
+    pbg.updateProblems();
 
     return 0;
   } catch (std::exception& e) {

diff --git a/src/cpp/full_run/FullRunOptionsParser.cpp b/src/cpp/full_run/FullRunOptionsParser.cpp
@@ -14,7 +14,7 @@ FullRunOptionsParser::FullRunOptionsParser() : ProblemGenerationExeOptions() {
       "path to json solution file");
 }
 void FullRunOptionsParser::Parse(unsigned int argc, const char* const* argv) {
-  OptionsParser::Parse(argc, argv);
+  ProblemGenerationExeOptions::Parse(argc, argv);
   SetMethod();
 }
 void FullRunOptionsParser::SetMethod() {

diff --git a/src/cpp/full_run/include/FullRunOptionsParser.h b/src/cpp/full_run/include/FullRunOptionsParser.h
@@ -8,7 +8,7 @@
 class FullRunOptionsParser : public ProblemGenerationExeOptions {
  public:
   FullRunOptionsParser();
-  virtual ~FullRunOptionsParser() = default;
+  ~FullRunOptionsParser() override = default;
   void Parse(unsigned int argc, const char* const* argv) override;
   class FullRunOptionInvalidMethod : public std::runtime_error {
    public:
@@ -18,11 +18,13 @@ class FullRunOptionsParser : public ProblemGenerationExeOptions {
               " is not supported! Please choose one of: " +
               FullRunOptionsParser::ListAvailalbleMethods()) {}
   };
-  BENDERSMETHOD Method() const { return method_; };
-  std::filesystem::path BendersOptionsFile() const {
+  [[nodiscard]] BENDERSMETHOD Method() const { return method_; };
+  [[nodiscard]] std::filesystem::path BendersOptionsFile() const {
     return benders_options_file_;
   }
-  std::filesystem::path SolutionFile() const { return solutionFile_; }
+  [[nodiscard]] std::filesystem::path SolutionFile() const {
+    return solutionFile_;
+  }
   static inline std::string ListAvailalbleMethods() {
     return "benders,\n benders_by_batch,\n mergeMPS\n";
   }

diff --git a/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp b/src/cpp/lpnamer/input_reader/CandidatesINIReader.cpp
@@ -29,9 +29,11 @@ std::vector<IntercoFileData> CandidatesINIReader::ReadAntaresIntercoFile(
     const std::filesystem::path &antaresIntercoFile) const {
   std::ifstream interco_filestream(antaresIntercoFile);
   if (!interco_filestream.good()) {
-    (*logger_)(LogUtils::LOGLEVEL::FATAL)
-        << LOGLOCATION << "unable to open " << antaresIntercoFile.string();
-    std::exit(1);
+    auto loglocation = LOGLOCATION;
+    using namespace std::string_literals;
+    auto message = "unable to open "s + antaresIntercoFile.string();
+    (*logger_)(LogUtils::LOGLEVEL::FATAL) << LOGLOCATION << message;
+    throw InvalidIntercoFile(message, loglocation);
   }
 
   return ReadLineByLineInterco(interco_filestream);

diff --git a/src/cpp/lpnamer/input_reader/CandidatesINIReader.h b/src/cpp/lpnamer/input_reader/CandidatesINIReader.h
@@ -41,6 +41,11 @@ class CandidatesINIReader {
   std::vector<CandidateData> readCandidateData(
       const std::filesystem::path& candidateFile) const;
 
+  class InvalidIntercoFile
+      : public LogUtils::XpansionError<std::runtime_error> {
+    using LogUtils::XpansionError<std::runtime_error>::XpansionError;
+  };
+
  private:
   bool checkArea(std::string const& areaName_p) const;
   CandidateData readCandidateSection(const std::filesystem::path& candidateFile,

diff --git a/src/cpp/lpnamer/input_reader/MpsTxtWriter.cpp b/src/cpp/lpnamer/input_reader/MpsTxtWriter.cpp
@@ -5,22 +5,69 @@
 
 #include "ArchiveReader.h"
 
+using namespace std::string_literals;
+
 void FilesMapper::FillMap() {
+  if (antares_archive_path_.empty())
+    FillMapFiles();
+  else
+    FillMapZip();
+}
+
+auto FileWithExtension(std::string_view ext,
+                       const std::filesystem::path& directory) {
+  std::vector<std::filesystem::path> vect_of_files_with_ext;
+  for (auto& p : std::filesystem::directory_iterator(directory)) {
+    if (p.path().extension().string() == ext)
+      vect_of_files_with_ext.push_back(p.path().filename());
+  }
+  return vect_of_files_with_ext;
+}
+
+void FilesMapper::FillMapFiles() {
+  year_week_and_files_dict_.clear();
+  FillMapWithMpsFiles(FileWithExtension(".mps"s, simulation_dir_));
+  std::vector<std::filesystem::path> variables_files;
+  std::vector<std::filesystem::path> constraints_files;
+
+  auto vect_of_files_with_txt_ext = FileWithExtension(".txt"s, simulation_dir_);
+
+  for (const auto& file : vect_of_files_with_txt_ext) {
+    auto filename_str = file.stem().string();
+    auto is_it_variables_file =
+        filename_str.find("variables"s) != std::string::npos;
+    auto is_it_constraints_file =
+        filename_str.find("constraints"s) != std::string::npos;
+    if (is_it_variables_file && is_it_constraints_file) {
+      // case file : ***variables**constraints***.txt or
+      // ***constraints**variables***.txt
+      continue;
+    } else if (is_it_variables_file) {
+      variables_files.emplace_back(file);
+    } else if (is_it_constraints_file) {
+      constraints_files.emplace_back(file);
+    }
+  }
+  FillMapWithVariablesFiles(variables_files);
+  FillMapWithConstraintsFiles(constraints_files);
+}
+
+void FilesMapper::FillMapZip() {
   auto zip_reader = ArchiveReader(antares_archive_path_);
   zip_reader.Open();
   zip_reader.LoadEntriesPath();
   year_week_and_files_dict_.clear();
-  FillMapWithMpsFiles(zip_reader.GetEntriesPathWithExtension(".mps"));
+  FillMapWithMpsFiles(zip_reader.GetEntriesPathWithExtension(".mps"s));
   std::vector<std::filesystem::path> variables_files;
   std::vector<std::filesystem::path> constraints_files;
   auto vect_of_files_with_txt_ext =
-      zip_reader.GetEntriesPathWithExtension(".txt");
+      zip_reader.GetEntriesPathWithExtension(".txt"s);
   for (const auto& file : vect_of_files_with_txt_ext) {
     auto filename_str = file.stem().string();
     auto is_it_variables_file =
-        filename_str.find("variables") != std::string::npos;
+        filename_str.find("variables"s) != std::string::npos;
     auto is_it_constraints_file =
-        filename_str.find("constraints") != std::string::npos;
+        filename_str.find("constraints"s) != std::string::npos;
     if (is_it_variables_file && is_it_constraints_file) {
       // case file : ***variables**constraints***.txt or
       // ***constraints**variables***.txt
@@ -67,7 +114,7 @@ void FilesMapper::FillMapWithConstraintsFiles(
 YearAndWeek FilesMapper::YearAndWeekFromFileName(
     const std::filesystem::path& file_name) const {
   auto split_file_name =
-      StringManip::split(StringManip::trim(file_name.string()), '-');
+      StringManip::split(StringManip::trim(file_name.filename().string()), '-');
   return {std::stoi(split_file_name[1]), std::stoi(split_file_name[2])};
 }
 

diff --git a/src/cpp/lpnamer/input_reader/MpsTxtWriter.h b/src/cpp/lpnamer/input_reader/MpsTxtWriter.h
@@ -28,8 +28,10 @@ struct ProblemData {
 
 class FilesMapper {
  public:
-  explicit FilesMapper(const std::filesystem::path& antares_archive_path)
-      : antares_archive_path_(antares_archive_path) {}
+  explicit FilesMapper(std::filesystem::path antares_archive_path,
+                       std::filesystem::path simulation_dir = {})
+      : antares_archive_path_(std::move(antares_archive_path)),
+        simulation_dir_(std::move(simulation_dir)) {}
   YearWeekAndFilesDict FilesMap() {
     FillMap();
     return year_week_and_files_dict_;
@@ -39,6 +41,7 @@ class FilesMapper {
  private:
   YearWeekAndFilesDict year_week_and_files_dict_;
   std::filesystem::path antares_archive_path_;
+  std::filesystem::path simulation_dir_;
   void FillMap();
   void FillMapWithMpsFiles(const std::vector<std::filesystem::path>& mps_files);
   void FillMapWithVariablesFiles(
@@ -47,5 +50,7 @@ class FilesMapper {
       const std::vector<std::filesystem::path>& variables_files);
   YearAndWeek YearAndWeekFromFileName(
       const std::filesystem::path& file_name) const;
+  void FillMapFiles();
+  void FillMapZip();
 };
 #endif  // SRC_CPP_LPNAMER_INPUTREADER_MPSTXTWRITER_H