From 9e1f5a78f0b098600b08dbe8cc9e85e65793bdce Mon Sep 17 00:00:00 2001 From: kathvargasr <113925565+kathvargasr@users.noreply.github.com> Date: Wed, 14 Dec 2022 14:23:09 +0100 Subject: [PATCH 1/9] [DEV] Rework parallelization computation and remove duplication --- src/libs/antares/CMakeLists.txt | 2 + src/libs/antares/study/load.cpp | 9 +- src/libs/antares/study/parallel-years.cpp | 422 ++++++++++++++++++ src/libs/antares/study/parallel-years.h | 159 +++++++ src/libs/antares/study/study.cpp | 297 +----------- src/libs/antares/study/study.h | 19 +- src/solver/simulation/solver.h | 2 +- src/solver/simulation/solver.hxx | 131 +----- src/solver/simulation/solver.utils.h | 45 +- src/tests/end-to-end/simple-study.cpp | 2 +- .../src/libs/antares/study/CMakeLists.txt | 1 + .../study/parallel-years/CMakeLists.txt | 35 ++ .../parallel-years/parallel-years-wrapper.h | 55 +++ .../test-parallel-years-calculator.cpp | 291 ++++++++++++ src/ui/simulator/application/study.cpp | 2 +- .../windows/options/advanced/advanced.cpp | 2 +- 16 files changed, 1015 insertions(+), 459 deletions(-) create mode 100644 src/libs/antares/study/parallel-years.cpp create mode 100644 src/libs/antares/study/parallel-years.h create mode 100644 src/tests/src/libs/antares/study/parallel-years/CMakeLists.txt create mode 100644 src/tests/src/libs/antares/study/parallel-years/parallel-years-wrapper.h create mode 100644 src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp diff --git a/src/libs/antares/CMakeLists.txt b/src/libs/antares/CMakeLists.txt index 3421188970..7549a701e3 100644 --- a/src/libs/antares/CMakeLists.txt +++ b/src/libs/antares/CMakeLists.txt @@ -447,6 +447,8 @@ set(SRC study.h study/estimate.memory-footprint.cpp study/study.cpp + study/parallel-years.cpp + study/parallel-years.h study/correlation-updater.hxx study/study.importprepro.cpp study/memory-usage.h diff --git a/src/libs/antares/study/load.cpp b/src/libs/antares/study/load.cpp index 2409173268..7c3dc4b96e 100644 --- a/src/libs/antares/study/load.cpp +++ b/src/libs/antares/study/load.cpp @@ -148,15 +148,8 @@ bool Study::internalLoadFromFolder(const String& path, const StudyLoadOptions& o // ------------------------- // Getting the number of logical cores to use before loading and creating the areas : // Areas need this number to be up-to-date at construction. - getNumberOfCores(options.forceParallel, options.maxNbYearsInParallel); - // In case the study is run in the draft mode, only 1 core is allowed - if (parameters.mode == Data::stdmAdequacyDraft) - maxNbYearsInParallel = 1; - - // In case parallel mode was not chosen, only 1 core is allowed - if (!options.enableParallel && !options.forceParallel) - maxNbYearsInParallel = 1; + getNumberOfCores(options.forceParallel, options.enableParallel, options.maxNbYearsInParallel); // End logical core -------- diff --git a/src/libs/antares/study/parallel-years.cpp b/src/libs/antares/study/parallel-years.cpp new file mode 100644 index 0000000000..2c5bc3eb5d --- /dev/null +++ b/src/libs/antares/study/parallel-years.cpp @@ -0,0 +1,422 @@ +/* +** Copyright 2007-2022 RTE +** Authors: Antares_Simulator Team +** +** This file is part of Antares_Simulator. +** +** Antares_Simulator is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** There are special exceptions to the terms and conditions of the +** license as they are applied to this software. View the full text of +** the exceptions in file COPYING.txt in the directory of this software +** distribution +** +** Antares_Simulator is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with Antares_Simulator. If not, see . +** +** SPDX-License-Identifier: licenceRef-GPL3_WITH_RTE-Exceptions +*/ + +#include +#include +#include + +#include // For use of Yuni::System::CPU::Count() +#include +#include +#include + +#include +#include +#include "parameters.h" +#include "parallel-years.h" + +namespace Antares{ + +bool TempAreaListHolder::checkThermalTSGeneration(YString folder_) +{ + inputFolder = folder_; + loadAreaList(); + + return std::any_of(areaNames.begin(), areaNames.end(),[this](Yuni::String areaName){ + + auto folder = inputFolder; + Yuni::Clob thermalPlant = folder << SEP << "thermal" << SEP << "clusters" << SEP << areaName << SEP << "list.ini"; + + IniFile ini; + if (ini.open(thermalPlant)) + { + if(ini.firstSection) + { + for (IniFile::Section* section = ini.firstSection; section; section = section->next) + { + for (IniFile::Property* property = section->firstProperty; property; property = property->next) + { + return property->key == "gen-ts"; + } + } + + } + return false; + } + logs.error() << "Thermal Cluster Ini file cannot be opened: " << thermalPlant.c_str(); + return false; + + } + ); + +} + +void TempAreaListHolder::loadAreaList() +{ + auto folder = inputFolder; + Yuni::Clob filename = folder << SEP << "areas" << SEP << "list.txt"; + + Yuni::IO::File::Stream file; + Yuni::String buffer; + buffer.reserve(1024 /* to force the allocation */); + + if (!file.open(filename)) + { + logs.error() << "I/O error: " << filename << ": Impossible to open the file"; + return; + } + while(file.readline(buffer)) + { + // The area name + Yuni::String name; + Yuni::String lname; + name = buffer; + name.trim(" \t\n\r"); + TransformNameIntoID(name, lname); + areaNames.push_back(lname); + } + +} + + +void SetsOfParallelYearCalculator::computeRawNbParallelYear() +{ + if (forceParallel) + return; + + std::map numberOfMCYearThreads; + uint nbLogicalCores = Yuni::System::CPU::Count(); + + numberOfMCYearThreads[ncMin] = 1; + switch (nbLogicalCores) + { + case 0: + logs.fatal() << "Number of logical cores available is 0."; + break; + case 1: + numberOfMCYearThreads[ncLow] = 1; + numberOfMCYearThreads[ncAvg] = 1; + numberOfMCYearThreads[ncHigh] = 1; + numberOfMCYearThreads[ncMax] = 1; + break; + case 2: + numberOfMCYearThreads[ncLow] = 1; + numberOfMCYearThreads[ncAvg] = 1; + numberOfMCYearThreads[ncHigh] = 2; + numberOfMCYearThreads[ncMax] = 2; + break; + case 3: + numberOfMCYearThreads[ncLow] = 2; + numberOfMCYearThreads[ncAvg] = 2; + numberOfMCYearThreads[ncHigh] = 2; + numberOfMCYearThreads[ncMax] = 3; + break; + case 4: + numberOfMCYearThreads[ncLow] = 2; + numberOfMCYearThreads[ncAvg] = 2; + numberOfMCYearThreads[ncHigh] = 3; + numberOfMCYearThreads[ncMax] = 4; + break; + case 5: + numberOfMCYearThreads[ncLow] = 2; + numberOfMCYearThreads[ncAvg] = 3; + numberOfMCYearThreads[ncHigh] = 4; + numberOfMCYearThreads[ncMax] = 5; + break; + case 6: + numberOfMCYearThreads[ncLow] = 2; + numberOfMCYearThreads[ncAvg] = 3; + numberOfMCYearThreads[ncHigh] = 4; + numberOfMCYearThreads[ncMax] = 6; + break; + case 7: + numberOfMCYearThreads[ncLow] = 2; + numberOfMCYearThreads[ncAvg] = 3; + numberOfMCYearThreads[ncHigh] = 5; + numberOfMCYearThreads[ncMax] = 7; + break; + case 8: + numberOfMCYearThreads[ncLow] = 2; + numberOfMCYearThreads[ncAvg] = 4; + numberOfMCYearThreads[ncHigh] = 6; + numberOfMCYearThreads[ncMax] = 8; + break; + case 9: + numberOfMCYearThreads[ncLow] = 3; + numberOfMCYearThreads[ncAvg] = 5; + numberOfMCYearThreads[ncHigh] = 7; + numberOfMCYearThreads[ncMax] = 8; + break; + case 10: + numberOfMCYearThreads[ncLow] = 3; + numberOfMCYearThreads[ncAvg] = 5; + numberOfMCYearThreads[ncHigh] = 8; + numberOfMCYearThreads[ncMax] = 9; + break; + case 11: + numberOfMCYearThreads[ncLow] = 3; + numberOfMCYearThreads[ncAvg] = 6; + numberOfMCYearThreads[ncHigh] = 8; + numberOfMCYearThreads[ncMax] = 10; + break; + case 12: + numberOfMCYearThreads[ncLow] = 3; + numberOfMCYearThreads[ncAvg] = 6; + numberOfMCYearThreads[ncHigh] = 9; + numberOfMCYearThreads[ncMax] = 11; + break; + default: + numberOfMCYearThreads[ncLow] = (uint)std::ceil(nbLogicalCores / 4.); + numberOfMCYearThreads[ncAvg] = (uint)std::ceil(nbLogicalCores / 2.); + numberOfMCYearThreads[ncHigh] = (uint)std::ceil(3 * nbLogicalCores / 4.); + numberOfMCYearThreads[ncMax] = nbLogicalCores - 1; + break; + } + + /* + Getting the number of parallel years based on the number + of cores level. + This number is limited by the smallest refresh span (if at least + one type of time series is generated) + */ + + try + { + forcedNbOfParallelYears = numberOfMCYearThreads.at(p.nbCores.ncMode); + } + catch(const std::out_of_range& e) + { + logs.fatal() << "Simulation cores level not correct : " << (int)p.nbCores.ncMode; + } + +} + + +void SetsOfParallelYearCalculator::limitNbOfParallelYearsbyMinRefreshSpan() +{ + uint TSlimit = UINT_MAX; + if ((p.timeSeriesToGenerate & timeSeriesLoad) && (p.timeSeriesToRefresh & timeSeriesLoad)) + TSlimit = p.refreshIntervalLoad; + if ((p.timeSeriesToGenerate & timeSeriesSolar) && (p.timeSeriesToRefresh & timeSeriesSolar)) + TSlimit = (p.refreshIntervalSolar < TSlimit) ? p.refreshIntervalSolar : TSlimit; + if ((p.timeSeriesToGenerate & timeSeriesHydro) && (p.timeSeriesToRefresh & timeSeriesHydro)) + TSlimit = (p.refreshIntervalHydro < TSlimit) ? p.refreshIntervalHydro : TSlimit; + if ((p.timeSeriesToGenerate & timeSeriesWind) && (p.timeSeriesToRefresh & timeSeriesWind)) + TSlimit = (p.refreshIntervalWind < TSlimit) ? p.refreshIntervalWind : TSlimit; + if ((p.timeSeriesToGenerate & timeSeriesThermal) && (p.timeSeriesToRefresh & timeSeriesThermal)) + TSlimit = (p.refreshIntervalThermal < TSlimit) ? p.refreshIntervalThermal : TSlimit; + + if (TSlimit < forcedNbOfParallelYears) + forcedNbOfParallelYears = TSlimit; + + // Limiting the number of parallel years by the total number of years + if (p.nbYears < forcedNbOfParallelYears) + forcedNbOfParallelYears = p.nbYears; +} + +bool SetsOfParallelYearCalculator::isRefreshNeededForCurrentYear(uint y) +{ + bool refreshing = false; + refreshing = (p.timeSeriesToGenerate & timeSeriesLoad) + && (p.timeSeriesToRefresh & timeSeriesLoad) + && (!y || ((y % p.refreshIntervalLoad) == 0)); + refreshing = refreshing + || ((p.timeSeriesToGenerate & timeSeriesSolar) + && (p.timeSeriesToRefresh & timeSeriesSolar) + && (!y || ((y % p.refreshIntervalSolar) == 0))); + refreshing = refreshing + || ((p.timeSeriesToGenerate & timeSeriesWind) + && (p.timeSeriesToRefresh & timeSeriesWind) + && (!y || ((y % p.refreshIntervalWind) == 0))); + refreshing = refreshing + || ((p.timeSeriesToGenerate & timeSeriesHydro) + && (p.timeSeriesToRefresh & timeSeriesHydro) + && (!y || ((y % p.refreshIntervalHydro) == 0))); + + bool haveToRefreshTSThermal + = ((p.timeSeriesToGenerate & timeSeriesThermal) + && (p.timeSeriesToRefresh & timeSeriesThermal)) || thermalTSRefresh; + refreshing + = refreshing || (haveToRefreshTSThermal && (y % p.refreshIntervalThermal == 0)); + + return refreshing; + +} + +void SetsOfParallelYearCalculator::buildSetsOfParallelYears() +{ + setOfParallelYears* set = nullptr; + bool buildNewSet = true; + bool foundFirstPerformedYearOfCurrentSet = false; + // Gets information on each parallel years set + for (uint y = 0; y < p.nbYears; ++y) + { + unsigned int indexSpace = 999999; + bool performCalculations = true; + + if(p.userPlaylist) + performCalculations = p.yearsFilter[y]; + + bool refreshing = isRefreshNeededForCurrentYear(y); + buildNewSet = buildNewSet || refreshing; + + if (buildNewSet) + { + setOfParallelYears setToCreate; + setsOfParallelYears.push_back(setToCreate); + set = &(setsOfParallelYears.back()); + + // Initializations + set->nbPerformedYears = 0; + set->nbYears = 0; + set->regenerateTS = false; + set->yearForTSgeneration = 999999; + + // In case we have to regenerate times series before run the current set of parallel + // years + if (refreshing) + { + set->regenerateTS = true; + set->yearForTSgeneration = y; + } + } + + set->yearsIndices.push_back(y); + set->nbYears++; + set->yearFailed[y] = true; + set->isFirstPerformedYearOfASet[y] = false; + + if (performCalculations) + { + set->setsSizes.push_back(y); + // Another year performed + ++nbYearsReallyPerformed; + + // Number of actually performed years in the current set (up to now). + set->nbPerformedYears++; + // Index of the MC year's space (useful if this year is actually run) + indexSpace = set->nbPerformedYears - 1; + + set->isYearPerformed[y] = true; + set->performedYearToSpace[y] = indexSpace; + set->spaceToPerformedYear[indexSpace] = y; + + if (!foundFirstPerformedYearOfCurrentSet) + { + set->isFirstPerformedYearOfASet[y] = true; + foundFirstPerformedYearOfCurrentSet = true; + } + } + else + { + set->isYearPerformed[y] = false; + } + + // Do we build a new set at next iteration (for years to be executed or not) ? + + // In case the study is run in the draft mode, only 1 core is allowed + if (p.mode == Antares::Data::stdmAdequacyDraft){ + forcedNbOfParallelYears = 1; + } + + // In case parallel mode was not chosen, only 1 core is allowed + if (!enableParallel && !forceParallel){ + forcedNbOfParallelYears = 1; + } + + if (indexSpace == forcedNbOfParallelYears - 1 || y == p.nbYears - 1) + { + buildNewSet = true; + foundFirstPerformedYearOfCurrentSet = false; + if (set->nbPerformedYears > forcedNbOfParallelYears) + forcedNbOfParallelYears = set->nbPerformedYears; + } + else + buildNewSet = false; + + // End of loop over years + + } + +} + + +bool SetsOfParallelYearCalculator::allSetsParallelYearsHaveSameSize() +{ + if (p.initialReservoirLevels.iniLevels == Antares::Data::irlHotStart + && !setsOfParallelYears.empty() && forcedNbOfParallelYears > 1) + { + uint currentSetSize = (uint)setsOfParallelYears[0].setsSizes.size(); + return all_of(setsOfParallelYears.begin(), setsOfParallelYears.end(), + [currentSetSize](const setOfParallelYears& v){ return v.setsSizes.size() == currentSetSize; } ); + } // End if hot start + return true; + // parameters.allSetsHaveSameSize takes this result; +} + +uint SetsOfParallelYearCalculator::computeMinNbParallelYears() const +{ + // Now finding the smallest size among all sets. + uint minNbYearsInParallel = forcedNbOfParallelYears; + for (uint s = 0; s < setsOfParallelYears.size(); s++) + { + uint setSize = (uint)setsOfParallelYears[s].setsSizes.size(); + // Empty sets are not taken into account because, on the solver side, + // they will contain only skipped years + if (setSize && (setSize < minNbYearsInParallel)) + minNbYearsInParallel = setSize; + } + return minNbYearsInParallel; +} + +void SetsOfParallelYearCalculator::computeForcedNbYearsInParallelYearSet() +{ + + uint maxNbYearsOverAllSets = 0; + for (uint s = 0; s < setsOfParallelYears.size(); s++) + { + if (setsOfParallelYears[s].setsSizes.size() > maxNbYearsOverAllSets) + maxNbYearsOverAllSets = (uint)setsOfParallelYears[s].setsSizes.size(); + } + + forcedNbOfParallelYears = maxNbYearsOverAllSets; //j'aime pas + +} + +void SetsOfParallelYearCalculator::build() +{ + + computeRawNbParallelYear(); + limitNbOfParallelYearsbyMinRefreshSpan(); + + buildSetsOfParallelYears(); + + computeForcedNbYearsInParallelYearSet(); + +} + +} // namespace Antares diff --git a/src/libs/antares/study/parallel-years.h b/src/libs/antares/study/parallel-years.h new file mode 100644 index 0000000000..9e85bd0aa1 --- /dev/null +++ b/src/libs/antares/study/parallel-years.h @@ -0,0 +1,159 @@ +/* +** Copyright 2007-2022 RTE +** Authors: Antares_Simulator Team +** +** This file is part of Antares_Simulator. +** +** Antares_Simulator is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** There are special exceptions to the terms and conditions of the +** license as they are applied to this software. View the full text of +** the exceptions in file COPYING.txt in the directory of this software +** distribution +** +** Antares_Simulator is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with Antares_Simulator. If not, see . +** +** SPDX-License-Identifier: licenceRef-GPL3_WITH_RTE-Exceptions +*/ +#pragma once + +#include +#include "../../../solver/simulation/solver.utils.h" // KVR se debarraser de ce include + +namespace Antares +{ + +using namespace Data; +// using namespace Solver::Simulation; + +struct setOfParallelYears +{ + // Un lot d'ann�e � ex�cuter en parall�le. + // En fonction d'une �ventuelle play-list, certaines seront jou�es et d'autres non. + +public: + std::vector setsSizes; + // Numeros des annees en parallele pour ce lot (certaines ne seront pas jou�es en cas de + // play-list "trou�e") + std::vector yearsIndices; + + // Une annee doit-elle �tre rejou�e ? + std::map yearFailed; + + // Associe le numero d'une ann�e jou�e � l'indice de l'espace + std::map performedYearToSpace; + + // L'inverse : pour une ann�e jou�e, associe l'indice de l'espace au numero de l'ann�e + std::map spaceToPerformedYear; + + // Pour chaque ann�e, est-elle la premi�re � devoir �tre jou�e dans son lot d'ann�es ? + std::map isFirstPerformedYearOfASet; + + // Pour chaque ann�e du lot, est-elle jou�e ou non ? + std::map isYearPerformed; + + // Nbre d'ann�es en parallele vraiment jou�es pour ce lot + unsigned int nbPerformedYears; + + // Nbre d'ann�es en parallele jou�es ou non pour ce lot + unsigned int nbYears; + + // Regenere-t-on des times series avant de jouer les annees du lot courant + bool regenerateTS; + + // Annee a passer a la fonction "regenerateTimeSeries(y)" (si regenerateTS is "true") + unsigned int yearForTSgeneration; +}; + +class TempAreaListHolder +{ +public: + TempAreaListHolder() = default; + + bool checkThermalTSGeneration(YString folder_); + +private: + void loadAreaList(); + YString inputFolder; + std::vector areaNames; + +}; + +class SetsOfParallelYearCalculator +{ +public: + + friend class SetsOfParallelYearCalculatorWrapper; + + SetsOfParallelYearCalculator( + bool forceParallel_, + bool enableParallel_, + uint forcedNbOfParallelYears_, + bool thermalTSRefresh_, // Temporary parameter needed. To be removed once TSGen will be detached from simulator + Parameters& params_) + : forceParallel{forceParallel_}, + enableParallel {enableParallel_}, + forcedNbOfParallelYears{forcedNbOfParallelYears_}, + thermalTSRefresh{thermalTSRefresh_}, + p{params_}{ + this->build(); + } + + + bool allSetsParallelYearsHaveSameSize(); + + uint getForcedNbOfParallelYears() const + { + return forcedNbOfParallelYears; + } + + uint getMinNbParallelYearsForGUI() const + { + return computeMinNbParallelYears(); + } + + std::vector getSetsOfParallelYears() const + { + return setsOfParallelYears; + } + + uint getNbYearsReallyPerformed() const + { + return nbYearsReallyPerformed; + } + +private: + + void build(); + + void computeRawNbParallelYear(); + void limitNbOfParallelYearsbyMinRefreshSpan(); + + void buildSetsOfParallelYears(); + + uint computeMinNbParallelYears() const; + void computeForcedNbYearsInParallelYearSet(); + + bool isRefreshNeededForCurrentYear(uint y); + + std::vector setsOfParallelYears; + + bool forceParallel; + bool enableParallel; + uint forcedNbOfParallelYears; + bool thermalTSRefresh; + Parameters& p; + uint nbYearsReallyPerformed{0}; + +}; + +} // namespace Anatares \ No newline at end of file diff --git a/src/libs/antares/study/study.cpp b/src/libs/antares/study/study.cpp index 29184733e7..bf8250f8d6 100644 --- a/src/libs/antares/study/study.cpp +++ b/src/libs/antares/study/study.cpp @@ -224,284 +224,25 @@ uint64 Study::memoryUsage() const + (uiinfo ? uiinfo->memoryUsage() : 0); } -std::map Study::getRawNumberCoresPerLevel() -{ - std::map table; - - uint nbLogicalCores = Yuni::System::CPU::Count(); - if (!nbLogicalCores) - logs.fatal() << "Number of logical cores available is 0."; - - switch (nbLogicalCores) - { - case 1: - table["min"] = 1; - table["low"] = 1; - table["med"] = 1; - table["high"] = 1; - table["max"] = 1; - break; - case 2: - table["min"] = 1; - table["low"] = 1; - table["med"] = 1; - table["high"] = 2; - table["max"] = 2; - break; - case 3: - table["min"] = 1; - table["low"] = 2; - table["med"] = 2; - table["high"] = 2; - table["max"] = 3; - break; - case 4: - table["min"] = 1; - table["low"] = 2; - table["med"] = 2; - table["high"] = 3; - table["max"] = 4; - break; - case 5: - table["min"] = 1; - table["low"] = 2; - table["med"] = 3; - table["high"] = 4; - table["max"] = 5; - break; - case 6: - table["min"] = 1; - table["low"] = 2; - table["med"] = 3; - table["high"] = 4; - table["max"] = 6; - break; - case 7: - table["min"] = 1; - table["low"] = 2; - table["med"] = 3; - table["high"] = 5; - table["max"] = 7; - break; - case 8: - table["min"] = 1; - table["low"] = 2; - table["med"] = 4; - table["high"] = 6; - table["max"] = 8; - break; - case 9: - table["min"] = 1; - table["low"] = 3; - table["med"] = 5; - table["high"] = 7; - table["max"] = 8; - break; - case 10: - table["min"] = 1; - table["low"] = 3; - table["med"] = 5; - table["high"] = 8; - table["max"] = 9; - break; - case 11: - table["min"] = 1; - table["low"] = 3; - table["med"] = 6; - table["high"] = 8; - table["max"] = 10; - break; - case 12: - table["min"] = 1; - table["low"] = 3; - table["med"] = 6; - table["high"] = 9; - table["max"] = 11; - break; - default: - table["min"] = 1; - table["low"] = (uint)std::ceil(nbLogicalCores / 4.); - table["med"] = (uint)std::ceil(nbLogicalCores / 2.); - table["high"] = (uint)std::ceil(3 * nbLogicalCores / 4.); - table["max"] = nbLogicalCores - 1; - break; - } - - return table; -} - -void Study::getNumberOfCores(const bool forceParallel, const uint nbYearsParallelForced) -{ - /* - Getting the number of parallel years based on the number - of cores level. - This number is limited by the smallest refresh span (if at least - one type of time series is generated) - */ - - std::map table = getRawNumberCoresPerLevel(); - - // Getting the number of parallel years based on the number of cores level. - switch (parameters.nbCores.ncMode) - { - case ncMin: - nbYearsParallelRaw = table["min"]; - break; - case ncLow: - nbYearsParallelRaw = table["low"]; - break; - case ncAvg: - nbYearsParallelRaw = table["med"]; - break; - case ncHigh: - nbYearsParallelRaw = table["high"]; - break; - case ncMax: - nbYearsParallelRaw = table["max"]; - break; - default: - logs.fatal() << "Simulation cores level not correct : " << (int)parameters.nbCores.ncMode; - break; - } - - maxNbYearsInParallel = nbYearsParallelRaw; - - // In case solver option '--force-parallel n' is used, previous computation is overridden. - if (forceParallel) - maxNbYearsInParallel = nbYearsParallelForced; - - // Limiting the number of parallel years by the smallest refresh span - auto& p = parameters; - uint TSlimit = UINT_MAX; - if ((p.timeSeriesToGenerate & timeSeriesLoad) && (p.timeSeriesToRefresh & timeSeriesLoad)) - TSlimit = p.refreshIntervalLoad; - if ((p.timeSeriesToGenerate & timeSeriesSolar) && (p.timeSeriesToRefresh & timeSeriesSolar)) - TSlimit = (p.refreshIntervalSolar < TSlimit) ? p.refreshIntervalSolar : TSlimit; - if ((p.timeSeriesToGenerate & timeSeriesHydro) && (p.timeSeriesToRefresh & timeSeriesHydro)) - TSlimit = (p.refreshIntervalHydro < TSlimit) ? p.refreshIntervalHydro : TSlimit; - if ((p.timeSeriesToGenerate & timeSeriesWind) && (p.timeSeriesToRefresh & timeSeriesWind)) - TSlimit = (p.refreshIntervalWind < TSlimit) ? p.refreshIntervalWind : TSlimit; - if ((p.timeSeriesToGenerate & timeSeriesThermal) && (p.timeSeriesToRefresh & timeSeriesThermal)) - TSlimit = (p.refreshIntervalThermal < TSlimit) ? p.refreshIntervalThermal : TSlimit; - - if (TSlimit < maxNbYearsInParallel) - maxNbYearsInParallel = TSlimit; - - // Limiting the number of parallel years by the total number of years - if (p.nbYears < maxNbYearsInParallel) - maxNbYearsInParallel = p.nbYears; - - // Getting the minimum number of years in a set of parallel years. - // To get this number, we have to divide all years into sets of parallel - // years and pick the size of the smallest set. - - std::vector* set = nullptr; - bool buildNewSet = true; - std::vector> setsOfParallelYears; - - for (uint y = 0; y < p.nbYears; ++y) - { - bool performCalculations = true; - if (p.userPlaylist) - performCalculations = p.yearsFilter[y]; - - // Do we have to refresh ? - bool refreshing = false; - refreshing = (p.timeSeriesToGenerate & timeSeriesLoad) - && (p.timeSeriesToRefresh & timeSeriesLoad) - && (!y || ((y % p.refreshIntervalLoad) == 0)); - refreshing = refreshing - || ((p.timeSeriesToGenerate & timeSeriesSolar) - && (p.timeSeriesToRefresh & timeSeriesSolar) - && (!y || ((y % p.refreshIntervalSolar) == 0))); - refreshing = refreshing - || ((p.timeSeriesToGenerate & timeSeriesWind) - && (p.timeSeriesToRefresh & timeSeriesWind) - && (!y || ((y % p.refreshIntervalWind) == 0))); - refreshing = refreshing - || ((p.timeSeriesToGenerate & timeSeriesHydro) - && (p.timeSeriesToRefresh & timeSeriesHydro) - && (!y || ((y % p.refreshIntervalHydro) == 0))); - refreshing = refreshing - || ((p.timeSeriesToGenerate & timeSeriesThermal) - && (p.timeSeriesToRefresh & timeSeriesThermal) - && (!y || ((y % p.refreshIntervalThermal) == 0))); - - buildNewSet = buildNewSet || refreshing; - - // We build a new set of parallel years if one of these conditions is fulfilled : - // - We have to refresh (or regenerate) some or all time series before running the - // current year - // - This is the first year after the previous set is full with years to be actually - // executed (not skipped). That is : in the previous set filled, the max number of - // years to be actually run is reached. - if (buildNewSet) - { - std::vector setToCreate; - setsOfParallelYears.push_back(setToCreate); - set = &(setsOfParallelYears.back()); - } - - if (performCalculations) - set->push_back(y); - - // Do we build a new set at next iteration (for years to be executed or not) ? - if (set->size() == maxNbYearsInParallel) - buildNewSet = true; - else - buildNewSet = false; - } // End of loop over years - - // Now finding the smallest size among all sets. - minNbYearsInParallel = maxNbYearsInParallel; - for (uint s = 0; s < setsOfParallelYears.size(); s++) - { - uint setSize = (uint)setsOfParallelYears[s].size(); - // Empty sets are not taken into account because, on the solver side, - // they will contain only skipped years - if (setSize && (setSize < minNbYearsInParallel)) - minNbYearsInParallel = setSize; - } - - // GUI : storing minimum number of parallel years (in a set of parallel years). - // Useful in the run window's simulation cores field in case parallel mode is enabled - // by user. - minNbYearsInParallel_save = minNbYearsInParallel; - - // The max number of years to run in parallel is limited by the max number years in a set of - // parallel years. This latter number can be limited by the smallest interval between 2 refresh - // points and determined by the unrun MC years in case of play-list. - uint maxNbYearsOverAllSets = 0; - for (uint s = 0; s < setsOfParallelYears.size(); s++) - { - if (setsOfParallelYears[s].size() > maxNbYearsOverAllSets) - maxNbYearsOverAllSets = (uint)setsOfParallelYears[s].size(); - } - maxNbYearsInParallel = maxNbYearsOverAllSets; - - // GUI : storing max nb of parallel years (in a set of parallel years) in case parallel mode is - // enabled. - // Useful for RAM estimation. - maxNbYearsInParallel_save = maxNbYearsInParallel; - - // Here we answer the question (useful only if hydro hot start is asked) : do all sets of - // parallel years have the same size ? - if (parameters.initialReservoirLevels.iniLevels == Antares::Data::irlHotStart - && setsOfParallelYears.size() && maxNbYearsInParallel > 1) - { - uint currentSetSize = (uint)setsOfParallelYears[0].size(); - if (setsOfParallelYears.size() > 1) - { - for (uint s = 1; s < setsOfParallelYears.size(); s++) - { - if (setsOfParallelYears[s].size() != currentSetSize) - { - parameters.allSetsHaveSameSize = false; - break; - } - } - } - } // End if hot start +void Study::getNumberOfCores(const bool forceParallel, const bool enableParallel, const uint nbYearsParallelForced) { + + TempAreaListHolder holder{}; + bool thermalTSRefresh = holder.checkThermalTSGeneration(folderInput); + + SetsOfParallelYearCalculator setsBuilder(forceParallel, + enableParallel, + nbYearsParallelForced, + thermalTSRefresh, + parameters); + // For GUI + minNbYearsInParallel_save = setsBuilder.getMinNbParallelYearsForGUI(); + maxNbYearsInParallel_save = setsBuilder.getForcedNbOfParallelYears(); + + // For the solver + maxNbYearsInParallel = setsBuilder.getForcedNbOfParallelYears(); + parameters.allSetsHaveSameSize = setsBuilder.allSetsParallelYearsHaveSameSize(); + setsOfParallelYears = setsBuilder.getSetsOfParallelYears(); + pNbYearsReallyPerformed = setsBuilder.getNbYearsReallyPerformed(); } bool Study::checkHydroHotStart() diff --git a/src/libs/antares/study/study.h b/src/libs/antares/study/study.h index ea81886ff0..3bbb0c1edb 100644 --- a/src/libs/antares/study/study.h +++ b/src/libs/antares/study/study.h @@ -51,6 +51,7 @@ #include "load-options.h" #include "../date.h" #include "layerdata.h" +#include "parallel-years.h" #include @@ -442,21 +443,12 @@ class Study final : public Yuni::NonCopyable, public IObject, public Laye //@{ /*! - ** \brief Computes a raw number of cores table. - ** - ** The table associetes a raw number of cores to each level ("min", "low", "med", "high", - *"max"). - ** - */ - std::map getRawNumberCoresPerLevel(); - - /*! - ** \brief Computes number of cores + ** \brief Gets the number of cores calculated in SetsOfParallelYearCalculator class ** ** From the "Number of Cores" level (in GUI --> Advanced parameters), computes ** the real numbers of logical cores to be involved in the MC years parallelisation. */ - void getNumberOfCores(const bool forceParallel, const uint nbYearsParallelForced); + void getNumberOfCores(const bool forceParallel, const bool enableParallel, const uint nbYearsParallelForced); /*! ** \brief In case hydro hot start is enabled, checking all conditions are met. @@ -619,6 +611,11 @@ class Study final : public Yuni::NonCopyable, public IObject, public Laye // Useful to populate the run window's simulation cores field. uint minNbYearsInParallel_save; + // Used in solver + std::vector setsOfParallelYears; + + uint pNbYearsReallyPerformed; + //! Parameters Parameters parameters; diff --git a/src/solver/simulation/solver.h b/src/solver/simulation/solver.h index a394bcee98..47058e1bfe 100644 --- a/src/solver/simulation/solver.h +++ b/src/solver/simulation/solver.h @@ -158,7 +158,7 @@ class ISimulation : public Impl ** \param firstYear The first real MC year ** \param endYear The last MC year */ - void loopThroughYears(uint firstYear, uint endYear, std::vector& state); + void loopThroughYears(uint endYear, std::vector& state); private: //! Some temporary to avoid performing useless complex checks diff --git a/src/solver/simulation/solver.hxx b/src/solver/simulation/solver.hxx index 62a3566d52..d22c5a438b 100644 --- a/src/solver/simulation/solver.hxx +++ b/src/solver/simulation/solver.hxx @@ -39,6 +39,7 @@ #include "apply-scenario.h" #include #include "../ts-generator/generator.h" +#include "../../libs/antares/study/parallel-years.h" #include "../hydro/management.h" // Added for use of randomReservoirLevel(...) @@ -284,6 +285,7 @@ template void ISimulation::run() { pNbMaxPerformedYearsInParallel = study.maxNbYearsInParallel; + pNbYearsReallyPerformed = study.pNbYearsReallyPerformed; // Initialize all data ImplementationType::variables.initializeFromStudy(study); @@ -362,10 +364,10 @@ void ISimulation::run() ImplementationType::initializeState(state[numSpace], numSpace); logs.info() << " Starting the simulation"; - uint finalYear = 1 + study.runtime->rangeLimits.year[Data::rangeEnd]; { Benchmarking::Timer timer; - loopThroughYears(0, finalYear, state); + uint finalYear = 1 + study.runtime->rangeLimits.year[Data::rangeEnd]; + loopThroughYears(finalYear, state); timer.stop(); pDurationCollector->addDuration("mc_years", timer.get_duration()); } @@ -1062,120 +1064,6 @@ void ISimulation::regenerateTimeSeries(uint year) } } -template -uint ISimulation::buildSetsOfParallelYears( - uint firstYear, - uint endYear, - std::vector& setsOfParallelYears) -{ - // Filter on the years - const auto& yearsFilter = study.parameters.yearsFilter; - - // number max of years (to be executed or not) in a set of parallel years - uint maxNbYearsPerformed = 0; - - setOfParallelYears* set = nullptr; - bool buildNewSet = true; - bool foundFirstPerformedYearOfCurrentSet = false; - - // Gets information on each parallel years set - for (uint y = firstYear; y < endYear; ++y) - { - unsigned int indexSpace = 999999; - bool performCalculations = yearsFilter[y]; - - // Do we refresh just before this year ? If yes a new set of parallel years has to be - // created - bool refreshing = false; - refreshing = pData.haveToRefreshTSLoad && (y % pData.refreshIntervalLoad == 0); - refreshing - = refreshing || (pData.haveToRefreshTSSolar && (y % pData.refreshIntervalSolar == 0)); - refreshing - = refreshing || (pData.haveToRefreshTSWind && (y % pData.refreshIntervalWind == 0)); - refreshing - = refreshing || (pData.haveToRefreshTSHydro && (y % pData.refreshIntervalHydro == 0)); - - // Some thermal clusters may override the global parameter. - // Therefore, we may want to refresh TS even if pData.haveToRefreshTSThermal == false - bool haveToRefreshTSThermal - = pData.haveToRefreshTSThermal || study.runtime->thermalTSRefresh; - refreshing - = refreshing || (haveToRefreshTSThermal && (y % pData.refreshIntervalThermal == 0)); - - // We build a new set of parallel years if one of these conditions is fulfilled : - // - We have to refresh (or regenerate) some or all time series before running the - // current year - // - This is the first year (to be executed or not) after the previous set is full with - // years to be executed. That is : in the previous set filled, the max number of - // years to be actually run is reached. - buildNewSet = buildNewSet || refreshing; - - if (buildNewSet) - { - setOfParallelYears setToCreate; - setsOfParallelYears.push_back(setToCreate); - set = &(setsOfParallelYears.back()); - - // Initializations - set->nbPerformedYears = 0; - set->nbYears = 0; - set->regenerateTS = false; - set->yearForTSgeneration = 999999; - - // In case we have to regenerate times series before run the current set of parallel - // years - if (refreshing) - { - set->regenerateTS = true; - set->yearForTSgeneration = y; - } - } - - set->yearsIndices.push_back(y); - set->nbYears++; - set->yearFailed[y] = true; - set->isFirstPerformedYearOfASet[y] = false; - - if (performCalculations) - { - // Another year performed - ++pNbYearsReallyPerformed; - - // Number of actually performed years in the current set (up to now). - set->nbPerformedYears++; - // Index of the MC year's space (useful if this year is actually run) - indexSpace = set->nbPerformedYears - 1; - - set->isYearPerformed[y] = true; - set->performedYearToSpace[y] = indexSpace; - set->spaceToPerformedYear[indexSpace] = y; - - if (!foundFirstPerformedYearOfCurrentSet) - { - set->isFirstPerformedYearOfASet[y] = true; - foundFirstPerformedYearOfCurrentSet = true; - } - } - else - { - set->isYearPerformed[y] = false; - } - - // Do we build a new set at next iteration (for years to be executed or not) ? - if (indexSpace == pNbMaxPerformedYearsInParallel - 1 || y == endYear - 1) - { - buildNewSet = true; - foundFirstPerformedYearOfCurrentSet = false; - if (set->nbPerformedYears > maxNbYearsPerformed) - maxNbYearsPerformed = set->nbPerformedYears; - } - else - buildNewSet = false; - - } // End of loop over years - - return maxNbYearsPerformed; -} template void ISimulation::allocateMemoryForRandomNumbers(randomNumbers& randomForParallelYears) @@ -1492,21 +1380,22 @@ static void logPerformedYearsInAset(setOfParallelYears& set) } template -void ISimulation::loopThroughYears(uint firstYear, - uint endYear, +void ISimulation::loopThroughYears(uint endYear, std::vector& state) { assert(endYear <= study.parameters.nbYears); // List of parallel years sets - std::vector setsOfParallelYears; + + std::vector setsOfParallelYears = study.setsOfParallelYears; // Gets information on each set of parallel years and returns the max number of years performed // in a set The variable "maxNbYearsPerformedInAset" is the maximum numbers of years to be // actually executed in a set. A set contains some years to be actually executed (at most // "pNbMaxPerformedYearsInParallel" years) and some others to skip. - uint maxNbYearsPerformedInAset - = buildSetsOfParallelYears(firstYear, endYear, setsOfParallelYears); + + uint maxNbYearsPerformedInAset = pNbMaxPerformedYearsInParallel; + // Related to annual costs statistics (printed in output into separate files) pAnnualCostsStatistics.setNbPerformedYears(pNbYearsReallyPerformed); diff --git a/src/solver/simulation/solver.utils.h b/src/solver/simulation/solver.utils.h index 64287285f3..6afeebbbac 100644 --- a/src/solver/simulation/solver.utils.h +++ b/src/solver/simulation/solver.utils.h @@ -45,43 +45,6 @@ namespace Solver { namespace Simulation { -struct setOfParallelYears -{ - // Un lot d'année à exécuter en parallèle. - // En fonction d'une éventuelle play-list, certaines seront jouées et d'autres non. - -public: - // Numeros des annees en parallele pour ce lot (certaines ne seront pas jouées en cas de - // play-list "trouée") - std::vector yearsIndices; - - // Une annee doit-elle être rejouée ? - std::map yearFailed; - - // Associe le numero d'une année jouée à l'indice de l'espace - std::map performedYearToSpace; - - // L'inverse : pour une année jouée, associe l'indice de l'espace au numero de l'année - std::map spaceToPerformedYear; - - // Pour chaque année, est-elle la première à devoir être jouée dans son lot d'années ? - std::map isFirstPerformedYearOfASet; - - // Pour chaque année du lot, est-elle jouée ou non ? - std::map isYearPerformed; - - // Nbre d'années en parallele vraiment jouées pour ce lot - unsigned int nbPerformedYears; - - // Nbre d'années en parallele jouées ou non pour ce lot - unsigned int nbYears; - - // Regenere-t-on des times series avant de jouer les annees du lot courant - bool regenerateTS; - - // Annee a passer a la fonction "regenerateTimeSeries(y)" (si regenerateTS is "true") - unsigned int yearForTSgeneration; -}; class costStatistics { @@ -95,6 +58,11 @@ class costStatistics { } + uint getNbPerformedYears() const + { + return nbPerformedYears; + } + void setNbPerformedYears(uint n) { assert(n); @@ -211,11 +179,14 @@ class annualCostsStatistics buffer << to_scientific(criterionCost1.costStdDeviation) << "\n"; buffer << to_scientific(criterionCost1.costMin) << "\n"; buffer << to_scientific(criterionCost1.costMax) << "\n"; + // buffer << to_scientific(criterionCost1.getNbPerformedYears()) << "\n"; + buffer << to_scientific(criterionCost2.costAverage) << "\n"; buffer << to_scientific(criterionCost2.costStdDeviation) << "\n"; buffer << to_scientific(criterionCost2.costMin) << "\n"; buffer << to_scientific(criterionCost2.costMax) << "\n"; + // buffer << to_scientific(criterionCost2.getNbPerformedYears()) << "\n"; writer->addEntryFromBuffer(criterionsCostsFilename, buffer); } diff --git a/src/tests/end-to-end/simple-study.cpp b/src/tests/end-to-end/simple-study.cpp index c71c966d6f..516e9f81af 100644 --- a/src/tests/end-to-end/simple-study.cpp +++ b/src/tests/end-to-end/simple-study.cpp @@ -61,7 +61,7 @@ void prepareStudy(Study::Ptr pStudy, int nbYears) // ------------------------- // Getting the number of logical cores to use before loading and creating the areas : // Areas need this number to be up-to-date at construction. - pStudy->getNumberOfCores(false, 0); + pStudy->getNumberOfCores(false, false, 0); // Define as current study Data::Study::Current::Set(pStudy); diff --git a/src/tests/src/libs/antares/study/CMakeLists.txt b/src/tests/src/libs/antares/study/CMakeLists.txt index c5c0bcfb12..3ea4e8a414 100644 --- a/src/tests/src/libs/antares/study/CMakeLists.txt +++ b/src/tests/src/libs/antares/study/CMakeLists.txt @@ -3,3 +3,4 @@ set(src_tests_src_libs_antares_study "${CMAKE_CURRENT_SOURCE_DIR}") add_subdirectory(area) add_subdirectory(scenario-builder) add_subdirectory(output-folder) +add_subdirectory(parallel-years) \ No newline at end of file diff --git a/src/tests/src/libs/antares/study/parallel-years/CMakeLists.txt b/src/tests/src/libs/antares/study/parallel-years/CMakeLists.txt new file mode 100644 index 0000000000..cca96e077a --- /dev/null +++ b/src/tests/src/libs/antares/study/parallel-years/CMakeLists.txt @@ -0,0 +1,35 @@ +# Useful variables definitions +set(src_libs_antares_study "${CMAKE_SOURCE_DIR}/libs/antares/study") + +# =========================================== +# Tests on calculating forced parallel years +# =========================================== +set(SRC_PARALLEL_YEAR_CALC + test-parallel-years-calculator.cpp +) +add_executable(test-parallel-years-calculator ${SRC_PARALLEL_YEAR_CALC}) + +target_include_directories(test-parallel-years-calculator + PRIVATE + "${src_libs_antares_study}" + "${src_libs_antares_study}/parallel-years" +) + +target_link_libraries(test-parallel-years-calculator + PRIVATE + Boost::unit_test_framework + libantares-core + libmodel_antares +) +# Linux +if(UNIX AND NOT APPLE) +target_link_libraries(test-parallel-years-calculator PRIVATE stdc++fs) +endif() + + +# Storing test-parallel-years-calculator under the folder Unit-tests in the IDE +set_target_properties(test-parallel-years-calculator PROPERTIES FOLDER Unit-tests/parallel-years) + +add_test(NAME parallel-years-calculator COMMAND test-parallel-years-calculator) + +set_property(TEST parallel-years-calculator PROPERTY LABELS unit) \ No newline at end of file diff --git a/src/tests/src/libs/antares/study/parallel-years/parallel-years-wrapper.h b/src/tests/src/libs/antares/study/parallel-years/parallel-years-wrapper.h new file mode 100644 index 0000000000..92c11704bf --- /dev/null +++ b/src/tests/src/libs/antares/study/parallel-years/parallel-years-wrapper.h @@ -0,0 +1,55 @@ +/* +** Copyright 2007-2022 RTE +** Authors: Antares_Simulator Team +** +** This file is part of Antares_Simulator. +** +** Antares_Simulator is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** There are special exceptions to the terms and conditions of the +** license as they are applied to this software. View the full text of +** the exceptions in file COPYING.txt in the directory of this software +** distribution +** +** Antares_Simulator is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with Antares_Simulator. If not, see . +** +** SPDX-License-Identifier: licenceRef-GPL3_WITH_RTE-Exceptions +*/ +#pragma once + +#include + +namespace Antares +{ + + +class SetsOfParallelYearCalculatorWrapper +{ +public: + SetsOfParallelYearCalculatorWrapper(SetsOfParallelYearCalculator& builder_) + : builder(builder_){} + + void build(){ builder.build(); } + + void computeRawNbParallelYear(){ builder.computeRawNbParallelYear(); } + void limitNbOfParallelYearsbyMinRefreshSpan() {builder.limitNbOfParallelYearsbyMinRefreshSpan();} + + void buildSetsOfParallelYears(){builder.buildSetsOfParallelYears();} + + uint computeMinNbParallelYears() const {return builder.computeMinNbParallelYears();} + void computeForcedNbYearsInParallelYearSet() {builder.computeForcedNbYearsInParallelYearSet();} + + SetsOfParallelYearCalculator builder; + +}; + +} // namespace Anatares \ No newline at end of file diff --git a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp new file mode 100644 index 0000000000..008aeebd36 --- /dev/null +++ b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp @@ -0,0 +1,291 @@ +#define BOOST_TEST_MODULE test-end-to-end tests + +#define WIN32_LEAN_AND_MEAN + +#include + +#include +#include + +#include +#include + +namespace utf = boost::unit_test; +namespace tt = boost::test_tools; + +using namespace Yuni; +using namespace Antares::Data; + + +uint nbLogicalCores_{2}; + +namespace Yuni +{ +namespace System +{ +namespace CPU +{ + uint Count() + { + // The number of processors online (capable of running processes) + return nbLogicalCores_; + } + +} +} +} + + +struct Fixture +{ + Fixture(const Fixture& f) = delete; + Fixture(const Fixture&& f) = delete; + Fixture& operator= (const Fixture& f) = delete; + Fixture& operator= (const Fixture&& f) = delete; + Fixture() + { + // using default params + params.reset(); + } + + ~Fixture(){} + + //creating a study, maybe not needed + Study::Ptr study = std::make_shared(); + Parameters params{}; + Data::StudyLoadOptions options; + +}; + +BOOST_FIXTURE_TEST_SUITE(s, Fixture) + +BOOST_AUTO_TEST_CASE(default_params_no_force) +{ + bool thermalTSRefresh = false; + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 1); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 1); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 1); +} + +BOOST_AUTO_TEST_CASE(hundred_years_no_force) +{ + bool thermalTSRefresh = false; + params.nbYears = 100; + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 1); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 1); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); +} + +BOOST_AUTO_TEST_CASE(four_mc_years_force_parallel) +{ + + bool thermalTSRefresh = false; + options.forceParallel = true; + options.enableParallel = true; + options.maxNbYearsInParallel = 2; + params.nbYears = 4; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 2); + BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 2); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 4); +} + +BOOST_AUTO_TEST_CASE(hundred_mc_years_force_parallel_sets_size_five_thermal_refresh_on) +{ + + bool thermalTSRefresh = true; + options.forceParallel = true; + options.enableParallel = true; + options.maxNbYearsInParallel = 5; + params.nbYears = 100; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 5); + BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 5); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); +} + + +BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_thermal_refresh_on) +{ + + bool thermalTSRefresh = true; + options.forceParallel = false; + options.enableParallel = false; + params.nbYears = 100; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 1); + // BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 1); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); +} + +BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel) +{ + + bool thermalTSRefresh = false; + options.forceParallel = false; + options.enableParallel = true; + + // Shouldn't have any effect since force parallel is off + options.maxNbYearsInParallel = 10; + + params.nbYears = 100; + + nbLogicalCores_ = 16; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 12); + BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 4); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); +} + +BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel_thermal_refresh_on) +{ + + bool thermalTSRefresh = true; + options.forceParallel = false; + options.enableParallel = true; + + // Shouldn't have any effect since force parallel is off + options.maxNbYearsInParallel = 10; + + params.nbYears = 100; + + nbLogicalCores_ = 8; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 4); + BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 4); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); +} + + +BOOST_AUTO_TEST_CASE(hundred_mc_years_force_10_parallel_user_playlist_thermal_refresh_on) +{ + + bool thermalTSRefresh = true; + options.forceParallel = true; + options.enableParallel = true; + + options.maxNbYearsInParallel = 10; + + params.nbYears = 100; + + // Create playlist + params.userPlaylist = true; + params.yearsFilter.reserve(params.nbYears); + for (uint i = 0; i != params.nbYears; ++i) + params.yearsFilter[i] = false; + params.yearsFilter[2] = true; + params.resetYearsWeigth(); + params.setYearWeight(0,4); + params.setYearWeight(1,10); + params.setYearWeight(2,3); + + // Override number of raw cores + nbLogicalCores_ = 4; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncLow; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 1); + BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 1); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 1); +} + +BOOST_AUTO_TEST_CASE(hundred_mc_years_enable_parallel_user_playlist_thermal_refresh_on) +{ + + bool thermalTSRefresh = true; + options.forceParallel = false; + options.enableParallel = true; + + // Shouldn't have any effect since force parallel is off + options.maxNbYearsInParallel = 10; + + params.nbYears = 100; + + // Create playlist + params.userPlaylist = true; + params.yearsFilter.reserve(params.nbYears); + for (uint i = 0; i != params.nbYears; ++i) + { + if(i%2 == 0) + params.yearsFilter[i] = true; + else + params.yearsFilter[i] = false; + } + + params.resetYearsWeigth(); + params.setYearWeight(0,4); + params.setYearWeight(1,10); + params.setYearWeight(2,3); + + // Override number of raw cores + nbLogicalCores_ = 6; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 4); + BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 2); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 50); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/src/ui/simulator/application/study.cpp b/src/ui/simulator/application/study.cpp index 798d9b0c3a..b1d0753f5c 100644 --- a/src/ui/simulator/application/study.cpp +++ b/src/ui/simulator/application/study.cpp @@ -319,7 +319,7 @@ class JobSaveStudy final : public Toolbox::Jobs::Job // Updating the number of logical cores to use when saving the study // so that the run window is up to date. - study->getNumberOfCores(false, 0); + study->getNumberOfCores(false, false, 0); if (pSaveAs || pShouldInvalidateStudy) { diff --git a/src/ui/simulator/windows/options/advanced/advanced.cpp b/src/ui/simulator/windows/options/advanced/advanced.cpp index 331c011ef3..8d6ce3926f 100644 --- a/src/ui/simulator/windows/options/advanced/advanced.cpp +++ b/src/ui/simulator/windows/options/advanced/advanced.cpp @@ -916,7 +916,7 @@ void AdvancedParameters::onSelectNumberOfCoresLevel(Data::NumberOfCoresMode ncMo { study->parameters.nbCores.ncMode = ncMode; // Force refresh for study->nbYearsParallelRaw - study->getNumberOfCores(false, 1 /* ignored */); + study->getNumberOfCores(false, false, 1 /* ignored */); MarkTheStudyAsModified(); refresh(); } From b2e33ee60f550f05ef174e181fb612410eb53fe7 Mon Sep 17 00:00:00 2001 From: kathvargasr <113925565+kathvargasr@users.noreply.github.com> Date: Mon, 19 Dec 2022 17:22:15 +0100 Subject: [PATCH 2/9] Add more UTs and resolve code review comments --- src/libs/antares/study/parallel-years.cpp | 53 +++++--- src/libs/antares/study/parallel-years.h | 39 +++--- src/libs/antares/study/study.h | 2 + src/solver/simulation/solver.utils.h | 3 - .../parallel-years/parallel-years-wrapper.h | 55 --------- .../test-parallel-years-calculator.cpp | 116 +++++++++++++++--- 6 files changed, 160 insertions(+), 108 deletions(-) delete mode 100644 src/tests/src/libs/antares/study/parallel-years/parallel-years-wrapper.h diff --git a/src/libs/antares/study/parallel-years.cpp b/src/libs/antares/study/parallel-years.cpp index 2c5bc3eb5d..578a191c5e 100644 --- a/src/libs/antares/study/parallel-years.cpp +++ b/src/libs/antares/study/parallel-years.cpp @@ -27,7 +27,7 @@ #include #include -#include +#include #include // For use of Yuni::System::CPU::Count() #include @@ -39,7 +39,16 @@ #include "parameters.h" #include "parallel-years.h" -namespace Antares{ +#define SEP Yuni::IO::Separator + +namespace Antares +{ + +/*! +** \brief Checks if clusters have the "gen-ts" param +** +** \warning TODO: REMOVE THIS METHOD ONCE THE TSGEN HAS BEEN DECOUPLED FROM SOLVER +*/ bool TempAreaListHolder::checkThermalTSGeneration(YString folder_) { @@ -52,6 +61,7 @@ bool TempAreaListHolder::checkThermalTSGeneration(YString folder_) Yuni::Clob thermalPlant = folder << SEP << "thermal" << SEP << "clusters" << SEP << areaName << SEP << "list.ini"; IniFile ini; + bool ret = false; if (ini.open(thermalPlant)) { if(ini.firstSection) @@ -60,12 +70,12 @@ bool TempAreaListHolder::checkThermalTSGeneration(YString folder_) { for (IniFile::Property* property = section->firstProperty; property; property = property->next) { - return property->key == "gen-ts"; + ret = ret || (property->key == "gen-ts"); } } } - return false; + return ret; } logs.error() << "Thermal Cluster Ini file cannot be opened: " << thermalPlant.c_str(); return false; @@ -75,6 +85,12 @@ bool TempAreaListHolder::checkThermalTSGeneration(YString folder_) } +/*! +** \brief Loads Area List to later check Thermal Cluster's refresh +** +** \warning TODO: REMOVE THIS METHOD ONCE THE TSGEN HAS BEEN DECOUPLED FROM SOLVER +*/ + void TempAreaListHolder::loadAreaList() { auto folder = inputFolder; @@ -105,11 +121,13 @@ void TempAreaListHolder::loadAreaList() void SetsOfParallelYearCalculator::computeRawNbParallelYear() { + // In case solver option '--force-parallel n' is used, this computation is not needed + // and n will remain the forcedNbOfParallelYears if (forceParallel) return; std::map numberOfMCYearThreads; - uint nbLogicalCores = Yuni::System::CPU::Count(); + const uint nbLogicalCores = Yuni::System::CPU::Count(); numberOfMCYearThreads[ncMin] = 1; switch (nbLogicalCores) @@ -222,20 +240,15 @@ void SetsOfParallelYearCalculator::limitNbOfParallelYearsbyMinRefreshSpan() if ((p.timeSeriesToGenerate & timeSeriesLoad) && (p.timeSeriesToRefresh & timeSeriesLoad)) TSlimit = p.refreshIntervalLoad; if ((p.timeSeriesToGenerate & timeSeriesSolar) && (p.timeSeriesToRefresh & timeSeriesSolar)) - TSlimit = (p.refreshIntervalSolar < TSlimit) ? p.refreshIntervalSolar : TSlimit; + TSlimit = std::min(p.refreshIntervalSolar, TSlimit); if ((p.timeSeriesToGenerate & timeSeriesHydro) && (p.timeSeriesToRefresh & timeSeriesHydro)) - TSlimit = (p.refreshIntervalHydro < TSlimit) ? p.refreshIntervalHydro : TSlimit; + TSlimit = std::min(p.refreshIntervalHydro, TSlimit); if ((p.timeSeriesToGenerate & timeSeriesWind) && (p.timeSeriesToRefresh & timeSeriesWind)) - TSlimit = (p.refreshIntervalWind < TSlimit) ? p.refreshIntervalWind : TSlimit; + TSlimit = std::min(p.refreshIntervalWind, TSlimit); if ((p.timeSeriesToGenerate & timeSeriesThermal) && (p.timeSeriesToRefresh & timeSeriesThermal)) - TSlimit = (p.refreshIntervalThermal < TSlimit) ? p.refreshIntervalThermal : TSlimit; - - if (TSlimit < forcedNbOfParallelYears) - forcedNbOfParallelYears = TSlimit; + TSlimit = std::min(p.refreshIntervalThermal, TSlimit); - // Limiting the number of parallel years by the total number of years - if (p.nbYears < forcedNbOfParallelYears) - forcedNbOfParallelYears = p.nbYears; + forcedNbOfParallelYears = std::min({p.nbYears, TSlimit, forcedNbOfParallelYears}); } bool SetsOfParallelYearCalculator::isRefreshNeededForCurrentYear(uint y) @@ -243,19 +256,19 @@ bool SetsOfParallelYearCalculator::isRefreshNeededForCurrentYear(uint y) bool refreshing = false; refreshing = (p.timeSeriesToGenerate & timeSeriesLoad) && (p.timeSeriesToRefresh & timeSeriesLoad) - && (!y || ((y % p.refreshIntervalLoad) == 0)); + && ((y % p.refreshIntervalLoad) == 0); refreshing = refreshing || ((p.timeSeriesToGenerate & timeSeriesSolar) && (p.timeSeriesToRefresh & timeSeriesSolar) - && (!y || ((y % p.refreshIntervalSolar) == 0))); + && (y % p.refreshIntervalSolar) == 0); refreshing = refreshing || ((p.timeSeriesToGenerate & timeSeriesWind) && (p.timeSeriesToRefresh & timeSeriesWind) - && (!y || ((y % p.refreshIntervalWind) == 0))); + && (y % p.refreshIntervalWind) == 0); refreshing = refreshing || ((p.timeSeriesToGenerate & timeSeriesHydro) && (p.timeSeriesToRefresh & timeSeriesHydro) - && (!y || ((y % p.refreshIntervalHydro) == 0))); + && (y % p.refreshIntervalHydro) == 0); bool haveToRefreshTSThermal = ((p.timeSeriesToGenerate & timeSeriesThermal) @@ -403,7 +416,7 @@ void SetsOfParallelYearCalculator::computeForcedNbYearsInParallelYearSet() maxNbYearsOverAllSets = (uint)setsOfParallelYears[s].setsSizes.size(); } - forcedNbOfParallelYears = maxNbYearsOverAllSets; //j'aime pas + forcedNbOfParallelYears = maxNbYearsOverAllSets; } diff --git a/src/libs/antares/study/parallel-years.h b/src/libs/antares/study/parallel-years.h index 9e85bd0aa1..1206d4d2e0 100644 --- a/src/libs/antares/study/parallel-years.h +++ b/src/libs/antares/study/parallel-years.h @@ -27,44 +27,42 @@ #pragma once #include -#include "../../../solver/simulation/solver.utils.h" // KVR se debarraser de ce include namespace Antares { using namespace Data; -// using namespace Solver::Simulation; struct setOfParallelYears { - // Un lot d'ann�e � ex�cuter en parall�le. - // En fonction d'une �ventuelle play-list, certaines seront jou�es et d'autres non. + // Un lot d'année à exécuter en parallèle. + // En fonction d'une éventuelle play-list, certaines seront jouées et d'autres non. public: std::vector setsSizes; - // Numeros des annees en parallele pour ce lot (certaines ne seront pas jou�es en cas de - // play-list "trou�e") + // Numeros des annees en parallele pour ce lot (certaines ne seront pas jouées en cas de + // play-list "trouée") std::vector yearsIndices; - // Une annee doit-elle �tre rejou�e ? + // Une annee doit-elle être rejouée ? std::map yearFailed; - // Associe le numero d'une ann�e jou�e � l'indice de l'espace + // Associe le numero d'une année jouée à l'indice de l'espace std::map performedYearToSpace; - // L'inverse : pour une ann�e jou�e, associe l'indice de l'espace au numero de l'ann�e + // L'inverse : pour une année jouée, associe l'indice de l'espace au numero de l'année std::map spaceToPerformedYear; - // Pour chaque ann�e, est-elle la premi�re � devoir �tre jou�e dans son lot d'ann�es ? + // Pour chaque année, est-elle la première à devoir être jouée dans son lot d'années ? std::map isFirstPerformedYearOfASet; - // Pour chaque ann�e du lot, est-elle jou�e ou non ? + // Pour chaque année du lot, est-elle jouée ou non ? std::map isYearPerformed; - // Nbre d'ann�es en parallele vraiment jou�es pour ce lot + // Nbre d'années en parallele vraiment jouées pour ce lot unsigned int nbPerformedYears; - // Nbre d'ann�es en parallele jou�es ou non pour ce lot + // Nbre d'années en parallele jouées ou non pour ce lot unsigned int nbYears; // Regenere-t-on des times series avant de jouer les annees du lot courant @@ -74,6 +72,19 @@ struct setOfParallelYears unsigned int yearForTSgeneration; }; +/*! +** \brief Temporary Area List Holder +** +** \warning TODO: REMOVE THIS CLASS ONCE THE TSGEN HAS BEEN DECOUPLED FROM SOLVER +** +** To compute the sets of parallel years while using the Time Series Generator +** we need to know in advance if the area's thermal clusters will be refresehd during generation +** Although, since the max number of parallel years is needed to create the areas, this info +** is not available yet (and tht's why, originally, this calculation was done twice) +** +** This temporary holder will load the area list and will check for each one if +** their clusters will require refreshing during generation +*/ class TempAreaListHolder { public: @@ -91,8 +102,6 @@ class TempAreaListHolder class SetsOfParallelYearCalculator { public: - - friend class SetsOfParallelYearCalculatorWrapper; SetsOfParallelYearCalculator( bool forceParallel_, diff --git a/src/libs/antares/study/study.h b/src/libs/antares/study/study.h index 3bbb0c1edb..a3594f7e3c 100644 --- a/src/libs/antares/study/study.h +++ b/src/libs/antares/study/study.h @@ -612,6 +612,8 @@ class Study final : public Yuni::NonCopyable, public IObject, public Laye uint minNbYearsInParallel_save; // Used in solver + // ----------------- + // Stores the sets of parallel years to be used by the solver std::vector setsOfParallelYears; uint pNbYearsReallyPerformed; diff --git a/src/solver/simulation/solver.utils.h b/src/solver/simulation/solver.utils.h index 6afeebbbac..e8376ae2f2 100644 --- a/src/solver/simulation/solver.utils.h +++ b/src/solver/simulation/solver.utils.h @@ -179,14 +179,11 @@ class annualCostsStatistics buffer << to_scientific(criterionCost1.costStdDeviation) << "\n"; buffer << to_scientific(criterionCost1.costMin) << "\n"; buffer << to_scientific(criterionCost1.costMax) << "\n"; - // buffer << to_scientific(criterionCost1.getNbPerformedYears()) << "\n"; - buffer << to_scientific(criterionCost2.costAverage) << "\n"; buffer << to_scientific(criterionCost2.costStdDeviation) << "\n"; buffer << to_scientific(criterionCost2.costMin) << "\n"; buffer << to_scientific(criterionCost2.costMax) << "\n"; - // buffer << to_scientific(criterionCost2.getNbPerformedYears()) << "\n"; writer->addEntryFromBuffer(criterionsCostsFilename, buffer); } diff --git a/src/tests/src/libs/antares/study/parallel-years/parallel-years-wrapper.h b/src/tests/src/libs/antares/study/parallel-years/parallel-years-wrapper.h deleted file mode 100644 index 92c11704bf..0000000000 --- a/src/tests/src/libs/antares/study/parallel-years/parallel-years-wrapper.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -** Copyright 2007-2022 RTE -** Authors: Antares_Simulator Team -** -** This file is part of Antares_Simulator. -** -** Antares_Simulator is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** There are special exceptions to the terms and conditions of the -** license as they are applied to this software. View the full text of -** the exceptions in file COPYING.txt in the directory of this software -** distribution -** -** Antares_Simulator is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with Antares_Simulator. If not, see . -** -** SPDX-License-Identifier: licenceRef-GPL3_WITH_RTE-Exceptions -*/ -#pragma once - -#include - -namespace Antares -{ - - -class SetsOfParallelYearCalculatorWrapper -{ -public: - SetsOfParallelYearCalculatorWrapper(SetsOfParallelYearCalculator& builder_) - : builder(builder_){} - - void build(){ builder.build(); } - - void computeRawNbParallelYear(){ builder.computeRawNbParallelYear(); } - void limitNbOfParallelYearsbyMinRefreshSpan() {builder.limitNbOfParallelYearsbyMinRefreshSpan();} - - void buildSetsOfParallelYears(){builder.buildSetsOfParallelYears();} - - uint computeMinNbParallelYears() const {return builder.computeMinNbParallelYears();} - void computeForcedNbYearsInParallelYearSet() {builder.computeForcedNbYearsInParallelYearSet();} - - SetsOfParallelYearCalculator builder; - -}; - -} // namespace Anatares \ No newline at end of file diff --git a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp index 008aeebd36..94508d3d3d 100644 --- a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp +++ b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp @@ -7,9 +7,6 @@ #include #include -#include -#include - namespace utf = boost::unit_test; namespace tt = boost::test_tools; @@ -19,11 +16,7 @@ using namespace Antares::Data; uint nbLogicalCores_{2}; -namespace Yuni -{ -namespace System -{ -namespace CPU +namespace Yuni::System::CPU { uint Count() { @@ -32,9 +25,6 @@ namespace CPU } } -} -} - struct Fixture { @@ -62,6 +52,10 @@ BOOST_FIXTURE_TEST_SUITE(s, Fixture) BOOST_AUTO_TEST_CASE(default_params_no_force) { bool thermalTSRefresh = false; + + nbLogicalCores_ = 1; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; + SetsOfParallelYearCalculator builder(options.forceParallel, options.enableParallel, options.maxNbYearsInParallel, @@ -73,10 +67,44 @@ BOOST_AUTO_TEST_CASE(default_params_no_force) BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 1); } +BOOST_AUTO_TEST_CASE(default_params_no_force_bad_number_of_cores, *utf::expected_failures(1)) +{ + bool thermalTSRefresh = false; + + nbLogicalCores_ = 0; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncUnknown; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + +} + +BOOST_AUTO_TEST_CASE(default_params_no_force_bad_ncmode, *utf::expected_failures(1)) +{ + bool thermalTSRefresh = false; + + nbLogicalCores_ = 1; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncUnknown; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + +} + BOOST_AUTO_TEST_CASE(hundred_years_no_force) { bool thermalTSRefresh = false; params.nbYears = 100; + + nbLogicalCores_ = 12; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; + SetsOfParallelYearCalculator builder(options.forceParallel, options.enableParallel, options.maxNbYearsInParallel, @@ -88,6 +116,7 @@ BOOST_AUTO_TEST_CASE(hundred_years_no_force) BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); } + BOOST_AUTO_TEST_CASE(four_mc_years_force_parallel) { @@ -139,6 +168,9 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_thermal_refresh_on) options.enableParallel = false; params.nbYears = 100; + nbLogicalCores_ = 5; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; + SetsOfParallelYearCalculator builder(options.forceParallel, options.enableParallel, options.maxNbYearsInParallel, @@ -205,6 +237,60 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel_thermal_refresh_o BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); } +BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel_thermal_refresh_on_draft_mode) +{ + + bool thermalTSRefresh = true; + options.forceParallel = false; + options.enableParallel = true; + + params.mode = Antares::Data::stdmAdequacyDraft; + + params.nbYears = 100; + + nbLogicalCores_ = 3; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 1); + BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 1); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); +} + +BOOST_AUTO_TEST_CASE(hundred_mc_years_force_parallel_enable_parallel_thermal_refresh_on_hot_start) +{ + + bool thermalTSRefresh = true; + options.forceParallel = true; + options.enableParallel = true; + + options.maxNbYearsInParallel = 5; + + params.initialReservoirLevels.iniLevels = Antares::Data::irlHotStart; + + params.nbYears = 100; + + nbLogicalCores_ = 4; + params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; + + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, + thermalTSRefresh, + params); + + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 5); + BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 5); + BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 100); +} + BOOST_AUTO_TEST_CASE(hundred_mc_years_force_10_parallel_user_playlist_thermal_refresh_on) { @@ -229,7 +315,7 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_10_parallel_user_playlist_thermal_re params.setYearWeight(2,3); // Override number of raw cores - nbLogicalCores_ = 4; + nbLogicalCores_ = 7; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncLow; SetsOfParallelYearCalculator builder(options.forceParallel, @@ -273,7 +359,7 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_enable_parallel_user_playlist_thermal_refr params.setYearWeight(2,3); // Override number of raw cores - nbLogicalCores_ = 6; + nbLogicalCores_ = 9; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; SetsOfParallelYearCalculator builder(options.forceParallel, @@ -282,9 +368,9 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_enable_parallel_user_playlist_thermal_refr thermalTSRefresh, params); - BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 4); + BOOST_CHECK_EQUAL(builder.getForcedNbOfParallelYears(), 7); BOOST_CHECK(builder.allSetsParallelYearsHaveSameSize()); - BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 2); + BOOST_CHECK_EQUAL(builder.getMinNbParallelYearsForGUI(), 1); BOOST_CHECK_EQUAL(builder.getNbYearsReallyPerformed(), 50); } From d8cba1d1d27c33bbe8dae6da1a09af499dda469d Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Mon, 23 Jan 2023 11:36:58 +0100 Subject: [PATCH 3/9] Explain why call to loadAreaList is necessary --- src/libs/antares/study/parallel-years.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/libs/antares/study/parallel-years.cpp b/src/libs/antares/study/parallel-years.cpp index 578a191c5e..245d84ebd4 100644 --- a/src/libs/antares/study/parallel-years.cpp +++ b/src/libs/antares/study/parallel-years.cpp @@ -25,6 +25,7 @@ ** SPDX-License-Identifier: licenceRef-GPL3_WITH_RTE-Exceptions */ +#include #include #include #include @@ -52,14 +53,12 @@ namespace Antares bool TempAreaListHolder::checkThermalTSGeneration(YString folder_) { - inputFolder = folder_; - loadAreaList(); + inputFolder = std::move(folder_); + loadAreaList(); //Build areaNames - return std::any_of(areaNames.begin(), areaNames.end(),[this](Yuni::String areaName){ - + return std::any_of(areaNames.begin(), areaNames.end(),[this](const Yuni::String& areaName){ auto folder = inputFolder; Yuni::Clob thermalPlant = folder << SEP << "thermal" << SEP << "clusters" << SEP << areaName << SEP << "list.ini"; - IniFile ini; bool ret = false; if (ini.open(thermalPlant)) @@ -73,16 +72,13 @@ bool TempAreaListHolder::checkThermalTSGeneration(YString folder_) ret = ret || (property->key == "gen-ts"); } } - } return ret; } logs.error() << "Thermal Cluster Ini file cannot be opened: " << thermalPlant.c_str(); return false; - } ); - } /*! From 3d36a4c0c9aef36220f3ccd1b5627a23c21faa9e Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Mon, 23 Jan 2023 16:01:34 +0100 Subject: [PATCH 4/9] Inject the number of core to compute parallel years --- src/libs/antares/study/parallel-years.cpp | 2 +- src/libs/antares/study/parallel-years.h | 20 ++--- src/libs/antares/study/study.cpp | 2 +- .../test-parallel-years-calculator.cpp | 87 +++++++------------ 4 files changed, 43 insertions(+), 68 deletions(-) diff --git a/src/libs/antares/study/parallel-years.cpp b/src/libs/antares/study/parallel-years.cpp index 245d84ebd4..e3fb99e1c7 100644 --- a/src/libs/antares/study/parallel-years.cpp +++ b/src/libs/antares/study/parallel-years.cpp @@ -123,7 +123,7 @@ void SetsOfParallelYearCalculator::computeRawNbParallelYear() return; std::map numberOfMCYearThreads; - const uint nbLogicalCores = Yuni::System::CPU::Count(); + const uint nbLogicalCores = number_of_cores_; numberOfMCYearThreads[ncMin] = 1; switch (nbLogicalCores) diff --git a/src/libs/antares/study/parallel-years.h b/src/libs/antares/study/parallel-years.h index 1206d4d2e0..46d2daa76e 100644 --- a/src/libs/antares/study/parallel-years.h +++ b/src/libs/antares/study/parallel-years.h @@ -103,15 +103,12 @@ class SetsOfParallelYearCalculator { public: - SetsOfParallelYearCalculator( - bool forceParallel_, - bool enableParallel_, - uint forcedNbOfParallelYears_, - bool thermalTSRefresh_, // Temporary parameter needed. To be removed once TSGen will be detached from simulator - Parameters& params_) + SetsOfParallelYearCalculator(bool forceParallel_, bool enableParallel_, uint forcedNbOfParallelYears_, + uint number_of_cores, bool thermalTSRefresh_, Parameters ¶ms_) : forceParallel{forceParallel_}, enableParallel {enableParallel_}, forcedNbOfParallelYears{forcedNbOfParallelYears_}, + number_of_cores_{number_of_cores}, thermalTSRefresh{thermalTSRefresh_}, p{params_}{ this->build(); @@ -120,22 +117,22 @@ class SetsOfParallelYearCalculator bool allSetsParallelYearsHaveSameSize(); - uint getForcedNbOfParallelYears() const + [[nodiscard]] uint getForcedNbOfParallelYears() const { return forcedNbOfParallelYears; } - uint getMinNbParallelYearsForGUI() const + [[nodiscard]] uint getMinNbParallelYearsForGUI() const { return computeMinNbParallelYears(); } - std::vector getSetsOfParallelYears() const + [[nodiscard]] std::vector getSetsOfParallelYears() const { return setsOfParallelYears; } - uint getNbYearsReallyPerformed() const + [[nodiscard]] uint getNbYearsReallyPerformed() const { return nbYearsReallyPerformed; } @@ -149,7 +146,7 @@ class SetsOfParallelYearCalculator void buildSetsOfParallelYears(); - uint computeMinNbParallelYears() const; + [[nodiscard]] uint computeMinNbParallelYears() const; void computeForcedNbYearsInParallelYearSet(); bool isRefreshNeededForCurrentYear(uint y); @@ -159,6 +156,7 @@ class SetsOfParallelYearCalculator bool forceParallel; bool enableParallel; uint forcedNbOfParallelYears; + uint number_of_cores_; bool thermalTSRefresh; Parameters& p; uint nbYearsReallyPerformed{0}; diff --git a/src/libs/antares/study/study.cpp b/src/libs/antares/study/study.cpp index bf8250f8d6..e0d9f9dbde 100644 --- a/src/libs/antares/study/study.cpp +++ b/src/libs/antares/study/study.cpp @@ -231,7 +231,7 @@ void Study::getNumberOfCores(const bool forceParallel, const bool enableParallel SetsOfParallelYearCalculator setsBuilder(forceParallel, enableParallel, - nbYearsParallelForced, + nbYearsParallelForced, Yuni::System::CPU::Count(), thermalTSRefresh, parameters); // For GUI diff --git a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp index 94508d3d3d..d6578a2c0c 100644 --- a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp +++ b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp @@ -13,19 +13,6 @@ namespace tt = boost::test_tools; using namespace Yuni; using namespace Antares::Data; - -uint nbLogicalCores_{2}; - -namespace Yuni::System::CPU -{ - uint Count() - { - // The number of processors online (capable of running processes) - return nbLogicalCores_; - } - -} - struct Fixture { Fixture(const Fixture& f) = delete; @@ -53,12 +40,11 @@ BOOST_AUTO_TEST_CASE(default_params_no_force) { bool thermalTSRefresh = false; - nbLogicalCores_ = 1; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, 1, thermalTSRefresh, params); @@ -71,12 +57,11 @@ BOOST_AUTO_TEST_CASE(default_params_no_force_bad_number_of_cores, *utf::expected { bool thermalTSRefresh = false; - nbLogicalCores_ = 0; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncUnknown; SetsOfParallelYearCalculator builder(options.forceParallel, options.enableParallel, - options.maxNbYearsInParallel, + options.maxNbYearsInParallel, 0, thermalTSRefresh, params); @@ -86,28 +71,27 @@ BOOST_AUTO_TEST_CASE(default_params_no_force_bad_ncmode, *utf::expected_failures { bool thermalTSRefresh = false; - nbLogicalCores_ = 1; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncUnknown; SetsOfParallelYearCalculator builder(options.forceParallel, options.enableParallel, - options.maxNbYearsInParallel, + options.maxNbYearsInParallel, 1, thermalTSRefresh, params); } +constexpr uint default_number_of_core = 12; BOOST_AUTO_TEST_CASE(hundred_years_no_force) { bool thermalTSRefresh = false; params.nbYears = 100; - nbLogicalCores_ = 12; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, default_number_of_core, thermalTSRefresh, params); @@ -126,9 +110,9 @@ BOOST_AUTO_TEST_CASE(four_mc_years_force_parallel) options.maxNbYearsInParallel = 2; params.nbYears = 4; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, default_number_of_core, thermalTSRefresh, params); @@ -147,9 +131,9 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_parallel_sets_size_five_thermal_refr options.maxNbYearsInParallel = 5; params.nbYears = 100; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, default_number_of_core, thermalTSRefresh, params); @@ -168,12 +152,11 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_thermal_refresh_on) options.enableParallel = false; params.nbYears = 100; - nbLogicalCores_ = 5; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, 5, thermalTSRefresh, params); @@ -195,12 +178,11 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel) params.nbYears = 100; - nbLogicalCores_ = 16; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, 16, thermalTSRefresh, params); @@ -222,12 +204,11 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel_thermal_refresh_o params.nbYears = 100; - nbLogicalCores_ = 8; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, 8, thermalTSRefresh, params); @@ -248,12 +229,11 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel_thermal_refresh_o params.nbYears = 100; - nbLogicalCores_ = 3; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; SetsOfParallelYearCalculator builder(options.forceParallel, options.enableParallel, - options.maxNbYearsInParallel, + options.maxNbYearsInParallel, 3, thermalTSRefresh, params); @@ -276,12 +256,11 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_parallel_enable_parallel_thermal_ref params.nbYears = 100; - nbLogicalCores_ = 4; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; SetsOfParallelYearCalculator builder(options.forceParallel, options.enableParallel, - options.maxNbYearsInParallel, + options.maxNbYearsInParallel, 4, thermalTSRefresh, params); @@ -315,12 +294,11 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_10_parallel_user_playlist_thermal_re params.setYearWeight(2,3); // Override number of raw cores - nbLogicalCores_ = 7; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncLow; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, 7, thermalTSRefresh, params); @@ -359,12 +337,11 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_enable_parallel_user_playlist_thermal_refr params.setYearWeight(2,3); // Override number of raw cores - nbLogicalCores_ = 9; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, + SetsOfParallelYearCalculator builder(options.forceParallel, + options.enableParallel, + options.maxNbYearsInParallel, 9, thermalTSRefresh, params); From dc8aba10e5e5b0d86e9f24e4006cee7e93051446 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Tue, 24 Jan 2023 14:56:40 +0100 Subject: [PATCH 5/9] Remove dependency to Data::StudyLoadOptions for test --- .../test-parallel-years-calculator.cpp | 132 +++++++++--------- 1 file changed, 67 insertions(+), 65 deletions(-) diff --git a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp index d6578a2c0c..f942883519 100644 --- a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp +++ b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp @@ -30,7 +30,9 @@ struct Fixture //creating a study, maybe not needed Study::Ptr study = std::make_shared(); Parameters params{}; - Data::StudyLoadOptions options; + bool forceParallel = false; + bool enableParallel = false; + uint maxNbYearsInParallel = 0; }; @@ -42,9 +44,9 @@ BOOST_AUTO_TEST_CASE(default_params_no_force) params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 1, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 1, thermalTSRefresh, params); @@ -59,9 +61,9 @@ BOOST_AUTO_TEST_CASE(default_params_no_force_bad_number_of_cores, *utf::expected params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncUnknown; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 0, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 0, thermalTSRefresh, params); @@ -73,9 +75,9 @@ BOOST_AUTO_TEST_CASE(default_params_no_force_bad_ncmode, *utf::expected_failures params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncUnknown; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 1, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 1, thermalTSRefresh, params); @@ -89,9 +91,9 @@ BOOST_AUTO_TEST_CASE(hundred_years_no_force) params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, default_number_of_core, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, default_number_of_core, thermalTSRefresh, params); @@ -105,14 +107,14 @@ BOOST_AUTO_TEST_CASE(four_mc_years_force_parallel) { bool thermalTSRefresh = false; - options.forceParallel = true; - options.enableParallel = true; - options.maxNbYearsInParallel = 2; + forceParallel = true; + enableParallel = true; + maxNbYearsInParallel = 2; params.nbYears = 4; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, default_number_of_core, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, default_number_of_core, thermalTSRefresh, params); @@ -126,14 +128,14 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_parallel_sets_size_five_thermal_refr { bool thermalTSRefresh = true; - options.forceParallel = true; - options.enableParallel = true; - options.maxNbYearsInParallel = 5; + forceParallel = true; + enableParallel = true; + maxNbYearsInParallel = 5; params.nbYears = 100; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, default_number_of_core, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, default_number_of_core, thermalTSRefresh, params); @@ -148,15 +150,15 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_thermal_refresh_on) { bool thermalTSRefresh = true; - options.forceParallel = false; - options.enableParallel = false; + forceParallel = false; + enableParallel = false; params.nbYears = 100; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 5, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 5, thermalTSRefresh, params); @@ -170,19 +172,19 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel) { bool thermalTSRefresh = false; - options.forceParallel = false; - options.enableParallel = true; + forceParallel = false; + enableParallel = true; // Shouldn't have any effect since force parallel is off - options.maxNbYearsInParallel = 10; + maxNbYearsInParallel = 10; params.nbYears = 100; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 16, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 16, thermalTSRefresh, params); @@ -196,19 +198,19 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel_thermal_refresh_o { bool thermalTSRefresh = true; - options.forceParallel = false; - options.enableParallel = true; + forceParallel = false; + enableParallel = true; // Shouldn't have any effect since force parallel is off - options.maxNbYearsInParallel = 10; + maxNbYearsInParallel = 10; params.nbYears = 100; params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 8, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 8, thermalTSRefresh, params); @@ -222,8 +224,8 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel_thermal_refresh_o { bool thermalTSRefresh = true; - options.forceParallel = false; - options.enableParallel = true; + forceParallel = false; + enableParallel = true; params.mode = Antares::Data::stdmAdequacyDraft; @@ -231,9 +233,9 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_no_force_enable_parallel_thermal_refresh_o params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 3, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 3, thermalTSRefresh, params); @@ -247,10 +249,10 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_parallel_enable_parallel_thermal_ref { bool thermalTSRefresh = true; - options.forceParallel = true; - options.enableParallel = true; + forceParallel = true; + enableParallel = true; - options.maxNbYearsInParallel = 5; + maxNbYearsInParallel = 5; params.initialReservoirLevels.iniLevels = Antares::Data::irlHotStart; @@ -258,9 +260,9 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_parallel_enable_parallel_thermal_ref params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncAvg; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 4, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 4, thermalTSRefresh, params); @@ -275,10 +277,10 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_10_parallel_user_playlist_thermal_re { bool thermalTSRefresh = true; - options.forceParallel = true; - options.enableParallel = true; + forceParallel = true; + enableParallel = true; - options.maxNbYearsInParallel = 10; + maxNbYearsInParallel = 10; params.nbYears = 100; @@ -296,9 +298,9 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_force_10_parallel_user_playlist_thermal_re // Override number of raw cores params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncLow; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 7, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 7, thermalTSRefresh, params); @@ -312,11 +314,11 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_enable_parallel_user_playlist_thermal_refr { bool thermalTSRefresh = true; - options.forceParallel = false; - options.enableParallel = true; + forceParallel = false; + enableParallel = true; // Shouldn't have any effect since force parallel is off - options.maxNbYearsInParallel = 10; + maxNbYearsInParallel = 10; params.nbYears = 100; @@ -339,9 +341,9 @@ BOOST_AUTO_TEST_CASE(hundred_mc_years_enable_parallel_user_playlist_thermal_refr // Override number of raw cores params.nbCores.ncMode = Antares::Data::NumberOfCoresMode::ncHigh; - SetsOfParallelYearCalculator builder(options.forceParallel, - options.enableParallel, - options.maxNbYearsInParallel, 9, + SetsOfParallelYearCalculator builder(forceParallel, + enableParallel, + maxNbYearsInParallel, 9, thermalTSRefresh, params); From 2da0d632d17e3319bae6051e7fb5646997e62909 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Tue, 24 Jan 2023 17:12:18 +0100 Subject: [PATCH 6/9] Remove unused fixture member --- .../study/parallel-years/test-parallel-years-calculator.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp index f942883519..ff24dbaba2 100644 --- a/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp +++ b/src/tests/src/libs/antares/study/parallel-years/test-parallel-years-calculator.cpp @@ -27,8 +27,6 @@ struct Fixture ~Fixture(){} - //creating a study, maybe not needed - Study::Ptr study = std::make_shared(); Parameters params{}; bool forceParallel = false; bool enableParallel = false; From 813cc77794f7000ead3a1c0720ae50de465989a8 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Tue, 24 Jan 2023 17:20:59 +0100 Subject: [PATCH 7/9] Changes setSizes to a simple counter --- src/libs/antares/study/parallel-years.cpp | 12 ++++++------ src/libs/antares/study/parallel-years.h | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libs/antares/study/parallel-years.cpp b/src/libs/antares/study/parallel-years.cpp index e3fb99e1c7..87e249347d 100644 --- a/src/libs/antares/study/parallel-years.cpp +++ b/src/libs/antares/study/parallel-years.cpp @@ -321,7 +321,7 @@ void SetsOfParallelYearCalculator::buildSetsOfParallelYears() if (performCalculations) { - set->setsSizes.push_back(y); + set->setsSizes++; // Another year performed ++nbYearsReallyPerformed; @@ -379,9 +379,9 @@ bool SetsOfParallelYearCalculator::allSetsParallelYearsHaveSameSize() if (p.initialReservoirLevels.iniLevels == Antares::Data::irlHotStart && !setsOfParallelYears.empty() && forcedNbOfParallelYears > 1) { - uint currentSetSize = (uint)setsOfParallelYears[0].setsSizes.size(); + uint currentSetSize = (uint)setsOfParallelYears[0].setsSizes; return all_of(setsOfParallelYears.begin(), setsOfParallelYears.end(), - [currentSetSize](const setOfParallelYears& v){ return v.setsSizes.size() == currentSetSize; } ); + [currentSetSize](const setOfParallelYears& v){ return v.setsSizes == currentSetSize; } ); } // End if hot start return true; // parameters.allSetsHaveSameSize takes this result; @@ -393,7 +393,7 @@ uint SetsOfParallelYearCalculator::computeMinNbParallelYears() const uint minNbYearsInParallel = forcedNbOfParallelYears; for (uint s = 0; s < setsOfParallelYears.size(); s++) { - uint setSize = (uint)setsOfParallelYears[s].setsSizes.size(); + uint setSize = (uint)setsOfParallelYears[s].setsSizes; // Empty sets are not taken into account because, on the solver side, // they will contain only skipped years if (setSize && (setSize < minNbYearsInParallel)) @@ -408,8 +408,8 @@ void SetsOfParallelYearCalculator::computeForcedNbYearsInParallelYearSet() uint maxNbYearsOverAllSets = 0; for (uint s = 0; s < setsOfParallelYears.size(); s++) { - if (setsOfParallelYears[s].setsSizes.size() > maxNbYearsOverAllSets) - maxNbYearsOverAllSets = (uint)setsOfParallelYears[s].setsSizes.size(); + if (setsOfParallelYears[s].setsSizes > maxNbYearsOverAllSets) + maxNbYearsOverAllSets = (uint)setsOfParallelYears[s].setsSizes; } forcedNbOfParallelYears = maxNbYearsOverAllSets; diff --git a/src/libs/antares/study/parallel-years.h b/src/libs/antares/study/parallel-years.h index 46d2daa76e..0e946bfd2e 100644 --- a/src/libs/antares/study/parallel-years.h +++ b/src/libs/antares/study/parallel-years.h @@ -38,8 +38,7 @@ struct setOfParallelYears // Un lot d'année à exécuter en parallèle. // En fonction d'une éventuelle play-list, certaines seront jouées et d'autres non. -public: - std::vector setsSizes; + unsigned int setsSizes = 0; // Numeros des annees en parallele pour ce lot (certaines ne seront pas jouées en cas de // play-list "trouée") std::vector yearsIndices; From 813e2c088edb77fdf9dcd141b9c7ecdea12c25b2 Mon Sep 17 00:00:00 2001 From: Jason Marechal Date: Tue, 24 Jan 2023 17:22:12 +0100 Subject: [PATCH 8/9] Rename some member to follow conventions --- src/libs/antares/study/parallel-years.cpp | 26 +++++++++++------------ src/libs/antares/study/parallel-years.h | 24 ++++++++++----------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/libs/antares/study/parallel-years.cpp b/src/libs/antares/study/parallel-years.cpp index 87e249347d..a5a10fb200 100644 --- a/src/libs/antares/study/parallel-years.cpp +++ b/src/libs/antares/study/parallel-years.cpp @@ -119,7 +119,7 @@ void SetsOfParallelYearCalculator::computeRawNbParallelYear() { // In case solver option '--force-parallel n' is used, this computation is not needed // and n will remain the forcedNbOfParallelYears - if (forceParallel) + if (forceParallel_) return; std::map numberOfMCYearThreads; @@ -220,7 +220,7 @@ void SetsOfParallelYearCalculator::computeRawNbParallelYear() try { - forcedNbOfParallelYears = numberOfMCYearThreads.at(p.nbCores.ncMode); + forcedNbOfParallelYears_ = numberOfMCYearThreads.at(p.nbCores.ncMode); } catch(const std::out_of_range& e) { @@ -244,7 +244,7 @@ void SetsOfParallelYearCalculator::limitNbOfParallelYearsbyMinRefreshSpan() if ((p.timeSeriesToGenerate & timeSeriesThermal) && (p.timeSeriesToRefresh & timeSeriesThermal)) TSlimit = std::min(p.refreshIntervalThermal, TSlimit); - forcedNbOfParallelYears = std::min({p.nbYears, TSlimit, forcedNbOfParallelYears}); + forcedNbOfParallelYears_ = std::min({p.nbYears, TSlimit, forcedNbOfParallelYears_}); } bool SetsOfParallelYearCalculator::isRefreshNeededForCurrentYear(uint y) @@ -268,7 +268,7 @@ bool SetsOfParallelYearCalculator::isRefreshNeededForCurrentYear(uint y) bool haveToRefreshTSThermal = ((p.timeSeriesToGenerate & timeSeriesThermal) - && (p.timeSeriesToRefresh & timeSeriesThermal)) || thermalTSRefresh; + && (p.timeSeriesToRefresh & timeSeriesThermal)) || thermalTSRefresh_; refreshing = refreshing || (haveToRefreshTSThermal && (y % p.refreshIntervalThermal == 0)); @@ -349,20 +349,20 @@ void SetsOfParallelYearCalculator::buildSetsOfParallelYears() // In case the study is run in the draft mode, only 1 core is allowed if (p.mode == Antares::Data::stdmAdequacyDraft){ - forcedNbOfParallelYears = 1; + forcedNbOfParallelYears_ = 1; } // In case parallel mode was not chosen, only 1 core is allowed - if (!enableParallel && !forceParallel){ - forcedNbOfParallelYears = 1; + if (!enableParallel_ && !forceParallel_){ + forcedNbOfParallelYears_ = 1; } - if (indexSpace == forcedNbOfParallelYears - 1 || y == p.nbYears - 1) + if (indexSpace == forcedNbOfParallelYears_ - 1 || y == p.nbYears - 1) { buildNewSet = true; foundFirstPerformedYearOfCurrentSet = false; - if (set->nbPerformedYears > forcedNbOfParallelYears) - forcedNbOfParallelYears = set->nbPerformedYears; + if (set->nbPerformedYears > forcedNbOfParallelYears_) + forcedNbOfParallelYears_ = set->nbPerformedYears; } else buildNewSet = false; @@ -377,7 +377,7 @@ void SetsOfParallelYearCalculator::buildSetsOfParallelYears() bool SetsOfParallelYearCalculator::allSetsParallelYearsHaveSameSize() { if (p.initialReservoirLevels.iniLevels == Antares::Data::irlHotStart - && !setsOfParallelYears.empty() && forcedNbOfParallelYears > 1) + && !setsOfParallelYears.empty() && forcedNbOfParallelYears_ > 1) { uint currentSetSize = (uint)setsOfParallelYears[0].setsSizes; return all_of(setsOfParallelYears.begin(), setsOfParallelYears.end(), @@ -390,7 +390,7 @@ bool SetsOfParallelYearCalculator::allSetsParallelYearsHaveSameSize() uint SetsOfParallelYearCalculator::computeMinNbParallelYears() const { // Now finding the smallest size among all sets. - uint minNbYearsInParallel = forcedNbOfParallelYears; + uint minNbYearsInParallel = forcedNbOfParallelYears_; for (uint s = 0; s < setsOfParallelYears.size(); s++) { uint setSize = (uint)setsOfParallelYears[s].setsSizes; @@ -412,7 +412,7 @@ void SetsOfParallelYearCalculator::computeForcedNbYearsInParallelYearSet() maxNbYearsOverAllSets = (uint)setsOfParallelYears[s].setsSizes; } - forcedNbOfParallelYears = maxNbYearsOverAllSets; + forcedNbOfParallelYears_ = maxNbYearsOverAllSets; } diff --git a/src/libs/antares/study/parallel-years.h b/src/libs/antares/study/parallel-years.h index 0e946bfd2e..de23b9bfea 100644 --- a/src/libs/antares/study/parallel-years.h +++ b/src/libs/antares/study/parallel-years.h @@ -102,14 +102,14 @@ class SetsOfParallelYearCalculator { public: - SetsOfParallelYearCalculator(bool forceParallel_, bool enableParallel_, uint forcedNbOfParallelYears_, - uint number_of_cores, bool thermalTSRefresh_, Parameters ¶ms_) - : forceParallel{forceParallel_}, - enableParallel {enableParallel_}, - forcedNbOfParallelYears{forcedNbOfParallelYears_}, + SetsOfParallelYearCalculator(bool forceParallel, bool enableParallel, uint forcedNbOfParallelYears, + uint number_of_cores, bool thermalTSRefresh, Parameters ¶ms) + : forceParallel_{forceParallel}, + enableParallel_ {enableParallel}, + forcedNbOfParallelYears_{forcedNbOfParallelYears}, number_of_cores_{number_of_cores}, - thermalTSRefresh{thermalTSRefresh_}, - p{params_}{ + thermalTSRefresh_{thermalTSRefresh}, + p{params}{ this->build(); } @@ -118,7 +118,7 @@ class SetsOfParallelYearCalculator [[nodiscard]] uint getForcedNbOfParallelYears() const { - return forcedNbOfParallelYears; + return forcedNbOfParallelYears_; } [[nodiscard]] uint getMinNbParallelYearsForGUI() const @@ -152,11 +152,11 @@ class SetsOfParallelYearCalculator std::vector setsOfParallelYears; - bool forceParallel; - bool enableParallel; - uint forcedNbOfParallelYears; + bool forceParallel_; + bool enableParallel_; + uint forcedNbOfParallelYears_; uint number_of_cores_; - bool thermalTSRefresh; + bool thermalTSRefresh_; Parameters& p; uint nbYearsReallyPerformed{0}; From 11a85647288d36396dcd4f79d392a205fa6456b1 Mon Sep 17 00:00:00 2001 From: Florian OMNES Date: Tue, 7 Feb 2023 18:28:57 +0100 Subject: [PATCH 9/9] Fix Simulation Cores in UI --- src/libs/antares/study/parallel-years.cpp | 8 +++----- src/libs/antares/study/parallel-years.h | 6 ++++++ src/libs/antares/study/study.cpp | 6 ++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/libs/antares/study/parallel-years.cpp b/src/libs/antares/study/parallel-years.cpp index a5a10fb200..11cd0ba1f8 100644 --- a/src/libs/antares/study/parallel-years.cpp +++ b/src/libs/antares/study/parallel-years.cpp @@ -220,13 +220,12 @@ void SetsOfParallelYearCalculator::computeRawNbParallelYear() try { - forcedNbOfParallelYears_ = numberOfMCYearThreads.at(p.nbCores.ncMode); + rawNbOfParallelYears_ = numberOfMCYearThreads.at(p.nbCores.ncMode); } catch(const std::out_of_range& e) { - logs.fatal() << "Simulation cores level not correct : " << (int)p.nbCores.ncMode; + logs.fatal() << "Simulation cores level not correct : " << static_cast(p.nbCores.ncMode); } - } @@ -244,7 +243,7 @@ void SetsOfParallelYearCalculator::limitNbOfParallelYearsbyMinRefreshSpan() if ((p.timeSeriesToGenerate & timeSeriesThermal) && (p.timeSeriesToRefresh & timeSeriesThermal)) TSlimit = std::min(p.refreshIntervalThermal, TSlimit); - forcedNbOfParallelYears_ = std::min({p.nbYears, TSlimit, forcedNbOfParallelYears_}); + forcedNbOfParallelYears_ = std::min({p.nbYears, TSlimit, rawNbOfParallelYears_}); } bool SetsOfParallelYearCalculator::isRefreshNeededForCurrentYear(uint y) @@ -370,7 +369,6 @@ void SetsOfParallelYearCalculator::buildSetsOfParallelYears() // End of loop over years } - } diff --git a/src/libs/antares/study/parallel-years.h b/src/libs/antares/study/parallel-years.h index de23b9bfea..b81c5e0de0 100644 --- a/src/libs/antares/study/parallel-years.h +++ b/src/libs/antares/study/parallel-years.h @@ -121,6 +121,11 @@ class SetsOfParallelYearCalculator return forcedNbOfParallelYears_; } + [[nodiscard]] uint getRawNbParallelYearsForGUI() const + { + return rawNbOfParallelYears_; + } + [[nodiscard]] uint getMinNbParallelYearsForGUI() const { return computeMinNbParallelYears(); @@ -155,6 +160,7 @@ class SetsOfParallelYearCalculator bool forceParallel_; bool enableParallel_; uint forcedNbOfParallelYears_; + uint rawNbOfParallelYears_; uint number_of_cores_; bool thermalTSRefresh_; Parameters& p; diff --git a/src/libs/antares/study/study.cpp b/src/libs/antares/study/study.cpp index e0d9f9dbde..a53d8c8560 100644 --- a/src/libs/antares/study/study.cpp +++ b/src/libs/antares/study/study.cpp @@ -226,16 +226,18 @@ uint64 Study::memoryUsage() const void Study::getNumberOfCores(const bool forceParallel, const bool enableParallel, const uint nbYearsParallelForced) { - TempAreaListHolder holder{}; + TempAreaListHolder holder; bool thermalTSRefresh = holder.checkThermalTSGeneration(folderInput); SetsOfParallelYearCalculator setsBuilder(forceParallel, enableParallel, - nbYearsParallelForced, Yuni::System::CPU::Count(), + nbYearsParallelForced, + Yuni::System::CPU::Count(), thermalTSRefresh, parameters); // For GUI minNbYearsInParallel_save = setsBuilder.getMinNbParallelYearsForGUI(); + nbYearsParallelRaw = setsBuilder.getRawNbParallelYearsForGUI(); maxNbYearsInParallel_save = setsBuilder.getForcedNbOfParallelYears(); // For the solver