Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Manage multithreads logs #1764

Closed
wants to merge 30 commits into from
Closed
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
78790e6
set multi threads log dir
a-zakir Nov 12, 2023
151dedd
[skip ci] will not work thank u @guilpier-code
a-zakir Nov 16, 2023
6da7fde
fstream-->ostream ptr
a-zakir Nov 16, 2023
d0697c4
[skip ci] move streams creation
a-zakir Nov 16, 2023
b6e4720
dirty
a-zakir Nov 16, 2023
1586ab5
[skip] better
a-zakir Nov 16, 2023
81ef7b0
suggestion by @flomnes
a-zakir Nov 16, 2023
c935389
direct call
a-zakir Nov 17, 2023
b2a612a
delete deprecated method
a-zakir Nov 17, 2023
2775642
[skip ci] set field width for incoming msg
a-zakir Nov 17, 2023
a69f2df
test SiriusCallback
a-zakir Nov 17, 2023
9756b55
Merge branch 'feature/multi-threads-solver-logs' of https://github.co…
a-zakir Nov 17, 2023
5769816
[skip ci] fix
a-zakir Nov 17, 2023
8c615ae
test Sirius callback: ok
a-zakir Nov 17, 2023
c8d8f82
adaptation for COIN solvers
a-zakir Nov 23, 2023
1535d8c
bug
a-zakir Nov 23, 2023
b1dce69
set log files per thread
a-zakir Nov 23, 2023
30853d1
tmp Revert "bug"
a-zakir Nov 23, 2023
9d1db92
Merge branch 'feature/multi-threads-solver-logs' of https://github.co…
a-zakir Nov 23, 2023
df4a671
copy log files
a-zakir Nov 23, 2023
3a24ba1
remove &
a-zakir Nov 23, 2023
14b1150
fix
a-zakir Nov 23, 2023
00926bc
ok
a-zakir Nov 23, 2023
0a44ba7
clean&clear
a-zakir Nov 23, 2023
568e8f0
update log handler
a-zakir Dec 11, 2023
ae4c29f
update log handler (#1793)
a-zakir Dec 14, 2023
9f44c53
Merge branch 'feature/multi-threads-solver-logs' of https://github.co…
a-zakir Dec 15, 2023
f1cb6aa
remove old code & update
a-zakir Dec 15, 2023
e63468c
use numSpace and thread number
a-zakir Dec 19, 2023
becdf7f
update & fix
a-zakir Dec 19, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -49,7 +49,8 @@ AdequacyPatchOptimization::AdequacyPatchOptimization(const Antares::Data::Study&
void AdequacyPatchOptimization::solve()
{
problemeHebdo_->adequacyPatchRuntimeData->AdequacyFirstStep = true;
OPT_OptimisationHebdomadaire(options_, problemeHebdo_, adqPatchParams_, writer_);
OPT_OptimisationHebdomadaire(
options_, problemeHebdo_, adqPatchParams_, writer_, thread_number_);
problemeHebdo_->adequacyPatchRuntimeData->AdequacyFirstStep = false;

for (uint32_t pays = 0; pays < problemeHebdo_->NombreDePays; ++pays)
@@ -63,7 +64,8 @@ void AdequacyPatchOptimization::solve()
problemeHebdo_->ResultatsHoraires[pays].ValeursHorairesDENS.end(), 0);
}

OPT_OptimisationHebdomadaire(options_, problemeHebdo_, adqPatchParams_, writer_);
OPT_OptimisationHebdomadaire(
options_, problemeHebdo_, adqPatchParams_, writer_, thread_number_);
}

} // namespace Antares::Solver::Optimization
68 changes: 47 additions & 21 deletions src/solver/optimisation/opt_appel_solveur_lineaire.cpp
Original file line number Diff line number Diff line change
@@ -36,7 +36,6 @@ extern "C"
{
#include "spx_definition_arguments.h"
#include "spx_fonctions.h"

#include "srs_api.h"
}

@@ -96,16 +95,15 @@ struct SimplexResult
long long solveTime = 0;
mpsWriterFactory mps_writer_factory;
};

static SimplexResult OPT_TryToCallSimplex(
const OptimizationOptions& options,
PROBLEME_HEBDO* problemeHebdo,
Optimization::PROBLEME_SIMPLEXE_NOMME& Probleme,
const int NumIntervalle,
const int optimizationNumber,
const OptPeriodStringGenerator& optPeriodStringGenerator,
bool PremierPassage,
IResultWriter& writer)
static SimplexResult OPT_TryToCallSimplex(const OptimizationOptions& options,
PROBLEME_HEBDO* problemeHebdo,
Optimization::PROBLEME_SIMPLEXE_NOMME& Probleme,
const int NumIntervalle,
const int optimizationNumber,
const OptPeriodStringGenerator& optPeriodStringGenerator,
bool PremierPassage,
IResultWriter& writer,
OrtoolsLogHandler& ortools_logger)
a-zakir marked this conversation as resolved.
Show resolved Hide resolved
{
const auto& ProblemeAResoudre = problemeHebdo->ProblemeAResoudre;
auto ProbSpx
@@ -225,7 +223,8 @@ static SimplexResult OPT_TryToCallSimplex(
{
solver = ORTOOLS_ConvertIfNeeded(options.solverName, &Probleme, solver);
}
const std::string filename = createMPSfilename(optPeriodStringGenerator, optimizationNumber);

const std::string filename = createMPSfilename(optPeriodStringGenerator, optimizationNumber);

mpsWriterFactory mps_writer_factory(problemeHebdo->ExportMPS,
problemeHebdo->exportMPSOnError,
@@ -241,6 +240,11 @@ static SimplexResult OPT_TryToCallSimplex(
if (options.useOrtools)
{
const bool keepBasis = (optimizationNumber == PREMIERE_OPTIMISATION);

if (Probleme.SolverLogs())
{
solver->EnableOutput(&ortools_logger);
}
solver = ORTOOLS_Simplexe(&Probleme, solver, keepBasis);
if (solver != nullptr)
{
@@ -296,7 +300,8 @@ bool OPT_AppelDuSimplexe(const OptimizationOptions& options,
int NumIntervalle,
const int optimizationNumber,
const OptPeriodStringGenerator& optPeriodStringGenerator,
IResultWriter& writer)
IResultWriter& writer,
uint thread_number)
{
const auto& ProblemeAResoudre = problemeHebdo->ProblemeAResoudre;
Optimization::PROBLEME_SIMPLEXE_NOMME Probleme(ProblemeAResoudre->NomDesVariables,
@@ -308,16 +313,31 @@ bool OPT_AppelDuSimplexe(const OptimizationOptions& options,
problemeHebdo->solverLogs);

bool PremierPassage = true;

struct SimplexResult simplexResult =
OPT_TryToCallSimplex(options, problemeHebdo, Probleme, NumIntervalle, optimizationNumber,
optPeriodStringGenerator, PremierPassage, writer);
std::filesystem::path log_file = logs.logfile().c_str();
auto log_directory = log_file.parent_path();
OrtoolsLogHandler ortools_logger(options.solverName, log_directory, thread_number);
struct SimplexResult simplexResult = OPT_TryToCallSimplex(options,
problemeHebdo,
Probleme,
NumIntervalle,
optimizationNumber,
optPeriodStringGenerator,
PremierPassage,
writer,
ortools_logger);

if (!simplexResult.success)
{
PremierPassage = false;
simplexResult = OPT_TryToCallSimplex(options, problemeHebdo, Probleme, NumIntervalle, optimizationNumber,
optPeriodStringGenerator, PremierPassage, writer);
simplexResult = OPT_TryToCallSimplex(options,
problemeHebdo,
Probleme,
NumIntervalle,
optimizationNumber,
optPeriodStringGenerator,
PremierPassage,
writer,
ortools_logger);
}

long long solveTime = simplexResult.solveTime;
@@ -372,8 +392,13 @@ bool OPT_AppelDuSimplexe(const OptimizationOptions& options,

Probleme.SetUseNamedProblems(true);

auto MPproblem = std::shared_ptr<MPSolver>(ProblemSimplexeNommeConverter(options.solverName, &Probleme).Convert());
auto MPproblem = std::shared_ptr<MPSolver>(
ProblemSimplexeNommeConverter(options.solverName, &Probleme).Convert());

if (Probleme.SolverLogs())
{
MPproblem->EnableOutput(&ortools_logger);
}
auto analyzer = makeUnfeasiblePbAnalyzer();
analyzer->run(MPproblem.get());
analyzer->printReport();
@@ -384,6 +409,7 @@ bool OPT_AppelDuSimplexe(const OptimizationOptions& options,

return false;
}

// copy log file(s)
ortools_logger.copy_log(writer);
return true;
}
12 changes: 8 additions & 4 deletions src/solver/optimisation/opt_fonctions.h
Original file line number Diff line number Diff line change
@@ -41,7 +41,8 @@ using OptimizationOptions = Antares::Solver::Optimization::OptimizationOptions;
void OPT_OptimisationHebdomadaire(const OptimizationOptions& options,
PROBLEME_HEBDO*,
const AdqPatchParams&,
Antares::Solver::IResultWriter& writer);
Antares::Solver::IResultWriter& writer,
uint thread_number);
void OPT_NumeroDeJourDuPasDeTemps(PROBLEME_HEBDO*);
void OPT_NumeroDIntervalleOptimiseDuPasDeTemps(PROBLEME_HEBDO*);
void OPT_ConstruireLaListeDesVariablesOptimiseesDuProblemeLineaire(PROBLEME_HEBDO*);
@@ -75,7 +76,8 @@ bool ADQ_PATCH_CSR(PROBLEME_ANTARES_A_RESOUDRE&,
bool OPT_PilotageOptimisationLineaire(const OptimizationOptions& options,
PROBLEME_HEBDO*,
const AdqPatchParams&,
Antares::Solver::IResultWriter& writer);
Antares::Solver::IResultWriter& writer,
uint thread_number);
void OPT_VerifierPresenceReserveJmoins1(PROBLEME_HEBDO*);
bool OPT_PilotageOptimisationQuadratique(PROBLEME_HEBDO*);

@@ -89,13 +91,15 @@ bool OPT_AppelDuSimplexe(const OptimizationOptions& options,
int,
const int,
const OptPeriodStringGenerator&,
Antares::Solver::IResultWriter& writer);
Antares::Solver::IResultWriter& writer,
uint thread_number);
void OPT_LiberationProblemesSimplexe(const OptimizationOptions& options, const PROBLEME_HEBDO*);

bool OPT_OptimisationLineaire(const OptimizationOptions& options,
PROBLEME_HEBDO*,
const AdqPatchParams&,
Antares::Solver::IResultWriter& writer);
Antares::Solver::IResultWriter& writer,
uint thread_number);
void OPT_RestaurerLesDonnees(const PROBLEME_HEBDO*);
/*------------------------------*/

6 changes: 4 additions & 2 deletions src/solver/optimisation/opt_optimisation_hebdo.cpp
Original file line number Diff line number Diff line change
@@ -48,11 +48,13 @@ using Antares::Solver::Optimization::OptimizationOptions;
void OPT_OptimisationHebdomadaire(const OptimizationOptions& options,
PROBLEME_HEBDO* pProblemeHebdo,
const AdqPatchParams& adqPatchParams,
Solver::IResultWriter& writer)
Solver::IResultWriter& writer,
uint thread_number)
{
if (pProblemeHebdo->TypeDOptimisation == OPTIMISATION_LINEAIRE)
{
if (!OPT_PilotageOptimisationLineaire(options, pProblemeHebdo, adqPatchParams, writer))
if (!OPT_PilotageOptimisationLineaire(
options, pProblemeHebdo, adqPatchParams, writer, thread_number))
{
logs.error() << "Linear optimization failed";
throw UnfeasibleProblemError("Linear optimization failed");
14 changes: 8 additions & 6 deletions src/solver/optimisation/opt_optimisation_lineaire.cpp
Original file line number Diff line number Diff line change
@@ -66,7 +66,8 @@ bool runWeeklyOptimization(const OptimizationOptions& options,
PROBLEME_HEBDO* problemeHebdo,
const AdqPatchParams& adqPatchParams,
Solver::IResultWriter& writer,
int optimizationNumber)
int optimizationNumber,
uint thread_number)
{
const int NombreDePasDeTempsPourUneOptimisation
= problemeHebdo->NombreDePasDeTempsPourUneOptimisation;
@@ -107,7 +108,8 @@ bool runWeeklyOptimization(const OptimizationOptions& options,
numeroDeLIntervalle,
optimizationNumber,
*optPeriodStringGenerator,
writer))
writer,
thread_number))
return false;

if (problemeHebdo->ExportMPS != Data::mpsExportStatus::NO_EXPORT
@@ -135,11 +137,11 @@ void runThermalHeuristic(PROBLEME_HEBDO* problemeHebdo)
}
} // namespace


bool OPT_OptimisationLineaire(const OptimizationOptions& options,
PROBLEME_HEBDO* problemeHebdo,
const AdqPatchParams& adqPatchParams,
Solver::IResultWriter& writer)
Solver::IResultWriter& writer,
uint thread_number)
{
if (!problemeHebdo->OptimisationAuPasHebdomadaire)
{
@@ -162,7 +164,7 @@ bool OPT_OptimisationLineaire(const OptimizationOptions& options,
OPT_ConstruireLaMatriceDesContraintesDuProblemeLineaire(problemeHebdo, writer);

bool ret = runWeeklyOptimization(
options, problemeHebdo, adqPatchParams, writer, PREMIERE_OPTIMISATION);
options, problemeHebdo, adqPatchParams, writer, PREMIERE_OPTIMISATION, thread_number);

// We only need the 2nd optimization when NOT solving with integer variables
// We also skip the 2nd optimization in the hidden 'Expansion' mode
@@ -172,7 +174,7 @@ bool OPT_OptimisationLineaire(const OptimizationOptions& options,
// We need to adjust some stuff before running the 2nd optimisation
runThermalHeuristic(problemeHebdo);
return runWeeklyOptimization(
options, problemeHebdo, adqPatchParams, writer, DEUXIEME_OPTIMISATION);
options, problemeHebdo, adqPatchParams, writer, DEUXIEME_OPTIMISATION, thread_number);
}
return ret;
}
Original file line number Diff line number Diff line change
@@ -37,7 +37,8 @@ using Antares::Solver::Optimization::OptimizationOptions;
bool OPT_PilotageOptimisationLineaire(const OptimizationOptions& options,
PROBLEME_HEBDO* problemeHebdo,
const AdqPatchParams& adqPatchParams,
Solver::IResultWriter& writer)
Solver::IResultWriter& writer,
uint thread_number)
{
if (!problemeHebdo->LeProblemeADejaEteInstancie)
{
@@ -83,5 +84,5 @@ bool OPT_PilotageOptimisationLineaire(const OptimizationOptions& options,
OPT_InitialiserNombreMinEtMaxDeGroupesCoutsDeDemarrage(problemeHebdo);
}

return OPT_OptimisationLineaire(options, problemeHebdo, adqPatchParams, writer);
return OPT_OptimisationLineaire(options, problemeHebdo, adqPatchParams, writer, thread_number);
}
3 changes: 2 additions & 1 deletion src/solver/optimisation/weekly_optimization.cpp
Original file line number Diff line number Diff line change
@@ -41,7 +41,8 @@ DefaultWeeklyOptimization::DefaultWeeklyOptimization(const OptimizationOptions&

void DefaultWeeklyOptimization::solve()
{
OPT_OptimisationHebdomadaire(options_, problemeHebdo_, adqPatchParams_, writer_);
OPT_OptimisationHebdomadaire(
options_, problemeHebdo_, adqPatchParams_, writer_, thread_number_);
}

} // namespace Antares::Solver::Optimization
5 changes: 3 additions & 2 deletions src/solver/simulation/adequacy.cpp
Original file line number Diff line number Diff line change
@@ -199,7 +199,8 @@ bool Adequacy::year(Progression::Task& progression,
OPT_OptimisationHebdomadaire(createOptimizationOptions(study),
&pProblemesHebdo[numSpace],
study.parameters.adqPatchParams,
resultWriter);
resultWriter,
numSpace);

computingHydroLevels(study.areas, pProblemesHebdo[numSpace], false);

@@ -382,7 +383,7 @@ void Adequacy::simulationEnd()
if (!preproOnly && study.runtime->interconnectionsCount() > 0)
{
auto balance = retrieveBalance(study, variables);
ComputeFlowQuad(study, pProblemesHebdo[0], balance, pNbWeeks);
ComputeFlowQuad(study, pProblemesHebdo[0], balance, pNbWeeks, 0);
}
}

14 changes: 10 additions & 4 deletions src/solver/simulation/common-eco-adq.cpp
Original file line number Diff line number Diff line change
@@ -44,7 +44,8 @@ namespace Antares::Solver::Simulation
static void RecalculDesEchangesMoyens(Data::Study& study,
PROBLEME_HEBDO& problem,
const std::vector<AvgExchangeResults*>& balance,
int PasDeTempsDebut)
int PasDeTempsDebut,
uint thread_number)
{
for (uint i = 0; i < (uint)problem.NombreDePasDeTemps; i++)
{
@@ -95,7 +96,11 @@ static void RecalculDesEchangesMoyens(Data::Study& study,
try
{
NullResultWriter resultWriter;
OPT_OptimisationHebdomadaire(createOptimizationOptions(study), &problem, study.parameters.adqPatchParams, resultWriter);
OPT_OptimisationHebdomadaire(createOptimizationOptions(study),
&problem,
study.parameters.adqPatchParams,
resultWriter,
thread_number);
}
catch (Data::UnfeasibleProblemError&)
{
@@ -197,7 +202,8 @@ bool ShouldUseQuadraticOptimisation(const Data::Study& study)
void ComputeFlowQuad(Data::Study& study,
PROBLEME_HEBDO& problem,
const std::vector<AvgExchangeResults*>& balance,
uint nbWeeks)
uint nbWeeks,
uint thread_number)
{
uint startTime = study.calendar.days[study.parameters.simulationDays.first].hours.first;

@@ -210,7 +216,7 @@ void ComputeFlowQuad(Data::Study& study,
for (uint w = 0; w != nbWeeks; ++w)
{
int PasDeTempsDebut = startTime + (w * problem.NombreDePasDeTemps);
RecalculDesEchangesMoyens(study, problem, balance, PasDeTempsDebut);
RecalculDesEchangesMoyens(study, problem, balance, PasDeTempsDebut, thread_number);
}
}
else
3 changes: 2 additions & 1 deletion src/solver/simulation/common-eco-adq.h
Original file line number Diff line number Diff line change
@@ -89,7 +89,8 @@ bool ShouldUseQuadraticOptimisation(const Data::Study& study);
void ComputeFlowQuad(Data::Study& study,
PROBLEME_HEBDO& problem,
const std::vector<AvgExchangeResults*>& balance,
unsigned int nbWeeks);
unsigned int nbWeeks,
uint thread_number);

/*!
** \brief Hydro Remix
2 changes: 1 addition & 1 deletion src/solver/simulation/economy.cpp
Original file line number Diff line number Diff line change
@@ -260,7 +260,7 @@ void Economy::simulationEnd()
if (!preproOnly && study.runtime->interconnectionsCount() > 0)
{
auto balance = retrieveBalance(study, variables);
ComputeFlowQuad(study, pProblemesHebdo[0], balance, pNbWeeks);
ComputeFlowQuad(study, pProblemesHebdo[0], balance, pNbWeeks, 0);
}
}

Loading