Add dynawo logs (#354)

* Add logLevelFilter parameter * Add specific logs * Add reporters * Add specific logs in integration tests * Add documentation * Remove setDefaultDumpFileParameters Signed-off-by: lisrte <[email protected]>
powsybl · Jul 2, 2024 · 8be2853 · 8be2853
1 parent bcce6b1
commit 8be2853
Show file tree

Hide file tree

Showing 14 changed files with 199 additions and 37 deletions.
diff --git a/docs/dynamic_simulation/dynawo-configuration.md b/docs/dynamic_simulation/dynawo-configuration.md
@@ -98,6 +98,25 @@ The default value is `TXT`.
 `precision` defines the simulation step precision.  
 The default value is `1e-6`.
 
+**log.levelFilter**  
+`log.levelFilter` defines the log level for Dynawo log.  
+The default value is `INFO`.
+The available `com.powsybl.dynawaltz.DynaWaltzParameters.LogLevel` values are:
+- `DEBUG`
+- `INFO`
+- `WARN`
+- `ERROR`
+
+**log.specificLogs**  
+`log.specificLogs` defines as a list the specifics logs to return besides the regular Dynawo log.  
+The default value is an empty list.
+The available `com.powsybl.dynawaltz.DynaWaltzParameters.SpecificLog` values are:
+- `NETWORK`
+- `MODELER`
+- `PARAMETERS`
+- `VARIABLES`
+- `EQUATIONS`
+
 ### Examples
 
 **YAML configuration:**
@@ -117,6 +136,10 @@ dynawo-default-parameters:
   mergeLoads: false
   timeline.exportMode: XML
   precision: 10e-6
+  log.levelFilter: INFO
+  log.specificLogs:
+    - NETWORK
+    - PARAMETERS
 ```
 
 **XML configuration:**
@@ -136,5 +159,7 @@ dynawo-default-parameters:
   <mergeLoads>false</mergeLoads>
   <timeline.exportMode>XML</timeline.exportMode>
   <precision>10e-6</precision>
+  <log.levelFilter>INFO</log.levelFilter>
+  <log.specificLogs>NETWORK, PARAMETERS</log.specificLogs>
 </dynawo-default-parameters>
 ```
diff --git a/dynawaltz-dsl/src/test/java/com/powsybl/dynawaltz/dsl/ieee/AbstractIeeeTest.java b/dynawaltz-dsl/src/test/java/com/powsybl/dynawaltz/dsl/ieee/AbstractIeeeTest.java
@@ -100,8 +100,7 @@ protected void setup(String parametersFile, String networkParametersFile, String
         dynaWaltzParameters.setModelsParameters(ParametersXml.load(getClass().getResourceAsStream(parametersFile)))
                 .setNetworkParameters(ParametersXml.load(getClass().getResourceAsStream(networkParametersFile), networkParametersId))
                 .setSolverParameters(ParametersXml.load(getClass().getResourceAsStream(solverParametersFile), solverParametersId))
-                .setSolverType(DynaWaltzParameters.SolverType.IDA)
-                .setDefaultDumpFileParameters();
+                .setSolverType(DynaWaltzParameters.SolverType.IDA);
     }
 
     protected DynaWaltzParameters getDynaWaltzSimulationParameters(DynamicSimulationParameters parameters) {

diff --git a/dynawaltz/src/main/java/com/powsybl/dynawaltz/DumpFileParameters.java b/dynawaltz/src/main/java/com/powsybl/dynawaltz/DumpFileParameters.java
@@ -26,7 +26,7 @@ public record DumpFileParameters(boolean exportDumpFile, boolean useDumpFile, Pa
     public static final boolean DEFAULT_USE_DUMP = false;
     public static final String DEFAULT_DUMP_FOLDER = null;
     public static final String DEFAULT_DUMP_NAME = null;
-    public static final DumpFileParameters DEFAULT_DUMP_FILE_PARAMETERS = new DumpFileParameters(DEFAULT_EXPORT_DUMP, DEFAULT_USE_DUMP, null, DEFAULT_DUMP_NAME);
+    private static final DumpFileParameters DEFAULT_DUMP_FILE_PARAMETERS = new DumpFileParameters(DEFAULT_EXPORT_DUMP, DEFAULT_USE_DUMP, null, DEFAULT_DUMP_NAME);
 
     public DumpFileParameters {
         if (useDumpFile) {

diff --git a/dynawaltz/src/main/java/com/powsybl/dynawaltz/DynaWaltzParameters.java b/dynawaltz/src/main/java/com/powsybl/dynawaltz/DynaWaltzParameters.java
@@ -41,6 +41,7 @@ public class DynaWaltzParameters extends AbstractExtension<DynamicSimulationPara
     public static final boolean DEFAULT_USE_MODEL_SIMPLIFIERS = false;
     public static final double DEFAULT_PRECISION = 1e-6;
     public static final ExportMode DEFAULT_TIMELINE_EXPORT_MODE = ExportMode.TXT;
+    public static final LogLevel DEFAULT_LOG_LEVEL_FILTER = LogLevel.INFO;
 
     /**
      * Information about the solver to use in the simulation
@@ -72,16 +73,46 @@ public String getFileExtension() {
         }
     }
 
+    public enum LogLevel {
+        DEBUG,
+        INFO,
+        WARN,
+        ERROR
+    }
+
+    public enum SpecificLog {
+
+        NETWORK("network"),
+        MODELER("modeler"),
+        PARAMETERS("param"),
+        VARIABLES("variables"),
+        EQUATIONS("equations");
+
+        private static final String DEFAULT_FILE_EXTENSION = ExportMode.TXT.getFileExtension();
+
+        private final String fileName;
+
+        SpecificLog(String fileName) {
+            this.fileName = fileName;
+        }
+
+        public String getFileName() {
+            return fileName + DEFAULT_FILE_EXTENSION;
+        }
+    }
+
     private Map<String, ParametersSet> modelsParameters = new LinkedHashMap<>();
     private ParametersSet networkParameters;
     private ParametersSet solverParameters;
     private SolverType solverType = DEFAULT_SOLVER_TYPE;
     private boolean mergeLoads = DEFAULT_MERGE_LOADS;
     private boolean writeFinalState = DEFAULT_WRITE_FINAL_STATE;
     private boolean useModelSimplifiers = DEFAULT_USE_MODEL_SIMPLIFIERS;
-    private DumpFileParameters dumpFileParameters = DumpFileParameters.DEFAULT_DUMP_FILE_PARAMETERS;
+    private DumpFileParameters dumpFileParameters = DumpFileParameters.createDefaultDumpFileParameters();
     private double precision = DEFAULT_PRECISION;
     private ExportMode timelineExportMode = DEFAULT_TIMELINE_EXPORT_MODE;
+    private LogLevel logLevelFilter = DEFAULT_LOG_LEVEL_FILTER;
+    private EnumSet<SpecificLog> specificLogs = EnumSet.noneOf(SpecificLog.class);
 
     /**
      * Loads parameters from the default platform configuration.
@@ -122,6 +153,8 @@ public static DynaWaltzParameters load(PlatformConfig platformConfig, FileSystem
             c.getOptionalBooleanProperty("useModelSimplifiers").ifPresent(parameters::setUseModelSimplifiers);
             c.getOptionalDoubleProperty("precision").ifPresent(parameters::setPrecision);
             c.getOptionalEnumProperty("timeline.exportMode", ExportMode.class).ifPresent(parameters::setTimelineExportMode);
+            c.getOptionalEnumProperty("log.levelFilter", LogLevel.class).ifPresent(parameters::setLogLevelFilter);
+            c.getOptionalEnumSetProperty("log.specificLogs", SpecificLog.class).ifPresent(parameters::setSpecificLogs);
         });
         return parameters;
     }
@@ -222,11 +255,6 @@ public DynaWaltzParameters setDumpFileParameters(DumpFileParameters dumpFilePara
         return this;
     }
 
-    public DynaWaltzParameters setDefaultDumpFileParameters() {
-        this.dumpFileParameters = DumpFileParameters.createDefaultDumpFileParameters();
-        return this;
-    }
-
     public double getPrecision() {
         return precision;
     }
@@ -244,4 +272,27 @@ public DynaWaltzParameters setTimelineExportMode(ExportMode timelineExportMode)
         this.timelineExportMode = timelineExportMode;
         return this;
     }
+
+    public LogLevel getLogLevelFilter() {
+        return logLevelFilter;
+    }
+
+    public DynaWaltzParameters setLogLevelFilter(LogLevel logLevelFilter) {
+        this.logLevelFilter = logLevelFilter;
+        return this;
+    }
+
+    public Set<SpecificLog> getSpecificLogs() {
+        return specificLogs;
+    }
+
+    public DynaWaltzParameters setSpecificLogs(Set<SpecificLog> specificLogs) {
+        this.specificLogs = EnumSet.copyOf(specificLogs);
+        return this;
+    }
+
+    public DynaWaltzParameters addSpecificLog(SpecificLog specificLog) {
+        specificLogs.add(specificLog);
+        return this;
+    }
 }
diff --git a/dynawaltz/src/main/java/com/powsybl/dynawaltz/DynaWaltzProvider.java b/dynawaltz/src/main/java/com/powsybl/dynawaltz/DynaWaltzProvider.java
@@ -194,7 +194,7 @@ public DynamicSimulationResult after(Path workingDir, ExecutionReport report) th
             DynaWaltzParameters parameters = context.getDynaWaltzParameters();
             DumpFileParameters dumpFileParameters = parameters.getDumpFileParameters();
 
-            setDynawoLog(outputsFolder);
+            setDynawoLog(outputsFolder, parameters.getSpecificLogs());
             // Error file
             Path errorFile = workingDir.resolve(ERROR_FILENAME);
             if (Files.exists(errorFile)) {
@@ -224,11 +224,21 @@ public DynamicSimulationResult after(Path workingDir, ExecutionReport report) th
             return new DynamicSimulationResultImpl(status, statusText, curves, timeline);
         }
 
-        private void setDynawoLog(Path outputsFolder) {
-            Path logFile = outputsFolder.resolve(LOGS_FOLDER).resolve(LOGS_FILENAME);
-            if (Files.exists(logFile)) {
-                ReportNode logReportNode = CommonReports.createDynawoLogReportNode(reportNode);
-                new CsvLogParser().parse(logFile).forEach(e -> CommonReports.reportLogEntry(logReportNode, e));
+        private void setDynawoLog(Path outputsFolder, Set<DynaWaltzParameters.SpecificLog> specificLogs) throws IOException {
+            Path logFolder = outputsFolder.resolve(LOGS_FOLDER);
+            if (Files.exists(logFolder)) {
+                Path logFile = logFolder.resolve(LOGS_FILENAME);
+                if (Files.exists(logFile)) {
+                    ReportNode logReportNode = CommonReports.createDynawoLogReportNode(reportNode);
+                    new CsvLogParser().parse(logFile).forEach(e -> CommonReports.reportLogEntry(logReportNode, e));
+                }
+                for (DynaWaltzParameters.SpecificLog specificLog : specificLogs) {
+                    Path specificLogFile = logFolder.resolve(specificLog.getFileName());
+                    if (Files.exists(specificLogFile)) {
+                        ReportNode logReport = DynawaltzReports.createDynawoSpecificLogReportNode(reportNode, specificLog);
+                        DynawaltzReports.reportSpecificLogEntry(logReport, Files.readString(specificLogFile));
+                    }
+                }
             } else {
                 LOGGER.warn("Dynawo logs file not found");
             }

diff --git a/dynawaltz/src/main/java/com/powsybl/dynawaltz/DynawaltzReports.java b/dynawaltz/src/main/java/com/powsybl/dynawaltz/DynawaltzReports.java
@@ -9,6 +9,7 @@
 
 import com.powsybl.commons.report.ReportNode;
 import com.powsybl.commons.report.TypedValue;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * @author Laurent Issertial {@literal <laurent.issertial at rte-france.com>}
@@ -72,4 +73,19 @@ public static void reportEmptyListAutomaton(ReportNode reportNode, String automa
                 .withSeverity(TypedValue.WARN_SEVERITY)
                 .add();
     }
+
+    public static ReportNode createDynawoSpecificLogReportNode(ReportNode reportNode, DynaWaltzParameters.SpecificLog logType) {
+        String logTypeName = StringUtils.capitalize(logType.toString().toLowerCase());
+        return reportNode.newReportNode()
+                .withMessageTemplate("dynawo" + logTypeName + "Log", logTypeName + " log")
+                .add();
+
+    }
+
+    public static void reportSpecificLogEntry(ReportNode reportNode, String logEntry) {
+        reportNode.newReportNode()
+                .withMessageTemplate("dynawoSpecificLogEntry", "${message}")
+                .withUntypedValue("message", logEntry)
+                .add();
+    }
 }
diff --git a/dynawaltz/src/main/java/com/powsybl/dynawaltz/xml/JobsXml.java b/dynawaltz/src/main/java/com/powsybl/dynawaltz/xml/JobsXml.java
@@ -87,6 +87,7 @@ private static void writeSimulation(XMLStreamWriter writer, DynaWaltzContext con
     }
 
     private static void writeOutput(XMLStreamWriter writer, DynaWaltzContext context) throws XMLStreamException {
+        DynaWaltzParameters parameters = context.getDynaWaltzParameters();
         writer.writeStartElement(DYN_URI, "outputs");
         writer.writeAttribute("directory", "outputs");
 
@@ -95,29 +96,39 @@ private static void writeOutput(XMLStreamWriter writer, DynaWaltzContext context
         writer.writeAttribute("global", Boolean.toString(false));
 
         writer.writeEmptyElement(DYN_URI, "timeline");
-        writer.writeAttribute("exportMode", context.getDynaWaltzParameters().getTimelineExportMode().name());
+        writer.writeAttribute("exportMode", parameters.getTimelineExportMode().toString());
 
         writer.writeEmptyElement(DYN_URI, "finalState");
-        writer.writeAttribute("exportIIDMFile", Boolean.toString(context.getDynaWaltzParameters().isWriteFinalState()));
-        writer.writeAttribute("exportDumpFile", Boolean.toString(context.getDynaWaltzParameters().getDumpFileParameters().exportDumpFile()));
+        writer.writeAttribute("exportIIDMFile", Boolean.toString(parameters.isWriteFinalState()));
+        writer.writeAttribute("exportDumpFile", Boolean.toString(parameters.getDumpFileParameters().exportDumpFile()));
 
         if (context.withCurves()) {
             writer.writeEmptyElement(DYN_URI, "curves");
             writer.writeAttribute("inputFile", DynaWaltzConstants.CRV_FILENAME);
-            writer.writeAttribute("exportMode", DynaWaltzParameters.ExportMode.CSV.name());
+            writer.writeAttribute("exportMode", DynaWaltzParameters.ExportMode.CSV.toString());
         }
 
         writer.writeStartElement(DYN_URI, "logs");
-        writeAppender(writer);
+        writeAppender(writer, parameters);
         writer.writeEndElement();
 
         writer.writeEndElement();
     }
 
-    private static void writeAppender(XMLStreamWriter writer) throws XMLStreamException {
+    private static void writeAppender(XMLStreamWriter writer, DynaWaltzParameters parameters) throws XMLStreamException {
         writer.writeEmptyElement(DYN_URI, "appender");
         writer.writeAttribute("tag", "");
         writer.writeAttribute("file", "dynawaltz.log");
-        writer.writeAttribute("lvlFilter", "DEBUG");
+        writer.writeAttribute("lvlFilter", parameters.getLogLevelFilter().toString());
+        for (DynaWaltzParameters.SpecificLog log : parameters.getSpecificLogs()) {
+            writeSpecificAppender(writer, log);
+        }
+    }
+
+    private static void writeSpecificAppender(XMLStreamWriter writer, DynaWaltzParameters.SpecificLog log) throws XMLStreamException {
+        writer.writeEmptyElement(DYN_URI, "appender");
+        writer.writeAttribute("tag", log.toString());
+        writer.writeAttribute("file", log.getFileName());
+        writer.writeAttribute("lvlFilter", DynaWaltzParameters.LogLevel.DEBUG.toString());
     }
 }
diff --git a/dynawaltz/src/test/java/com/powsybl/dynawaltz/DynaWaltzParametersTest.java b/dynawaltz/src/test/java/com/powsybl/dynawaltz/DynaWaltzParametersTest.java
@@ -13,7 +13,9 @@
 import com.powsybl.dynamicsimulation.DynamicSimulationParameters;
 import com.powsybl.dynamicsimulation.json.JsonDynamicSimulationParameters;
 import com.powsybl.dynawaltz.DynaWaltzParameters.ExportMode;
+import com.powsybl.dynawaltz.DynaWaltzParameters.LogLevel;
 import com.powsybl.dynawaltz.DynaWaltzParameters.SolverType;
+import com.powsybl.dynawaltz.DynaWaltzParameters.SpecificLog;
 import com.powsybl.dynawaltz.parameters.Parameter;
 import com.powsybl.dynawaltz.parameters.ParameterType;
 import com.powsybl.dynawaltz.xml.ParametersXml;
@@ -24,8 +26,7 @@
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -61,7 +62,9 @@ void checkParameters() throws IOException {
         boolean useModelSimplifiers = true;
         double precision = 1e-8;
         ExportMode timelinExportMode = ExportMode.XML;
-        initPlatformConfig(networkParametersId, solverType, solverParametersId, mergeLoads, useModelSimplifiers, precision, timelinExportMode);
+        LogLevel logLevel = LogLevel.WARN;
+        Set<SpecificLog> specificLogs = EnumSet.of(SpecificLog.MODELER, SpecificLog.EQUATIONS);
+        initPlatformConfig(networkParametersId, solverType, solverParametersId, mergeLoads, useModelSimplifiers, precision, timelinExportMode, logLevel, specificLogs);
 
         DynaWaltzParameters parameters = DynaWaltzParameters.load(platformConfig, fileSystem);
 
@@ -94,6 +97,8 @@ void checkParameters() throws IOException {
         assertEquals(useModelSimplifiers, parameters.isUseModelSimplifiers());
         assertEquals(precision, parameters.getPrecision());
         assertEquals(timelinExportMode, parameters.getTimelineExportMode());
+        assertEquals(logLevel, parameters.getLogLevelFilter());
+        assertEquals(specificLogs, parameters.getSpecificLogs());
     }
 
     @Test
@@ -116,7 +121,7 @@ void roundTripParametersSerializing() throws IOException {
         SolverType solverType = SolverType.IDA;
         String solverParametersId = "solverParametersId";
         boolean mergeLoads = false;
-        initPlatformConfig(networkParametersId, solverType, solverParametersId, mergeLoads, false, 1e-7, ExportMode.TXT);
+        initPlatformConfig(networkParametersId, solverType, solverParametersId, mergeLoads, false, 1e-7, ExportMode.TXT, LogLevel.INFO, Set.of(SpecificLog.PARAMETERS, SpecificLog.VARIABLES));
 
         DynamicSimulationParameters dynamicSimulationParameters = new DynamicSimulationParameters()
                 .setStartTime(0)
@@ -127,7 +132,9 @@ void roundTripParametersSerializing() throws IOException {
                 JsonDynamicSimulationParameters::read, "/DynaWaltzParameters.json");
     }
 
-    private void initPlatformConfig(String networkParametersId, SolverType solverType, String solverParametersId, boolean mergeLoads, boolean useModelSimplifiers, double precision, ExportMode timelineExportMode) throws IOException {
+    private void initPlatformConfig(String networkParametersId, SolverType solverType, String solverParametersId,
+                                    boolean mergeLoads, boolean useModelSimplifiers, double precision, ExportMode timelineExportMode,
+                                    LogLevel logLevel, Set<SpecificLog> specificLogs) throws IOException {
         String parametersFile = USER_HOME + "parametersFile";
         String networkParametersFile = USER_HOME + "networkParametersFile";
         String solverParametersFile = USER_HOME + "solverParametersFile";
@@ -143,6 +150,8 @@ private void initPlatformConfig(String networkParametersId, SolverType solverTyp
         moduleConfig.setStringProperty("useModelSimplifiers", String.valueOf(useModelSimplifiers));
         moduleConfig.setStringProperty("precision", Double.toString(precision));
         moduleConfig.setStringProperty("timeline.exportMode", String.valueOf(timelineExportMode));
+        moduleConfig.setStringProperty("log.levelFilter", logLevel.toString());
+        moduleConfig.setStringListProperty("log.specificLogs", specificLogs.stream().map(SpecificLog::toString).toList());
 
         Files.createDirectories(fileSystem.getPath(USER_HOME));
         copyFile("/parametersSet/models.par", parametersFile);

diff --git a/dynawaltz/src/test/java/com/powsybl/dynawaltz/xml/JobsXmlTest.java b/dynawaltz/src/test/java/com/powsybl/dynawaltz/xml/JobsXmlTest.java
@@ -15,6 +15,7 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.EnumSet;
 
 /**
  * @author Marcos de Miguel {@literal <demiguelm at aia.es>}
@@ -42,4 +43,15 @@ void writeJobWithDumpFile() throws SAXException, IOException {
         validate("jobs.xsd", "jobsWithDump.xml", tmpDir.resolve(DynaWaltzConstants.JOBS_FILENAME));
     }
 
+    @Test
+    void writeJobWithSpecificLogs() throws SAXException, IOException {
+        DynamicSimulationParameters parameters = DynamicSimulationParameters.load();
+        DynaWaltzParameters dynawoParameters = DynaWaltzParameters.load()
+                .setSpecificLogs(EnumSet.allOf(DynaWaltzParameters.SpecificLog.class));
+        DynaWaltzContext context = new DynaWaltzContext(network, network.getVariantManager().getWorkingVariantId(), dynamicModels, eventModels, curves, parameters, dynawoParameters);
+
+        JobsXml.write(tmpDir, context);
+        validate("jobs.xsd", "jobsWithSpecificLogs.xml", tmpDir.resolve(DynaWaltzConstants.JOBS_FILENAME));
+    }
+
 }
diff --git a/dynawaltz/src/test/resources/DynaWaltzParameters.json b/dynawaltz/src/test/resources/DynaWaltzParameters.json
@@ -48,6 +48,8 @@
       },
       "precision" : 1.0E-7,
       "timelineExportMode" : "TXT",
+      "logLevelFilter" : "INFO",
+      "specificLogs" : [ "PARAMETERS", "VARIABLES" ],
       "modelsParameters" : [ {
         "id" : "test",
         "parameters" : {

diff --git a/dynawaltz/src/test/resources/jobs.xml b/dynawaltz/src/test/resources/jobs.xml
@@ -15,7 +15,7 @@
             <dyn:finalState exportIIDMFile="true" exportDumpFile="false"/>
             <dyn:curves inputFile="powsybl_dynawaltz.crv" exportMode="CSV"/>
             <dyn:logs>
-                <dyn:appender tag="" file="dynawaltz.log" lvlFilter="DEBUG"/>
+                <dyn:appender tag="" file="dynawaltz.log" lvlFilter="INFO"/>
             </dyn:logs>
         </dyn:outputs>
     </dyn:job>