Parse Fsv CSV file

Add Fsv handling in integration test Curve renaming 2 Signed-off-by: lisrte <[email protected]>
powsybl · Sep 23, 2024 · 88d9654 · 88d9654
1 parent 3a2eb5c
commit 88d9654
Show file tree

Hide file tree

Showing 30 changed files with 243 additions and 136 deletions.
diff --git a/README.md b/README.md
@@ -28,7 +28,7 @@ To run a dynamic simulation, you need:
 - a case file
 - a `DynamicModelsSupplier` to associate dynamic models to the equipment of the network
 - an `EventModelsSuppler` to configure events simulated during the simulation (optional)
-- a `CurvesSupplier` to follow the evolution of dynamic variables during the simulation (optional)
+- an `OutputVariablesSupplier` to follow the evolution of dynamic variables during the simulation (optional)
 - a set of parameters to configure the simulation (optional)
 
 Thanks to `powsybl-dynawo-dsl`, the inputs can be easily configured using Groovy scripts.

diff --git a/commons/src/main/java/com/powsybl/dynawo/commons/AbstractCsvParser.java b/commons/src/main/java/com/powsybl/dynawo/commons/AbstractCsvParser.java
@@ -28,36 +28,30 @@ public abstract class AbstractCsvParser<T> {
 
     protected static final char DEFAULT_SEPARATOR = '|';
 
-    private final char separator;
+    private final CsvParser csvParser;
 
-    protected AbstractCsvParser(char separator) {
-        this.separator = separator;
+    protected AbstractCsvParser(char separator, boolean skipHeader) {
+        CsvParserSettings settings = new CsvParserSettings();
+        settings.getFormat().setDelimiter(separator);
+        settings.getFormat().setQuoteEscape('"');
+        settings.getFormat().setLineSeparator(System.lineSeparator());
+        settings.setMaxColumns(getNbColumns());
+        settings.setHeaderExtractionEnabled(skipHeader);
+        csvParser = new CsvParser(settings);
     }
 
     public List<T> parse(Path file) {
         if (!Files.exists(file)) {
             return Collections.emptyList();
         }
-
         try (BufferedReader reader = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
-            return parse(reader);
+            Objects.requireNonNull(reader);
+            return read(csvParser.iterate(reader).iterator());
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
     }
 
-    private List<T> parse(BufferedReader reader) {
-        Objects.requireNonNull(reader);
-        CsvParserSettings settings = new CsvParserSettings();
-        settings.getFormat().setDelimiter(separator);
-        settings.getFormat().setQuoteEscape('"');
-        settings.getFormat().setLineSeparator(System.lineSeparator());
-        settings.setMaxColumns(getNbColumns());
-        CsvParser csvParser = new CsvParser(settings);
-        ResultIterator<String[], ParsingContext> iterator = csvParser.iterate(reader).iterator();
-        return read(iterator);
-    }
-
     protected List<T> read(ResultIterator<String[], ParsingContext> iterator) {
         List<T> logs = new ArrayList<>();
         int iLine = 0;

diff --git a/commons/src/main/java/com/powsybl/dynawo/commons/dynawologs/CsvLogParser.java b/commons/src/main/java/com/powsybl/dynawo/commons/dynawologs/CsvLogParser.java
@@ -23,7 +23,7 @@ public CsvLogParser() {
     }
 
     public CsvLogParser(char separator) {
-        super(separator);
+        super(separator, false);
     }
 
     @Override

diff --git a/commons/src/main/java/com/powsybl/dynawo/commons/timeline/CsvTimeLineParser.java b/commons/src/main/java/com/powsybl/dynawo/commons/timeline/CsvTimeLineParser.java
@@ -23,7 +23,7 @@ public CsvTimeLineParser() {
     }
 
     public CsvTimeLineParser(char separator) {
-        super(separator);
+        super(separator, false);
     }
 
     @Override

diff --git a/docs/dynamic_simulation/index.md b/docs/dynamic_simulation/index.md
@@ -5,7 +5,7 @@
 configuration.md
 dynamic-models-dsl.md
 event-models-dsl.md
-curves-dsl.md
+output-variables-dsl.md
 ```
 
 PowSyBl provides an implementation of the [DynamicSimulation API from powsybl-core](inv:powsyblcore:*:*#simulation/dynamic/index) with [Dynaωo](https://dynawo.github.io), a tool for long-term stability simulation.

diff --git a/docs/dynamic_simulation/curves-dsl.md → ...ynamic_simulation/output-variables-dsl.md b/docs/dynamic_simulation/curves-dsl.md → ...ynamic_simulation/output-variables-dsl.md
@@ -1,10 +1,19 @@
-# Curves DSL
-The curves domain specific language allow a user to configure the curves Dynawo will export at the end of the simulation. This DSL defines the `curve` keywords.
+# Output Variables DSL
+The output variables domain specific language allow a user to configure the curves or final state values Dynawo will export at the end of the simulation.
+This DSL defines the `curve` and `fsv`keywords.
 
 The `curve` keyword combined with the `variable` field create a single curve for a dynamic model. One identifies a dynamic model by its ID, the same as the one used in the [Dynamic Models DSL](dynamic-models-dsl). The variable to plot is identified by its name.
 ```groovy
 curve {
-    dynamicModelId load.id
+    dynamicModelId "dynamicId"
+    variable "load_PPu"
+}
+```
+
+If the only information needed is the variable final value, the `curve` keyword can be replaced by `fsv` keyword:
+```groovy
+fsv {
+    dynamicModelId "dynamicId"
     variable "load_PPu"
 }
 ```

diff --git a/dynawo-dsl/src/main/groovy/com/powsybl/dynawo/dsl/DynawoOutputVariableGroovyExtension.groovy b/dynawo-dsl/src/main/groovy/com/powsybl/dynawo/dsl/DynawoOutputVariableGroovyExtension.groovy
@@ -31,11 +31,10 @@ class DynawoOutputVariableGroovyExtension implements OutputVariableGroovyExtensi
     void load(Binding binding, Consumer<OutputVariable> consumer, ReportNode reportNode) {
         Closure<Void> closure = { Closure<Void> closure, OutputVariable.OutputType type ->
             def cloned = closure.clone()
-            DynawoOutputVariablesBuilder curvesBuilder = new DynawoOutputVariablesBuilder(reportNode)
-            curvesBuilder.outputType(type)
-            cloned.delegate = curvesBuilder
+            DynawoOutputVariablesBuilder variablesBuilder = new DynawoOutputVariablesBuilder(reportNode).outputType(type)
+            cloned.delegate = variablesBuilder
             cloned()
-            curvesBuilder.add(consumer)
+            variablesBuilder.add(consumer)
         }
         binding.curve = c -> closure(c, OutputVariable.OutputType.CURVE)
         binding.fsv = c -> closure(c, OutputVariable.OutputType.FSV)

diff --git a/dynawo-dsl/src/test/java/com/powsybl/dynawo/dsl/ieee/AbstractIeeeTest.java b/dynawo-dsl/src/test/java/com/powsybl/dynawo/dsl/ieee/AbstractIeeeTest.java
@@ -54,7 +54,7 @@ void tearDown() throws IOException {
     public abstract String getWorkingDirName();
 
     protected void setup(String parametersFile, String networkParametersFile, String networkParametersId, String solverParametersFile, String solverParametersId, String networkFile,
-                         String dynamicModelsFile, String eventModelsFile, String curvesFile, int startTime, int stopTime) throws IOException {
+                         String dynamicModelsFile, String eventModelsFile, String outputVariablesFile, int startTime, int stopTime) throws IOException {
 
         // The parameter files are copied into the PlatformConfig filesystem,
         // that filesystem is the one that DynawoSimulationContext and ParametersXml will use to read the parameters
@@ -83,11 +83,11 @@ protected void setup(String parametersFile, String networkParametersFile, String
             eventModelsSupplier = EventModelsSupplier.empty();
         }
 
-        // Curves
-        if (curvesFile != null) {
-            Files.copy(Objects.requireNonNull(getClass().getResourceAsStream(curvesFile)), workingDir.resolve("outputVariables.groovy"));
-            List<OutputVariableGroovyExtension> curveGroovyExtensions = GroovyExtension.find(OutputVariableGroovyExtension.class, DynawoSimulationProvider.NAME);
-            outputVariablesSupplier = new GroovyOutputVariablesSupplier(workingDir.resolve("outputVariables.groovy"), curveGroovyExtensions);
+        // Output Variables
+        if (outputVariablesFile != null) {
+            Files.copy(Objects.requireNonNull(getClass().getResourceAsStream(outputVariablesFile)), workingDir.resolve("outputVariables.groovy"));
+            List<OutputVariableGroovyExtension> variableGroovyExtensions = GroovyExtension.find(OutputVariableGroovyExtension.class, DynawoSimulationProvider.NAME);
+            outputVariablesSupplier = new GroovyOutputVariablesSupplier(workingDir.resolve("outputVariables.groovy"), variableGroovyExtensions);
         } else {
             outputVariablesSupplier = OutputVariablesSupplier.empty();
         }

diff --git a/dynawo-integration-tests/src/test/java/com/powsybl/dynawo/it/DynawoSimulationTest.java b/dynawo-integration-tests/src/test/java/com/powsybl/dynawo/it/DynawoSimulationTest.java
@@ -73,7 +73,7 @@ void testIeee14() {
                 GroovyExtension.find(EventModelGroovyExtension.class, DynawoSimulationProvider.NAME));
 
         GroovyOutputVariablesSupplier outputVariablesSupplier = new GroovyOutputVariablesSupplier(
-                getResourceAsStream("/ieee14/disconnectline/curves.groovy"),
+                getResourceAsStream("/ieee14/disconnectline/outputVariables.groovy"),
                 GroovyExtension.find(OutputVariableGroovyExtension.class, DynawoSimulationProvider.NAME));
 
         List<ParametersSet> modelsParameters = ParametersXml.load(getResourceAsStream("/ieee14/disconnectline/models.par"));
@@ -91,10 +91,12 @@ void testIeee14() {
 
         assertEquals(DynamicSimulationResult.Status.SUCCESS, result.getStatus());
         assertTrue(result.getStatusText().isEmpty());
-        assertEquals(41, result.getCurves().size());
+        assertEquals(27, result.getCurves().size());
         DoubleTimeSeries ts1 = result.getCurve("_GEN____1_SM_generator_UStatorPu");
         assertEquals("_GEN____1_SM_generator_UStatorPu", ts1.getMetadata().getName());
         assertEquals(587, ts1.toArray().length);
+        assertEquals(14, result.getFinalStateValues().size());
+        assertEquals(1.046227, result.getFinalStateValues().get("NETWORK__BUS___10_TN_Upu_value"));
         List<TimelineEvent> timeLine = result.getTimeLine();
         assertEquals(23, timeLine.size());
         checkFirstTimeLineEvent(timeLine.get(0), 0, "_GEN____8_SM", "PMIN : activation");
@@ -113,7 +115,7 @@ void testIeee14WithDump() throws IOException {
                 GroovyExtension.find(EventModelGroovyExtension.class, DynawoSimulationProvider.NAME));
 
         GroovyOutputVariablesSupplier outputVariablesSupplier = new GroovyOutputVariablesSupplier(
-                getResourceAsStream("/ieee14/disconnectline/curves.groovy"),
+                getResourceAsStream("/ieee14/disconnectline/outputVariables.groovy"),
                 GroovyExtension.find(OutputVariableGroovyExtension.class, DynawoSimulationProvider.NAME));
 
         Path dumpDir = Files.createDirectory(localDir.resolve("dumpFiles"));

diff --git a/...urces/ieee14/disconnectline/curves.groovy → ...e14/disconnectline/outputVariables.groovy b/...urces/ieee14/disconnectline/curves.groovy → ...e14/disconnectline/outputVariables.groovy
@@ -5,12 +5,14 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
+package ieee14.disconnectline
+
 import com.powsybl.iidm.network.Bus
 import com.powsybl.iidm.network.Generator
 import com.powsybl.iidm.network.Load
 
 for (Bus bus : network.busBreakerView.buses) {
-    curve {
+    fsv {
         staticId bus.id
         variable "Upu_value"
     }

diff --git a/dynawo-simulation/src/main/java/com/powsybl/dynawo/DynawoSimulationHandler.java b/dynawo-simulation/src/main/java/com/powsybl/dynawo/DynawoSimulationHandler.java
@@ -15,6 +15,7 @@
 import com.powsybl.dynamicsimulation.DynamicSimulationResult;
 import com.powsybl.dynamicsimulation.DynamicSimulationResultImpl;
 import com.powsybl.dynamicsimulation.TimelineEvent;
+import com.powsybl.dynawo.outputvariables.CsvFsvParser;
 import com.powsybl.dynawo.xml.OutputVariablesXml;
 import com.powsybl.dynawo.xml.DydXml;
 import com.powsybl.dynawo.xml.JobsXml;
@@ -190,7 +191,7 @@ private void setTimeline(Path outputsFolder) {
     }
 
     private void setCurves(Path workingDir) {
-        Path curvesPath = workingDir.resolve(CURVES_OUTPUT_PATH).toAbsolutePath().resolve(CURVES_FILENAME);
+        Path curvesPath = workingDir.resolve(CURVES_OUTPUT_PATH).resolve(CURVES_FILENAME);
         if (Files.exists(curvesPath)) {
             TimeSeries.parseCsv(curvesPath, new TimeSeriesCsvConfig(TimeSeriesConstants.DEFAULT_SEPARATOR, false, TimeSeries.TimeFormat.FRACTIONS_OF_SECOND))
                     .values().forEach(l -> l.forEach(curve -> curves.put(curve.getMetadata().getName(), (DoubleTimeSeries) curve)));
@@ -201,12 +202,10 @@ private void setCurves(Path workingDir) {
         }
     }
 
-    //TODO parse FSV
     private void setFinalStateValues(Path workingDir) {
-        Path fsvPath = workingDir.resolve(FSV_OUTPUT_PATH).toAbsolutePath().resolve(FSV_OUTPUT_FILENAME);
+        Path fsvPath = workingDir.resolve(FSV_OUTPUT_PATH).resolve(FSV_OUTPUT_FILENAME);
         if (Files.exists(fsvPath)) {
-            TimeSeries.parseCsv(fsvPath, new TimeSeriesCsvConfig(TimeSeriesConstants.DEFAULT_SEPARATOR, false, TimeSeries.TimeFormat.FRACTIONS_OF_SECOND))
-                    .values().forEach(l -> l.forEach(curve -> curves.put(curve.getMetadata().getName(), (DoubleTimeSeries) curve)));
+            new CsvFsvParser(';').parse(fsvPath).forEach(e -> fsv.put(e.model() + "_" + e.variable(), e.value()));
         } else {
             LOGGER.warn("Final state values folder not found");
             status = DynamicSimulationResult.Status.FAILURE;
@@ -235,9 +234,13 @@ private void writeInputFiles(Path workingDir) throws IOException {
     }
 
     private static void deleteExistingFile(Path basePath, String... elements) throws IOException {
-        Path finalPath = Arrays.stream(elements).map(Path::of).reduce(basePath, Path::resolve);
-        if (Files.exists(finalPath)) {
-            Files.delete(finalPath);
+        Path finalPath = basePath;
+        for (String element : elements) {
+            finalPath = finalPath.resolve(element);
+            if (!Files.exists(finalPath)) {
+                return;
+            }
         }
+        Files.delete(finalPath);
     }
 }
diff --git a/dynawo-simulation/src/main/java/com/powsybl/dynawo/builders/BuilderReports.java b/dynawo-simulation/src/main/java/com/powsybl/dynawo/builders/BuilderReports.java
@@ -55,9 +55,9 @@ public static void reportModelInstantiationFailure(ReportNode reportNode, String
                 .add();
     }
 
-    public static void reportCurveInstantiationFailure(ReportNode reportNode, String id) {
+    public static void reportOutputVariableInstantiationFailure(ReportNode reportNode, String id) {
         reportNode.newReportNode()
-                .withMessageTemplate("curveInstantiationError", "Curve ${id} cannot be instantiated")
+                .withMessageTemplate("outputVariableInstantiationError", "Output variable ${id} cannot be instantiated")
                 .withUntypedValue("id", id)
                 .withSeverity(TypedValue.WARN_SEVERITY)
                 .add();

diff --git a/dynawo-simulation/src/main/java/com/powsybl/dynawo/outputvariables/CsvFsvParser.java b/dynawo-simulation/src/main/java/com/powsybl/dynawo/outputvariables/CsvFsvParser.java
@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) 2023, RTE (http://www.rte-france.com/)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+package com.powsybl.dynawo.outputvariables;
+
+import com.powsybl.dynawo.commons.AbstractCsvParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Optional;
+
+/**
+ * @author Laurent Issertial {@literal <laurent.issertial at rte-france.com>}
+ */
+public final class CsvFsvParser extends AbstractCsvParser<FsvEntry> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(CsvFsvParser.class);
+
+    private static final int NB_COLUMNS = 4;
+
+    public CsvFsvParser() {
+        this(DEFAULT_SEPARATOR);
+    }
+
+    public CsvFsvParser(char separator) {
+        super(separator, true);
+    }
+
+    @Override
+    protected Optional<FsvEntry> createEntry(String[] tokens) {
+        String model = tokens[0];
+        String variable = tokens[1];
+        String value = tokens[2];
+        if (model == null || variable == null || value == null) {
+            LOGGER.warn("Inconsistent FSV entry (model: '{}', variable: '{}', value: '{}')", model, variable, value);
+        } else {
+            try {
+                double valueD = Double.parseDouble(value);
+                return Optional.of(new FsvEntry(model, variable, valueD));
+            } catch (NumberFormatException e) {
+                LOGGER.warn("Inconsistent value entry '{}'", value);
+            }
+        }
+        return Optional.empty();
+    }
+
+    @Override
+    protected boolean hasCorrectNbColumns(int tokensSize) {
+        return tokensSize == NB_COLUMNS;
+    }
+
+    @Override
+    protected int getNbColumns() {
+        return NB_COLUMNS;
+    }
+}
diff --git a/...lation/src/main/java/com/powsybl/dynawo/outputvariables/DynawoOutputVariablesBuilder.java b/...lation/src/main/java/com/powsybl/dynawo/outputvariables/DynawoOutputVariablesBuilder.java
@@ -89,23 +89,23 @@ private void checkData() {
     private boolean isInstantiable() {
         checkData();
         if (!isInstantiable) {
-            BuilderReports.reportCurveInstantiationFailure(reportNode, getId());
+            BuilderReports.reportOutputVariableInstantiationFailure(reportNode, getId());
         }
         return isInstantiable;
     }
 
-    public void add(Consumer<OutputVariable> curveConsumer) {
+    public void add(Consumer<OutputVariable> outputVariableConsumer) {
         if (isInstantiable()) {
             boolean hasDynamicModelId = dynamicModelId != null;
             String id = hasDynamicModelId ? dynamicModelId : DEFAULT_DYNAMIC_MODEL_ID;
             variables.forEach(v ->
-                    curveConsumer.accept(new DynawoOutputVariable(id, hasDynamicModelId ? v : staticId + "_" + v, type)));
+                    outputVariableConsumer.accept(new DynawoOutputVariable(id, hasDynamicModelId ? v : staticId + "_" + v, type)));
         }
     }
 
     public List<OutputVariable> build() {
-        List<OutputVariable> curves = new ArrayList<>();
-        add(curves::add);
-        return curves;
+        List<OutputVariable> outputVariables = new ArrayList<>();
+        add(outputVariables::add);
+        return outputVariables;
     }
 }
diff --git a/dynawo-simulation/src/main/java/com/powsybl/dynawo/outputvariables/FsvEntry.java b/dynawo-simulation/src/main/java/com/powsybl/dynawo/outputvariables/FsvEntry.java
@@ -0,0 +1,14 @@
+/**
+ * Copyright (c) 2024, RTE (http://www.rte-france.com/)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+package com.powsybl.dynawo.outputvariables;
+
+/**
+ * @author Laurent Issertial {@literal <laurent.issertial at rte-france.com>}
+ */
+public record FsvEntry(String model, String variable, double value) {
+}
diff --git a/...in/java/com/powsybl/dynawo/suppliers/outputvariables/OutputVariablesJsonDeserializer.java b/...in/java/com/powsybl/dynawo/suppliers/outputvariables/OutputVariablesJsonDeserializer.java
@@ -40,10 +40,10 @@ public List<OutputVariable> deserialize(JsonParser parser, DeserializationContex
         List<DynawoOutputVariablesBuilder> modelConfigList = new ArrayList<>();
         JsonUtil.parseObject(parser, name -> {
             if (name.equals("curves")) {
-                JsonUtil.parseObjectArray(parser, modelConfigList::add, p -> parseCurvesBuilder(p, OutputVariable.OutputType.CURVE));
+                JsonUtil.parseObjectArray(parser, modelConfigList::add, p -> parseOutputVariablesBuilder(p, OutputVariable.OutputType.CURVE));
                 return true;
             } else if (name.equals("fsv")) {
-                JsonUtil.parseObjectArray(parser, modelConfigList::add, p -> parseCurvesBuilder(p, OutputVariable.OutputType.FSV));
+                JsonUtil.parseObjectArray(parser, modelConfigList::add, p -> parseOutputVariablesBuilder(p, OutputVariable.OutputType.FSV));
                 return true;
             }
             return false;
@@ -54,7 +54,7 @@ public List<OutputVariable> deserialize(JsonParser parser, DeserializationContex
                 .toList();
     }
 
-    private DynawoOutputVariablesBuilder parseCurvesBuilder(JsonParser parser, OutputVariable.OutputType outputType) {
+    private DynawoOutputVariablesBuilder parseOutputVariablesBuilder(JsonParser parser, OutputVariable.OutputType outputType) {
         DynawoOutputVariablesBuilder variablesBuilder = builderConstructor.get();
         variablesBuilder.outputType(outputType);
         JsonUtil.parseObject(parser, name -> {

diff --git a/dynawo-simulation/src/main/java/com/powsybl/dynawo/xml/JobsXml.java b/dynawo-simulation/src/main/java/com/powsybl/dynawo/xml/JobsXml.java
@@ -111,6 +111,7 @@ private static void writeOutput(XMLStreamWriter writer, DynawoSimulationContext
         if (context.withFsvVariables()) {
             writer.writeEmptyElement(DYN_URI, "finalStateValues");
             writer.writeAttribute("inputFile", FSV_FILENAME);
+            writer.writeAttribute("exportMode", DynawoSimulationParameters.ExportMode.CSV.toString());
         }
 
         writer.writeStartElement(DYN_URI, "logs");