From e4cf98cf8acb7a85edfae2367d53d3de690f708f Mon Sep 17 00:00:00 2001 From: Benjamin Gaidioz Date: Thu, 6 Jul 2023 15:00:12 +0200 Subject: [PATCH] RD-9178: CsvWriter option for Windows friendly output (#22) Added a test ensuring the line separator is indeed changed depending on the config. --- .../rql2/truffle/Rql2TruffleCompiler.scala | 11 ++++++++--- .../rql2/tests/CompilerTestContext.scala | 17 ++++++++++++----- .../rql2/tests/output/CsvOutputTest.scala | 14 ++++++++++++++ .../ast/csv/writer/CsvIterableWriterNode.java | 9 +++++---- .../ast/csv/writer/CsvListWriterNode.java | 7 ++++--- 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/raw-compiler-rql2-truffle/src/main/scala/raw/compiler/rql2/truffle/Rql2TruffleCompiler.scala b/raw-compiler-rql2-truffle/src/main/scala/raw/compiler/rql2/truffle/Rql2TruffleCompiler.scala index 8d86b490b..e7a5e8324 100644 --- a/raw-compiler-rql2-truffle/src/main/scala/raw/compiler/rql2/truffle/Rql2TruffleCompiler.scala +++ b/raw-compiler-rql2-truffle/src/main/scala/raw/compiler/rql2/truffle/Rql2TruffleCompiler.scala @@ -177,8 +177,11 @@ class Rql2TruffleCompiler(implicit compilerContext: CompilerContext) val frameDescriptor = emitter.dropScope() // Wrap output node + val rootNode: RootNode = programContext.settings.getString(ProgramSettings.output_format) match { - case "csv" => tree.analyzer.tipe(me.get) match { + case "csv" => + val lineSeparator = if (programContext.settings.getBoolean("raw.compiler.windows-line-ending")) "\r\n" else "\n" + tree.analyzer.tipe(me.get) match { case Rql2IterableType(Rql2RecordType(atts, rProps), iProps) => assert(rProps.isEmpty) assert(iProps.isEmpty) @@ -188,7 +191,8 @@ class Rql2TruffleCompiler(implicit compilerContext: CompilerContext) new CsvIterableWriterNode( bodyExpNode, CsvWriter(atts.map(_.tipe)), - atts.map(_.idn).toArray + atts.map(_.idn).toArray, + lineSeparator ) ) case Rql2ListType(Rql2RecordType(atts, rProps), iProps) => @@ -200,7 +204,8 @@ class Rql2TruffleCompiler(implicit compilerContext: CompilerContext) new CsvListWriterNode( bodyExpNode, CsvWriter(atts.map(_.tipe)), - atts.map(_.idn).toArray + atts.map(_.idn).toArray, + lineSeparator ) ) } diff --git a/raw-compiler-rql2/src/test/scala/raw/compiler/rql2/tests/CompilerTestContext.scala b/raw-compiler-rql2/src/test/scala/raw/compiler/rql2/tests/CompilerTestContext.scala index d83ca5269..9c039bd3d 100644 --- a/raw-compiler-rql2/src/test/scala/raw/compiler/rql2/tests/CompilerTestContext.scala +++ b/raw-compiler-rql2/src/test/scala/raw/compiler/rql2/tests/CompilerTestContext.scala @@ -511,14 +511,21 @@ trait CompilerTestContext } } - def saveTo(path: Path, executionLogger: ExecutionLogger = DebugExecutionLogger): SaveTo = - new SaveTo(path, Map.empty, executionLogger) + def saveTo( + path: Path, + executionLogger: ExecutionLogger = DebugExecutionLogger, + options: Map[String, String] = Map.empty + ): SaveTo = new SaveTo(path, options, executionLogger) - def saveToInFormat(path: Path, format: String, executionLogger: ExecutionLogger = DebugExecutionLogger): SaveTo = - new SaveTo(path, Map("output-format" -> format), executionLogger) + def saveToInFormat( + path: Path, + format: String, + executionLogger: ExecutionLogger = DebugExecutionLogger, + options: Map[String, String] = Map.empty + ): SaveTo = new SaveTo(path, options + ("output-format" -> format), executionLogger) // a Matcher[Path] that compares the content of the file at the given path to the given string. - protected def contain(content: String) = be(content) compose { p: Path => + protected def contain(content: String): Matcher[Path] = be(content) compose { p: Path => val bufferedSource = Source.fromFile(p.toFile) val fileContent = bufferedSource.mkString bufferedSource.close() diff --git a/raw-compiler-rql2/src/test/scala/raw/compiler/rql2/tests/output/CsvOutputTest.scala b/raw-compiler-rql2/src/test/scala/raw/compiler/rql2/tests/output/CsvOutputTest.scala index 3b200516a..aacb90f64 100644 --- a/raw-compiler-rql2/src/test/scala/raw/compiler/rql2/tests/output/CsvOutputTest.scala +++ b/raw-compiler-rql2/src/test/scala/raw/compiler/rql2/tests/output/CsvOutputTest.scala @@ -134,4 +134,18 @@ trait CsvOutputTest extends CompilerTestContext { RawUtils.deleteTestPath(path) } } + + test("""[{a: 1, b: 2}, {a: 3, b: 4}]""") { it => + it should run + val path = Files.createTempFile("", "") + try { + it should saveToInFormat(path, "csv", options = Map("windows-line-ending" -> "false")) + path should contain("a,b\n1,2\n3,4\n") + it should saveToInFormat(path, "csv", options = Map("windows-line-ending" -> "true")) + path should contain("a,b\r\n1,2\r\n3,4\r\n") + } finally { + Files.deleteIfExists(path) + } + } + } diff --git a/raw-runtime-rql2-truffle/src/main/java/raw/runtime/truffle/ast/csv/writer/CsvIterableWriterNode.java b/raw-runtime-rql2-truffle/src/main/java/raw/runtime/truffle/ast/csv/writer/CsvIterableWriterNode.java index 1680c6d1e..47fa99d19 100644 --- a/raw-runtime-rql2-truffle/src/main/java/raw/runtime/truffle/ast/csv/writer/CsvIterableWriterNode.java +++ b/raw-runtime-rql2-truffle/src/main/java/raw/runtime/truffle/ast/csv/writer/CsvIterableWriterNode.java @@ -23,7 +23,6 @@ import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawContext; import raw.runtime.truffle.StatementNode; -import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; import raw.runtime.truffle.runtime.exceptions.csv.CsvWriterRawTruffleException; import raw.runtime.truffle.runtime.generator.GeneratorLibrary; import raw.runtime.truffle.runtime.iterable.IterableLibrary; @@ -47,11 +46,13 @@ public class CsvIterableWriterNode extends StatementNode { @Child private GeneratorLibrary generators = GeneratorLibrary.getFactory().createDispatched(3); - private String[] columnNames; + private final String[] columnNames; + private final String lineSeparator; - public CsvIterableWriterNode(ExpressionNode dataNode, RootNode writerNode, String[] columnNames) { + public CsvIterableWriterNode(ExpressionNode dataNode, RootNode writerNode, String[] columnNames, String lineSeparator) { this.dataNode = dataNode; this.columnNames = columnNames; + this.lineSeparator = lineSeparator; itemWriter = DirectCallNode.create(writerNode.getCallTarget()); } @@ -82,7 +83,7 @@ private CsvGenerator createGenerator(OutputStream os) { } schemaBuilder.setColumnSeparator(','); schemaBuilder.setUseHeader(true); - schemaBuilder.setLineSeparator('\n'); + schemaBuilder.setLineSeparator(lineSeparator); schemaBuilder.setQuoteChar('"'); schemaBuilder.setNullValue(""); generator.setSchema(schemaBuilder.build()); diff --git a/raw-runtime-rql2-truffle/src/main/java/raw/runtime/truffle/ast/csv/writer/CsvListWriterNode.java b/raw-runtime-rql2-truffle/src/main/java/raw/runtime/truffle/ast/csv/writer/CsvListWriterNode.java index 014a4b50f..ac5928262 100644 --- a/raw-runtime-rql2-truffle/src/main/java/raw/runtime/truffle/ast/csv/writer/CsvListWriterNode.java +++ b/raw-runtime-rql2-truffle/src/main/java/raw/runtime/truffle/ast/csv/writer/CsvListWriterNode.java @@ -23,7 +23,6 @@ import raw.runtime.truffle.ExpressionNode; import raw.runtime.truffle.RawContext; import raw.runtime.truffle.StatementNode; -import raw.runtime.truffle.runtime.exceptions.RawTruffleRuntimeException; import raw.runtime.truffle.runtime.exceptions.csv.CsvWriterRawTruffleException; import raw.runtime.truffle.runtime.list.ListLibrary; import raw.runtime.truffle.runtime.list.ObjectList; @@ -45,11 +44,13 @@ public class CsvListWriterNode extends StatementNode { private ListLibrary lists = ListLibrary.getFactory().createDispatched(3); private String[] columnNames; + private final String lineSeparator; - public CsvListWriterNode(ExpressionNode dataNode, RootNode writerNode, String[] columnNames) { + public CsvListWriterNode(ExpressionNode dataNode, RootNode writerNode, String[] columnNames, String lineSeparator) { this.dataNode = dataNode; itemWriter = DirectCallNode.create(writerNode.getCallTarget()); this.columnNames = columnNames; + this.lineSeparator = lineSeparator; } @Override @@ -78,7 +79,7 @@ private CsvGenerator createGenerator(OutputStream os) { } schemaBuilder.setColumnSeparator(','); schemaBuilder.setUseHeader(true); - schemaBuilder.setLineSeparator('\n'); + schemaBuilder.setLineSeparator(lineSeparator); schemaBuilder.setQuoteChar('"'); schemaBuilder.setNullValue(""); generator.setSchema(schemaBuilder.build());