diff --git a/extensions/csv/src/main/java/io/deephaven/csv/CsvTools.java b/extensions/csv/src/main/java/io/deephaven/csv/CsvTools.java index 0b9bdf4c27e..9b52b133e14 100644 --- a/extensions/csv/src/main/java/io/deephaven/csv/CsvTools.java +++ b/extensions/csv/src/main/java/io/deephaven/csv/CsvTools.java @@ -5,20 +5,9 @@ import io.deephaven.api.ColumnName; import io.deephaven.api.Pair; -import io.deephaven.chunk.ByteChunk; -import io.deephaven.chunk.CharChunk; -import io.deephaven.chunk.Chunk; -import io.deephaven.chunk.DoubleChunk; -import io.deephaven.chunk.FloatChunk; -import io.deephaven.chunk.IntChunk; -import io.deephaven.chunk.LongChunk; -import io.deephaven.chunk.ObjectChunk; -import io.deephaven.chunk.ShortChunk; -import io.deephaven.chunk.WritableByteChunk; -import io.deephaven.chunk.WritableChunk; -import io.deephaven.chunk.WritableIntChunk; -import io.deephaven.chunk.WritableLongChunk; -import io.deephaven.chunk.WritableShortChunk; +import io.deephaven.base.verify.Assert; +import io.deephaven.chunk.*; +import io.deephaven.chunk.attributes.Any; import io.deephaven.chunk.attributes.Values; import io.deephaven.csv.CsvSpecs.Builder; import io.deephaven.csv.reading.CsvReader; @@ -34,31 +23,17 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.TrackingRowSet; -import io.deephaven.engine.table.ChunkSink; -import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.WritableColumnSource; +import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.InMemoryTable; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; -import io.deephaven.engine.table.impl.sources.BooleanArraySource; -import io.deephaven.engine.table.impl.sources.ByteArraySource; -import io.deephaven.engine.table.impl.sources.CharacterArraySource; -import io.deephaven.engine.table.impl.sources.DoubleArraySource; -import io.deephaven.engine.table.impl.sources.FloatArraySource; -import io.deephaven.engine.table.impl.sources.InstantArraySource; -import io.deephaven.engine.table.impl.sources.IntegerArraySource; -import io.deephaven.engine.table.impl.sources.LongArraySource; -import io.deephaven.engine.table.impl.sources.ObjectArraySource; -import io.deephaven.engine.table.impl.sources.ShortArraySource; +import io.deephaven.engine.table.impl.sources.*; import io.deephaven.engine.util.PathUtil; -import io.deephaven.engine.util.TableTools; import io.deephaven.io.streams.BzipFileOutputStream; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.BooleanUtils; -import io.deephaven.util.QueryConstants; import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.ScriptApi; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.BufferedWriter; @@ -85,6 +60,10 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Stream; + +import static io.deephaven.engine.util.TableTools.NULL_STRING; +import static io.deephaven.util.QueryConstants.*; /** * Utilities for reading and writing CSV files to and from {@link Table}s @@ -123,7 +102,6 @@ public static Builder builder() { * * @param path the path * @return the table - * @throws IOException if an I/O exception occurs * @see #readCsv(String, CsvSpecs) */ @ScriptApi @@ -137,7 +115,6 @@ public static Table readCsv(String path) throws CsvReaderException { * * @param stream an InputStream providing access to the CSV data. * @return a Deephaven Table object - * @throws IOException if the InputStream cannot be read * @see #readCsv(InputStream, CsvSpecs) */ @ScriptApi @@ -150,7 +127,6 @@ public static Table readCsv(InputStream stream) throws CsvReaderException { * * @param url the url * @return the table - * @throws IOException if an I/O exception occurs * @see #readCsv(URL, CsvSpecs) */ @ScriptApi @@ -167,7 +143,6 @@ public static Table readCsv(URL url) throws CsvReaderException { * * @param path the file path * @return the table - * @throws IOException if an I/O exception occurs * @see #readCsv(Path, CsvSpecs) */ @ScriptApi @@ -188,7 +163,6 @@ public static Table readCsv(Path path) throws CsvReaderException { * @param path the path * @param specs the csv specs * @return the table - * @throws IOException if an I/O exception occurs * @see #readCsv(URL, CsvSpecs) * @see #readCsv(Path, CsvSpecs) */ @@ -237,7 +211,6 @@ public static Table readCsv(InputStream stream, CsvSpecs specs) throws CsvReader * @param specs the csv specs * @return the table * @throws CsvReaderException If some CSV reading error occurs. - * @throws IOException if the URL cannot be opened. */ @ScriptApi public static Table readCsv(URL url, CsvSpecs specs) throws CsvReaderException { @@ -259,7 +232,6 @@ public static Table readCsv(URL url, CsvSpecs specs) throws CsvReaderException { * @param specs the csv specs * @return the table * @throws CsvReaderException If some CSV reading error occurs. - * @throws IOException if an I/O exception occurs * @see PathUtil#open(Path) */ @ScriptApi @@ -327,7 +299,6 @@ public static Table readHeaderlessCsv(String filePath, String... columnNames) th * @param format an Apache Commons CSV format name to be used to parse the CSV, or a single non-newline character to * use as a delimiter. * @return a Deephaven Table object - * @throws IOException if the InputStream cannot be read * @deprecated See {@link #readCsv(InputStream, CsvSpecs)} */ @ScriptApi @@ -347,7 +318,6 @@ public static Table readCsv(InputStream is, final String format) throws CsvReade * @param is an InputStream providing access to the CSV data. * @param separator a char to use as the delimiter value when parsing the file. * @return a Deephaven Table object - * @throws IOException if the InputStream cannot be read * @deprecated See {@link #readCsv(InputStream, CsvSpecs)} */ @ScriptApi @@ -676,6 +646,7 @@ public static void writeCsv(Table source, Writer out, ZoneId timeZone, writeCsvHeader(out, separator, columns); writeCsvContents(source, out, timeZone, progress, nullsAsEmpty, separator, columns); + out.write(System.lineSeparator()); out.close(); } @@ -702,12 +673,13 @@ public static void writeCsvHeader(Writer out, String... columns) throws IOExcept */ @ScriptApi public static void writeCsvHeader(Writer out, char separator, String... columns) throws IOException { - for (int i = 0; i < columns.length; i++) { - String column = columns[i]; - if (i > 0) { + final String separatorStr = String.valueOf(separator); + for (int ci = 0; ci < columns.length; ci++) { + String column = columns[ci]; + if (ci > 0) { out.write(separator); } - out.write(column); + out.write(separatorCsvEscape(column, separatorStr)); } } @@ -913,35 +885,321 @@ private static void writeCsvContentsSeq( final boolean nullsAsEmpty, final char separator, @Nullable final BiConsumer progress) throws IOException { - final long size = rows.size(); + if (rows.isEmpty()) { + return; + } try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getNugget("CsvTools.writeCsvContentsSeq()"); - final RowSet.Iterator rowsIter = rows.iterator()) { - String separatorStr = String.valueOf(separator); - for (long ri = 0; ri < size; ri++) { - final long rowKey = rowsIter.nextLong(); - for (int ci = 0; ci < cols.length; ci++) { - if (ci > 0) { - out.write(separatorStr); - } else { - out.write("\n"); + final CsvRowFormatter formatter = new CsvRowFormatter(timeZone, nullsAsEmpty, + String.valueOf(separator), System.lineSeparator(), cols)) { + formatter.writeRows(out, rows, progress); + } + } + + private static final class CsvRowFormatter implements Context { + + private static final int CHUNK_CAPACITY = ArrayBackedColumnSource.BLOCK_SIZE; + + private final ZoneId timeZone; + private final String separator; + private final String lineSeparator; + private final String nullValue; + private final SharedContext sharedContext; + private final ColumnFormatter[] columnFormatters; + + private CsvRowFormatter( + @NotNull final ZoneId timeZone, + final boolean nullsAsEmpty, + final String separator, + final String lineSeparator, + @NotNull final ColumnSource... columns) { + this.timeZone = timeZone; + this.separator = separator; + this.lineSeparator = lineSeparator; + nullValue = separatorCsvEscape(nullsAsEmpty ? "" : NULL_STRING, separator); + sharedContext = columns.length > 1 ? SharedContext.makeSharedContext() : null; + columnFormatters = Arrays.stream(columns).map(this::makeColumnFormatter).toArray(ColumnFormatter[]::new); + } + + private synchronized void writeRows( + @NotNull final Writer writer, + @NotNull final RowSequence rows, + @Nullable final BiConsumer progress) throws IOException { + final long totalSize = rows.size(); + long rowsWritten = 0; + try (final RowSequence.Iterator rowsIterator = rows.getRowSequenceIterator()) { + while (rowsIterator.hasMore()) { + final RowSequence sliceRows = rowsIterator.getNextRowSequenceWithLength(CHUNK_CAPACITY); + final int sliceSize = sliceRows.intSize(); + for (final ColumnFormatter columnFormatter : columnFormatters) { + columnFormatter.fill(sliceRows); + } + for (int sri = 0; sri < sliceSize; ++sri) { + for (int ci = 0; ci < columnFormatters.length; ++ci) { + if (ci == 0) { + // Writing our line separator at the beginning, rather than the end, seems like an + // unusual choice, but the code has behaved this way for some time. Maybe related to + // the multi-table writing functionality and `tableSeparator`? + writer.write(lineSeparator); + } else { + writer.write(separator); + } + columnFormatters[ci].write(writer, sri); + } } - final Object o = cols[ci].get(rowKey); - if (o instanceof String) { - out.write("" + separatorCsvEscape((String) o, separatorStr)); - } else if (o instanceof Instant) { - final ZonedDateTime zdt = ZonedDateTime.ofInstant((Instant) o, timeZone); - out.write(separatorCsvEscape( - zdt.toLocalDateTime().toString() + zdt.getOffset().toString(), separatorStr)); - } else { - out.write(nullsAsEmpty - ? separatorCsvEscape(o == null ? "" : o.toString(), separatorStr) - : separatorCsvEscape(TableTools.nullToNullString(o), separatorStr)); + if (sharedContext != null) { + sharedContext.reset(); } + if (progress != null) { + progress.accept(rowsWritten += sliceSize, totalSize); + } + } + } + } + + @Override + public void close() { + SafeCloseable.closeAll(Stream.concat(Stream.of(sharedContext), Arrays.stream(columnFormatters))); + } + + private ColumnFormatter makeColumnFormatter(@NotNull final ColumnSource source) { + final Class type = source.getType(); + if (type == char.class || type == Character.class) { + Assert.eq(source.getChunkType(), "source.getChunkType()", ChunkType.Char, "ChunkType.Char"); + return new CharColumnFormatter(source); + } + if (type == byte.class || type == Byte.class) { + Assert.eq(source.getChunkType(), "source.getChunkType()", ChunkType.Byte, "ChunkType.Byte"); + return new ByteColumnFormatter(source); + } + if (type == short.class || type == Short.class) { + Assert.eq(source.getChunkType(), "source.getChunkType()", ChunkType.Short, "ChunkType.Short"); + return new ShortColumnFormatter(source); + } + if (type == int.class || type == Integer.class) { + Assert.eq(source.getChunkType(), "source.getChunkType()", ChunkType.Int, "ChunkType.Int"); + return new IntColumnFormatter(source); + } + if (type == long.class || type == Long.class) { + Assert.eq(source.getChunkType(), "source.getChunkType()", ChunkType.Long, "ChunkType.Long"); + return new LongColumnFormatter(source); + } + if (type == float.class || type == Float.class) { + Assert.eq(source.getChunkType(), "source.getChunkType()", ChunkType.Float, "ChunkType.Float"); + return new FloatColumnFormatter(source); + } + if (type == double.class || type == Double.class) { + Assert.eq(source.getChunkType(), "source.getChunkType()", ChunkType.Double, "ChunkType.Double"); + return new DoubleColumnFormatter(source); + } + Assert.eq(source.getChunkType(), "source.getChunkType()", ChunkType.Object, "ChunkType.Object"); + if (type == Instant.class) { + return new InstantColumnFormatter(source); + } + if (type == ZonedDateTime.class) { + return new ZonedDateTimeColumnFormatter(source); + } + return new ObjectColumnFormatter(source); + } + + private abstract class ColumnFormatter> implements Context { + + private final ChunkSource source; + private final ChunkSource.GetContext getContext; + + CHUNK_CLASS values; + + private ColumnFormatter(@NotNull final ChunkSource source) { + this.source = source; + this.getContext = source.makeGetContext(CHUNK_CAPACITY, sharedContext); + } + + @Override + public void close() { + getContext.close(); + } + + private void fill(@NotNull final RowSequence rows) { + // noinspection unchecked + values = (CHUNK_CLASS) source.getChunk(getContext, rows); + } + + abstract void write(@NotNull Writer writer, int offset) throws IOException; + + void writeNull(@NotNull final Writer writer) throws IOException { + writer.write(nullValue); + } + } + + private final class CharColumnFormatter extends ColumnFormatter> { + + private CharColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final char value = values.get(offset); + if (value == NULL_CHAR) { + writeNull(writer); + return; + } + writer.write(separatorCsvEscape(String.valueOf(value), separator)); + } + } + + private final class ByteColumnFormatter extends ColumnFormatter> { + + private ByteColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final byte value = values.get(offset); + if (value == NULL_BYTE) { + writeNull(writer); + return; + } + writer.write(separatorCsvEscape(Byte.toString(value), separator)); + } + } + + private final class ShortColumnFormatter extends ColumnFormatter> { + + private ShortColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final short value = values.get(offset); + if (value == NULL_SHORT) { + writeNull(writer); + return; + } + writer.write(separatorCsvEscape(Short.toString(value), separator)); + } + } + + private final class IntColumnFormatter extends ColumnFormatter> { + + private IntColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final int value = values.get(offset); + if (value == NULL_INT) { + writeNull(writer); + return; } - if (progress != null) { - progress.accept(ri, size); + writer.write(separatorCsvEscape(Integer.toString(value), separator)); + } + } + + private final class LongColumnFormatter extends ColumnFormatter> { + + private LongColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final long value = values.get(offset); + if (value == NULL_LONG) { + writeNull(writer); + return; + } + writer.write(separatorCsvEscape(Long.toString(value), separator)); + } + } + + private final class FloatColumnFormatter extends ColumnFormatter> { + + private FloatColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final float value = values.get(offset); + if (value == NULL_FLOAT) { + writeNull(writer); + return; + } + writer.write(separatorCsvEscape(Float.toString(value), separator)); + } + } + + private final class DoubleColumnFormatter extends ColumnFormatter> { + + private DoubleColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final double value = values.get(offset); + if (value == NULL_DOUBLE) { + writeNull(writer); + return; + } + writer.write(separatorCsvEscape(Double.toString(value), separator)); + } + } + + private final class ObjectColumnFormatter extends ColumnFormatter> { + + private ObjectColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final Object value = values.get(offset); + if (value == null) { + writeNull(writer); + return; + } + writer.write(separatorCsvEscape(value.toString(), separator)); + } + } + + private final class InstantColumnFormatter extends ColumnFormatter> { + + private InstantColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final Instant value = values.get(offset); + if (value == null) { + writeNull(writer); + return; + } + final ZonedDateTime zdt = ZonedDateTime.ofInstant(value, timeZone); + writer.write(separatorCsvEscape(zdt.toLocalDateTime().toString() + zdt.getOffset(), separator)); + } + } + + private final class ZonedDateTimeColumnFormatter + extends ColumnFormatter> { + + private ZonedDateTimeColumnFormatter(@NotNull final ChunkSource source) { + super(source); + } + + @Override + void write(@NotNull final Writer writer, final int offset) throws IOException { + final ZonedDateTime value = values.get(offset); + if (value == null) { + writeNull(writer); + return; } + writer.write(separatorCsvEscape(value.toLocalDateTime().toString() + value.getOffset(), separator)); } } } @@ -1054,7 +1312,7 @@ public MyCharSink(int columnIndex) { protected void nullFlagsToValues(final char[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii < size; ++ii) { if (isNull[ii]) { - values[ii] = QueryConstants.NULL_CHAR; + values[ii] = NULL_CHAR; } } } @@ -1084,7 +1342,7 @@ public MyByteSink(int columnIndex) { protected void nullFlagsToValues(final byte[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii != size; ++ii) { if (isNull[ii]) { - values[ii] = QueryConstants.NULL_BYTE; + values[ii] = NULL_BYTE; } } } @@ -1092,7 +1350,7 @@ protected void nullFlagsToValues(final byte[] values, final boolean[] isNull, fi @Override protected void valuesToNullFlags(final byte[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii < size; ++ii) { - isNull[ii] = values[ii] == QueryConstants.NULL_BYTE; + isNull[ii] = values[ii] == NULL_BYTE; } } } @@ -1106,7 +1364,7 @@ public MyShortSink(int columnIndex) { protected void nullFlagsToValues(final short[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii != size; ++ii) { if (isNull[ii]) { - values[ii] = QueryConstants.NULL_SHORT; + values[ii] = NULL_SHORT; } } } @@ -1114,7 +1372,7 @@ protected void nullFlagsToValues(final short[] values, final boolean[] isNull, f @Override protected void valuesToNullFlags(final short[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii < size; ++ii) { - isNull[ii] = values[ii] == QueryConstants.NULL_SHORT; + isNull[ii] = values[ii] == NULL_SHORT; } } } @@ -1128,7 +1386,7 @@ public MyIntSink(int columnIndex) { protected void nullFlagsToValues(final int[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii != size; ++ii) { if (isNull[ii]) { - values[ii] = QueryConstants.NULL_INT; + values[ii] = NULL_INT; } } } @@ -1136,7 +1394,7 @@ protected void nullFlagsToValues(final int[] values, final boolean[] isNull, fin @Override protected void valuesToNullFlags(final int[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii < size; ++ii) { - isNull[ii] = values[ii] == QueryConstants.NULL_INT; + isNull[ii] = values[ii] == NULL_INT; } } } @@ -1150,7 +1408,7 @@ public MyLongSink(int columnIndex) { protected void nullFlagsToValues(final long[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii != size; ++ii) { if (isNull[ii]) { - values[ii] = QueryConstants.NULL_LONG; + values[ii] = NULL_LONG; } } } @@ -1158,7 +1416,7 @@ protected void nullFlagsToValues(final long[] values, final boolean[] isNull, fi @Override protected void valuesToNullFlags(final long[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii < size; ++ii) { - isNull[ii] = values[ii] == QueryConstants.NULL_LONG; + isNull[ii] = values[ii] == NULL_LONG; } } @@ -1173,7 +1431,7 @@ public MyFloatSink(int columnIndex) { protected void nullFlagsToValues(final float[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii != size; ++ii) { if (isNull[ii]) { - values[ii] = QueryConstants.NULL_FLOAT; + values[ii] = NULL_FLOAT; } } } @@ -1188,7 +1446,7 @@ public MyDoubleSink(int columnIndex) { protected void nullFlagsToValues(final double[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii != size; ++ii) { if (isNull[ii]) { - values[ii] = QueryConstants.NULL_DOUBLE; + values[ii] = NULL_DOUBLE; } } } @@ -1218,7 +1476,7 @@ public MyInstantAsLongSink(int columnIndex) { protected void nullFlagsToValues(final long[] values, final boolean[] isNull, final int size) { for (int ii = 0; ii != size; ++ii) { if (isNull[ii]) { - values[ii] = QueryConstants.NULL_LONG; + values[ii] = NULL_LONG; } } } @@ -1226,16 +1484,16 @@ protected void nullFlagsToValues(final long[] values, final boolean[] isNull, fi private static SinkFactory makeMySinkFactory() { return SinkFactory.of( - MyByteSink::new, QueryConstants.NULL_BYTE_BOXED, - MyShortSink::new, QueryConstants.NULL_SHORT_BOXED, - MyIntSink::new, QueryConstants.NULL_INT_BOXED, - MyLongSink::new, QueryConstants.NULL_LONG_BOXED, - MyFloatSink::new, QueryConstants.NULL_FLOAT_BOXED, - MyDoubleSink::new, QueryConstants.NULL_DOUBLE_BOXED, + MyByteSink::new, NULL_BYTE_BOXED, + MyShortSink::new, NULL_SHORT_BOXED, + MyIntSink::new, NULL_INT_BOXED, + MyLongSink::new, NULL_LONG_BOXED, + MyFloatSink::new, NULL_FLOAT_BOXED, + MyDoubleSink::new, NULL_DOUBLE_BOXED, MyBooleanAsByteSink::new, - MyCharSink::new, QueryConstants.NULL_CHAR, + MyCharSink::new, NULL_CHAR, MyStringSink::new, null, - MyInstantAsLongSink::new, QueryConstants.NULL_LONG, - MyInstantAsLongSink::new, QueryConstants.NULL_LONG); + MyInstantAsLongSink::new, NULL_LONG, + MyInstantAsLongSink::new, NULL_LONG); } } diff --git a/extensions/csv/src/test/java/io/deephaven/csv/TestCsvTools.java b/extensions/csv/src/test/java/io/deephaven/csv/TestCsvTools.java index aef7108115c..5e073244548 100644 --- a/extensions/csv/src/test/java/io/deephaven/csv/TestCsvTools.java +++ b/extensions/csv/src/test/java/io/deephaven/csv/TestCsvTools.java @@ -11,7 +11,6 @@ import io.deephaven.engine.testutil.TstUtils; import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.test.types.OutOfBandTest; -import io.deephaven.time.DateTimeUtils; import io.deephaven.util.QueryConstants; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; @@ -24,10 +23,11 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.time.Instant; +import java.time.ZonedDateTime; import java.util.List; -import static io.deephaven.util.QueryConstants.NULL_DOUBLE; -import static io.deephaven.util.QueryConstants.NULL_INT; +import static io.deephaven.time.DateTimeUtils.*; +import static io.deephaven.util.QueryConstants.*; /** * Unit tests for {@link CsvTools}. @@ -209,7 +209,8 @@ public void testLoadCsv() throws Exception { @Test public void testWriteCsv() throws Exception { final File csvFile = new File(tmpDir, "tmp.csv"); - final String[] colNames = {"StringKeys", "GroupedInts", "Doubles", "DateTime"}; + final String[] colNames = {"Strings", "Chars", "Bytes", "Shorts", "Ints", "Longs", "Floats", "Doubles", + "Instants", "ZonedDateTimes", "Booleans"}; final Table tableToTest = new InMemoryTable( colNames, new Object[] { @@ -217,38 +218,77 @@ public void testWriteCsv() throws Exception { "key11", "key11", "key21", "key21", "key22", null, "ABCDEFGHIJK", "\"", "123", "456", "789", ",", "8" }, + new char[] { + 'a', 'b', 'b', NULL_CHAR, 'c', '\n', ',', MIN_CHAR, MAX_CHAR, 'Z', 'Y', '~', '0' + }, + new byte[] { + 1, 2, 2, NULL_BYTE, 3, -99, -100, MIN_BYTE, MAX_BYTE, 5, 6, 7, 8 + }, + new short[] { + 1, 2, 2, NULL_SHORT, 3, -99, -100, MIN_SHORT, MAX_SHORT, 5, 6, 7, 8 + }, new int[] { - 1, 2, 2, NULL_INT, 3, -99, -100, Integer.MIN_VALUE + 1, Integer.MAX_VALUE, - 5, 6, 7, 8 + 1, 2, 2, NULL_INT, 3, -99, -100, MIN_INT, MAX_INT, 5, 6, 7, 8 + }, + new long[] { + 1, 2, 2, NULL_LONG, 3, -99, -100, MIN_LONG, MAX_LONG, 5, 6, 7, 8 + }, + new float[] { + 2.342f, 0.0932f, 10000000, NULL_FLOAT, 3, MIN_FINITE_FLOAT, MAX_FINITE_FLOAT, + Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, -1.00f, 0.0f, -0.001f, Float.NaN }, new double[] { - 2.342, 0.0932, 10000000, NULL_DOUBLE, 3, Double.MIN_VALUE, Double.MAX_VALUE, + 2.342, 0.0932, 10000000, NULL_DOUBLE, 3, MIN_FINITE_DOUBLE, MAX_FINITE_DOUBLE, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, -1.00, 0.0, -0.001, Double.NaN }, new Instant[] { - DateTimeUtils.epochNanosToInstant(100), - DateTimeUtils.epochNanosToInstant(10000), + epochNanosToInstant(100), + epochNanosToInstant(10000), null, - DateTimeUtils.epochNanosToInstant(100000), - DateTimeUtils.epochNanosToInstant(1000000), - DateTimeUtils.parseInstant("2022-11-06T02:00:00.000000000-04:00"), - DateTimeUtils.parseInstant("2022-11-06T02:00:00.000000000-05:00"), - DateTimeUtils.parseInstant("2022-11-06T02:00:01.000000001-04:00"), - DateTimeUtils.parseInstant("2022-11-06T02:00:01.000000001-05:00"), - DateTimeUtils.parseInstant("2022-11-06T02:59:59.999999999-04:00"), - DateTimeUtils.parseInstant("2022-11-06T02:59:59.999999999-05:00"), - DateTimeUtils.parseInstant("2022-11-06T03:00:00.000000000-04:00"), - DateTimeUtils.parseInstant("2022-11-06T03:00:00.000000000-05:00") + epochNanosToInstant(100000), + epochNanosToInstant(1000000), + parseInstant("2022-11-06T02:00:00.000000000-04:00"), + parseInstant("2022-11-06T02:00:00.000000000-05:00"), + parseInstant("2022-11-06T02:00:01.000000001-04:00"), + parseInstant("2022-11-06T02:00:01.000000001-05:00"), + parseInstant("2022-11-06T02:59:59.999999999-04:00"), + parseInstant("2022-11-06T02:59:59.999999999-05:00"), + parseInstant("2022-11-06T03:00:00.000000000-04:00"), + parseInstant("2022-11-06T03:00:00.000000000-05:00") + }, + new ZonedDateTime[] { + epochNanosToZonedDateTime(100, timeZone("America/New_York")), + epochNanosToZonedDateTime(10000, timeZone("America/New_York")), + null, + epochNanosToZonedDateTime(100000, timeZone("America/New_York")), + epochNanosToZonedDateTime(1000000, timeZone("America/New_York")), + parseZonedDateTime("2022-11-06T02:00:00.000000000 America/New_York"), + parseZonedDateTime("2022-11-06T02:00:00.000000000 America/New_York"), + parseZonedDateTime("2022-11-06T02:00:01.000000001 America/New_York"), + parseZonedDateTime("2022-11-06T02:00:01.000000001 America/New_York"), + parseZonedDateTime("2022-11-06T02:59:59.999999999 America/New_York"), + parseZonedDateTime("2022-11-06T02:59:59.999999999 America/New_York"), + parseZonedDateTime("2022-11-06T03:00:00.000000000 America/New_York"), + parseZonedDateTime("2022-11-06T03:00:00.000000000 America/New_York") + }, + new Boolean[] { + null, false, true, true, false, false, false, false, true, false, null, null, null } }); - - final String allSeparators = ",|\tzZ- 9@"; + final String[] casts = { + "Bytes = (byte) Bytes", "Shorts = (short) Shorts", "Floats = (float) Floats", + "ZonedDateTimes = toZonedDateTime(ZonedDateTimes, 'America/New_York')"}; + final String allSeparators = ",|\tzZ- 90@"; for (final char separator : allSeparators.toCharArray()) { - CsvTools.writeCsv( - tableToTest, csvFile.getPath(), false, DateTimeUtils.timeZone(), false, separator, colNames); - final Table result = CsvTools.readCsv(csvFile.getPath(), - CsvSpecs.builder().delimiter(separator).nullValueLiterals(List.of("(null)")).build()); - TstUtils.assertTableEquals(tableToTest, result); + for (final boolean nullAsEmpty : new boolean[] {false, true}) { + CsvTools.writeCsv( + tableToTest, csvFile.getPath(), false, timeZone(), nullAsEmpty, separator, colNames); + final Table result = CsvTools.readCsv(csvFile.getPath(), CsvSpecs.builder() + .delimiter(separator) + .nullValueLiterals(List.of(nullAsEmpty ? "" : "(null)")) + .build()); + TstUtils.assertTableEquals(tableToTest, result.updateView(casts)); + } } } }