-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add FirehoseSink and FirehoseSinkBuilder. Uses directPut * support schema 2.0.4 * add CSV report mapping support. from schema functions are generated taking care about the order of the columns
- Loading branch information
1 parent
714d5c0
commit f2e68c1
Showing
53 changed files
with
1,599 additions
and
60 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
151 changes: 151 additions & 0 deletions
151
src/main/java/org/observertc/observer/common/CsvRecordMapper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| package org.observertc.observer.common; | ||
|
|
||
| import java.util.function.Function; | ||
|
|
||
| /** | ||
| * This class is based on the CSVWriter class in open-csv project. The project is available at http://opencsv.sourceforge.net/licenses.html | ||
| */ | ||
| public class CsvRecordMapper implements Function<Iterable<?>, String> { | ||
|
|
||
| public static final int INITIAL_STRING_SIZE = 1024; | ||
| /** | ||
| * The character used for escaping quotes. | ||
| */ | ||
| public static final char DEFAULT_ESCAPE_CHARACTER = '"'; | ||
| /** | ||
| * The default separator to use if none is supplied to the constructor. | ||
| */ | ||
| public static final char DEFAULT_SEPARATOR = ','; | ||
| /** | ||
| * The default quote character to use if none is supplied to the | ||
| * constructor. | ||
| */ | ||
| public static final char DEFAULT_QUOTE_CHARACTER = '"'; | ||
| /** | ||
| * The quote constant to use when you wish to suppress all quoting. | ||
| */ | ||
| public static final char NO_QUOTE_CHARACTER = '\u0000'; | ||
| /** | ||
| * The escape constant to use when you wish to suppress all escaping. | ||
| */ | ||
| public static final char NO_ESCAPE_CHARACTER = '\u0000'; | ||
| /** | ||
| * Default line terminator. | ||
| */ | ||
| public static final String DEFAULT_LINE_END = "\n"; | ||
| /** | ||
| * RFC 4180 compliant line terminator. | ||
| */ | ||
| public static final String RFC4180_LINE_END = "\r\n"; | ||
|
|
||
| /** | ||
| * Checks to see if the line contains special characters. | ||
| * @param line Element of data to check for special characters. | ||
| * @return True if the line contains the quote, escape, separator, newline, or return. | ||
| */ | ||
|
|
||
| private char separator = DEFAULT_SEPARATOR; | ||
| private char quotechar = DEFAULT_QUOTE_CHARACTER; | ||
| private char escapechar = DEFAULT_ESCAPE_CHARACTER; | ||
| private boolean addLineSeparator = false; | ||
| private String lineSeparator = DEFAULT_LINE_END; | ||
|
|
||
| public static Builder builder() { | ||
| return new Builder(); | ||
| } | ||
|
|
||
| private CsvRecordMapper() { | ||
|
|
||
| } | ||
|
|
||
| public String apply(Iterable<?> record) { | ||
| var line = new StringBuilder(); | ||
| var it = record.iterator(); | ||
| for (var firstColumn = true; it.hasNext(); firstColumn = false) { | ||
| if (!firstColumn) { | ||
| line.append(separator); | ||
| } | ||
| var column = it.next(); | ||
|
|
||
| if (column == null) { | ||
| continue; | ||
| } | ||
| var value = column.toString(); | ||
|
|
||
| if (!this.hasSpecialCharacter(value)) { | ||
| line.append(value); | ||
| continue; | ||
| } | ||
|
|
||
| if (quotechar != NO_QUOTE_CHARACTER) { | ||
| line.append(quotechar); | ||
| } | ||
|
|
||
| for (int j = 0; j < value.length(); j++) { | ||
| char nextChar = value.charAt(j); | ||
| if (escapechar != NO_ESCAPE_CHARACTER && isEscapeCharacter(nextChar)) { | ||
| line.append(escapechar); | ||
| } | ||
| line.append(nextChar); | ||
| } | ||
|
|
||
| if (quotechar != NO_QUOTE_CHARACTER) { | ||
| line.append(quotechar); | ||
| } | ||
| } | ||
| if (this.addLineSeparator) { | ||
| line.append(this.lineSeparator); | ||
| } | ||
| return line.toString(); | ||
| } | ||
|
|
||
| protected boolean isEscapeCharacter(char nextChar) { | ||
| return quotechar == NO_QUOTE_CHARACTER | ||
| ? (nextChar == quotechar || nextChar == escapechar || nextChar == separator || nextChar == '\n') | ||
| : (nextChar == quotechar || nextChar == escapechar); | ||
| } | ||
|
|
||
| private boolean hasSpecialCharacter(String value) { | ||
| return value.indexOf(quotechar) != -1 | ||
| || value.indexOf(escapechar) != -1 | ||
| || value.indexOf(separator) != -1 | ||
| || value.contains(DEFAULT_LINE_END) | ||
| || value.contains("\r"); | ||
| } | ||
|
|
||
| public static class Builder { | ||
| private CsvRecordMapper result = new CsvRecordMapper(); | ||
| Builder() { | ||
|
|
||
| } | ||
|
|
||
| public Builder addLineSeparator(boolean value) { | ||
| this.result.addLineSeparator = value; | ||
| return this; | ||
| } | ||
|
|
||
| public Builder setSeparator(char value) { | ||
| this.result.separator = value; | ||
| return this; | ||
| } | ||
|
|
||
| public Builder setQuoteCharacter(char value) { | ||
| this.result.quotechar = value; | ||
| return this; | ||
| } | ||
|
|
||
| public Builder setEscapeCharacter(char value) { | ||
| this.result.escapechar = value; | ||
| return this; | ||
| } | ||
|
|
||
| public Builder setRfc4180LineEnd() { | ||
| this.result.lineSeparator = RFC4180_LINE_END; | ||
| return this; | ||
| } | ||
|
|
||
| public CsvRecordMapper build() { | ||
| return this.result; | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.