Skip to content
This repository has been archived by the owner on Oct 18, 2023. It is now read-only.

Commit

Permalink
Add support for unix_ms timestamps in csv parser.
Browse files Browse the repository at this point in the history
  • Loading branch information
danielnelson committed Nov 29, 2018
1 parent f9113b6 commit a26aaa5
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 7 deletions.
8 changes: 4 additions & 4 deletions plugins/parsers/csv/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,10 @@ time using the JSON document you can use the `csv_timestamp_column` and
`csv_timestamp_format` options together to set the time to a value in the parsed
document.

The `csv_timestamp_column` option specifies the column name containing the
time value and `csv_timestamp_format` must be set to a Go "reference time"
which is defined to be the specific time: `Mon Jan 2 15:04:05 MST 2006`,
it can also be `unix` (for epoch in ms format like 1257894000 )
The `csv_timestamp_column` option specifies the key containing the time value and
`csv_timestamp_format` must be set to `unix`, `unix_ms`, or a format string in
using the Go "reference time" which is defined to be the **specific time**:
`Mon Jan 2 15:04:05 MST 2006`.

Consult the Go [time][time parse] package for details and additional examples
on how to set the time format.
Expand Down
16 changes: 13 additions & 3 deletions plugins/parsers/csv/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -219,9 +219,12 @@ outer:
return m, nil
}

// ParseTimestamp return a timestamp, if there is no timestamp on the csv it will be the current timestamp, else it will try to parse the time according to the format
// if the format is "unix" it tries to parse assuming that on the csv it will find an epoch in ms.
func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{}, timestampColumn, timestampFormat string) (metricTime time.Time, err error) {
// ParseTimestamp return a timestamp, if there is no timestamp on the csv it
// will be the current timestamp, else it will try to parse the time according
// to the format.
func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{},
timestampColumn, timestampFormat string,
) (metricTime time.Time, err error) {
metricTime = timeFunc()

if timestampColumn != "" {
Expand All @@ -243,6 +246,13 @@ func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface
return
}
metricTime = time.Unix(unixTime, 0)
case "unix_ms":
var unixTime int64
unixTime, err = strconv.ParseInt(tStr, 10, 64)
if err != nil {
return
}
metricTime = time.Unix(unixTime/1000, (unixTime%1000)*1e6)
default:
metricTime, err = time.Parse(timestampFormat, tStr)
if err != nil {
Expand Down
18 changes: 18 additions & 0 deletions plugins/parsers/csv/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,24 @@ func TestTimestampUnixFormat(t *testing.T) {
require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906000000000))
}

func TestTimestampUnixMSFormat(t *testing.T) {
p := Parser{
HeaderRowCount: 1,
ColumnNames: []string{"first", "second", "third"},
MeasurementColumn: "third",
TimestampColumn: "first",
TimestampFormat: "unix_ms",
TimeFunc: DefaultTime,
}
testCSV := `line1,line2,line3
1243094706123,70,test_name
1257609906123,80,test_name2`
metrics, err := p.Parse([]byte(testCSV))
require.NoError(t, err)
require.Equal(t, metrics[0].Time().UnixNano(), int64(1243094706123000000))
require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906123000000))
}

func TestQuotedCharacter(t *testing.T) {
p := Parser{
HeaderRowCount: 1,
Expand Down

0 comments on commit a26aaa5

Please sign in to comment.