From f267f342aec81655eefab9d3d817ab0c0bee2fd5 Mon Sep 17 00:00:00 2001 From: Nicolas Filotto Date: Tue, 23 Mar 2021 18:45:29 +0100 Subject: [PATCH] Add support of the time module in Starlark Processor (#9004) --- go.mod | 6 ++--- go.sum | 14 +++++++---- plugins/processors/starlark/README.md | 5 +++- plugins/processors/starlark/starlark.go | 5 ++++ .../starlark/testdata/time_date.star | 19 +++++++++++++++ .../starlark/testdata/time_duration.star | 17 ++++++++++++++ .../starlark/testdata/time_timestamp.star | 23 +++++++++++++++++++ 7 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 plugins/processors/starlark/testdata/time_date.star create mode 100644 plugins/processors/starlark/testdata/time_duration.star create mode 100644 plugins/processors/starlark/testdata/time_timestamp.star diff --git a/go.mod b/go.mod index 1dfe1d9a2d712..54eef23f3daf2 100644 --- a/go.mod +++ b/go.mod @@ -141,11 +141,11 @@ require ( github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a // indirect github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4 // indirect - go.starlark.net v0.0.0-20200901195727-6e684ef5eeee - golang.org/x/net v0.0.0-20201110031124-69a78807bb2b + go.starlark.net v0.0.0-20210312235212-74c10e2c17dc + golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 - golang.org/x/sys v0.0.0-20201112073958-5cba982894dd + golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 golang.org/x/text v0.3.4 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200205215550-e35592f146e4 google.golang.org/api v0.20.0 diff --git a/go.sum b/go.sum index 26aea881ce100..4c5884da2485b 100644 --- a/go.sum +++ b/go.sum @@ -335,6 +335,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -353,6 +354,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -742,8 +744,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.starlark.net v0.0.0-20200901195727-6e684ef5eeee h1:N4eRtIIYHZE5Mw/Km/orb+naLdwAe+lv2HCxRR5rEBw= -go.starlark.net v0.0.0-20200901195727-6e684ef5eeee/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= +go.starlark.net v0.0.0-20210312235212-74c10e2c17dc h1:pVkptfeOTFfx+zXZo7HEHN3d5LmhatBFvHdm/f2QnpY= +go.starlark.net v0.0.0-20210312235212-74c10e2c17dc/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -838,8 +840,9 @@ golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -891,11 +894,12 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/plugins/processors/starlark/README.md b/plugins/processors/starlark/README.md index 7e1015674df7c..1f5adbec1f472 100644 --- a/plugins/processors/starlark/README.md +++ b/plugins/processors/starlark/README.md @@ -227,7 +227,10 @@ def apply(metric): - [ratio](/plugins/processors/starlark/testdata/ratio.star) - Compute the ratio of two integer fields - [rename](/plugins/processors/starlark/testdata/rename.star) - Rename tags or fields using a name mapping. - [scale](/plugins/processors/starlark/testdata/scale.star) - Multiply any field by a number -- [value filter](/plugins/processors/starlark/testdata/value_filter.star) - remove a metric based on a field value. +- [time date](/plugins/processors/starlark/testdata/time_date.star) - Parse a date and extract the year, month and day from it. +- [time duration](/plugins/processors/starlark/testdata/time_duration.star) - Parse a duration and convert it into a total amount of seconds. +- [time timestamp](/plugins/processors/starlark/testdata/time_timestamp.star) - Filter metrics based on the timestamp. +- [value filter](/plugins/processors/starlark/testdata/value_filter.star) - Remove a metric based on a field value. - [logging](/plugins/processors/starlark/testdata/logging.star) - Log messages with the logger of Telegraf - [multiple metrics](/plugins/processors/starlark/testdata/multiple_metrics.star) - Return multiple metrics by using [a list](https://docs.bazel.build/versions/master/skylark/lib/list.html) of metrics. - [multiple metrics from json array](/plugins/processors/starlark/testdata/multiple_metrics_with_json.star) - Builds a new metric from each element of a json array then returns all the created metrics. diff --git a/plugins/processors/starlark/starlark.go b/plugins/processors/starlark/starlark.go index ffd13680f88e5..968908c6589a6 100644 --- a/plugins/processors/starlark/starlark.go +++ b/plugins/processors/starlark/starlark.go @@ -7,6 +7,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/processors" + "go.starlark.net/lib/time" "go.starlark.net/resolve" "go.starlark.net/starlark" "go.starlark.net/starlarkjson" @@ -252,6 +253,10 @@ func loadFunc(module string, logger telegraf.Logger) (starlark.StringDict, error return starlark.StringDict{ "log": LogModule(logger), }, nil + case "time.star": + return starlark.StringDict{ + "time": time.Module, + }, nil default: return nil, errors.New("module " + module + " is not available") } diff --git a/plugins/processors/starlark/testdata/time_date.star b/plugins/processors/starlark/testdata/time_date.star new file mode 100644 index 0000000000000..7be7f8fa7fcf2 --- /dev/null +++ b/plugins/processors/starlark/testdata/time_date.star @@ -0,0 +1,19 @@ +# Example of parsing a date out of a field and modifying the metric to inject the year, month and day. +# +# Example Input: +# time value="2009-06-12T12:06:10.000000099" 1465839830100400201 +# +# Example Output: +# time year=2009i,month=6i,day=12i 1465839830100400201 + +load('time.star', 'time') +# loads time.parse_duration(), time.is_valid_timezone(), time.now(), time.time(), +# time.parse_time() and time.from_timestamp() + +def apply(metric): + date = time.parse_time(metric.fields.get('value'), format="2006-01-02T15:04:05.999999999", location="UTC") + metric.fields.pop('value') + metric.fields["year"] = date.year + metric.fields["month"] = date.month + metric.fields["day"] = date.day + return metric diff --git a/plugins/processors/starlark/testdata/time_duration.star b/plugins/processors/starlark/testdata/time_duration.star new file mode 100644 index 0000000000000..773e20744cce6 --- /dev/null +++ b/plugins/processors/starlark/testdata/time_duration.star @@ -0,0 +1,17 @@ +# Example of parsing a duration out of a field and modifying the metric to inject the equivalent in seconds. +# +# Example Input: +# time value="3m35s" 1465839830100400201 +# +# Example Output: +# time seconds=215 1465839830100400201 + +load('time.star', 'time') +# loads time.parse_duration(), time.is_valid_timezone(), time.now(), time.time(), +# time.parse_time() and time.from_timestamp() + +def apply(metric): + duration = time.parse_duration(metric.fields.get('value')) + metric.fields.pop('value') + metric.fields["seconds"] = duration.seconds + return metric diff --git a/plugins/processors/starlark/testdata/time_timestamp.star b/plugins/processors/starlark/testdata/time_timestamp.star new file mode 100644 index 0000000000000..dc1cbaea0296d --- /dev/null +++ b/plugins/processors/starlark/testdata/time_timestamp.star @@ -0,0 +1,23 @@ +# Example of filtering metrics based on the timestamp. Beware the built-in function from_timestamp +# only supports timestamps in seconds. +# +# Example Input: +# time result="KO" 1616020365100400201 +# time result="OK" 1616150517100400201 +# +# Example Output: +# time result="OK" 1616150517100400201 + +load('time.star', 'time') +# loads time.parse_duration(), time.is_valid_timezone(), time.now(), time.time(), +# time.parse_time() and time.from_timestamp() + +def apply(metric): + # 1616198400 sec = Saturday, March 20, 2021 0:00:00 GMT + refDate = time.from_timestamp(1616198400) + # 1616020365 sec = Wednesday, March 17, 2021 22:32:45 GMT + # 1616150517 sec = Friday, March 19, 2021 10:41:57 GMT + metric_date = time.from_timestamp(int(metric.time / 1e9)) + # Only keep metrics with a timestamp that is not more than 24 hours before the reference date + if refDate - time.parse_duration("24h") < metric_date: + return metric