From f57c3ebecf99f0d7fe546c058d4086e2454075ba Mon Sep 17 00:00:00 2001 From: James Riley Wilburn Date: Tue, 22 Oct 2024 15:58:33 -0400 Subject: [PATCH] feat: Add Time configtype (#1905) #### Summary This adds a new Time type to be used in plugin specs that can represent either fixed times or relative times. A future PR should extend the duration parsing to handle days, etc. ```yml kind: source spec: name: "orb" registry: "grpc" path: "localhost:7777" version: "v1.0.0" tables: ["*"] destinations: - "sqlite" spec: api_key: "${CQ_ORB_API_KEY}" timeframe_start: 10 days ago timeframe_end: "2024-09-25T03:52:40.14157935-04:00" ``` Here `timeframe_start` and `timeframe_end` are both configtype.Time types. Using mixed relative/fixed timestamps for timeframes isn't really useful here as the relative one will be relative to the run-time of the sync but in this example having ```yml timeframe_start: 10 days ago timeframe_end: now ``` or even omitting `timeframe_end` entirely is more useful ```yml timeframe_start: 10 days ago ``` The plugin is free to define default like so ```go type Spec struct { // TimeframeStart is the beginning of the timeframe in which to fetch cost data. TimeframeStart configtype.Time `json:"timeframe_start"` // TimeframeEnd is the beginning of the timeframe in which to fetch cost data. TimeframeEnd configtype.Time `json:"timeframe_end"` // ... } func (s *Spec) SetDefaults() { if s.TimeframeStart.IsZero() { s.TimeframeStart = configtype.NewRelativeTime(-1 * 365 * 24 * time.Hour) } if s.TimeframeEnd.IsZero() { s.TimeframeEnd = configtype.NewRelativeTime(0) } } ``` --- configtype/time.go | 259 ++++++++++++++++++++++++++++++++++++++++ configtype/time_test.go | 153 ++++++++++++++++++++++++ configtype/util.go | 7 ++ configtype/util_test.go | 19 +++ 4 files changed, 438 insertions(+) create mode 100644 configtype/time.go create mode 100644 configtype/time_test.go create mode 100644 configtype/util.go create mode 100644 configtype/util_test.go diff --git a/configtype/time.go b/configtype/time.go new file mode 100644 index 0000000000..966e1f8d82 --- /dev/null +++ b/configtype/time.go @@ -0,0 +1,259 @@ +package configtype + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "time" + + "github.com/invopop/jsonschema" +) + +// Time is a wrapper around time.Time that should be used in config +// when a time type is required. We wrap the time.Time type so that +// the spec can be extended in the future to support other types of times +type Time struct { + input string + time time.Time + duration *timeDuration +} + +func ParseTime(s string) (Time, error) { + var t Time + t.input = s + + var err error + switch { + case timeNowRegexp.MatchString(s): + t.duration = new(timeDuration) + *t.duration = newTimeDuration(0) + case timeRFC3339Regexp.MatchString(s): + t.time, err = time.Parse(time.RFC3339, s) + case dateRegexp.MatchString(s): + t.time, err = time.Parse(time.DateOnly, s) + case baseDurationRegexp.MatchString(s), humanRelativeDurationRegexp.MatchString(s): + t.duration = new(timeDuration) + *t.duration, err = parseTimeDuration(s) + default: + return t, fmt.Errorf("invalid time format: %s", s) + } + + return t, err +} + +var ( + timeRFC3339Pattern = `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.(\d{1,9}))?(Z|((-|\+)\d{2}:\d{2}))$` + timeRFC3339Regexp = regexp.MustCompile(timeRFC3339Pattern) + + timeNowPattern = `^now$` + timeNowRegexp = regexp.MustCompile(timeNowPattern) + + datePattern = `^\d{4}-\d{2}-\d{2}$` + dateRegexp = regexp.MustCompile(datePattern) + + numberRegexp = regexp.MustCompile(`^[0-9]+$`) + + baseDurationSegmentPattern = `[-+]?([0-9]*(\.[0-9]*)?[a-z]+)+` // copied from time.ParseDuration + baseDurationPattern = fmt.Sprintf(`^%s$`, baseDurationSegmentPattern) + baseDurationRegexp = regexp.MustCompile(baseDurationPattern) + + humanDurationSignsPattern = `ago|from\s+now` + + humanDurationUnitsPattern = `nanoseconds?|ns|microseconds?|us|µs|μs|milliseconds?|ms|seconds?|s|minutes?|m|hours?|h|days?|d|months?|M|years?|Y` + humanDurationUnitsRegex = regexp.MustCompile(fmt.Sprintf(`^%s$`, humanDurationUnitsPattern)) + + humanDurationSegmentPattern = fmt.Sprintf(`(([0-9]+\s+(%[1]s)|%[2]s))`, humanDurationUnitsPattern, baseDurationSegmentPattern) + + humanRelativeDurationPattern = fmt.Sprintf(`^%[1]s(\s+%[1]s)*\s+(%[2]s)$`, humanDurationSegmentPattern, humanDurationSignsPattern) + humanRelativeDurationRegexp = regexp.MustCompile(humanRelativeDurationPattern) + + whitespaceRegexp = regexp.MustCompile(`\s+`) + + timePattern = patternCases( + timeNowPattern, + timeRFC3339Pattern, + datePattern, + baseDurationPattern, + humanRelativeDurationPattern, + ) +) + +func (Time) JSONSchema() *jsonschema.Schema { + return &jsonschema.Schema{ + Type: "string", + Pattern: timePattern, + Title: "CloudQuery configtype.Time", + } +} + +func (t *Time) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + tim, err := ParseTime(s) + if err != nil { + return err + } + + *t = tim + return nil +} + +func (t Time) MarshalJSON() ([]byte, error) { + return json.Marshal(t.input) +} + +func (t Time) AsTime(now time.Time) time.Time { + if t.duration != nil { + sign := t.duration.sign + return now.Add( + t.duration.duration*time.Duration(sign), + ).AddDate( + t.duration.years*sign, + t.duration.months*sign, + t.duration.days*sign, + ) + } + + return t.time +} + +func (t Time) IsZero() bool { + return t.duration == nil && t.time.IsZero() +} + +func (t Time) String() string { + return t.input +} + +type timeDuration struct { + input string + + relative bool + sign int + duration time.Duration + days int + months int + years int +} + +func newTimeDuration(d time.Duration) timeDuration { + return timeDuration{ + input: d.String(), + sign: 1, + duration: d, + } +} + +func parseTimeDuration(s string) (timeDuration, error) { + var d timeDuration + d.input = s + + var inValue bool + var value int64 + + var inSign bool + + parts := whitespaceRegexp.Split(s, -1) + + var err error + + for _, part := range parts { + switch { + case inSign: + if part != "now" { + return timeDuration{}, fmt.Errorf("invalid duration format: invalid sign specifier: %q", part) + } + + d.sign = 1 + inSign = false + case inValue: + if !humanDurationUnitsRegex.MatchString(part) { + return timeDuration{}, fmt.Errorf("invalid duration format: invalid unit specifier: %q", part) + } + + err = d.addUnit(part, value) + if err != nil { + return timeDuration{}, fmt.Errorf("invalid duration format: %w", err) + } + + value = 0 + inValue = false + case part == "ago": + if d.sign != 0 { + return timeDuration{}, fmt.Errorf("invalid duration format: more than one sign specifier") + } + + d.sign = -1 + case part == "from": + if d.sign != 0 { + return timeDuration{}, fmt.Errorf("invalid duration format: more than one sign specifier") + } + + inSign = true + case numberRegexp.MatchString(part): + value, err = strconv.ParseInt(part, 10, 64) + if err != nil { + return timeDuration{}, fmt.Errorf("invalid duration format: invalid value specifier: %q", part) + } + + inValue = true + case baseDurationRegexp.MatchString(part): + duration, err := time.ParseDuration(part) + if err != nil { + return timeDuration{}, fmt.Errorf("invalid duration format: invalid value specifier: %q", part) + } + + d.duration += duration + default: + return timeDuration{}, fmt.Errorf("invalid duration format: invalid value: %q", part) + } + } + + d.relative = d.sign != 0 + + if !d.relative { + d.sign = 1 + } + + return d, nil +} + +func (d *timeDuration) addUnit(unit string, number int64) error { + switch unit { + case "nanosecond", "nanoseconds", "ns": + d.duration += time.Nanosecond * time.Duration(number) + case "microsecond", "microseconds", "us", "μs", "µs": + d.duration += time.Microsecond * time.Duration(number) + case "millisecond", "milliseconds": + d.duration += time.Millisecond * time.Duration(number) + case "second", "seconds": + d.duration += time.Second * time.Duration(number) + case "minute", "minutes": + d.duration += time.Minute * time.Duration(number) + case "hour", "hours": + d.duration += time.Hour * time.Duration(number) + case "day", "days": + d.days += int(number) + case "month", "months": + d.months += int(number) + case "year", "years": + d.years += int(number) + default: + return fmt.Errorf("invalid unit: %q", unit) + } + + return nil +} + +func (d timeDuration) Duration() time.Duration { + duration := d.duration + duration += time.Duration(d.days) * 24 * time.Hour + duration += time.Duration(d.months) * 30 * 24 * time.Hour + duration += time.Duration(d.years) * 365 * 24 * time.Hour + duration *= time.Duration(d.sign) + return duration +} diff --git a/configtype/time_test.go b/configtype/time_test.go new file mode 100644 index 0000000000..245b350d30 --- /dev/null +++ b/configtype/time_test.go @@ -0,0 +1,153 @@ +package configtype_test + +import ( + "encoding/json" + "math/rand" + "testing" + "time" + + "github.com/cloudquery/plugin-sdk/v4/configtype" + "github.com/cloudquery/plugin-sdk/v4/plugin" + "github.com/invopop/jsonschema" + "github.com/stretchr/testify/require" +) + +func TestTime(t *testing.T) { + now, _ := time.Parse(time.RFC3339Nano, time.RFC3339Nano) + + cases := []struct { + give string + want time.Time + }{ + {"1ns", now.Add(1 * time.Nanosecond)}, + {"20s", now.Add(20 * time.Second)}, + {"-50m30s", now.Add(-50*time.Minute - 30*time.Second)}, + {"2021-09-01T00:00:00Z", time.Date(2021, 9, 1, 0, 0, 0, 0, time.UTC)}, + {"2021-09-01T00:00:00.123Z", time.Date(2021, 9, 1, 0, 0, 0, 123000000, time.UTC)}, + {"2021-09-01T00:00:00.123456Z", time.Date(2021, 9, 1, 0, 0, 0, 123456000, time.UTC)}, + {"2021-09-01T00:00:00.123456789Z", time.Date(2021, 9, 1, 0, 0, 0, 123456789, time.UTC)}, + {"2021-09-01T00:00:00.123+02:00", time.Date(2021, 9, 1, 0, 0, 0, 123000000, time.FixedZone("CET", 2*60*60))}, + {"2021-09-01T00:00:00.123456+02:00", time.Date(2021, 9, 1, 0, 0, 0, 123456000, time.FixedZone("CET", 2*60*60))}, + {"2021-09-01T00:00:00.123456789+02:00", time.Date(2021, 9, 1, 0, 0, 0, 123456789, time.FixedZone("CET", 2*60*60))}, + {"2024-09-26T10:18:07.37338-04:00", time.Date(2024, 9, 26, 10, 18, 7, 373380000, time.FixedZone("EDT", -4*60*60))}, + {"2021-09-01", time.Date(2021, 9, 1, 0, 0, 0, 0, time.UTC)}, + {"now", now}, + {"2 days from now", now.AddDate(0, 0, 2)}, + {"5 months ago", now.AddDate(0, -5, 0)}, + {"10 months 3 days 4h20m from now", now.AddDate(0, 10, 3).Add(4 * time.Hour).Add(20 * time.Minute)}, + {"10 months 3 days 4 hours 20 minutes from now", now.AddDate(0, 10, 3).Add(4 * time.Hour).Add(20 * time.Minute)}, + } + for _, tc := range cases { + var d configtype.Time + err := json.Unmarshal([]byte(`"`+tc.give+`"`), &d) + if err != nil { + t.Fatalf("error calling Unmarshal(%q): %v", tc.give, err) + } + computedTime := d.AsTime(now) + if !computedTime.Equal(tc.want) { + t.Errorf("Unmarshal(%q) = %v, want %v", tc.give, computedTime, tc.want) + } + } +} + +func TestTime_JSONSchema(t *testing.T) { + sc := (&jsonschema.Reflector{RequiredFromJSONSchemaTags: true}).Reflect(configtype.Time{}) + schema, err := json.MarshalIndent(sc, "", " ") + require.NoError(t, err) + + validator, err := plugin.JSONSchemaValidator(string(schema)) + require.NoError(t, err) + + type testCase struct { + Name string + Spec string + Err bool + } + + for _, tc := range append([]testCase{ + { + Name: "empty", + Err: true, + Spec: `""`, + }, + { + Name: "null", + Err: true, + Spec: `null`, + }, + { + Name: "bad type", + Err: true, + Spec: `false`, + }, + { + Name: "bad format", + Err: true, + Spec: `false`, + }, + { + Name: "not relative duration", + Err: true, + Spec: `"10 days"`, + }, + { + Name: "relative duration", + Err: false, + Spec: `"10 months from now"`, + }, + { + Name: "complex relative duration", + Err: false, + Spec: `"10 months 3 days 4h20m from now"`, + }, + { + Name: "complex relative duration human", + Err: false, + Spec: `"10 months 3 days 4 hours 20 minutes from now"`, + }, + }, + func() []testCase { + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + const ( + cases = 20 + maxDur = int64(100 * time.Hour) + maxDurHalf = maxDur / 2 + ) + now := time.Now() + var result []testCase + for i := 0; i < cases; i++ { + val := rnd.Int63n(maxDur) - maxDurHalf + dur := must(configtype.ParseTime(time.Duration(val).String())) + + durationData, err := dur.MarshalJSON() + require.NoError(t, err) + result = append(result, testCase{ + Name: string(durationData), + Spec: string(durationData), + }) + + tim := must(configtype.ParseTime(must(marshalString(now.Add(time.Duration(val)))))) + + timeData, err := tim.MarshalJSON() + require.NoError(t, err) + result = append(result, testCase{ + Name: string(timeData), + Spec: string(timeData), + }) + } + + return result + }()..., + ) { + t.Run(tc.Name, func(t *testing.T) { + var val any + err := json.Unmarshal([]byte(tc.Spec), &val) + require.NoError(t, err) + if tc.Err { + require.Error(t, validator.Validate(val)) + } else { + require.NoError(t, validator.Validate(val)) + } + }) + } +} diff --git a/configtype/util.go b/configtype/util.go new file mode 100644 index 0000000000..0cd4444223 --- /dev/null +++ b/configtype/util.go @@ -0,0 +1,7 @@ +package configtype + +import "strings" + +func patternCases(cases ...string) string { + return "(" + strings.Join(cases, "|") + ")" +} diff --git a/configtype/util_test.go b/configtype/util_test.go new file mode 100644 index 0000000000..6d35298944 --- /dev/null +++ b/configtype/util_test.go @@ -0,0 +1,19 @@ +package configtype_test + +import "encoding/json" + +func must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} + +func marshalString[T any](v T) (string, error) { + b, err := json.Marshal(v) + if err != nil { + return "", err + } + + return string(b)[1 : len(b)-1], nil +}