Eval 'now-1d' date expressions (#143)

Here is an alternative date math expression renderer. This renderer
evaluates expressions on Quesma side. Evaluated
times are passed to the database as literals. 

Tests rely on the original (it uses "interval") renderer. We don't have
a "current time provider" component right now.


Code is simple, but plumbing is ..... 

@trzysiek Please take a look

http_requests/agg-with-now.http

@@ -0,0 +1,68 @@
+POST http://localhost:8080/kibana_sample_data_ecommerce/_search
+Content-Type: application/json
+
+{
+			"_source": {
+				"excludes": []
+			},
+			"aggs": {
+				"2": {
+					"date_range": {
+						"field": "timestamp",
+						"ranges": [
+							{
+								"to": "now"
+							},
+							{
+								"from": "now-3w/d",
+								"to": "now"
+							},
+							{
+								"from": "2024-04-14"
+							}
+						],
+						"time_zone": "Europe/Warsaw"
+					}
+				}
+			},
+			"docvalue_fields": [
+				{
+					"field": "customer_birth_date",
+					"format": "date_time"
+				},
+				{
+					"field": "order_date",
+					"format": "date_time"
+				},
+				{
+					"field": "products.created_on",
+					"format": "date_time"
+				}
+			],
+			"query": {
+				"bool": {
+					"filter": [
+						{
+							"match_all": {}
+						},
+						{
+							"range": {
+								"timestamp": {
+									"format": "strict_date_optional_time",
+									"gte": "2024-04-06T07:28:50.059Z",
+									"lte": "2024-04-16T17:28:50.059Z"
+								}
+							}
+						}
+					],
+					"must": [],
+					"must_not": [],
+					"should": []
+				}
+			},
+			"script_fields": {},
+			"size": 0,
+			"stored_fields": [
+				"*"
+			]
+		}

quesma/queryparser/aggregation_date_range_parser.go

@@ -87,7 +87,7 @@ func (cw *ClickhouseQueryTranslator) parseDateTimeInClickhouseMathLanguage(dateT
 		return "'" + dateTime + "'", nil
 	}
 	// 2. expressions like now() or now()-1d
-	res, err := parseDateMathExpression(dateTime)
+	res, err := cw.parseDateMathExpression(dateTime)
 	if err != nil {
 		return "", err
 	}

quesma/queryparser/date_expr.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 )
 
 type timeUnit string
@@ -99,6 +100,25 @@ type DateMathExpressionRenderer interface {
 	RenderSQL(expression *DateMathExpression) (string, error)
 }
 
+const DateMathExpressionFormatLiteral = "literal"
+const DateMathExpressionFormatClickhouse = "clickhouse_intervals"
+const DateMathExpressionFormatLiteralTest = "test"
+
+func DateMathExpressionRendererFactory(format string) DateMathExpressionRenderer {
+	switch format {
+	case "":
+		return &DateMathAsClickhouseIntervals{}
+	case DateMathExpressionFormatClickhouse:
+		return &DateMathAsClickhouseIntervals{}
+	case DateMathExpressionFormatLiteral:
+		return &DateMathExpressionAsLiteral{now: time.Now()}
+	case DateMathExpressionFormatLiteralTest:
+		return &DateMathExpressionAsLiteral{now: time.Date(2024, 5, 17, 12, 1, 2, 3, time.UTC)}
+	default:
+		return nil
+	}
+}
+
 type DateMathAsClickhouseIntervals struct{}
 
 func (b *DateMathAsClickhouseIntervals) RenderSQL(expression *DateMathExpression) (string, error) {
@@ -170,3 +190,70 @@ func (b *DateMathAsClickhouseIntervals) parseTimeUnit(timeUnit timeUnit) (string
 	}
 	return "", errors.New("unsupported time unit")
 }
+
+type DateMathExpressionAsLiteral struct {
+	now time.Time
+}
+
+func (b *DateMathExpressionAsLiteral) RenderSQL(expression *DateMathExpression) (string, error) {
+
+	const format = "2006-01-02 15:04:05"
+
+	result := b.now
+
+	for _, interval := range expression.intervals {
+
+		if interval.amount == 0 {
+			continue
+		}
+
+		amount := interval.amount
+
+		switch interval.unit {
+		case "m":
+			result = result.Add(time.Minute * time.Duration(amount))
+
+		case "s":
+			result = result.Add(time.Duration(amount) * time.Second)
+
+		case "h", "H":
+			result = result.Add(time.Duration(amount) * time.Hour)
+
+		case "d":
+			result = result.AddDate(0, 0, amount)
+
+		case "w":
+			result = result.AddDate(0, 0, amount*7)
+
+		case "M":
+			result = result.AddDate(0, amount, 0)
+
+		case "Y", "y":
+			result = result.AddDate(amount, 0, 0)
+
+		default:
+			return "", fmt.Errorf("unsupported time unit: %s", interval.unit)
+		}
+
+	}
+
+	switch expression.rounding {
+	case "":
+		// do nothing
+	case "d":
+		result = time.Date(result.Year(), result.Month(), result.Day(), 0, 0, 0, 0, result.Location())
+	case "w":
+		weekday := int(result.Weekday())
+		result = result.AddDate(0, 0, -weekday)
+		result = time.Date(result.Year(), result.Month(), result.Day(), 0, 0, 0, 0, result.Location())
+	case "M":
+		result = time.Date(result.Year(), result.Month(), 1, 0, 0, 0, 0, result.Location())
+	case "Y":
+		result = time.Date(result.Year(), 1, 1, 0, 0, 0, 0, result.Location())
+
+	default:
+		return "", fmt.Errorf("unsupported rounding unit: %s", expression.rounding)
+	}
+
+	return fmt.Sprintf("'%s'", result.Format(format)), nil
+}

quesma/queryparser/date_expr_test.go

@@ -30,6 +30,7 @@ func TestParseDateMathExpression(t *testing.T) {
 
 func Test_parseDateTimeInClickhouseMathLanguage(t *testing.T) {
 	exprs := map[string]string{
+		"now":          "now()",
 		"now-15m":      "subDate(now(), INTERVAL 15 minute)",
 		"now-15m+5s":   "addDate(subDate(now(), INTERVAL 15 minute), INTERVAL 5 second)",
 		"now-":         "now()",
@@ -63,3 +64,49 @@ func Test_parseDateTimeInClickhouseMathLanguage(t *testing.T) {
 		})
 	}
 }
+
+func Test_DateMathExpressionAsLiteral(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"now", "'2024-05-17 12:01:02'"},
+		{"now-15m", "'2024-05-17 11:46:02'"},
+		{"now-15m+5s", "'2024-05-17 11:46:07'"},
+		{"now-", "'2024-05-17 12:01:02'"},
+		{"now-15m+/M", "'2024-05-01 00:00:00'"},
+		{"now-15m/d", "'2024-05-17 00:00:00'"},
+		{"now-15m+5s/w", "'2024-05-12 00:00:00'"}, // week starts on Sunday here so 2024-05-12 is the start of the week
+		{"now-/Y", "'2024-01-01 00:00:00'"},
+		{"now-2M", "'2024-03-17 12:01:02'"},
+		{"now-1y", "'2023-05-17 12:01:02'"},
+		{"now-1w", "'2024-05-10 12:01:02'"},
+		{"now-1s", "'2024-05-17 12:01:01'"},
+		{"now-1m", "'2024-05-17 12:00:02'"},
+		{"now-1d", "'2024-05-16 12:01:02'"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.input, func(tt *testing.T) {
+
+			dt, err := ParseDateMathExpression(test.input)
+			assert.NoError(tt, err)
+
+			if err != nil {
+				return
+			}
+
+			// this renderer is single use, so we can't reuse it
+			renderer := DateMathExpressionRendererFactory(DateMathExpressionFormatLiteralTest)
+
+			resultExpr, err := renderer.RenderSQL(dt)
+			assert.NoError(t, err)
+
+			if err != nil {
+				return
+			}
+
+			assert.Equal(t, test.expected, resultExpr)
+		})
+	}
+}

quesma/queryparser/query_parser.go

@@ -678,7 +678,7 @@ func (cw *ClickhouseQueryTranslator) parseNested(queryMap QueryMap) SimpleQuery
 	return newSimpleQuery(NewSimpleStatement("no query in nested query"), false)
 }
 
-func parseDateMathExpression(expr string) (string, error) {
+func (cw *ClickhouseQueryTranslator) parseDateMathExpression(expr string) (string, error) {
 	expr = strings.ReplaceAll(expr, "'", "")
 
 	exp, err := ParseDateMathExpression(expr)
@@ -687,7 +687,10 @@ func parseDateMathExpression(expr string) (string, error) {
 		return "", err
 	}
 
-	builder := &DateMathAsClickhouseIntervals{}
+	builder := DateMathExpressionRendererFactory(cw.DateMathRenderer)
+	if builder == nil {
+		return "", fmt.Errorf("no date math expression renderer found: %s", cw.DateMathRenderer)
+	}
 
 	sql, err := builder.RenderSQL(exp)
 	if err != nil {
@@ -736,7 +739,7 @@ func (cw *ClickhouseQueryTranslator) parseRange(queryMap QueryMap) SimpleQuery {
 						if _, err := time.Parse(time.RFC3339Nano, dateTime); err == nil {
 							vToPrint = cw.parseDateTimeString(cw.Table, field, dateTime)
 						} else if op == "gte" || op == "lte" || op == "gt" || op == "lt" {
-							vToPrint, err = parseDateMathExpression(vToPrint)
+							vToPrint, err = cw.parseDateMathExpression(vToPrint)
 							if err != nil {
 								logger.WarnWithCtx(cw.Ctx).Msgf("error parsing date math expression: %s", vToPrint)
 								return newSimpleQuery(NewSimpleStatement("error parsing date math expression: "+vToPrint), false)

quesma/queryparser/query_parser_test.go

@@ -439,20 +439,6 @@ func TestOrAndAnd(t *testing.T) {
 	}
 }
 
-func TestQueryParseDateMathExpression(t *testing.T) {
-	exprs := map[string]string{
-		"now-15m":    "subDate(now(), INTERVAL 15 minute)",
-		"now-15m+5s": "addDate(subDate(now(), INTERVAL 15 minute), INTERVAL 5 second)",
-		"now-":       "now()",
-		"now-15m+":   "subDate(now(), INTERVAL 15 minute)",
-	}
-	for expr, expected := range exprs {
-		resultExpr, err := parseDateMathExpression(expr)
-		assert.Nil(t, err)
-		assert.Equal(t, expected, resultExpr)
-	}
-}
-
 func Test_parseSortFields(t *testing.T) {
 	tests := []struct {
 		name       string

quesma/queryparser/query_translator.go

@@ -24,6 +24,8 @@ type ClickhouseQueryTranslator struct {
 	Table             *clickhouse.Table
 	tokensToHighlight []string
 	Ctx               context.Context
+
+	DateMathRenderer string // "clickhouse_interval" or "literal"  if not set, we use "clickhouse_interval"
 }
 
 var completionStatusOK = func() *int { value := 200; return &value }()

quesma/quesma/query_translator.go

@@ -35,13 +35,13 @@ const (
 	QueryLanguageEQL     = "eql"
 )
 
-func NewQueryTranslator(ctx context.Context, language QueryLanguage, table *clickhouse.Table, logManager *clickhouse.LogManager) (queryTranslator IQueryTranslator) {
+func NewQueryTranslator(ctx context.Context, language QueryLanguage, table *clickhouse.Table, logManager *clickhouse.LogManager, dateMathRenderer string) (queryTranslator IQueryTranslator) {
 
 	switch language {
 	case QueryLanguageEQL:
 		queryTranslator = &eql.ClickhouseEQLQueryTranslator{ClickhouseLM: logManager, Table: table, Ctx: ctx}
 	default:
-		queryTranslator = &queryparser.ClickhouseQueryTranslator{ClickhouseLM: logManager, Table: table, Ctx: ctx}
+		queryTranslator = &queryparser.ClickhouseQueryTranslator{ClickhouseLM: logManager, Table: table, Ctx: ctx, DateMathRenderer: dateMathRenderer}
 	}
 
 	return queryTranslator

quesma/quesma/quesma.go

@@ -103,6 +103,12 @@ func NewQuesmaTcpProxy(phoneHomeAgent telemetry.PhoneHomeAgent, config config.Qu
 func NewHttpProxy(phoneHomeAgent telemetry.PhoneHomeAgent, logManager *clickhouse.LogManager, indexManager elasticsearch.IndexManagement, config config.QuesmaConfiguration, logChan <-chan tracing.LogWithLevel) *Quesma {
 	quesmaManagementConsole := ui.NewQuesmaManagementConsole(config, logManager, indexManager, logChan, phoneHomeAgent)
 	queryRunner := NewQueryRunner(logManager, config, indexManager, quesmaManagementConsole)
+
+	// not sure how we should configure our query translator ???
+	// is this a config option??
+
+	queryRunner.DateMathRenderer = queryparser.DateMathExpressionFormatLiteral
+
 	router := configureRouter(config, logManager, quesmaManagementConsole, phoneHomeAgent, queryRunner)
 	return &Quesma{
 		telemetryAgent:          phoneHomeAgent,

quesma/quesma/search.go

@@ -54,6 +54,11 @@ type QueryRunner struct {
 	cfg                     config.QuesmaConfiguration
 	im                      elasticsearch.IndexManagement
 	quesmaManagementConsole *ui.QuesmaManagementConsole
+
+	// configuration
+
+	// this is passed to the QueryTranslator to render date math expressions
+	DateMathRenderer string // "clickhouse_interval" or "literal"  if not set, we use "clickhouse_interval"
 }
 
 func NewQueryRunner(lm *clickhouse.LogManager, cfg config.QuesmaConfiguration, im elasticsearch.IndexManagement, qmc *ui.QuesmaManagementConsole) *QueryRunner {
@@ -208,7 +213,7 @@ func (q *QueryRunner) handleSearchCommon(ctx context.Context, indexPattern strin
 		}
 		var simpleQuery queryparser.SimpleQuery
 
-		queryTranslator = NewQueryTranslator(ctx, queryLanguage, table, q.logManager)
+		queryTranslator = NewQueryTranslator(ctx, queryLanguage, table, q.logManager, q.DateMathRenderer)
 
 		simpleQuery, queryInfo, highlighter, err = queryTranslator.ParseQuery(string(body))
 		if err != nil {