Issue #4: timestamp field

pkg/quickwit/quickwit.go

@@ -26,6 +26,110 @@ type QuickwitDatasource struct {
 	dsInfo es.DatasourceInfo
 }
 
+type FieldMappings struct {
+	Name          string          `json:"name"`
+	Type          string          `json:"type"`
+	OutputFormat  *string         `json:"output_format,omitempty"`
+	FieldMappings []FieldMappings `json:"field_mappings,omitempty"`
+}
+
+type QuickwitMapping struct {
+	IndexConfig struct {
+		DocMapping struct {
+			TimestampField string          `json:"timestamp_field"`
+			FieldMappings  []FieldMappings `json:"field_mappings"`
+		} `json:"doc_mapping"`
+	} `json:"index_config"`
+}
+
+type QuickwitCreationErrorPayload struct {
+	Message    string `json:"message"`
+	StatusCode int    `json:"status"`
+}
+
+func newErrorCreationPayload(statusCode int, message string) error {
+	var payload QuickwitCreationErrorPayload
+	payload.Message = message
+	payload.StatusCode = statusCode
+	json, err := json.Marshal(payload)
+	if nil != err {
+		return err
+	}
+
+	return errors.New(string(json))
+}
+
+func findTimeStampFormat(timestampFieldName string, parentName *string, fieldMappings []FieldMappings) *string {
+	if nil == fieldMappings {
+		return nil
+	}
+
+	for _, field := range fieldMappings {
+		fieldName := field.Name
+		if nil != parentName {
+			fieldName = fmt.Sprintf("%s.%s", *parentName, fieldName)
+		}
+
+		if field.Type == "datetime" && fieldName == timestampFieldName && nil != field.OutputFormat {
+			return field.OutputFormat
+		} else if field.Type == "object" && nil != field.FieldMappings {
+			format := findTimeStampFormat(timestampFieldName, &field.Name, field.FieldMappings)
+			if nil != format {
+				return format
+			}
+		}
+	}
+
+	return nil
+}
+
+func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (string, string, error) {
+	mappingEndpointUrl := qwUrl + "/indexes/" + index
+	qwlog.Info("Calling quickwit endpoint: " + mappingEndpointUrl)
+	r, err := cli.Get(mappingEndpointUrl)
+	if err != nil {
+		errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error())
+		qwlog.Error(errMsg)
+		return "", "", err
+	}
+
+	statusCode := r.StatusCode
+
+	if statusCode < 200 || statusCode >= 400 {
+		errMsg := fmt.Sprintf("Error when calling url = %s", mappingEndpointUrl)
+		qwlog.Error(errMsg)
+		return "", "", newErrorCreationPayload(statusCode, errMsg)
+	}
+
+	defer r.Body.Close()
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error())
+		qwlog.Error(errMsg)
+		return "", "", newErrorCreationPayload(statusCode, errMsg)
+	}
+
+	var payload QuickwitMapping
+	err = json.Unmarshal(body, &payload)
+	if err != nil {
+		errMsg := fmt.Sprintf("Unmarshalling body error: err = %s, body = %s", err.Error(), (body))
+		qwlog.Error(errMsg)
+		return "", "", newErrorCreationPayload(statusCode, errMsg)
+	}
+
+	timestampFieldName := payload.IndexConfig.DocMapping.TimestampField
+	timestampFieldFormat := findTimeStampFormat(timestampFieldName, nil, payload.IndexConfig.DocMapping.FieldMappings)
+
+	if nil == timestampFieldFormat {
+		errMsg := fmt.Sprintf("No format found for field: %s", string(timestampFieldName))
+		qwlog.Error(errMsg)
+		return timestampFieldName, "", newErrorCreationPayload(statusCode, errMsg)
+	}
+
+	qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, *timestampFieldFormat))
+	return timestampFieldName, *timestampFieldFormat, nil
+}
+
 // Creates a Quickwit datasource.
 func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
 	qwlog.Debug("Initializing new data source instance")
@@ -50,19 +154,8 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
 		return nil, err
 	}
 
-	timeField, ok := jsonData["timeField"].(string)
-	if !ok {
-		return nil, errors.New("timeField cannot be cast to string")
-	}
-
-	if timeField == "" {
-		return nil, errors.New("a time field name is required")
-	}
-
-	timeOutputFormat, ok := jsonData["timeOutputFormat"].(string)
-	if !ok {
-		return nil, errors.New("timeOutputFormat cannot be cast to string")
-	}
+	timeField, toOk := jsonData["timeField"].(string)
+	timeOutputFormat, tofOk := jsonData["timeOutputFormat"].(string)
 
 	logLevelField, ok := jsonData["logLevelField"].(string)
 	if !ok {
@@ -96,6 +189,13 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc
 		maxConcurrentShardRequests = 256
 	}
 
+	if !toOk || !tofOk {
+		timeField, timeOutputFormat, err = getTimestampFieldInfos(index, settings.URL, httpCli)
+		if nil != err {
+			return nil, err
+		}
+	}
+
 	configuredFields := es.ConfiguredFields{
 		TimeField:        timeField,
 		TimeOutputFormat: timeOutputFormat,

src/configuration/ConfigEditor.tsx

@@ -73,24 +73,6 @@ export const QuickwitDetails = ({ value, onChange }: DetailsProps) => {
               width={40}
             />
           </InlineField>
-          <InlineField label="Timestamp field" labelWidth={26} tooltip="Timestamp field of your index. Required.">
-            <Input
-              id="quickwit_index_timestamp_field"
-              value={value.jsonData.timeField}
-              onChange={(event) => onChange({ ...value, jsonData: {...value.jsonData, timeField: event.currentTarget.value}})}
-              placeholder="timestamp"
-              width={40}
-            />
-          </InlineField>
-          <InlineField label="Timestamp output format" labelWidth={26} tooltip="If you don't remember the output format, check the datasource and the error message will give you a hint.">
-            <Input
-              id="quickwit_index_timestamp_field_output_format"
-              value={value.jsonData.timeOutputFormat}
-              onChange={(event) => onChange({ ...value, jsonData: {...value.jsonData, timeOutputFormat: event.currentTarget.value}})}
-              placeholder="unix_timestamp_millisecs"
-              width={40}
-            />
-          </InlineField>
           <InlineField label="Message field name" labelWidth={26} tooltip="Field used to display a log line in the Explore view">
             <Input
               id="quickwit_log_message_field"

src/datasource.ts

@@ -51,6 +51,7 @@ import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregatio
 import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations';
 import ElasticsearchLanguageProvider from 'LanguageProvider';
 import { ReactNode } from 'react';
+import { extractJsonPayload } from 'utils';
 
 export const REF_ID_STARTER_LOG_VOLUME = 'log-volume-';
 
@@ -79,13 +80,41 @@ export class QuickwitDataSource
     super(instanceSettings);
     const settingsData = instanceSettings.jsonData || ({} as QuickwitOptions);
     this.index = settingsData.index || '';
-    this.timeField = settingsData.timeField || '';
-    this.timeOutputFormat = settingsData.timeOutputFormat || '';
-    this.logMessageField = settingsData.logMessageField || '';
-    this.logLevelField = settingsData.logLevelField || '';
+    this.timeField = ''
+    this.timeOutputFormat = ''
     this.queryBuilder = new ElasticQueryBuilder({
       timeField: this.timeField,
     });
+    from(this.getResource('indexes/' + this.index)).pipe(
+      map((indexMetadata) => {
+        let fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings);
+        let timestampFieldName = indexMetadata.index_config.doc_mapping.timestamp_field
+        let timestampField = fields.find((field) => field.json_path === timestampFieldName);
+        let timestampFormat = timestampField ? timestampField.field_mapping.output_format || '' : ''
+        let timestampFieldInfos = { 'field': timestampFieldName, 'format': timestampFormat }
+        return timestampFieldInfos
+      }),
+      catchError((err) => {
+        if (!err.data || !err.data.error) {
+          let err_source = extractJsonPayload(err.data.error)
+          if(!err_source) {
+            throw err
+          }
+        }
+
+        // the error will be handle in the testDatasource function
+        return of({'field': '', 'format': ''})
+      })
+    ).subscribe(result => {
+      this.timeField = result.field;
+      this.timeOutputFormat = result.format;
+      this.queryBuilder = new ElasticQueryBuilder({
+        timeField: this.timeField,
+      });
+    });
+
+    this.logMessageField = settingsData.logMessageField || '';
+    this.logLevelField = settingsData.logLevelField || '';
     this.dataLinks = settingsData.dataLinks || [];
     this.languageProvider = new ElasticsearchLanguageProvider(this);
   }
@@ -111,12 +140,7 @@ export class QuickwitDataSource
         message: 'Cannot save datasource, `index` is required',
       };
     }
-    if (this.timeField === '' ) {
-      return {
-        status: 'error',
-        message: 'Cannot save datasource, `timeField` is required',
-      };
-    }
+
     return lastValueFrom(
       from(this.getResource('indexes/' + this.index)).pipe(
         mergeMap((indexMetadata) => {
@@ -130,7 +154,14 @@ export class QuickwitDataSource
           return of({ status: 'success', message: `Index OK. Time field name OK` });
         }),
         catchError((err) => {
-          if (err.status === 404) {
+          if (err.data && err.data.error) {
+            let err_source = extractJsonPayload(err.data.error)
+            if (err_source) {
+              err = err_source
+            }
+          }
+
+          if (err.status && err.status === 404) {
             return of({ status: 'error', message: 'Index does not exists.' });
           } else if (err.message) {
             return of({ status: 'error', message: err.message });
@@ -147,21 +178,19 @@ export class QuickwitDataSource
     if (this.timeField === '') {
       return `Time field must not be empty`;
     }
-    if (indexMetadata.index_config.doc_mapping.timestamp_field !== this.timeField) {
-      return `No timestamp field named '${this.timeField}' found`;
-    }
+
     let fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings);
     let timestampField = fields.find((field) => field.json_path === this.timeField);
+
     // Should never happen.
     if (timestampField === undefined) {
       return `No field named '${this.timeField}' found in the doc mapping. This should never happen.`;
     }
-    if (timestampField.field_mapping.output_format !== this.timeOutputFormat) {
-      return `Timestamp output format is declared as '${timestampField.field_mapping.output_format}' in the doc mapping, not '${this.timeOutputFormat}'.`;
-    }
+
+    let timeOutputFormat = timestampField.field_mapping.output_format || 'unknown';
     const supportedTimestampOutputFormats = ['unix_timestamp_secs', 'unix_timestamp_millis', 'unix_timestamp_micros', 'unix_timestamp_nanos', 'iso8601', 'rfc3339'];
-    if (!supportedTimestampOutputFormats.includes(this.timeOutputFormat)) {
-      return `Timestamp output format '${this.timeOutputFormat} is not yet supported.`;
+    if (!supportedTimestampOutputFormats.includes(timeOutputFormat)) {
+      return `Timestamp output format '${timeOutputFormat} is not yet supported.`;
     }
     return;
   }
@@ -310,6 +339,7 @@ export class QuickwitDataSource
       ignore_unavailable: true,
       index: this.index,
     });
+
     let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef));
     esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf().toString());
     esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf().toString());
@@ -365,7 +395,7 @@ export class QuickwitDataSource
         return _map(filteredFields, (field) => {
           return {
             text: field.json_path,
-            value: typeMap[field.field_mapping.type],
+            value: typeMap[field.field_mapping.type]
           };
         });
       })

src/utils.ts

@@ -13,6 +13,20 @@ export const describeMetric = (metric: MetricAggregation) => {
   return `${metricAggregationConfig[metric.type].label} ${metric.field}`;
 };
 
+export const extractJsonPayload = (msg: string) => {
+  const match = msg.match(/{.*}/);
+
+  if (!match) {
+    return null;
+  }
+
+  try {
+      return JSON.parse(match[0]);
+  } catch (error) {
+      return null;
+  }
+}
+
 /**
  * Utility function to clean up aggregations settings objects.
  * It removes nullish values and empty strings, array and objects