From b70d8837c0ec2aca46cc0c3dd2a1ba12f5bb607b Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Fri, 8 Dec 2023 08:37:13 +0100 Subject: [PATCH 01/30] Issue #34: add jaeger to docker-compose --- docker-compose.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 75a87ab..e642a74 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,16 @@ version: '3.0' services: + jaeger: + image: jaegertracing/jaeger-query:1.51 + container_name: 'jaeger-quickwit' + environment: + - GRPC_STORAGE_SERVER=host.docker.internal:7280 + - SPAN_STORAGE_TYPE=grpc-plugin + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - quickwit grafana: container_name: 'grafana-quickwit-datasource' build: @@ -15,6 +25,12 @@ services: - gquickwit:/var/lib/grafana extra_hosts: - "host.docker.internal:host-gateway" + networks: + - quickwit + +networks: + quickwit: + driver: bridge volumes: gquickwit: From d9305e2e5181b19cd9dc6e70e64e9b0e930c9eef Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Fri, 8 Dec 2023 09:48:53 +0100 Subject: [PATCH 02/30] Issue #34: add DataLinks and DataLink webcomponent --- src/components/Divider.tsx | 25 ++++ src/configuration/ConfigEditor.tsx | 24 ++++ src/configuration/DataLink.tsx | 176 +++++++++++++++++++++++++++ src/configuration/DataLinks.test.tsx | 67 ++++++++++ src/configuration/DataLinks.tsx | 88 ++++++++++++++ 5 files changed, 380 insertions(+) create mode 100644 src/components/Divider.tsx create mode 100644 src/configuration/DataLink.tsx create mode 100644 src/configuration/DataLinks.test.tsx create mode 100644 src/configuration/DataLinks.tsx diff --git a/src/components/Divider.tsx b/src/components/Divider.tsx new file mode 100644 index 0000000..11f0239 --- /dev/null +++ b/src/components/Divider.tsx @@ -0,0 +1,25 @@ +import { css } from '@emotion/css'; +import React from 'react'; + +import { GrafanaTheme2 } from '@grafana/data'; +import { useStyles2 } from '@grafana/ui'; + +export const Divider = ({ hideLine = false }) => { + const styles = useStyles2(getStyles); + + if (hideLine) { + return
; + } + + return
; +}; + +const getStyles = (theme: GrafanaTheme2) => ({ + divider: css({ + margin: theme.spacing(4, 0), + }), + dividerHideLine: css({ + border: 'none', + margin: theme.spacing(3, 0), + }), +}); diff --git a/src/configuration/ConfigEditor.tsx b/src/configuration/ConfigEditor.tsx index efa011c..18e7dba 100644 --- a/src/configuration/ConfigEditor.tsx +++ b/src/configuration/ConfigEditor.tsx @@ -3,6 +3,8 @@ import { DataSourceHttpSettings, Input, InlineField, FieldSet } from '@grafana/u import { DataSourcePluginOptionsEditorProps, DataSourceSettings } from '@grafana/data'; import { QuickwitOptions } from 'quickwit'; import { coerceOptions } from './utils'; +import { Divider } from 'components/Divider'; +import { DataLinks } from './DataLinks'; interface Props extends DataSourcePluginOptionsEditorProps {} @@ -27,6 +29,7 @@ export const ConfigEditor = (props: Props) => { onChange={onOptionsChange} /> + ); }; @@ -35,6 +38,27 @@ type DetailsProps = { value: DataSourceSettings; onChange: (value: DataSourceSettings) => void; }; + +export const QuickwitDataLinks = ({ value, onChange }: DetailsProps) => { + return ( +
+ + { + onChange({ + ...value, + jsonData: { + ...value.jsonData, + dataLinks: newValue, + }, + }); + }} + /> +
+ ) +}; + export const QuickwitDetails = ({ value, onChange }: DetailsProps) => { return ( <> diff --git a/src/configuration/DataLink.tsx b/src/configuration/DataLink.tsx new file mode 100644 index 0000000..24eeccc --- /dev/null +++ b/src/configuration/DataLink.tsx @@ -0,0 +1,176 @@ +import { css } from '@emotion/css'; +import React, { Dispatch, SetStateAction, useEffect, useState } from 'react'; +import { usePrevious } from 'react-use'; + +import { DataSourceInstanceSettings, VariableSuggestion } from '@grafana/data'; +import { + Button, + DataLinkInput, + InlineField, + InlineSwitch, + InlineFieldRow, + InlineLabel, + Input, + useStyles2 +} from '@grafana/ui'; + +import { DataSourcePicker } from '@grafana/runtime' + +import { DataLinkConfig } from '../types'; + +interface Props { + value: DataLinkConfig; + onChange: (value: DataLinkConfig) => void; + onDelete: () => void; + suggestions: VariableSuggestion[]; + className?: string; +} + +export const DataLink = (props: Props) => { + const { value, onChange, onDelete, suggestions, className } = props; + const styles = useStyles2(getStyles); + const [showInternalLink, setShowInternalLink] = useInternalLink(value.datasourceUid); + + const handleChange = (field: keyof typeof value) => (event: React.ChangeEvent) => { + onChange({ + ...value, + [field]: event.currentTarget.value, + }); + }; + + return ( +
+
+ + + +
+ + +
+ + {showInternalLink ? 'Query' : 'URL'} + + + onChange({ + ...value, + url: newValue, + }) + } + suggestions={suggestions} + /> +
+ +
+ + + +
+ + +
+ + { + if (showInternalLink) { + onChange({ + ...value, + datasourceUid: undefined, + }); + } + setShowInternalLink(!showInternalLink); + }} + /> + + + {showInternalLink && ( + { + onChange({ + ...value, + datasourceUid: ds.uid, + }); + }} + current={value.datasourceUid} + /> + )} +
+
+ ); +}; + +function useInternalLink(datasourceUid?: string): [boolean, Dispatch>] { + const [showInternalLink, setShowInternalLink] = useState(!!datasourceUid); + const previousUid = usePrevious(datasourceUid); + + // Force internal link visibility change if uid changed outside of this component. + useEffect(() => { + if (!previousUid && datasourceUid && !showInternalLink) { + setShowInternalLink(true); + } + if (previousUid && !datasourceUid && showInternalLink) { + setShowInternalLink(false); + } + }, [previousUid, datasourceUid, showInternalLink]); + + return [showInternalLink, setShowInternalLink]; +} + +const getStyles = () => ({ + firstRow: css` + display: flex; + `, + nameField: css` + flex: 2; + `, + regexField: css` + flex: 3; + `, + row: css` + display: flex; + align-items: baseline; + `, + urlField: css` + display: flex; + flex: 1; + `, + urlDisplayLabelField: css` + flex: 1; + `, +}); diff --git a/src/configuration/DataLinks.test.tsx b/src/configuration/DataLinks.test.tsx new file mode 100644 index 0000000..a623d68 --- /dev/null +++ b/src/configuration/DataLinks.test.tsx @@ -0,0 +1,67 @@ +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import React from 'react'; + +import { DataLinkConfig } from '../types'; + +import { DataLinks, Props } from './DataLinks'; + +const setup = (propOverrides?: Partial) => { + const props: Props = { + value: [], + onChange: jest.fn(), + ...propOverrides, + }; + + return render(); +}; + +describe('DataLinks tests', () => { + it('should render correctly with no fields', async () => { + setup(); + + expect(screen.getByRole('heading', { name: 'Data links' })); + expect(screen.getByRole('button', { name: 'Add' })).toBeInTheDocument(); + expect(await screen.findAllByRole('button')).toHaveLength(1); + }); + + it('should render correctly when passed fields', async () => { + setup({ value: testValue }); + + expect(await screen.findAllByRole('button', { name: 'Remove field' })).toHaveLength(2); + expect(await screen.findAllByRole('checkbox', { name: 'Internal link' })).toHaveLength(2); + }); + + it('should call onChange to add a new field when the add button is clicked', async () => { + const onChangeMock = jest.fn(); + setup({ onChange: onChangeMock }); + + expect(onChangeMock).not.toHaveBeenCalled(); + const addButton = screen.getByRole('button', { name: 'Add' }); + await userEvent.click(addButton); + + expect(onChangeMock).toHaveBeenCalled(); + }); + + it('should call onChange to remove a field when the remove button is clicked', async () => { + const onChangeMock = jest.fn(); + setup({ value: testValue, onChange: onChangeMock }); + + expect(onChangeMock).not.toHaveBeenCalled(); + const removeButton = await screen.findAllByRole('button', { name: 'Remove field' }); + await userEvent.click(removeButton[0]); + + expect(onChangeMock).toHaveBeenCalled(); + }); +}); + +const testValue: DataLinkConfig[] = [ + { + field: 'regex1', + url: 'localhost1', + }, + { + field: 'regex2', + url: 'localhost2', + }, +]; diff --git a/src/configuration/DataLinks.tsx b/src/configuration/DataLinks.tsx new file mode 100644 index 0000000..2ece07a --- /dev/null +++ b/src/configuration/DataLinks.tsx @@ -0,0 +1,88 @@ +import { css } from '@emotion/css'; +import React from 'react'; + +import { GrafanaTheme2, VariableOrigin, DataLinkBuiltInVars } from '@grafana/data'; +import { ConfigSubSection } from '@grafana/experimental'; +import { Button, useStyles2 } from '@grafana/ui'; + +import { DataLinkConfig } from '../types'; + +import { DataLink } from './DataLink'; + +const getStyles = (theme: GrafanaTheme2) => { + return { + addButton: css` + margin-right: 10px; + `, + container: css` + margin-bottom: ${theme.spacing(2)}; + `, + dataLink: css` + margin-bottom: ${theme.spacing(1)}; + `, + }; +}; + +export type Props = { + value?: DataLinkConfig[]; + onChange: (value: DataLinkConfig[]) => void; +}; +export const DataLinks = (props: Props) => { + const { value, onChange } = props; + const styles = useStyles2(getStyles); + + return ( + +
+ {value && value.length > 0 && ( +
+ {value.map((field, index) => { + return ( + { + const newDataLinks = [...value]; + newDataLinks.splice(index, 1, newField); + onChange(newDataLinks); + }} + onDelete={() => { + const newDataLinks = [...value]; + newDataLinks.splice(index, 1); + onChange(newDataLinks); + }} + suggestions={[ + { + value: DataLinkBuiltInVars.valueRaw, + label: 'Raw value', + documentation: 'Raw value of the field', + origin: VariableOrigin.Value, + }, + ]} + /> + ); + })} +
+ )} + + +
+ + ); +}; From 31f68a95f0ab51ef552b179aa99a89e400cc17b4 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Fri, 8 Dec 2023 10:37:46 +0100 Subject: [PATCH 03/30] Issue #34: fix docker-compose config for jaegger --- docker-compose.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index e642a74..7a84d0a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,8 +5,10 @@ services: image: jaegertracing/jaeger-query:1.51 container_name: 'jaeger-quickwit' environment: - - GRPC_STORAGE_SERVER=host.docker.internal:7280 + - GRPC_STORAGE_SERVER=host.docker.internal:7281 - SPAN_STORAGE_TYPE=grpc-plugin + ports: + - 16686:16686 extra_hosts: - "host.docker.internal:host-gateway" networks: From d387f4cf2d2e7d68c619fdaf836a014aa80bfe8d Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Fri, 8 Dec 2023 15:25:31 +0100 Subject: [PATCH 04/30] Issue #34: add enhanceDataFrameWithDataLinks --- src/datasource.ts | 66 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/src/datasource.ts b/src/datasource.ts index 36cd0a2..6d9ade0 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -6,6 +6,7 @@ import { AbstractQuery, CoreApp, DataFrame, + DataLink, DataQueryError, DataQueryRequest, DataQueryResponse, @@ -32,7 +33,11 @@ import { TimeRange, } from '@grafana/data'; import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field, FieldMapping, IndexMetadata, Logs, TermsQuery } from './types'; -import { DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; +import { + DataSourceWithBackend, + getTemplateSrv, + TemplateSrv, + getDataSourceSrv } from '@grafana/runtime'; import { LogRowContextOptions, LogRowContextQueryDirection, QuickwitOptions } from 'quickwit'; import { ElasticQueryBuilder } from 'QueryBuilder'; import { colors } from '@grafana/ui'; @@ -83,15 +88,15 @@ export class QuickwitDataSource this.languageProvider = new ElasticsearchLanguageProvider(this); } - // /** - // * Ideally final -- any other implementation may not work as expected - // */ - // query(request: DataQueryRequest): Observable { - // return super.query(request) - // .pipe(map((response) => { - // return response; - // })); - // } + query(request: DataQueryRequest): Observable { + return super.query(request) + .pipe(map((response) => { + response.data.forEach((dataFrame) => { + enhanceDataFrameWithDataLinks(dataFrame, this.dataLinks); + }); + return response; + })); + } /** * Checks the plugin health @@ -816,6 +821,46 @@ function luceneEscape(value: string) { return value.replace(/([\!\*\+\-\=<>\s\&\|\(\)\[\]\{\}\^\~\?\:\\/"])/g, '\\$1'); } +export function enhanceDataFrameWithDataLinks(dataFrame: DataFrame, dataLinks: DataLinkConfig[]) { + if (!dataLinks.length) { + return; + } + + for (const field of dataFrame.fields) { + const linksToApply = dataLinks.filter((dataLink) => new RegExp(dataLink.field).test(field.name)); + + if (linksToApply.length === 0) { + continue; + } + + field.config = field.config || {}; + field.config.links = [...(field.config.links || [], linksToApply.map(generateDataLink))]; + } +} + +function generateDataLink(linkConfig: DataLinkConfig): DataLink { + const dataSourceSrv = getDataSourceSrv(); + + if (linkConfig.datasourceUid) { + const dsSettings = dataSourceSrv.getInstanceSettings(linkConfig.datasourceUid); + + return { + title: linkConfig.urlDisplayLabel || '', + url: '', + internal: { + query: { query: linkConfig.url }, + datasourceUid: linkConfig.datasourceUid, + datasourceName: dsSettings?.name ?? 'Data source not found', + }, + }; + } else { + return { + title: linkConfig.urlDisplayLabel || '', + url: linkConfig.url, + }; + } +} + function createContextTimeRange(rowTimeEpochMs: number, direction: string) { const offset = 7; // For log context, we want to request data from 7 subsequent/previous indices @@ -831,4 +876,3 @@ function createContextTimeRange(rowTimeEpochMs: number, direction: string) { }; } } - From ca0e60aff8eb9ba4daba794e91de418fd37aaa31 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Mon, 11 Dec 2023 11:30:09 +0100 Subject: [PATCH 05/30] Issue #34: replace regexp by equal comparison --- src/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasource.ts b/src/datasource.ts index 6d9ade0..c47d444 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -827,7 +827,7 @@ export function enhanceDataFrameWithDataLinks(dataFrame: DataFrame, dataLinks: D } for (const field of dataFrame.fields) { - const linksToApply = dataLinks.filter((dataLink) => new RegExp(dataLink.field).test(field.name)); + const linksToApply = dataLinks.filter((dataLink) => dataLink.field === field.name); if (linksToApply.length === 0) { continue; From 242324ffe8a985223c669c5d5b285612a7389381 Mon Sep 17 00:00:00 2001 From: fmassot Date: Tue, 12 Dec 2023 20:50:09 +0100 Subject: [PATCH 06/30] Add parameter to convert a traceId from base64 to hex string --- src/configuration/DataLink.tsx | 2 +- src/configuration/DataLinks.test.tsx | 2 ++ src/configuration/DataLinks.tsx | 2 +- src/datasource.ts | 48 +++++++++++++++++++++++++--- src/types.ts | 1 + 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/configuration/DataLink.tsx b/src/configuration/DataLink.tsx index 24eeccc..c2cb76c 100644 --- a/src/configuration/DataLink.tsx +++ b/src/configuration/DataLink.tsx @@ -11,7 +11,7 @@ import { InlineFieldRow, InlineLabel, Input, - useStyles2 + useStyles2, } from '@grafana/ui'; import { DataSourcePicker } from '@grafana/runtime' diff --git a/src/configuration/DataLinks.test.tsx b/src/configuration/DataLinks.test.tsx index a623d68..17cfb1d 100644 --- a/src/configuration/DataLinks.test.tsx +++ b/src/configuration/DataLinks.test.tsx @@ -59,9 +59,11 @@ const testValue: DataLinkConfig[] = [ { field: 'regex1', url: 'localhost1', + base64TraceId: false, }, { field: 'regex2', url: 'localhost2', + base64TraceId: true, }, ]; diff --git a/src/configuration/DataLinks.tsx b/src/configuration/DataLinks.tsx index 2ece07a..9f066cc 100644 --- a/src/configuration/DataLinks.tsx +++ b/src/configuration/DataLinks.tsx @@ -76,7 +76,7 @@ export const DataLinks = (props: Props) => { icon="plus" onClick={(event) => { event.preventDefault(); - const newDataLinks = [...(value || []), { field: '', url: '' }]; + const newDataLinks = [...(value || []), { field: '', url: '', base64TraceId: true }]; onChange(newDataLinks); }} > diff --git a/src/datasource.ts b/src/datasource.ts index c47d444..acba41e 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -4,6 +4,7 @@ import { catchError, mergeMap, map } from 'rxjs/operators'; import { AbstractQuery, + ArrayVector, CoreApp, DataFrame, DataLink, @@ -17,6 +18,7 @@ import { DataSourceWithQueryImportSupport, DataSourceWithSupplementaryQueriesSupport, dateTime, + Field, FieldColorModeId, FieldType, getDefaultTimeRange, @@ -32,7 +34,7 @@ import { SupplementaryQueryType, TimeRange, } from '@grafana/data'; -import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field, FieldMapping, IndexMetadata, Logs, TermsQuery } from './types'; +import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field as QuickwitField, FieldMapping, IndexMetadata, Logs, TermsQuery } from './types'; import { DataSourceWithBackend, getTemplateSrv, @@ -348,7 +350,7 @@ export class QuickwitDataSource }; return from(this.getResource('indexes/' + this.index)).pipe( map((index_metadata) => { - const shouldAddField = (field: Field) => { + const shouldAddField = (field: QuickwitField) => { const translated_type = typeMap[field.field_mapping.type]; if (type?.length === 0) { return true; @@ -578,8 +580,8 @@ export class QuickwitDataSource } // Returns a flatten array of fields and nested fields found in the given `FieldMapping` array. -function getAllFields(field_mappings: FieldMapping[]): Field[] { - const fields: Field[] = []; +function getAllFields(field_mappings: FieldMapping[]): QuickwitField[] { + const fields: QuickwitField[] = []; for (const field_mapping of field_mappings) { if (field_mapping.type === 'object' && field_mapping.field_mappings !== undefined) { for (const child_field_mapping of getAllFields(field_mapping.field_mappings)) { @@ -821,13 +823,51 @@ function luceneEscape(value: string) { return value.replace(/([\!\*\+\-\=<>\s\&\|\(\)\[\]\{\}\^\~\?\:\\/"])/g, '\\$1'); } +function base64ToHex(base64String: string) { + const binaryString = window.atob(base64String); + return Array.from(binaryString).map(char => { + const byte = char.charCodeAt(0); + return ('0' + byte.toString(16)).slice(-2); + }).join(''); +} + export function enhanceDataFrameWithDataLinks(dataFrame: DataFrame, dataLinks: DataLinkConfig[]) { if (!dataLinks.length) { return; } + let fields_to_fix_condition = (field: Field) => { + return dataLinks.filter((dataLink) => dataLink.field === field.name && dataLink.base64TraceId).length === 1; + }; + const fields_to_keep = dataFrame.fields.filter((field) => { + return !fields_to_fix_condition(field) + }); + let new_fields = dataFrame + .fields + .filter(fields_to_fix_condition) + .map((field) => { + let values = field.values.toArray().map((value) => { + try { + return base64ToHex(value); + } catch (e) { + console.warn("cannot convert value from base64 to hex", e); + return value; + }; + }); + return { + ...field, + values: new ArrayVector(values), + } + }); + + if (new_fields.length === 0) { + return; + } + + dataFrame.fields = [new_fields[0], ...fields_to_keep]; for (const field of dataFrame.fields) { const linksToApply = dataLinks.filter((dataLink) => dataLink.field === field.name); + console.log(linksToApply); if (linksToApply.length === 0) { continue; diff --git a/src/types.ts b/src/types.ts index 9ef2672..f2a9d9c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -100,6 +100,7 @@ export interface TermsQuery { export type DataLinkConfig = { field: string; + base64TraceId: boolean; url: string; urlDisplayLabel?: string; datasourceUid?: string; From d76a42d7c84ccf8005b0f12bc79270a1e76e3609 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Wed, 13 Dec 2023 09:57:47 +0100 Subject: [PATCH 07/30] Issue #34: add switch to enable/disable base64 traceId encoding --- src/configuration/DataLink.tsx | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/configuration/DataLink.tsx b/src/configuration/DataLink.tsx index c2cb76c..bfde386 100644 --- a/src/configuration/DataLink.tsx +++ b/src/configuration/DataLink.tsx @@ -30,6 +30,7 @@ export const DataLink = (props: Props) => { const { value, onChange, onDelete, suggestions, className } = props; const styles = useStyles2(getStyles); const [showInternalLink, setShowInternalLink] = useInternalLink(value.datasourceUid); + const [base64TraceId, setBase64TraceId] = useState(true) const handleChange = (field: keyof typeof value) => (event: React.ChangeEvent) => { onChange({ @@ -38,6 +39,11 @@ export const DataLink = (props: Props) => { }); }; + const handleBase64TraceId = (base64TraceId: boolean, config: DataLinkConfig) => { + setBase64TraceId(base64TraceId) + config = {...config, base64TraceId: base64TraceId }; + } + return (
@@ -131,6 +137,17 @@ export const DataLink = (props: Props) => { /> )}
+ +
+ + handleBase64TraceId(!base64TraceId, value)} + /> + +
); }; From 71578d4122fdce943a786985b12677dce0d0ffbf Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Wed, 13 Dec 2023 10:00:07 +0100 Subject: [PATCH 08/30] Issue #34: move title tooltip --- src/configuration/DataLink.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/configuration/DataLink.tsx b/src/configuration/DataLink.tsx index bfde386..1cc4565 100644 --- a/src/configuration/DataLink.tsx +++ b/src/configuration/DataLink.tsx @@ -139,9 +139,8 @@ export const DataLink = (props: Props) => {
- + handleBase64TraceId(!base64TraceId, value)} From 65ff78acb7f4ec1239a989b3fac371f12f7d632e Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Wed, 13 Dec 2023 11:54:00 +0100 Subject: [PATCH 09/30] Issue #34: design review --- src/configuration/DataLink.tsx | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/configuration/DataLink.tsx b/src/configuration/DataLink.tsx index 1cc4565..f5a28b1 100644 --- a/src/configuration/DataLink.tsx +++ b/src/configuration/DataLink.tsx @@ -31,6 +31,7 @@ export const DataLink = (props: Props) => { const styles = useStyles2(getStyles); const [showInternalLink, setShowInternalLink] = useInternalLink(value.datasourceUid); const [base64TraceId, setBase64TraceId] = useState(true) + const labelWidth = 24 const handleChange = (field: keyof typeof value) => (event: React.ChangeEvent) => { onChange({ @@ -50,7 +51,7 @@ export const DataLink = (props: Props) => { {
- + {showInternalLink ? 'Query' : 'URL'} {
- + + handleBase64TraceId(!base64TraceId, value)} + /> + +
+ +
+ { /> )}
- -
- - handleBase64TraceId(!base64TraceId, value)} - /> - -
); }; From 23229edab1dea366059489ffc58ed79e3232636b Mon Sep 17 00:00:00 2001 From: fmassot Date: Wed, 13 Dec 2023 16:50:43 +0100 Subject: [PATCH 10/30] Improve wording. --- src/configuration/DataLink.tsx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/configuration/DataLink.tsx b/src/configuration/DataLink.tsx index f5a28b1..48431fd 100644 --- a/src/configuration/DataLink.tsx +++ b/src/configuration/DataLink.tsx @@ -73,6 +73,15 @@ export const DataLink = (props: Props) => { />
+
+ + handleBase64TraceId(!base64TraceId, value)} + /> + +
+
@@ -108,15 +117,6 @@ export const DataLink = (props: Props) => {
-
- - handleBase64TraceId(!base64TraceId, value)} - /> - -
-
Date: Wed, 13 Dec 2023 16:54:20 +0100 Subject: [PATCH 11/30] Bump 0.3.0-beta.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f6166f6..09b117a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "quickwit-datasource", - "version": "0.3.0-beta.1", + "version": "0.3.0-beta.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "quickwit-datasource", - "version": "0.3.0-beta.1", + "version": "0.3.0-beta.2", "license": "AGPL-3.0", "dependencies": { "@emotion/css": "^11.1.3", diff --git a/package.json b/package.json index 1bc5c42..b819ee4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "quickwit-datasource", - "version": "0.3.0-beta.1", + "version": "0.3.0-beta.2", "description": "Quickwit datasource", "scripts": { "build": "webpack -c ./.config/webpack/webpack.config.ts --env production", From 696d779b95245d5d8443cec5bf97bde7bb7e3ec6 Mon Sep 17 00:00:00 2001 From: fmassot Date: Wed, 13 Dec 2023 17:06:09 +0100 Subject: [PATCH 12/30] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43c4aef..de8538d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.3.0-beta.1 + +This version works only with quickwit main version (or docker edge). + +- Add support for data links (possibility to create links to other datasources). + ## 0.3.0-beta.0 This version works only with quickwit main version (or docker edge). From 67f56ba2dfb8ed5bcbacaf76dc7e32b2ee0d5121 Mon Sep 17 00:00:00 2001 From: fmassot Date: Wed, 13 Dec 2023 17:27:31 +0100 Subject: [PATCH 13/30] Rm console.log --- src/datasource.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/datasource.ts b/src/datasource.ts index acba41e..3c309af 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -867,7 +867,6 @@ export function enhanceDataFrameWithDataLinks(dataFrame: DataFrame, dataLinks: D for (const field of dataFrame.fields) { const linksToApply = dataLinks.filter((dataLink) => dataLink.field === field.name); - console.log(linksToApply); if (linksToApply.length === 0) { continue; From ae3d633256cc24bce0a2f6f303e41b77030adc35 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Fri, 8 Dec 2023 14:53:13 +0100 Subject: [PATCH 14/30] Issue #4: find the timestamp field dynamically --- pkg/quickwit/quickwit.go | 87 +++++++++++++++++++++++++----- src/configuration/ConfigEditor.tsx | 18 ------- src/datasource.ts | 48 ++++++++++------- 3 files changed, 103 insertions(+), 50 deletions(-) diff --git a/pkg/quickwit/quickwit.go b/pkg/quickwit/quickwit.go index 0aa9f1b..36ba450 100644 --- a/pkg/quickwit/quickwit.go +++ b/pkg/quickwit/quickwit.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -26,6 +25,70 @@ type QuickwitDatasource struct { dsInfo es.DatasourceInfo } +type QuickwitMapping struct { + IndexConfig struct { + DocMapping struct { + TimestampField string `json:"timestamp_field"` + FieldMappings []struct { + Name string `json:"name"` + InputFormats []string `json:"input_formats"` + } `json:"field_mappings"` + } `json:"doc_mapping"` + } `json:"index_config"` +} + +func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (string, string, error) { + mappingEndpointUrl := qwUrl + "/indexes/" + index + qwlog.Info("Calling quickwit endpoint: " + mappingEndpointUrl) + r, err := cli.Get(mappingEndpointUrl) + if err != nil { + errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error()) + qwlog.Error(errMsg) + return "", "", err + } + + statusCode := r.StatusCode + if statusCode < 200 || statusCode >= 400 { + errMsg := fmt.Sprintf("Error when calling url = %s: statusCode = %d", mappingEndpointUrl, statusCode) + qwlog.Error(errMsg) + return "", "", fmt.Errorf(errMsg) + } + + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error()) + qwlog.Error(errMsg) + return "", "", err + } + + var payload QuickwitMapping + err = json.Unmarshal(body, &payload) + if err != nil { + errMsg := fmt.Sprintf("Unmarshalling body error: err = %s, body = %s", err.Error(), (body)) + qwlog.Error(errMsg) + return "", "", fmt.Errorf(errMsg) + } + + timestampFieldName := payload.IndexConfig.DocMapping.TimestampField + timestampFieldFormat := "undef" + for _, field := range payload.IndexConfig.DocMapping.FieldMappings { + if field.Name == timestampFieldName && len(field.InputFormats) > 0 { + timestampFieldFormat = field.InputFormats[0] + break + } + } + + if timestampFieldFormat == "undef" { + errMsg := fmt.Sprintf("No format found for field: %s", string(timestampFieldName)) + qwlog.Error(errMsg) + return timestampFieldName, "", fmt.Errorf(errMsg) + } + + qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, timestampFieldFormat)) + return timestampFieldName, timestampFieldFormat, nil +} + // Creates a Quickwit datasource. func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { qwlog.Debug("Initializing new data source instance") @@ -50,19 +113,8 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc return nil, err } - timeField, ok := jsonData["timeField"].(string) - if !ok { - return nil, errors.New("timeField cannot be cast to string") - } - - if timeField == "" { - return nil, errors.New("a time field name is required") - } - - timeOutputFormat, ok := jsonData["timeOutputFormat"].(string) - if !ok { - return nil, errors.New("timeOutputFormat cannot be cast to string") - } + timeField, toOk := jsonData["timeField"].(string) + timeOutputFormat, tofOk := jsonData["timeOutputFormat"].(string) logLevelField, ok := jsonData["logLevelField"].(string) if !ok { @@ -96,6 +148,13 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc maxConcurrentShardRequests = 256 } + if !toOk || !tofOk { + timeField, timeOutputFormat, err = getTimestampFieldInfos(index, settings.URL, httpCli) + if nil != err { + return nil, err + } + } + configuredFields := es.ConfiguredFields{ TimeField: timeField, TimeOutputFormat: timeOutputFormat, diff --git a/src/configuration/ConfigEditor.tsx b/src/configuration/ConfigEditor.tsx index 18e7dba..f756d3c 100644 --- a/src/configuration/ConfigEditor.tsx +++ b/src/configuration/ConfigEditor.tsx @@ -73,24 +73,6 @@ export const QuickwitDetails = ({ value, onChange }: DetailsProps) => { width={40} /> - - onChange({ ...value, jsonData: {...value.jsonData, timeField: event.currentTarget.value}})} - placeholder="timestamp" - width={40} - /> - - - onChange({ ...value, jsonData: {...value.jsonData, timeOutputFormat: event.currentTarget.value}})} - placeholder="unix_timestamp_millisecs" - width={40} - /> - { + let fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings); + let timestampFieldName = indexMetadata.index_config.doc_mapping.timestamp_field + let timestampField = fields.find((field) => field.json_path === timestampFieldName); + let timestampFormat = timestampField ? timestampField.field_mapping.output_format || '' : '' + let timestampFieldInfos = { 'field': timestampFieldName, 'format': timestampFormat } + console.log("timestampFieldInfos = " + JSON.stringify(timestampFieldInfos)) + return timestampFieldInfos + }) + ).subscribe(result => { + this.timeField = result.field; + this.timeOutputFormat = result.format; + this.queryBuilder = new ElasticQueryBuilder({ + timeField: this.timeField, + }); + }); + + this.logMessageField = settingsData.logMessageField || ''; + this.logLevelField = settingsData.logLevelField || ''; this.dataLinks = settingsData.dataLinks || []; this.languageProvider = new ElasticsearchLanguageProvider(this); } @@ -111,12 +129,7 @@ export class QuickwitDataSource message: 'Cannot save datasource, `index` is required', }; } - if (this.timeField === '' ) { - return { - status: 'error', - message: 'Cannot save datasource, `timeField` is required', - }; - } + return lastValueFrom( from(this.getResource('indexes/' + this.index)).pipe( mergeMap((indexMetadata) => { @@ -147,21 +160,19 @@ export class QuickwitDataSource if (this.timeField === '') { return `Time field must not be empty`; } - if (indexMetadata.index_config.doc_mapping.timestamp_field !== this.timeField) { - return `No timestamp field named '${this.timeField}' found`; - } + let fields = getAllFields(indexMetadata.index_config.doc_mapping.field_mappings); let timestampField = fields.find((field) => field.json_path === this.timeField); + // Should never happen. if (timestampField === undefined) { return `No field named '${this.timeField}' found in the doc mapping. This should never happen.`; } - if (timestampField.field_mapping.output_format !== this.timeOutputFormat) { - return `Timestamp output format is declared as '${timestampField.field_mapping.output_format}' in the doc mapping, not '${this.timeOutputFormat}'.`; - } + + let timeOutputFormat = timestampField.field_mapping.output_format || 'unknown'; const supportedTimestampOutputFormats = ['unix_timestamp_secs', 'unix_timestamp_millis', 'unix_timestamp_micros', 'unix_timestamp_nanos', 'iso8601', 'rfc3339']; - if (!supportedTimestampOutputFormats.includes(this.timeOutputFormat)) { - return `Timestamp output format '${this.timeOutputFormat} is not yet supported.`; + if (!supportedTimestampOutputFormats.includes(timeOutputFormat)) { + return `Timestamp output format '${timeOutputFormat} is not yet supported.`; } return; } @@ -310,6 +321,7 @@ export class QuickwitDataSource ignore_unavailable: true, index: this.index, }); + let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef)); esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf().toString()); esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf().toString()); From 022d331db027d8ab5d31fe1275478a1f2b7528b5 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Sat, 16 Dec 2023 12:22:34 +0100 Subject: [PATCH 15/30] Issue #4: error handling --- pkg/quickwit/quickwit.go | 29 ++++++++++++++++++++++++----- src/datasource.ts | 24 +++++++++++++++++++++--- src/utils.ts | 14 ++++++++++++++ 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/pkg/quickwit/quickwit.go b/pkg/quickwit/quickwit.go index 36ba450..7393069 100644 --- a/pkg/quickwit/quickwit.go +++ b/pkg/quickwit/quickwit.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -37,6 +38,23 @@ type QuickwitMapping struct { } `json:"index_config"` } +type QuickwitCreationErrorPayload struct { + Message string `json:"message"` + StatusCode int `json:"status"` +} + +func newErrorCreationPayload(statusCode int, message string) error { + var payload QuickwitCreationErrorPayload + payload.Message = message + payload.StatusCode = statusCode + json, err := json.Marshal(payload) + if nil != err { + return err + } + + return errors.New(string(json)) +} + func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (string, string, error) { mappingEndpointUrl := qwUrl + "/indexes/" + index qwlog.Info("Calling quickwit endpoint: " + mappingEndpointUrl) @@ -48,10 +66,11 @@ func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (strin } statusCode := r.StatusCode + if statusCode < 200 || statusCode >= 400 { - errMsg := fmt.Sprintf("Error when calling url = %s: statusCode = %d", mappingEndpointUrl, statusCode) + errMsg := fmt.Sprintf("Error when calling url = %s", mappingEndpointUrl) qwlog.Error(errMsg) - return "", "", fmt.Errorf(errMsg) + return "", "", newErrorCreationPayload(statusCode, errMsg) } defer r.Body.Close() @@ -59,7 +78,7 @@ func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (strin if err != nil { errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error()) qwlog.Error(errMsg) - return "", "", err + return "", "", newErrorCreationPayload(statusCode, errMsg) } var payload QuickwitMapping @@ -67,7 +86,7 @@ func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (strin if err != nil { errMsg := fmt.Sprintf("Unmarshalling body error: err = %s, body = %s", err.Error(), (body)) qwlog.Error(errMsg) - return "", "", fmt.Errorf(errMsg) + return "", "", newErrorCreationPayload(statusCode, errMsg) } timestampFieldName := payload.IndexConfig.DocMapping.TimestampField @@ -82,7 +101,7 @@ func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (strin if timestampFieldFormat == "undef" { errMsg := fmt.Sprintf("No format found for field: %s", string(timestampFieldName)) qwlog.Error(errMsg) - return timestampFieldName, "", fmt.Errorf(errMsg) + return timestampFieldName, "", newErrorCreationPayload(statusCode, errMsg) } qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, timestampFieldFormat)) diff --git a/src/datasource.ts b/src/datasource.ts index f7547dc..dc09b12 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -51,6 +51,7 @@ import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregatio import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations'; import ElasticsearchLanguageProvider from 'LanguageProvider'; import { ReactNode } from 'react'; +import { extractJsonPayload } from 'utils'; export const REF_ID_STARTER_LOG_VOLUME = 'log-volume-'; @@ -91,8 +92,18 @@ export class QuickwitDataSource let timestampField = fields.find((field) => field.json_path === timestampFieldName); let timestampFormat = timestampField ? timestampField.field_mapping.output_format || '' : '' let timestampFieldInfos = { 'field': timestampFieldName, 'format': timestampFormat } - console.log("timestampFieldInfos = " + JSON.stringify(timestampFieldInfos)) return timestampFieldInfos + }), + catchError((err) => { + if (!err.data || !err.data.error) { + let err_source = extractJsonPayload(err.data.error) + if(!err_source) { + throw err + } + } + + // the error will be handle in the testDatasource function + return of({'field': '', 'format': ''}) }) ).subscribe(result => { this.timeField = result.field; @@ -143,7 +154,14 @@ export class QuickwitDataSource return of({ status: 'success', message: `Index OK. Time field name OK` }); }), catchError((err) => { - if (err.status === 404) { + if (err.data && err.data.error) { + let err_source = extractJsonPayload(err.data.error) + if (err_source) { + err = err_source + } + } + + if (err.status && err.status === 404) { return of({ status: 'error', message: 'Index does not exists.' }); } else if (err.message) { return of({ status: 'error', message: err.message }); @@ -377,7 +395,7 @@ export class QuickwitDataSource return _map(filteredFields, (field) => { return { text: field.json_path, - value: typeMap[field.field_mapping.type], + value: typeMap[field.field_mapping.type] }; }); }) diff --git a/src/utils.ts b/src/utils.ts index 5b8d940..a6747c3 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -13,6 +13,20 @@ export const describeMetric = (metric: MetricAggregation) => { return `${metricAggregationConfig[metric.type].label} ${metric.field}`; }; +export const extractJsonPayload = (msg: string) => { + const match = msg.match(/{.*}/); + + if (!match) { + return null; + } + + try { + return JSON.parse(match[0]); + } catch (error) { + return null; + } +} + /** * Utility function to clean up aggregations settings objects. * It removes nullish values and empty strings, array and objects From 6780309b8f0dc19bb789c63d5d9ae47c4db18b77 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Mon, 18 Dec 2023 17:05:58 +0100 Subject: [PATCH 16/30] Issue #4: use output_format instead input format and make it optional --- pkg/quickwit/quickwit.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/quickwit/quickwit.go b/pkg/quickwit/quickwit.go index 7393069..5dfd7dd 100644 --- a/pkg/quickwit/quickwit.go +++ b/pkg/quickwit/quickwit.go @@ -31,8 +31,9 @@ type QuickwitMapping struct { DocMapping struct { TimestampField string `json:"timestamp_field"` FieldMappings []struct { - Name string `json:"name"` - InputFormats []string `json:"input_formats"` + Name string `json:"name"` + Type string `json:"type"` + OutputFormat *string `json:"output_format,omitempty"` } `json:"field_mappings"` } `json:"doc_mapping"` } `json:"index_config"` @@ -92,8 +93,8 @@ func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (strin timestampFieldName := payload.IndexConfig.DocMapping.TimestampField timestampFieldFormat := "undef" for _, field := range payload.IndexConfig.DocMapping.FieldMappings { - if field.Name == timestampFieldName && len(field.InputFormats) > 0 { - timestampFieldFormat = field.InputFormats[0] + if field.Type == "datetime" && field.Name == timestampFieldName && nil != field.OutputFormat { + timestampFieldFormat = *field.OutputFormat break } } From bc6373057ecca0d073d752a598a20652fdc8f76d Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Fri, 22 Dec 2023 15:02:56 +0100 Subject: [PATCH 17/30] Issue #4: recursivity with fieldmappings with an object type --- pkg/quickwit/quickwit.go | 53 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/pkg/quickwit/quickwit.go b/pkg/quickwit/quickwit.go index 5dfd7dd..e570bb5 100644 --- a/pkg/quickwit/quickwit.go +++ b/pkg/quickwit/quickwit.go @@ -26,15 +26,18 @@ type QuickwitDatasource struct { dsInfo es.DatasourceInfo } +type FieldMappings struct { + Name string `json:"name"` + Type string `json:"type"` + OutputFormat *string `json:"output_format,omitempty"` + FieldMappings []FieldMappings `json:"field_mappings,omitempty"` +} + type QuickwitMapping struct { IndexConfig struct { DocMapping struct { - TimestampField string `json:"timestamp_field"` - FieldMappings []struct { - Name string `json:"name"` - Type string `json:"type"` - OutputFormat *string `json:"output_format,omitempty"` - } `json:"field_mappings"` + TimestampField string `json:"timestamp_field"` + FieldMappings []FieldMappings `json:"field_mappings"` } `json:"doc_mapping"` } `json:"index_config"` } @@ -56,6 +59,30 @@ func newErrorCreationPayload(statusCode int, message string) error { return errors.New(string(json)) } +func findTimeStampFormat(timestampFieldName string, parentName *string, fieldMappings []FieldMappings) *string { + if nil == fieldMappings { + return nil + } + + for _, field := range fieldMappings { + fieldName := field.Name + if nil != parentName { + fieldName = fmt.Sprintf("%s.%s", *parentName, fieldName) + } + + if field.Type == "datetime" && fieldName == timestampFieldName && nil != field.OutputFormat { + return field.OutputFormat + } else if field.Type == "object" && nil != field.FieldMappings { + format := findTimeStampFormat(timestampFieldName, &field.Name, field.FieldMappings) + if nil != format { + return format + } + } + } + + return nil +} + func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (string, string, error) { mappingEndpointUrl := qwUrl + "/indexes/" + index qwlog.Info("Calling quickwit endpoint: " + mappingEndpointUrl) @@ -91,22 +118,16 @@ func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (strin } timestampFieldName := payload.IndexConfig.DocMapping.TimestampField - timestampFieldFormat := "undef" - for _, field := range payload.IndexConfig.DocMapping.FieldMappings { - if field.Type == "datetime" && field.Name == timestampFieldName && nil != field.OutputFormat { - timestampFieldFormat = *field.OutputFormat - break - } - } + timestampFieldFormat := findTimeStampFormat(timestampFieldName, nil, payload.IndexConfig.DocMapping.FieldMappings) - if timestampFieldFormat == "undef" { + if nil == timestampFieldFormat { errMsg := fmt.Sprintf("No format found for field: %s", string(timestampFieldName)) qwlog.Error(errMsg) return timestampFieldName, "", newErrorCreationPayload(statusCode, errMsg) } - qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, timestampFieldFormat)) - return timestampFieldName, timestampFieldFormat, nil + qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, *timestampFieldFormat)) + return timestampFieldName, *timestampFieldFormat, nil } // Creates a Quickwit datasource. From f3b7750282d4019536935087156fa5a15a68e843 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Wed, 27 Dec 2023 18:49:24 +0100 Subject: [PATCH 18/30] Issue #20: add _field_caps support --- pkg/quickwit/quickwit.go | 4 +++- src/datasource.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/quickwit/quickwit.go b/pkg/quickwit/quickwit.go index 0aa9f1b..45047c4 100644 --- a/pkg/quickwit/quickwit.go +++ b/pkg/quickwit/quickwit.go @@ -143,7 +143,9 @@ func (ds *QuickwitDatasource) CallResource(ctx context.Context, req *backend.Cal // - empty string for fetching db version // - ?/_mapping for fetching index mapping // - _msearch for executing getTerms queries - if req.Path != "" && !strings.Contains(req.Path, "indexes/") && req.Path != "_elastic/_msearch" { + // - _field_caps for getting all the aggregeables fields + var isFieldCaps = req.Path != "" && strings.Contains(req.Path, "_elastic") && strings.Contains(req.Path, "/_field_caps") + if req.Path != "" && !strings.Contains(req.Path, "indexes/") && req.Path != "_elastic/_msearch" && !isFieldCaps { return fmt.Errorf("invalid resource URL: %s", req.Path) } diff --git a/src/datasource.ts b/src/datasource.ts index 3c309af..33ac5f1 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -348,7 +348,7 @@ export class QuickwitDataSource datetime: 'date', text: 'string', }; - return from(this.getResource('indexes/' + this.index)).pipe( + return from(this.getResource('_elastic/' + this.index + '/_field_caps')).pipe( map((index_metadata) => { const shouldAddField = (field: QuickwitField) => { const translated_type = typeMap[field.field_mapping.type]; From 87913b1713e53bf9e3cda326b423afd708819d08 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Wed, 27 Dec 2023 20:26:13 +0100 Subject: [PATCH 19/30] Issue #20: parse the _field_caps payload --- src/datasource.ts | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/datasource.ts b/src/datasource.ts index 33ac5f1..44b9762 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -343,29 +343,55 @@ export class QuickwitDataSource // or fix the implementation. getFields(type?: string[], _range?: TimeRange): Observable { const typeMap: Record = { - u64: 'number', - i64: 'number', - datetime: 'date', + date: 'date', + date_nanos: 'date', + keyword: 'string', text: 'string', + binary: 'string', + byte: 'number', + long: 'number', + unsigned_long: 'number', + double: 'number', + integer: 'number', + short: 'number', + float: 'number', + scaled_float: 'number' }; + return from(this.getResource('_elastic/' + this.index + '/_field_caps')).pipe( map((index_metadata) => { - const shouldAddField = (field: QuickwitField) => { - const translated_type = typeMap[field.field_mapping.type]; + const shouldAddField = (field: any) => { + if (!field.aggregatable) { + return false + } + + const translated_type = typeMap[field.type]; if (type?.length === 0) { return true; } + return type?.includes(translated_type); }; - const fields = getAllFields(index_metadata.index_config.doc_mapping.field_mappings); + const fields = Object.entries(index_metadata.fields).flatMap(([key, value]) => { + let payload = JSON.parse(JSON.stringify(value)) + return Object.entries(payload).map(([subkey, subvalue]) => { + let subpayload = JSON.parse(JSON.stringify(subvalue)) + return { + text: key, + type: subkey, + aggregatable: subpayload["aggregatable"] + } + }) + }); + const filteredFields = fields.filter(shouldAddField); // transform to array return _map(filteredFields, (field) => { return { - text: field.json_path, - value: typeMap[field.field_mapping.type], + text: field.text, + value: typeMap[field.type], }; }); }) From 34a70c7ff052a8c14abc6508d35e0c10a36473d8 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Thu, 28 Dec 2023 10:10:23 +0100 Subject: [PATCH 20/30] Issue #4: add frontend unit tests --- src/utils.test.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/utils.test.ts diff --git a/src/utils.test.ts b/src/utils.test.ts new file mode 100644 index 0000000..1bcec2c --- /dev/null +++ b/src/utils.test.ts @@ -0,0 +1,20 @@ +import { extractJsonPayload } from "utils"; + +describe('Test utils.extractJsonPayload', () => { + it('Extract valid JSON', () => { + const result = extractJsonPayload('Hey {"foo": "bar"}') + expect(result).toEqual({ + foo: "bar" + }); + }); + + it('Extract non valid JSON', () => { + const result = extractJsonPayload('Hey {"foo": invalid}') + expect(result).toEqual(null); + }); + + it('Extract multiple valid JSONs (not supported)', () => { + const result = extractJsonPayload('Hey {"foo": "bar"} {"foo2": "bar2"}') + expect(result).toEqual(null); + }); +}); From 557cf6054a4f3bc72d6b497fa9e4d6d71cdf1857 Mon Sep 17 00:00:00 2001 From: fmassot Date: Thu, 28 Dec 2023 10:42:55 +0100 Subject: [PATCH 21/30] Minor refactoring. --- docker-compose.yaml | 10 +-- .../MetricEditor.test.tsx | 4 +- src/datasource.ts | 67 ++++++------------- src/hooks/useFields.test.tsx | 4 +- src/hooks/useFields.ts | 2 +- src/types.ts | 20 ++++++ src/utils.ts | 16 +++++ 7 files changed, 69 insertions(+), 54 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 7a84d0a..62dbe47 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,8 +11,9 @@ services: - 16686:16686 extra_hosts: - "host.docker.internal:host-gateway" - networks: - - quickwit + network_mode: host + # networks: + # - quickwit grafana: container_name: 'grafana-quickwit-datasource' build: @@ -27,8 +28,9 @@ services: - gquickwit:/var/lib/grafana extra_hosts: - "host.docker.internal:host-gateway" - networks: - - quickwit + network_mode: host + # networks: + # - quickwit networks: quickwit: diff --git a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx index d98b9a9..36eb32c 100644 --- a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx +++ b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx @@ -27,11 +27,11 @@ describe('Metric Editor', () => { bucketAggs: [defaultBucketAgg('2')], }; - const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); + const getFields: ElasticDatasource['getAggregatableFields'] = jest.fn(() => from([[]])); const wrapper = ({ children }: PropsWithChildren<{}>) => ( { - const typeMap: Record = { - date: 'date', - date_nanos: 'date', - keyword: 'string', - text: 'string', - binary: 'string', - byte: 'number', - long: 'number', - unsigned_long: 'number', - double: 'number', - integer: 'number', - short: 'number', - float: 'number', - scaled_float: 'number' - }; - + getAggregatableFields(type?: string[], _range?: TimeRange): Observable { + // TODO: use the time range. return from(this.getResource('_elastic/' + this.index + '/_field_caps')).pipe( - map((index_metadata) => { + map((field_capabilities_response: FieldCapabilitiesResponse) => { const shouldAddField = (field: any) => { if (!field.aggregatable) { return false } - const translated_type = typeMap[field.type]; + const translatedType = fieldTypeMap[field.type]; if (type?.length === 0) { return true; } - return type?.includes(translated_type); + return type?.includes(translatedType); }; - - const fields = Object.entries(index_metadata.fields).flatMap(([key, value]) => { - let payload = JSON.parse(JSON.stringify(value)) - return Object.entries(payload).map(([subkey, subvalue]) => { - let subpayload = JSON.parse(JSON.stringify(subvalue)) + const fieldCapabilities = Object.entries(field_capabilities_response.fields) + .flatMap(([field_name, field_capabilities]) => { + return Object.values(field_capabilities) + .map(field_capability => { + field_capability.field_name = field_name; + return field_capability; + }); + }) + .filter(shouldAddField) + .map(field_capability => { return { - text: key, - type: subkey, - aggregatable: subpayload["aggregatable"] + text: field_capability.field_name, + value: fieldTypeMap[field_capability.type], } - }) - }); - - const filteredFields = fields.filter(shouldAddField); - - // transform to array - return _map(filteredFields, (field) => { - return { - text: field.text, - value: typeMap[field.type], - }; - }); + }); + return fieldCapabilities; }) ); } @@ -402,7 +379,7 @@ export class QuickwitDataSource * Get tag keys for adhoc filters */ getTagKeys() { - return lastValueFrom(this.getFields()); + return lastValueFrom(this.getAggregatableFields()); } /** diff --git a/src/hooks/useFields.test.tsx b/src/hooks/useFields.test.tsx index d94a8da..a639413 100644 --- a/src/hooks/useFields.test.tsx +++ b/src/hooks/useFields.test.tsx @@ -23,11 +23,11 @@ describe('useFields hook', () => { bucketAggs: [defaultBucketAgg()], }; - const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); + const getFields: ElasticDatasource['getAggregatableFields'] = jest.fn(() => from([[]])); const wrapper = ({ children }: PropsWithChildren<{}>) => ( { return async (q?: string) => { // _mapping doesn't support filtering, we avoid sending a request everytime q changes if (!rawFields) { - rawFields = await lastValueFrom(datasource.getFields(filter, range)); + rawFields = await lastValueFrom(datasource.getAggregatableFields(filter, range)); } return rawFields.filter(({ text }) => q === undefined || text.includes(q)).map(toSelectableValue); diff --git a/src/types.ts b/src/types.ts index f2a9d9c..804673e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -150,3 +150,23 @@ export type Field = { path_segments: string[]; field_mapping: FieldMapping; } + +export type FieldCapabilityType = "long" | "keyword" | "text" | "date" | "date_nanos" | "binary" | "double" | "boolean" | "ip" | "nested" | "object" ; + +export type FieldCapability = { + field_name: string; // Field not present in response but added on the front side. + type: FieldCapabilityType; + metadata_field: boolean; + searchable: boolean; + aggregatable: boolean; + indices: Array; +} + +export type FieldCapabilitiesResponse = { + indices: Array; + fields: { + [key: string]: { + [key in FieldCapabilityType]: FieldCapability; + } + }; +} diff --git a/src/utils.ts b/src/utils.ts index 5b8d940..52d23cb 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -101,3 +101,19 @@ export const getScriptValue = (metric: MetricAggregationWithInlineScript) => export const unsupportedVersionMessage = 'Support for Elasticsearch versions after their end-of-life (currently versions < 7.16) was removed. Using unsupported version of Elasticsearch may lead to unexpected and incorrect results.'; + +export const fieldTypeMap: Record = { + date: 'date', + date_nanos: 'date', + keyword: 'string', + text: 'string', + binary: 'string', + byte: 'number', + long: 'number', + unsigned_long: 'number', + double: 'number', + integer: 'number', + short: 'number', + float: 'number', + scaled_float: 'number' +}; From cf483a6bacfa6b0d05eb6aef6d5dbcd4d3b9251b Mon Sep 17 00:00:00 2001 From: fmassot Date: Thu, 28 Dec 2023 10:44:53 +0100 Subject: [PATCH 22/30] Fix js lint. --- src/types.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types.ts b/src/types.ts index 804673e..1dba31c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -159,11 +159,11 @@ export type FieldCapability = { metadata_field: boolean; searchable: boolean; aggregatable: boolean; - indices: Array; + indices: String[]; } export type FieldCapabilitiesResponse = { - indices: Array; + indices: String[]; fields: { [key: string]: { [key in FieldCapabilityType]: FieldCapability; From e08064226b5177a873aa8b0f9620c406bebc596d Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Thu, 28 Dec 2023 10:56:44 +0100 Subject: [PATCH 23/30] Issue #20: fix pipeline docker.errors.InvalidArgument: host network_mode is incompatible with port_bindings --- docker-compose.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 62dbe47..4deb6b0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,9 +11,8 @@ services: - 16686:16686 extra_hosts: - "host.docker.internal:host-gateway" - network_mode: host - # networks: - # - quickwit + networks: + - quickwit grafana: container_name: 'grafana-quickwit-datasource' build: @@ -28,9 +27,8 @@ services: - gquickwit:/var/lib/grafana extra_hosts: - "host.docker.internal:host-gateway" - network_mode: host - # networks: - # - quickwit + networks: + - quickwit networks: quickwit: From c214d1b96be78dbe046b23818c268187663d3236 Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Thu, 28 Dec 2023 10:58:36 +0100 Subject: [PATCH 24/30] Issue #20: remove old comment because the function getField doesn't exists anymore --- src/datasource.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/datasource.ts b/src/datasource.ts index e0bc004..b64c2c3 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -526,12 +526,6 @@ export class QuickwitDataSource const range = options?.range; const parsedQuery = JSON.parse(query); if (query) { - // Interpolation of variables with a list of values for which we don't - // know the field name is not supported yet. - // if (parsedQuery.find === 'fields') { - // parsedQuery.type = this.interpolateLuceneQuery(parsedQuery.type); - // return lastValueFrom(this.getFields(parsedQuery.type, range)); - // } if (parsedQuery.find === 'terms') { parsedQuery.field = this.interpolateLuceneQuery(parsedQuery.field); parsedQuery.query = this.interpolateLuceneQuery(parsedQuery.query); From 29ac5693dc9e8d235fbbea6f0472519708bd04fb Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Thu, 28 Dec 2023 11:04:52 +0100 Subject: [PATCH 25/30] Issue #20: fix indent yml --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4deb6b0..7a84d0a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,7 +12,7 @@ services: extra_hosts: - "host.docker.internal:host-gateway" networks: - - quickwit + - quickwit grafana: container_name: 'grafana-quickwit-datasource' build: From 7f41c412653956ad2a6399645e6f6afa29ec9bac Mon Sep 17 00:00:00 2001 From: fmassot Date: Thu, 28 Dec 2023 11:35:48 +0100 Subject: [PATCH 26/30] Allow template variable defined by a 'find' fields query. --- .../MetricEditor.test.tsx | 4 ++-- src/datasource.ts | 15 ++++++++------- src/hooks/useFields.test.tsx | 4 ++-- src/hooks/useFields.ts | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx index 36eb32c..20455cf 100644 --- a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx +++ b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx @@ -27,11 +27,11 @@ describe('Metric Editor', () => { bucketAggs: [defaultBucketAgg('2')], }; - const getFields: ElasticDatasource['getAggregatableFields'] = jest.fn(() => from([[]])); + const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); const wrapper = ({ children }: PropsWithChildren<{}>) => ( { + getFields(aggregatable?: boolean, type?: string[], _range?: TimeRange): Observable { // TODO: use the time range. return from(this.getResource('_elastic/' + this.index + '/_field_caps')).pipe( map((field_capabilities_response: FieldCapabilitiesResponse) => { const shouldAddField = (field: any) => { - if (!field.aggregatable) { + if (aggregatable !== undefined && !field.aggregatable) { return false } - - const translatedType = fieldTypeMap[field.type]; if (type?.length === 0) { return true; } - - return type?.includes(translatedType); + return type?.includes(field.type) || type?.includes(fieldTypeMap[field.type]); }; const fieldCapabilities = Object.entries(field_capabilities_response.fields) .flatMap(([field_name, field_capabilities]) => { @@ -379,7 +376,7 @@ export class QuickwitDataSource * Get tag keys for adhoc filters */ getTagKeys() { - return lastValueFrom(this.getAggregatableFields()); + return lastValueFrom(this.getFields(true)); } /** @@ -526,6 +523,10 @@ export class QuickwitDataSource const range = options?.range; const parsedQuery = JSON.parse(query); if (query) { + if (parsedQuery.find === 'fields') { + parsedQuery.type = this.interpolateLuceneQuery(parsedQuery.type); + return lastValueFrom(this.getFields(true, parsedQuery.type, range)); + } if (parsedQuery.find === 'terms') { parsedQuery.field = this.interpolateLuceneQuery(parsedQuery.field); parsedQuery.query = this.interpolateLuceneQuery(parsedQuery.query); diff --git a/src/hooks/useFields.test.tsx b/src/hooks/useFields.test.tsx index a639413..4f12347 100644 --- a/src/hooks/useFields.test.tsx +++ b/src/hooks/useFields.test.tsx @@ -23,11 +23,11 @@ describe('useFields hook', () => { bucketAggs: [defaultBucketAgg()], }; - const getFields: ElasticDatasource['getAggregatableFields'] = jest.fn(() => from([[]])); + const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); const wrapper = ({ children }: PropsWithChildren<{}>) => ( { return async (q?: string) => { // _mapping doesn't support filtering, we avoid sending a request everytime q changes if (!rawFields) { - rawFields = await lastValueFrom(datasource.getAggregatableFields(filter, range)); + rawFields = await lastValueFrom(datasource.getFields(true, filter, range)); } return rawFields.filter(({ text }) => q === undefined || text.includes(q)).map(toSelectableValue); From 93a622a600783ea4e39b7a5cc5534403f1dc0d12 Mon Sep 17 00:00:00 2001 From: fmassot Date: Thu, 28 Dec 2023 11:40:38 +0100 Subject: [PATCH 27/30] Fix tests. --- src/hooks/useFields.test.tsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/hooks/useFields.test.tsx b/src/hooks/useFields.test.tsx index 4f12347..5ccc29e 100644 --- a/src/hooks/useFields.test.tsx +++ b/src/hooks/useFields.test.tsx @@ -48,12 +48,12 @@ describe('useFields hook', () => { { wrapper, initialProps: 'cardinality' } ); result.current(); - expect(getFields).toHaveBeenLastCalledWith([], timeRange); + expect(getFields).toHaveBeenLastCalledWith(true, [], timeRange); // All other metric aggregations only work on numbers rerender('avg'); result.current(); - expect(getFields).toHaveBeenLastCalledWith(['number'], timeRange); + expect(getFields).toHaveBeenLastCalledWith(true, ['number'], timeRange); // // BUCKET AGGREGATIONS @@ -61,26 +61,26 @@ describe('useFields hook', () => { // Date Histrogram only works on dates rerender('date_histogram'); result.current(); - expect(getFields).toHaveBeenLastCalledWith(['date'], timeRange); + expect(getFields).toHaveBeenLastCalledWith(true, ['date'], timeRange); // Histrogram only works on numbers rerender('histogram'); result.current(); - expect(getFields).toHaveBeenLastCalledWith(['number'], timeRange); + expect(getFields).toHaveBeenLastCalledWith(true, ['number'], timeRange); // Geohash Grid only works on geo_point data rerender('geohash_grid'); result.current(); - expect(getFields).toHaveBeenLastCalledWith(['geo_point'], timeRange); + expect(getFields).toHaveBeenLastCalledWith(true, ['geo_point'], timeRange); // All other bucket aggregation work on any kind of data rerender('terms'); result.current(); - expect(getFields).toHaveBeenLastCalledWith([], timeRange); + expect(getFields).toHaveBeenLastCalledWith(true, [], timeRange); // top_metrics work on only on numeric data in 7.7 rerender('top_metrics'); result.current(); - expect(getFields).toHaveBeenLastCalledWith(['number'], timeRange); + expect(getFields).toHaveBeenLastCalledWith(true, ['number'], timeRange); }); }); From debd8ae83c65e90ca91a9cc19ef3e5e575018d5c Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Thu, 28 Dec 2023 11:46:06 +0100 Subject: [PATCH 28/30] Issue #20: fix aggregatable condition --- src/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasource.ts b/src/datasource.ts index e60b94f..36c2a6c 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -344,7 +344,7 @@ export class QuickwitDataSource return from(this.getResource('_elastic/' + this.index + '/_field_caps')).pipe( map((field_capabilities_response: FieldCapabilitiesResponse) => { const shouldAddField = (field: any) => { - if (aggregatable !== undefined && !field.aggregatable) { + if (aggregatable !== undefined && field.aggregatable !== aggregatable) { return false } if (type?.length === 0) { From b66cce1ddc40dae05d76a85aebefd949f028d4bf Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Thu, 28 Dec 2023 12:25:33 +0100 Subject: [PATCH 29/30] Issue #20: create a new module to make it more easily testable --- pkg/quickwit/quickwit.go | 100 +--------------------------- pkg/quickwit/timestamp_infos.go | 111 ++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 99 deletions(-) create mode 100644 pkg/quickwit/timestamp_infos.go diff --git a/pkg/quickwit/quickwit.go b/pkg/quickwit/quickwit.go index 9deef8f..4937372 100644 --- a/pkg/quickwit/quickwit.go +++ b/pkg/quickwit/quickwit.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -33,103 +32,6 @@ type FieldMappings struct { FieldMappings []FieldMappings `json:"field_mappings,omitempty"` } -type QuickwitMapping struct { - IndexConfig struct { - DocMapping struct { - TimestampField string `json:"timestamp_field"` - FieldMappings []FieldMappings `json:"field_mappings"` - } `json:"doc_mapping"` - } `json:"index_config"` -} - -type QuickwitCreationErrorPayload struct { - Message string `json:"message"` - StatusCode int `json:"status"` -} - -func newErrorCreationPayload(statusCode int, message string) error { - var payload QuickwitCreationErrorPayload - payload.Message = message - payload.StatusCode = statusCode - json, err := json.Marshal(payload) - if nil != err { - return err - } - - return errors.New(string(json)) -} - -func findTimeStampFormat(timestampFieldName string, parentName *string, fieldMappings []FieldMappings) *string { - if nil == fieldMappings { - return nil - } - - for _, field := range fieldMappings { - fieldName := field.Name - if nil != parentName { - fieldName = fmt.Sprintf("%s.%s", *parentName, fieldName) - } - - if field.Type == "datetime" && fieldName == timestampFieldName && nil != field.OutputFormat { - return field.OutputFormat - } else if field.Type == "object" && nil != field.FieldMappings { - format := findTimeStampFormat(timestampFieldName, &field.Name, field.FieldMappings) - if nil != format { - return format - } - } - } - - return nil -} - -func getTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (string, string, error) { - mappingEndpointUrl := qwUrl + "/indexes/" + index - qwlog.Info("Calling quickwit endpoint: " + mappingEndpointUrl) - r, err := cli.Get(mappingEndpointUrl) - if err != nil { - errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error()) - qwlog.Error(errMsg) - return "", "", err - } - - statusCode := r.StatusCode - - if statusCode < 200 || statusCode >= 400 { - errMsg := fmt.Sprintf("Error when calling url = %s", mappingEndpointUrl) - qwlog.Error(errMsg) - return "", "", newErrorCreationPayload(statusCode, errMsg) - } - - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error()) - qwlog.Error(errMsg) - return "", "", newErrorCreationPayload(statusCode, errMsg) - } - - var payload QuickwitMapping - err = json.Unmarshal(body, &payload) - if err != nil { - errMsg := fmt.Sprintf("Unmarshalling body error: err = %s, body = %s", err.Error(), (body)) - qwlog.Error(errMsg) - return "", "", newErrorCreationPayload(statusCode, errMsg) - } - - timestampFieldName := payload.IndexConfig.DocMapping.TimestampField - timestampFieldFormat := findTimeStampFormat(timestampFieldName, nil, payload.IndexConfig.DocMapping.FieldMappings) - - if nil == timestampFieldFormat { - errMsg := fmt.Sprintf("No format found for field: %s", string(timestampFieldName)) - qwlog.Error(errMsg) - return timestampFieldName, "", newErrorCreationPayload(statusCode, errMsg) - } - - qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, *timestampFieldFormat)) - return timestampFieldName, *timestampFieldFormat, nil -} - // Creates a Quickwit datasource. func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { qwlog.Debug("Initializing new data source instance") @@ -190,7 +92,7 @@ func NewQuickwitDatasource(settings backend.DataSourceInstanceSettings) (instanc } if !toOk || !tofOk { - timeField, timeOutputFormat, err = getTimestampFieldInfos(index, settings.URL, httpCli) + timeField, timeOutputFormat, err = GetTimestampFieldInfos(index, settings.URL, httpCli) if nil != err { return nil, err } diff --git a/pkg/quickwit/timestamp_infos.go b/pkg/quickwit/timestamp_infos.go new file mode 100644 index 0000000..5a49bc5 --- /dev/null +++ b/pkg/quickwit/timestamp_infos.go @@ -0,0 +1,111 @@ +package quickwit + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" +) + +type QuickwitMapping struct { + IndexConfig struct { + DocMapping struct { + TimestampField string `json:"timestamp_field"` + FieldMappings []FieldMappings `json:"field_mappings"` + } `json:"doc_mapping"` + } `json:"index_config"` +} + +type QuickwitCreationErrorPayload struct { + Message string `json:"message"` + StatusCode int `json:"status"` +} + +func NewErrorCreationPayload(statusCode int, message string) error { + var payload QuickwitCreationErrorPayload + payload.Message = message + payload.StatusCode = statusCode + json, err := json.Marshal(payload) + if nil != err { + return err + } + + return errors.New(string(json)) +} + +func FindTimeStampFormat(timestampFieldName string, parentName *string, fieldMappings []FieldMappings) *string { + if nil == fieldMappings { + return nil + } + + for _, field := range fieldMappings { + fieldName := field.Name + if nil != parentName { + fieldName = fmt.Sprintf("%s.%s", *parentName, fieldName) + } + + if field.Type == "datetime" && fieldName == timestampFieldName && nil != field.OutputFormat { + return field.OutputFormat + } else if field.Type == "object" && nil != field.FieldMappings { + format := FindTimeStampFormat(timestampFieldName, &field.Name, field.FieldMappings) + if nil != format { + return format + } + } + } + + return nil +} + +func DecodeTimestampFieldInfos(statusCode int, body []byte) (string, string, error) { + var payload QuickwitMapping + err := json.Unmarshal(body, &payload) + + if err != nil { + errMsg := fmt.Sprintf("Unmarshalling body error: err = %s, body = %s", err.Error(), (body)) + qwlog.Error(errMsg) + return "", "", NewErrorCreationPayload(statusCode, errMsg) + } + + timestampFieldName := payload.IndexConfig.DocMapping.TimestampField + timestampFieldFormat := FindTimeStampFormat(timestampFieldName, nil, payload.IndexConfig.DocMapping.FieldMappings) + + if nil == timestampFieldFormat { + errMsg := fmt.Sprintf("No format found for field: %s", string(timestampFieldName)) + qwlog.Error(errMsg) + return timestampFieldName, "", NewErrorCreationPayload(statusCode, errMsg) + } + + qwlog.Info(fmt.Sprintf("Found timestampFieldName = %s, timestampFieldFormat = %s", timestampFieldName, *timestampFieldFormat)) + return timestampFieldName, *timestampFieldFormat, nil +} + +func GetTimestampFieldInfos(index string, qwUrl string, cli *http.Client) (string, string, error) { + mappingEndpointUrl := qwUrl + "/indexes/" + index + qwlog.Info("Calling quickwit endpoint: " + mappingEndpointUrl) + r, err := cli.Get(mappingEndpointUrl) + if err != nil { + errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error()) + qwlog.Error(errMsg) + return "", "", err + } + + statusCode := r.StatusCode + + if statusCode < 200 || statusCode >= 400 { + errMsg := fmt.Sprintf("Error when calling url = %s", mappingEndpointUrl) + qwlog.Error(errMsg) + return "", "", NewErrorCreationPayload(statusCode, errMsg) + } + + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + errMsg := fmt.Sprintf("Error when calling url = %s: err = %s", mappingEndpointUrl, err.Error()) + qwlog.Error(errMsg) + return "", "", NewErrorCreationPayload(statusCode, errMsg) + } + + return DecodeTimestampFieldInfos(statusCode, body) +} From 1d645408bc4413a47374313814d7a2db826fec3e Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Thu, 28 Dec 2023 12:59:11 +0100 Subject: [PATCH 30/30] Issue #20: add backend unit tests --- pkg/quickwit/timestamp_infos_test.go | 313 +++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 pkg/quickwit/timestamp_infos_test.go diff --git a/pkg/quickwit/timestamp_infos_test.go b/pkg/quickwit/timestamp_infos_test.go new file mode 100644 index 0000000..55f5916 --- /dev/null +++ b/pkg/quickwit/timestamp_infos_test.go @@ -0,0 +1,313 @@ +package quickwit + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodeTimestampFieldInfos(t *testing.T) { + t.Run("Test decode timestam field infos", func(t *testing.T) { + t.Run("Test decode simple fields", func(t *testing.T) { + // Given + query := []byte(` + { + "version": "0.6", + "index_uid": "myindex:01HG7ZZK3ZD7XF6BKQCZJHSJ5W", + "index_config": { + "version": "0.6", + "index_id": "myindex", + "index_uri": "s3://quickwit-indexes/myindex", + "doc_mapping": { + "field_mappings": [ + { + "name": "foo", + "type": "text", + "fast": false, + "fieldnorms": false, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "default" + }, + { + "name": "timestamp", + "type": "datetime", + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "output_format": "rfc3339", + "stored": true + } + ], + "tag_fields": [], + "store_source": true, + "index_field_presence": false, + "timestamp_field": "timestamp", + "mode": "dynamic", + "dynamic_mapping": {}, + "partition_key": "foo", + "max_num_partitions": 1, + "tokenizers": [] + }, + "indexing_settings": {}, + "search_settings": { + "default_search_fields": [ + "foo" + ] + }, + "retention": null + }, + "checkpoint": {}, + "create_timestamp": 1701075471, + "sources": [] + } + `) + + // When + timestampFieldName, timestampFieldFormat, err := DecodeTimestampFieldInfos(200, query) + + // Then + require.NoError(t, err) + require.Equal(t, timestampFieldName, "timestamp") + require.Equal(t, timestampFieldFormat, "rfc3339") + }) + + t.Run("Test decode nested fields", func(t *testing.T) { + // Given + query := []byte(` + { + "version": "0.6", + "index_uid": "myindex:01HG7ZZK3ZD7XF6BKQCZJHSJ5W", + "index_config": { + "version": "0.6", + "index_id": "myindex", + "index_uri": "s3://quickwit-indexes/myindex", + "doc_mapping": { + "field_mappings": [ + { + "name": "foo", + "type": "text", + "fast": false, + "fieldnorms": false, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "default" + }, + { + "name": "sub", + "type": "object", + "field_mappings": [ + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "timestamp", + "output_format": "rfc3339", + "stored": true, + "type": "datetime" + } + ] + } + ], + "tag_fields": [], + "store_source": true, + "index_field_presence": false, + "timestamp_field": "sub.timestamp", + "mode": "dynamic", + "dynamic_mapping": {}, + "partition_key": "foo", + "max_num_partitions": 1, + "tokenizers": [] + }, + "indexing_settings": {}, + "search_settings": { + "default_search_fields": [ + "foo" + ] + }, + "retention": null + }, + "checkpoint": {}, + "create_timestamp": 1701075471, + "sources": [] + } + `) + + // When + timestampFieldName, timestampFieldFormat, err := DecodeTimestampFieldInfos(200, query) + + // Then + require.NoError(t, err) + require.Equal(t, timestampFieldName, "sub.timestamp") + require.Equal(t, timestampFieldFormat, "rfc3339") + }) + + t.Run("The timestamp field is not at the expected path", func(t *testing.T) { + // Given + query := []byte(` + { + "version": "0.6", + "index_uid": "myindex:01HG7ZZK3ZD7XF6BKQCZJHSJ5W", + "index_config": { + "version": "0.6", + "index_id": "myindex", + "index_uri": "s3://quickwit-indexes/myindex", + "doc_mapping": { + "field_mappings": [ + { + "name": "foo", + "type": "text", + "fast": false, + "fieldnorms": false, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "default" + }, + { + "name": "sub", + "type": "object", + "field_mappings": [ + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "timestamp", + "output_format": "rfc3339", + "stored": true, + "type": "datetime" + } + ] + } + ], + "tag_fields": [], + "store_source": true, + "index_field_presence": false, + "timestamp_field": "timestamp", + "mode": "dynamic", + "dynamic_mapping": {}, + "partition_key": "foo", + "max_num_partitions": 1, + "tokenizers": [] + }, + "indexing_settings": {}, + "search_settings": { + "default_search_fields": [ + "foo" + ] + }, + "retention": null + }, + "checkpoint": {}, + "create_timestamp": 1701075471, + "sources": [] + } + `) + + // When + _, _, err := DecodeTimestampFieldInfos(200, query) + + // Then + require.Error(t, err) + }) + + t.Run("The timestamp field has not the right type", func(t *testing.T) { + // Given + query := []byte(` + { + "version": "0.6", + "index_uid": "myindex:01HG7ZZK3ZD7XF6BKQCZJHSJ5W", + "index_config": { + "version": "0.6", + "index_id": "myindex", + "index_uri": "s3://quickwit-indexes/myindex", + "doc_mapping": { + "field_mappings": [ + { + "name": "foo", + "type": "text", + "fast": false, + "fieldnorms": false, + "indexed": true, + "record": "basic", + "stored": true, + "tokenizer": "default" + }, + { + "name": "sub", + "type": "object", + "field_mappings": [ + { + "fast": true, + "fast_precision": "seconds", + "indexed": true, + "input_formats": [ + "rfc3339", + "unix_timestamp" + ], + "name": "timestamp", + "output_format": "rfc3339", + "stored": true, + "type": "whatever" + } + ] + } + ], + "tag_fields": [], + "store_source": true, + "index_field_presence": false, + "timestamp_field": "sub.timestamp", + "mode": "dynamic", + "dynamic_mapping": {}, + "partition_key": "foo", + "max_num_partitions": 1, + "tokenizers": [] + }, + "indexing_settings": {}, + "search_settings": { + "default_search_fields": [ + "foo" + ] + }, + "retention": null + }, + "checkpoint": {}, + "create_timestamp": 1701075471, + "sources": [] + } + `) + + // When + _, _, err := DecodeTimestampFieldInfos(200, query) + + // Then + require.Error(t, err) + }) + }) +} + +func TestNewErrorCreationPayload(t *testing.T) { + t.Run("Test marshall creation payload error", func(t *testing.T) { + // When + err := NewErrorCreationPayload(400, "No valid format") + + // Then + require.Error(t, err) + require.ErrorContains(t, err, "\"message\":\"No valid format\"") + require.ErrorContains(t, err, "\"status\":400") + }) +}