diff --git a/src/LogContext/LogContextProvider.ts b/src/LogContext/LogContextProvider.ts new file mode 100644 index 0000000..744eb94 --- /dev/null +++ b/src/LogContext/LogContextProvider.ts @@ -0,0 +1,131 @@ +import { ReactNode } from 'react'; +import { lastValueFrom } from 'rxjs'; +import { QuickwitDataSource } from 'datasource'; +import { catchError } from 'rxjs/operators'; + +import { + CoreApp, + DataFrame, + DataQueryError, + DataQueryRequest, + dateTime, + LogRowModel, + rangeUtil, +} from '@grafana/data'; + +import { ElasticsearchQuery, Logs} from '../types'; + +import { LogContextUI } from 'LogContext/components/LogContextUI'; + +export interface LogRowContextOptions { + direction?: LogRowContextQueryDirection; + limit?: number; +} +export enum LogRowContextQueryDirection { + Backward = 'BACKWARD', + Forward = 'FORWARD', +} + +function createContextTimeRange(rowTimeEpochMs: number, direction: string) { + const offset = 7; + // For log context, we want to request data from 7 subsequent/previous indices + if (direction === LogRowContextQueryDirection.Forward) { + return { + from: dateTime(rowTimeEpochMs).utc(), + to: dateTime(rowTimeEpochMs).add(offset, 'hours').utc(), + }; + } else { + return { + from: dateTime(rowTimeEpochMs).subtract(offset, 'hours').utc(), + to: dateTime(rowTimeEpochMs).utc(), + }; + } +} + +export class LogContextProvider { + datasource: QuickwitDataSource; + contextQuery: string | null; + + constructor(datasource: QuickwitDataSource) { + this.datasource = datasource; + this.contextQuery = null; + } + private makeLogContextDataRequest = ( + row: LogRowModel, + options?: LogRowContextOptions, + origQuery?: ElasticsearchQuery + ) => { + const direction = options?.direction || LogRowContextQueryDirection.Backward; + const searchAfter = row.dataFrame.fields.find((f) => f.name === 'sort')?.values.get(row.rowIndex) ?? [row.timeEpochNs] + + const logQuery: Logs = { + type: 'logs', + id: '1', + settings: { + limit: options?.limit ? options?.limit.toString() : '10', + // Sorting of results in the context query + sortDirection: direction === LogRowContextQueryDirection.Backward ? 'desc' : 'asc', + // Used to get the next log lines before/after the current log line using sort field of selected log line + searchAfter: searchAfter, + }, + }; + + const query: ElasticsearchQuery = { + refId: `log-context-${row.dataFrame.refId}-${direction}`, + metrics: [logQuery], + query: this.contextQuery == null ? origQuery?.query : this.contextQuery, + }; + + const timeRange = createContextTimeRange(row.timeEpochMs, direction); + const range = { + from: timeRange.from, + to: timeRange.to, + raw: timeRange, + }; + + const interval = rangeUtil.calculateInterval(range, 1); + + const contextRequest: DataQueryRequest = { + requestId: `log-context-request-${row.dataFrame.refId}-${options?.direction}`, + targets: [query], + interval: interval.interval, + intervalMs: interval.intervalMs, + range, + scopedVars: {}, + timezone: 'UTC', + app: CoreApp.Explore, + startTime: Date.now(), + hideFromInspector: true, + }; + return contextRequest; + }; + + getLogRowContext = async ( + row: LogRowModel, + options?: LogRowContextOptions, + origQuery?: ElasticsearchQuery + ): Promise<{ data: DataFrame[] }> => { + const contextRequest = this.makeLogContextDataRequest(row, options, origQuery); + + return lastValueFrom( + this.datasource.query(contextRequest).pipe( + catchError((err) => { + const error: DataQueryError = { + message: 'Error during context query. Please check JS console logs.', + status: err.status, + statusText: err.message, + }; + throw error; + }) + ) + ); + }; + + getLogRowContextUi( + row: LogRowModel, + runContextQuery?: (() => void), + origQuery?: ElasticsearchQuery + ): ReactNode { + return ( LogContextUI({row, runContextQuery, origQuery, updateQuery: query=>{this.contextQuery=query}, datasource:this.datasource})) + } +} diff --git a/src/LogContext/components/LogContextQueryBuilderSidebar.tsx b/src/LogContext/components/LogContextQueryBuilderSidebar.tsx new file mode 100644 index 0000000..34cba78 --- /dev/null +++ b/src/LogContext/components/LogContextQueryBuilderSidebar.tsx @@ -0,0 +1,191 @@ +import React, { useEffect, useMemo, useState } from "react"; +// import { Field } from '@grafana/data'; +import { useTheme2, CollapsableSection, Icon } from '@grafana/ui'; +import { LogContextProps } from "./LogContextUI"; +import { css, cx } from "@emotion/css"; +import { LuceneQuery } from "utils/lucene"; +import { LuceneQueryBuilder } from '@/QueryBuilder/lucene'; + + +// TODO : define sensible defaults here +const excludedFields = [ + '_source', + 'sort', + 'attributes', + 'attributes.message', + 'body', + 'body.message', + 'resource_attributes', + 'observed_timestamp_nanos', + 'timestamp_nanos', +]; + +function isPrimitive(valT: any) { + return ['string', 'number', "boolean", "undefined"].includes(valT) +} + +type FieldContingency = { [value: string]: { + count: number, pinned: boolean, active?: boolean +}}; +type Field = { + name: string, + contingency: FieldContingency +} + +function LogContextFieldSection(field: Field) { + const theme = useTheme2() + const hasActiveFilters = Object.entries(field.contingency).map(([_,entry])=>!!entry.active).reduce((a,b)=>a || b, false); + return( + + {hasActiveFilters && } + {field.name} + + ) +} + +type FieldItemProps = { + label: any, + contingency: { + count: number, + pinned: boolean + }, + active?: boolean, + onClick: () => void +} + +function LogContextFieldItem(props: FieldItemProps){ + const theme = useTheme2() + const lcAttributeItemStyle = css({ + display: "flex", + justifyContent: "space-between", + paddingLeft: "10px", + fontSize: theme.typography.bodySmall.fontSize, + "&[data-active=true]": { + backgroundColor: theme.colors.primary.transparent, + }, + "&:hover": { + backgroundColor: theme.colors.secondary.shade, + } + }); + + const formatLabel = (value: any)=> { + let shouldEmphasize = false; + let label = `${value}`; + + if (value === null || value === '' || value === undefined){ + shouldEmphasize = true; + } + if (value === '') { + label = '' + } + return (shouldEmphasize ? {label} : label); + } + + return ( + + { formatLabel(props.label) } + {props.contingency.pinned && }{props.contingency.count} + + ) +} + +const lcSidebarStyle = css` + width: 300px; + min-width: 300px; + flex-shrink: 0; + overflow-y: scroll; + padding-right: 1rem; +` + +type QueryBuilderProps = { + builder: LuceneQueryBuilder, + searchableFields: any[], + updateQuery: (query: LuceneQuery) => void +} + +export function LogContextQueryBuilderSidebar(props: LogContextProps & QueryBuilderProps) { + + const {row, builder, updateQuery, searchableFields} = props; + const [fields, setFields] = useState([]); + + const filteredFields = useMemo(() => { + const searchableFieldsNames = searchableFields.map(f=>f.text); + return row.dataFrame.fields + .filter(f=>searchableFieldsNames.includes(f.name)) + // exclude some low-filterability fields + .filter((f)=> !excludedFields.includes(f.name) && isPrimitive(f.type)) + // sort fields by name + .sort((f1, f2)=> (f1.name>f2.name ? 1 : -1)) + }, [row, searchableFields]); + + useEffect(() => { + const fields = filteredFields + .map((f) => { + const contingency: FieldContingency = {}; + f.values.forEach((value, i) => { + if (!contingency[value]) { + contingency[value] = { + count: 0, + pinned: false, + active: builder.parsedQuery ? !!builder.parsedQuery.findFilter(f.name, `${value}`) : false + } + } + contingency[value].count += 1; + if (i === row.rowIndex) { + contingency[value].pinned = true; + } + }); + return { name: f.name, contingency }; + }) + + setFields(fields); + }, [filteredFields, row.rowIndex, builder.parsedQuery]); + + + const selectQueryFilter = (key: string, value: string): void => { + // Compute mutation to apply to the query and send to parent + // check if that filter is in the query + if (!builder.parsedQuery) { return; } + + const newParsedQuery = ( + builder.parsedQuery.hasFilter(key, value) + ? builder.parsedQuery.removeFilter(key, value) + : builder.parsedQuery.addFilter(key, value) + ) + + if (newParsedQuery) { + updateQuery(newParsedQuery); + } + } + + const renderFieldSection = (field: Field)=>{ + return ( + div { flex-grow:1; }` } + isOpen={false} key="log-attribute-field-{field.name}" + contentClassName={cx(css`margin:0; padding:0`)}> +
+ + {field.contingency && Object.entries(field.contingency) + .sort(([na, ca], [nb, cb])=>(cb.count - ca.count)) + .map(([fieldValue, contingency], i) => ( + {selectQueryFilter(field.name, fieldValue)}} + active={contingency.active} + /> + )) + } +
+
+ ) + } + + return ( +
+ {fields && fields.map((field) => { + return( renderFieldSection(field) ); + }) }
+ ); +} diff --git a/src/LogContext/components/LogContextUI.tsx b/src/LogContext/components/LogContextUI.tsx new file mode 100644 index 0000000..e60fa0c --- /dev/null +++ b/src/LogContext/components/LogContextUI.tsx @@ -0,0 +1,72 @@ +import React, { useEffect, useState, useCallback, useMemo } from "react"; +import { LogRowModel } from '@grafana/data'; +import { ElasticsearchQuery as DataQuery } from '../../types'; +import { LuceneQueryEditor } from "../../components/LuceneQueryEditor"; + +import { css } from "@emotion/css"; +import { Button } from "@grafana/ui"; +import { useQueryBuilder } from '@/QueryBuilder/lucene'; +import { LogContextQueryBuilderSidebar } from "./LogContextQueryBuilderSidebar"; +import { DatasourceContext } from "components/QueryEditor/ElasticsearchQueryContext"; +import { QuickwitDataSource } from "datasource"; +import { useDatasourceFields } from "datasource.utils"; + +const logContextUiStyle = css` + display: flex; + gap: 1rem; + width: 100%; + height: 200px; +` + +export interface LogContextProps { + row: LogRowModel, + runContextQuery?: (() => void) + origQuery?: DataQuery +} +export interface LogContextUIProps extends LogContextProps { + datasource: QuickwitDataSource, + updateQuery: (query: string) => void +} + +export function LogContextUI(props: LogContextUIProps ){ + const builder = useQueryBuilder(); + const {query, parsedQuery, setQuery, setParsedQuery} = builder; + const [canRunQuery, setCanRunQuery] = useState(false); + const { origQuery, updateQuery, runContextQuery } = props; + const {fields, getSuggestions} = useDatasourceFields(props.datasource); + + useEffect(()=>{ + setQuery(origQuery?.query || '') + }, [setQuery, origQuery]) + + useEffect(()=>{ + setCanRunQuery(!parsedQuery.parseError) + }, [parsedQuery, setCanRunQuery]) + + const runQuery = useCallback(()=>{ + if (runContextQuery){ + updateQuery(query); + runContextQuery(); + } + }, [query, runContextQuery, updateQuery]) + + const ActionBar = useMemo(()=>( +
+ + + +
+ ), [setQuery, canRunQuery, origQuery, runQuery]) + + return ( +
+ + +
+ {ActionBar} + +
+
+
+ ); +} diff --git a/src/datasource.ts b/src/datasource.ts index 2899855..190f6a1 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -5,10 +5,8 @@ import { catchError, mergeMap, map } from 'rxjs/operators'; import { AbstractQuery, AdHocVariableFilter, - CoreApp, DataFrame, DataLink, - DataQueryError, DataQueryRequest, DataQueryResponse, DataSourceApi, @@ -17,7 +15,6 @@ import { DataSourceWithLogsContextSupport, DataSourceWithQueryImportSupport, DataSourceWithSupplementaryQueriesSupport, - dateTime, FieldColorModeId, FieldType, getDefaultTimeRange, @@ -28,18 +25,17 @@ import { LogsVolumeType, MetricFindValue, QueryFixAction, - rangeUtil, ScopedVars, SupplementaryQueryType, TimeRange, } from '@grafana/data'; -import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field as QuickwitField, FieldMapping, IndexMetadata, Logs, TermsQuery, FieldCapabilitiesResponse } from './types'; +import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field as QuickwitField, FieldMapping, IndexMetadata, TermsQuery, FieldCapabilitiesResponse } from './types'; import { DataSourceWithBackend, getTemplateSrv, TemplateSrv, getDataSourceSrv } from '@grafana/runtime'; -import { LogRowContextOptions, LogRowContextQueryDirection, QuickwitOptions } from 'quickwit'; +import { QuickwitOptions } from 'quickwit'; import { ElasticQueryBuilder } from 'QueryBuilder/elastic'; import { colors } from '@grafana/ui'; @@ -52,6 +48,7 @@ import ElasticsearchLanguageProvider from 'LanguageProvider'; import { ReactNode } from 'react'; import { extractJsonPayload, fieldTypeMap } from 'utils'; import { addAddHocFilter } from 'modifyQuery'; +import { LogContextProvider, LogRowContextOptions } from './LogContext/LogContextProvider'; export const REF_ID_STARTER_LOG_VOLUME = 'log-volume-'; @@ -80,6 +77,8 @@ export class QuickwitDataSource dataLinks: DataLinkConfig[]; languageProvider: ElasticsearchLanguageProvider; + private logContextProvider: LogContextProvider; + constructor( instanceSettings: DataSourceInstanceSettings, private readonly templateSrv: TemplateSrv = getTemplateSrv() @@ -124,6 +123,7 @@ export class QuickwitDataSource this.logLevelField = settingsData.logLevelField || ''; this.dataLinks = settingsData.dataLinks || []; this.languageProvider = new ElasticsearchLanguageProvider(this); + this.logContextProvider = new LogContextProvider(this); } query(request: DataQueryRequest): Observable { @@ -482,75 +482,27 @@ export class QuickwitDataSource return text; } - private makeLogContextDataRequest = (row: LogRowModel, options?: LogRowContextOptions) => { - const direction = options?.direction || LogRowContextQueryDirection.Backward; - const searchAfter = row.dataFrame.fields.find((f) => f.name === 'sort')?.values.get(row.rowIndex) ?? [row.timeEpochNs] - - const logQuery: Logs = { - type: 'logs', - id: '1', - settings: { - limit: options?.limit ? options?.limit.toString() : '10', - // Sorting of results in the context query - sortDirection: direction === LogRowContextQueryDirection.Backward ? 'desc' : 'asc', - // Used to get the next log lines before/after the current log line using sort field of selected log line - searchAfter: searchAfter, - }, - }; - - const query: ElasticsearchQuery = { - refId: `log-context-${row.dataFrame.refId}-${direction}`, - metrics: [logQuery], - query: '', - }; - - const timeRange = createContextTimeRange(row.timeEpochMs, direction); - const range = { - from: timeRange.from, - to: timeRange.to, - raw: timeRange, - }; - - const interval = rangeUtil.calculateInterval(range, 1); - - const contextRequest: DataQueryRequest = { - requestId: `log-context-request-${row.dataFrame.refId}-${options?.direction}`, - targets: [query], - interval: interval.interval, - intervalMs: interval.intervalMs, - range, - scopedVars: {}, - timezone: 'UTC', - app: CoreApp.Explore, - startTime: Date.now(), - hideFromInspector: true, - }; - return contextRequest; - }; - - getLogRowContext = async (row: LogRowModel, options?: LogRowContextOptions): Promise<{ data: DataFrame[] }> => { - const contextRequest = this.makeLogContextDataRequest(row, options); - - return lastValueFrom( - this.query(contextRequest).pipe( - catchError((err) => { - const error: DataQueryError = { - message: 'Error during context query. Please check JS console logs.', - status: err.status, - statusText: err.message, - }; - throw error; - }) - ) - ); - }; + // Log Context + // NOTE : deprecated since grafana-data 10.3 showContextToggle(row?: LogRowModel | undefined): boolean { return true; } - getLogRowContextUi?(row: LogRowModel, runContextQuery?: (() => void) | undefined): ReactNode { - return true; + getLogRowContext = async ( + row: LogRowModel, + options?: LogRowContextOptions, + origQuery?: ElasticsearchQuery + ): Promise<{ data: DataFrame[] }> => { + return await this.logContextProvider.getLogRowContext(row, options, origQuery); + } + + getLogRowContextUi( + row: LogRowModel, + runContextQuery?: (() => void), + origQuery?: ElasticsearchQuery + ): ReactNode { + return this.logContextProvider.getLogRowContextUi(row, runContextQuery, origQuery); } /** @@ -916,19 +868,3 @@ function generateDataLink(linkConfig: DataLinkConfig): DataLink { }; } } - -function createContextTimeRange(rowTimeEpochMs: number, direction: string) { - const offset = 7; - // For log context, we want to request data from 7 subsequent/previous indices - if (direction === LogRowContextQueryDirection.Forward) { - return { - from: dateTime(rowTimeEpochMs).utc(), - to: dateTime(rowTimeEpochMs).add(offset, 'hours').utc(), - }; - } else { - return { - from: dateTime(rowTimeEpochMs).subtract(offset, 'hours').utc(), - to: dateTime(rowTimeEpochMs).utc(), - }; - } -} diff --git a/src/quickwit.ts b/src/quickwit.ts index fa886db..94e3725 100644 --- a/src/quickwit.ts +++ b/src/quickwit.ts @@ -10,13 +10,3 @@ export interface QuickwitOptions extends DataSourceJsonData { dataLinks?: DataLinkConfig[]; index: string; } - -export interface LogRowContextOptions { - direction?: LogRowContextQueryDirection; - limit?: number; -} - -export enum LogRowContextQueryDirection { - Backward = 'BACKWARD', - Forward = 'FORWARD', -}