diff --git a/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx b/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx index 8b473ae..f8be73f 100644 --- a/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx +++ b/src/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/TermsSettingsEditor.test.tsx @@ -22,6 +22,7 @@ describe('Terms Settings Editor', () => { query: '', bucketAggs: [termsAgg], metrics: [avg, derivative, topMetrics], + filters: [], }; renderWithESProvider(, { providerProps: { query } }); diff --git a/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx b/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx index bc52130..4db8980 100644 --- a/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx +++ b/src/components/QueryEditor/ElasticsearchQueryContext.test.tsx @@ -13,6 +13,7 @@ const query: ElasticsearchQuery = { query: '', metrics: [{ id: '1', type: 'count' }], bucketAggs: [{ type: 'date_histogram', id: '2' }], + filters: [] }; describe('ElasticsearchQueryContext', () => { diff --git a/src/components/QueryEditor/ElasticsearchQueryContext.tsx b/src/components/QueryEditor/ElasticsearchQueryContext.tsx index 919357d..f6ab989 100644 --- a/src/components/QueryEditor/ElasticsearchQueryContext.tsx +++ b/src/components/QueryEditor/ElasticsearchQueryContext.tsx @@ -8,6 +8,7 @@ import { ElasticsearchQuery } from '@/types'; import { createReducer as createBucketAggsReducer } from './BucketAggregationsEditor/state/reducer'; import { reducer as metricsReducer } from './MetricAggregationsEditor/state/reducer'; +import { reducer as filtersReducer } from './FilterEditor/state/reducer'; import { aliasPatternReducer, queryReducer, initQuery, initExploreQuery } from './state'; import { getHook } from '@/utils/context'; import { Provider, useDispatch } from "react-redux"; @@ -64,10 +65,11 @@ export const ElasticsearchProvider = withStore(({ [onChange] ); - const reducer = combineReducers>({ + const reducer = combineReducers>({ query: queryReducer, alias: aliasPatternReducer, metrics: metricsReducer, + filters: filtersReducer, bucketAggs: createBucketAggsReducer(datasource.timeField), }); @@ -78,7 +80,7 @@ export const ElasticsearchProvider = withStore(({ reducer ); - const isUninitialized = !query.metrics || !query.bucketAggs || query.query === undefined; + const isUninitialized = !query.metrics || !query.filters || !query.bucketAggs || query.query === undefined; const [shouldRunInit, setShouldRunInit] = useState(isUninitialized); diff --git a/src/components/QueryEditor/FilterEditor/index.tsx b/src/components/QueryEditor/FilterEditor/index.tsx new file mode 100644 index 0000000..e30bdbc --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/index.tsx @@ -0,0 +1,149 @@ +import React, { useRef } from 'react'; + +import { useDispatch } from '@/hooks/useStatelessReducer'; +import { IconButton } from '../../IconButton'; +import { useQuery } from '../ElasticsearchQueryContext'; +import { QueryEditorRow } from '../QueryEditorRow'; + +import { QueryFilter } from '@/types'; +import { InlineSegmentGroup, Input, Segment, SegmentAsync, Tooltip } from '@grafana/ui'; +import { + addFilter, + removeFilter, + toggleFilterVisibility, + changeFilterField, + changeFilterOperation, + changeFilterValue, +} from '@/components/QueryEditor/FilterEditor/state/actions'; +import { segmentStyles } from '@/components/QueryEditor/styles'; +import { useFields } from '@/hooks/useFields'; +import { newFilterId } from '@/utils/uid'; +import { filterOperations } from '@/queryDef'; +import { hasWhiteSpace, isSet } from '@/utils'; + +interface FilterEditorProps { + onSubmit: () => void; +} + +function filterErrors(filter: QueryFilter): string[] { + const errors: string[] = []; + + if (!isSet(filter.filter.key)) { + errors.push('Field is not set'); + } + + if (!isSet(filter.filter.operator)) { + errors.push('Operator is not set'); + } + + if (!['exists', 'not exists'].includes(filter.filter.operator) && !isSet(filter.filter.value)) { + errors.push('Value is not set'); + } + + if (['term', 'not term'].includes(filter.filter.operator) && filter.filter.value && hasWhiteSpace(filter.filter.value)) { + errors.push('Term cannot have whitespace in value'); + } + + return errors; +} + +export const FilterEditor = ({ onSubmit }: FilterEditorProps) => { + const dispatch = useDispatch(); + const { filters } = useQuery(); + + return ( + <> + {filters?.map((filter, index) => { + const errors = filterErrors(filter) + return ( + 0 ? ( + + Filter + + ): 'Filter'} + hidden={filter.hide} + onHideClick={() => { + dispatch(toggleFilterVisibility(filter.id)); + onSubmit(); + }} + onRemoveClick={() => { + dispatch(removeFilter(filter.id)); + onSubmit(); + }} + > + + + {index === 0 && dispatch(addFilter(newFilterId()))} + />} + + ) + })} + + ); +}; + +interface FilterEditorRowProps { + value: QueryFilter; + onSubmit: () => void; +} + +export const FilterEditorRow = ({ value, onSubmit }: FilterEditorRowProps) => { + const dispatch = useDispatch(); + const getFields = useFields('filters', 'startsWith'); + const valueInputRef = useRef(null); + + return ( + <> + + { + dispatch(changeFilterField({ id: value.id, field: e.value ?? '' })); + if (['exists', 'not exists'].includes(value.filter.operator) || isSet(value.filter.value)) { + onSubmit(); + } + // Auto focus the value input when a field is selected + setTimeout(() => valueInputRef.current?.focus(), 100); + }} + placeholder="Select Field" + value={value.filter.key} + /> +
+ op.value === value.filter.operator)} + options={filterOperations} + onChange={(e) => { + let op = e.value ?? filterOperations[0].value; + dispatch(changeFilterOperation({ id: value.id, op: op })); + if (['exists', 'not exists'].includes(op) || isSet(value.filter.value)) { + onSubmit(); + } + }} + /> +
+ {!['exists', 'not exists'].includes(value.filter.operator) && ( + dispatch(changeFilterValue({ id: value.id, value: e.currentTarget.value }))} + onKeyUp={(e) => { + if (e.key === 'Enter') { + onSubmit(); + } + }} + /> + )} +
+ + ); +}; diff --git a/src/components/QueryEditor/FilterEditor/state/actions.ts b/src/components/QueryEditor/FilterEditor/state/actions.ts new file mode 100644 index 0000000..ae9b7b5 --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/state/actions.ts @@ -0,0 +1,10 @@ +import { createAction } from '@reduxjs/toolkit'; + +import { QueryFilter } from '@/types'; + +export const addFilter = createAction('@filters/add'); +export const removeFilter = createAction('@filters/remove'); +export const toggleFilterVisibility = createAction('@filters/toggle_visibility'); +export const changeFilterField = createAction<{ id: QueryFilter['id']; field: string }>('@filters/change_field'); +export const changeFilterValue = createAction<{ id: QueryFilter['id']; value: string }>('@filters/change_value'); +export const changeFilterOperation = createAction<{ id: QueryFilter['id']; op: string }>('@filters/change_operation'); diff --git a/src/components/QueryEditor/FilterEditor/state/reducer.ts b/src/components/QueryEditor/FilterEditor/state/reducer.ts new file mode 100644 index 0000000..ed8ed87 --- /dev/null +++ b/src/components/QueryEditor/FilterEditor/state/reducer.ts @@ -0,0 +1,100 @@ +import { Action } from '@reduxjs/toolkit'; +import { defaultFilter } from '@/queryDef'; +import { ElasticsearchQuery } from '@/types'; +import { initExploreQuery, initQuery } from '../../state'; + +import { + addFilter, + changeFilterField, + changeFilterOperation, + changeFilterValue, + removeFilter, + toggleFilterVisibility, +} from './actions'; + +export const reducer = (state: ElasticsearchQuery['filters'], action: Action): ElasticsearchQuery['filters'] => { + // console.log('Running filters reducer with action:', action, state); + + if (addFilter.match(action)) { + return [...state!, defaultFilter(action.payload)]; + } + + if (removeFilter.match(action)) { + const filterToRemove = state!.find((m) => m.id === action.payload)!; + const resultingFilters = state!.filter((filter) => filterToRemove.id !== filter.id); + if (resultingFilters.length === 0) { + return [defaultFilter()]; + } + return resultingFilters; + } + + if (changeFilterField.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + key: action.payload.field, + } + }; + }); + } + + if (changeFilterOperation.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + operator: action.payload.op, + } + }; + }); + } + + if (changeFilterValue.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload.id) { + return filter; + } + + return { + ...filter, + filter: { + ...filter.filter, + value: action.payload.value, + } + }; + }); + } + + if (toggleFilterVisibility.match(action)) { + return state!.map((filter) => { + if (filter.id !== action.payload) { + return filter; + } + + return { + ...filter, + hide: !filter.hide, + }; + }); + } + + if (initQuery.match(action) || initExploreQuery.match(action)) { + if (state && state.length > 0) { + return state; + } + return [defaultFilter()]; + } + + return state; +}; diff --git a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx index e174716..2c8575a 100644 --- a/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx +++ b/src/components/QueryEditor/MetricAggregationsEditor/MetricEditor.test.tsx @@ -24,6 +24,7 @@ describe('Metric Editor', () => { query: '', metrics: [avg], bucketAggs: [defaultBucketAgg('2')], + filters: [], }; const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); @@ -62,6 +63,7 @@ describe('Metric Editor', () => { query: '', metrics: [count], bucketAggs: [], + filters: [], }; const wrapper = ({ children }: PropsWithChildren<{}>) => ( diff --git a/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx b/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx index 5032c52..a0b26ad 100644 --- a/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx +++ b/src/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.test.tsx @@ -27,6 +27,7 @@ describe('Settings Editor', () => { }, ], bucketAggs: [], + filters: [], }; const onChange = jest.fn(); @@ -102,6 +103,7 @@ describe('Settings Editor', () => { }, ], bucketAggs: [], + filters: [], }; const onChange = jest.fn(); diff --git a/src/components/QueryEditor/index.test.tsx b/src/components/QueryEditor/index.test.tsx index f51cf03..c932a8a 100644 --- a/src/components/QueryEditor/index.test.tsx +++ b/src/components/QueryEditor/index.test.tsx @@ -20,6 +20,7 @@ describe('QueryEditor', () => { ], // Even if present, this shouldn't be shown in the UI bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; render(); @@ -38,6 +39,7 @@ describe('QueryEditor', () => { }, ], bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; render(); diff --git a/src/components/QueryEditor/index.tsx b/src/components/QueryEditor/index.tsx index 773c435..9059fee 100644 --- a/src/components/QueryEditor/index.tsx +++ b/src/components/QueryEditor/index.tsx @@ -25,6 +25,7 @@ import { QueryTypeSelector } from './QueryTypeSelector'; import { getHook } from '@/utils/context'; import { LuceneQueryEditor } from '@/components/LuceneQueryEditor'; import { useDatasourceFields } from '@/datasource/utils'; +import { FilterEditor } from '@/components/QueryEditor/FilterEditor'; export type ElasticQueryEditorProps = QueryEditorProps; @@ -133,6 +134,7 @@ const QueryEditorForm = ({ value, onRunQuery }: Props) => { value={value?.query} onSubmit={onSubmitCB}/> + {showBucketAggregationsEditor && } diff --git a/src/dataquery.gen.ts b/src/dataquery.gen.ts index bd5dc47..0b70b88 100644 --- a/src/dataquery.gen.ts +++ b/src/dataquery.gen.ts @@ -9,6 +9,7 @@ // Run 'make gen-cue' from repository root to regenerate. import { DataQuery } from '@grafana/schema'; +import { AdHocVariableFilter } from '@grafana/data'; export const DataQueryModelVersion = Object.freeze([0, 0]); @@ -128,6 +129,12 @@ export interface BaseMetricAggregation { type: MetricAggregationType; } +export interface QueryFilter { + hide?: boolean; + id: string; + filter: AdHocVariableFilter; +} + export interface PipelineVariable { name: string; pipelineAgg: string; @@ -392,6 +399,10 @@ export interface Elasticsearch extends DataQuery { * List of metric aggregations */ metrics?: MetricAggregation[]; + /** + * List of filters + */ + filters?: QueryFilter[]; /** * Lucene query */ @@ -405,4 +416,5 @@ export interface Elasticsearch extends DataQuery { export const defaultElasticsearch: Partial = { bucketAggs: [], metrics: [], + filters: [], }; diff --git a/src/datasource/base.ts b/src/datasource/base.ts index e90f47d..2f3d624 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -16,9 +16,9 @@ import { TimeRange, } from '@grafana/data'; import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery, FieldCapabilitiesResponse } from '@/types'; -import { - DataSourceWithBackend, - getTemplateSrv, +import { + DataSourceWithBackend, + getTemplateSrv, TemplateSrv } from '@grafana/runtime'; import { QuickwitOptions } from 'quickwit'; import { getDataQuery } from 'QueryBuilder/elastic'; @@ -28,15 +28,15 @@ import { isMetricAggregationWithField } from 'components/QueryEditor/MetricAggre import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregationsEditor/utils'; import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations'; import ElasticsearchLanguageProvider from 'LanguageProvider'; -import { fieldTypeMap } from 'utils'; +import { fieldTypeMap, hasWhiteSpace } from 'utils'; import { addAddHocFilter } from 'modifyQuery'; import { getQueryResponseProcessor } from 'datasource/processResponse'; import { SECOND } from 'utils/time'; import { GConstructor } from 'utils/mixins'; -import { LuceneQuery } from '@/utils/lucene'; -import { uidMaker } from "@/utils/uid" +import { newFilterId, uidMaker } from '@/utils/uid'; import { DefaultsConfigOverrides } from 'store/defaults/conf'; +import { isSet } from '@/utils'; export type BaseQuickwitDataSourceConstructor = GConstructor @@ -122,18 +122,39 @@ export class BaseQuickwitDataSource return query; } - let lquery = LuceneQuery.parse(query.query ?? '') - switch (action.type) { - case 'ADD_FILTER': { - lquery = lquery.addFilter(action.options.key, action.options.value) - break; - } - case 'ADD_FILTER_OUT': { - lquery = lquery.addFilter(action.options.key, action.options.value, '-') - break; + const operationsMap: Record = { + 'ADD_FILTER': '=', + 'ADD_FILTER_OUT': '!=', + }; + const operation = operationsMap[action.type]; + + if (operation) { + // If the user has not added any filter, we can simply modify the last one (which is empty) + const len = query.filters?.length ?? 0; + if (len > 0) { + const last = query.filters![len - 1]; + if (!isSet(last.filter.key) && !isSet(last.filter.value)) { + last.filter.key = action.options.key; + last.filter.operator = operation; + last.filter.value = action.options.value; + return query; + } } + + query.filters?.push({ + id: newFilterId(), + hide: false, + filter: { + key: action.options.key, + operator: operation, + value: action.options.value, + }, + }); + } else { + console.warn('unsupported operation', action.type); } - return { ...query, query: lquery.toString() }; + + return { ...query }; } getDataQueryRequest(queryDef: TermsQuery, range: TimeRange, requestId?: string) { @@ -198,7 +219,7 @@ export class BaseQuickwitDataSource .map(field_capability => { return { text: field_capability.field_name, - type: fieldTypeMap[field_capability.type], + type: fieldTypeMap[field_capability.type], } }); const uniquefieldCapabilities = fieldCapabilities.filter((field_capability, index, self) => @@ -336,14 +357,36 @@ export class BaseQuickwitDataSource return bucketAgg; }; + const renderedQuery = (() => { + let q = this.interpolateLuceneQuery(query.query || '', scopedVars); + const queryFilters = query.filters + ?.filter((f) => { + if (f.hide) { + return false; + } + const hasValidValue = ( + ['exists', 'not exists'].includes(f.filter.operator) || isSet(f.filter.value) + ) && ( + !['term', 'not term'].includes(f.filter.operator) || !hasWhiteSpace(f.filter.value) + ) + + return isSet(f.filter.key) && hasValidValue && isSet(f.filter.operator) + }) + .map((f) => f.filter); + q = this.addAdHocFilters(q, queryFilters) + q = this.addAdHocFilters(q, filters) + return q + })() + const expandedQuery = { ...query, datasource: this.getRef(), - query: this.addAdHocFilters(this.interpolateLuceneQuery(query.query || '', scopedVars), filters), + query: renderedQuery, bucketAggs: query.bucketAggs?.map(interpolateBucketAgg), }; const finalQuery = JSON.parse(this.templateSrv.replace(JSON.stringify(expandedQuery), scopedVars)); + console.log('Final query', finalQuery.query) return finalQuery; } diff --git a/src/datasource/supplementaryQueries.ts b/src/datasource/supplementaryQueries.ts index 582d069..bf4f63d 100644 --- a/src/datasource/supplementaryQueries.ts +++ b/src/datasource/supplementaryQueries.ts @@ -102,6 +102,7 @@ export function withSupplementaryQueries { query: '', metrics: [defaultMetricAgg()], bucketAggs: [defaultBucketAgg()], + filters: [] }; const getFields: ElasticDatasource['getFields'] = jest.fn(() => from([[]])); diff --git a/src/hooks/useFields.ts b/src/hooks/useFields.ts index 400e071..b4f2192 100644 --- a/src/hooks/useFields.ts +++ b/src/hooks/useFields.ts @@ -46,14 +46,17 @@ const toSelectableValue = ({ text }: MetricFindValue): SelectableValue = value: text, }); +type MatchType = 'contains' | 'startsWith' + /** * Returns a function to query the configured datasource for autocomplete values for the specified aggregation type or data types. * Each aggregation can be run on different types, for example avg only operates on numeric fields, geohash_grid only on geo_point fields. * If an aggregation type is provided, the promise will resolve with all fields suitable to be used as a field for the given aggregation. * If an array of types is providem the promise will resolve with all the fields matching the provided types. - * @param aggregationType the type of aggregation to get fields for + * @param type the type of aggregation to get fields for + * @param matchType the type of matching to use when filtering fields based on the query string. Defaults to 'contains'. */ -export const useFields = (type: AggregationType | string[]) => { +export const useFields = (type: AggregationType | string[], matchType: MatchType = 'contains') => { const datasource = useDatasource(); const range = useRange(); const filter = Array.isArray(type) ? type : getFilter(type); @@ -65,6 +68,13 @@ export const useFields = (type: AggregationType | string[]) => { rawFields = await lastValueFrom(datasource.getFields({aggregatable:true, type:filter, range:range})); } - return rawFields.filter(({ text }) => q === undefined || text.includes(q)).map(toSelectableValue); + return rawFields + .filter(({ text }) => { + if (q === undefined) { + return true; + } + return matchType === 'contains' ? text.includes(q) : text.startsWith(q) + }) + .map(toSelectableValue); }; }; diff --git a/src/hooks/useNextId.test.tsx b/src/hooks/useNextId.test.tsx index 1b0d287..0e76dcb 100644 --- a/src/hooks/useNextId.test.tsx +++ b/src/hooks/useNextId.test.tsx @@ -16,6 +16,7 @@ describe('useNextId', () => { query: '', metrics: [{ id: '1', type: 'avg' }], bucketAggs: [{ id: '2', type: 'date_histogram' }], + filters: [], }; const wrapper = ({ children }: PropsWithChildren<{}>) => { return ( diff --git a/src/modifyQuery.ts b/src/modifyQuery.ts index 495fc42..7f1d8a9 100644 --- a/src/modifyQuery.ts +++ b/src/modifyQuery.ts @@ -5,7 +5,8 @@ import { AdHocVariableFilter } from '@grafana/data'; * Adds a label:"value" expression to the query. */ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): string { - if (!filter.key || !filter.value) { + const hasValidValue = ['exists', 'not exists'].includes(filter.operator) || !!filter.value + if (!filter.key || !hasValidValue) { return query; } @@ -39,6 +40,18 @@ export function addAddHocFilter(query: string, filter: AdHocVariableFilter): str case '<': addHocFilter = `${key}:<${value}`; break; + case 'term': + addHocFilter = `${key}:${value}`; + break; + case 'not term': + addHocFilter = `-${key}:${value}`; + break; + case 'exists': + addHocFilter = `${key}:*`; + break; + case 'not exists': + addHocFilter = `-${key}:*`; + break; } return concatenate(query, addHocFilter); } diff --git a/src/queryDef.ts b/src/queryDef.ts index 9fa54bb..7655f6e 100644 --- a/src/queryDef.ts +++ b/src/queryDef.ts @@ -6,7 +6,9 @@ import { MovingAverageModelOption, MetricAggregationType, DateHistogram, + QueryFilter, } from './types'; +import { newFilterId } from '@/utils/uid'; export const extendedStats: ExtendedStat[] = [ { label: 'Avg', value: 'avg' }, @@ -35,6 +37,19 @@ export function defaultBucketAgg(id = '1'): DateHistogram { return { type: 'date_histogram', id, settings: { interval: 'auto' } }; } +export const filterOperations = [ + { label: 'phrase', value: '=' }, + { label: 'not phrase', value: '!=' }, + { label: 'term', value: 'term' }, + { label: 'not term', value: 'not term' }, + { label: 'exists', value: 'exists' }, + { label: 'not exists', value: 'not exists' }, +]; + +export function defaultFilter(id = newFilterId()): QueryFilter { + return { id, filter: { key: '', operator: filterOperations[0].value, value: '' } }; +} + export const findMetricById = (metrics: MetricAggregation[], id: MetricAggregation['id']) => metrics.find((metric) => metric.id === id); diff --git a/src/utils/index.ts b/src/utils/index.ts index 8e5a826..518cf43 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -131,3 +131,8 @@ export const fieldTypeMap: Record = { float: 'number', scaled_float: 'number' }; + +export const isSet = (v: string) => v !== '' && v !== undefined && v !== null; + +export const hasWhiteSpace = (s: string) => /\s/g.test(s); + diff --git a/src/utils/uid.ts b/src/utils/uid.ts index bf51aac..9f029c7 100644 --- a/src/utils/uid.ts +++ b/src/utils/uid.ts @@ -1,3 +1,5 @@ +import { QueryFilter } from '@/dataquery.gen'; + export function uidMaker(prefix: string){ let i = 1; return { @@ -10,3 +12,7 @@ export function uidMaker(prefix: string){ } } } + +export function newFilterId(): QueryFilter['id'] { + return Math.floor(Math.random() * 100_000_000).toString() +}