From 1b6c497101e1b46d97904538b97faa2e276fa31d Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 23 Oct 2024 19:05:41 +0100 Subject: [PATCH] [ML] AIOps: Fixes issue where some queries cause filters to not be applied (#196585) --- .../public/application/utils/search_utils.ts | 74 +++---------------- .../change_point_detection_context.tsx | 15 ++-- .../log_categorization_for_discover.tsx | 11 ++- .../log_categorization_for_embeddable.tsx | 11 ++- .../log_categorization_for_flyout.tsx | 10 ++- .../components/search_panel/search_panel.tsx | 10 +-- .../embeddable_chart_component_wrapper.tsx | 19 +++-- .../plugins/aiops/public/hooks/use_search.ts | 11 ++- 8 files changed, 65 insertions(+), 96 deletions(-) diff --git a/x-pack/plugins/aiops/public/application/utils/search_utils.ts b/x-pack/plugins/aiops/public/application/utils/search_utils.ts index 533303436fa92..442ce910dd828 100644 --- a/x-pack/plugins/aiops/public/application/utils/search_utils.ts +++ b/x-pack/plugins/aiops/public/application/utils/search_utils.ts @@ -13,23 +13,14 @@ import type { IUiSettingsClient } from '@kbn/core/public'; import { getEsQueryConfig, SearchSource } from '@kbn/data-plugin/common'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; import type { FilterManager } from '@kbn/data-plugin/public'; -import { isQuery, mapAndFlattenFilters } from '@kbn/data-plugin/public'; -import type { Query, Filter, AggregateQuery } from '@kbn/es-query'; -import { - fromKueryExpression, - toElasticsearchQuery, - buildQueryFromFilters, - buildEsQuery, -} from '@kbn/es-query'; +import { mapAndFlattenFilters } from '@kbn/data-plugin/public'; +import type { Query, Filter } from '@kbn/es-query'; +import { buildEsQuery } from '@kbn/es-query'; import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { DataView } from '@kbn/data-views-plugin/public'; import type { SimpleSavedObject } from '@kbn/core/public'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; -import { - getDefaultDSLQuery, - type SearchQueryLanguage, - SEARCH_QUERY_LANGUAGE, -} from '@kbn/ml-query-utils'; +import { getDefaultDSLQuery, type SearchQueryLanguage } from '@kbn/ml-query-utils'; export type SavedSearchSavedObject = SimpleSavedObject; @@ -67,51 +58,6 @@ export function getQueryFromSavedSearchObject(savedSearch: SavedSearchSavedObjec return parsed; } -/** - * Create an Elasticsearch query that combines both lucene/kql query string and filters - * Should also form a valid query if only the query or filters is provided - */ -export function createMergedEsQuery( - query?: Query | AggregateQuery, - filters?: Filter[], - dataView?: DataView, - uiSettings?: IUiSettingsClient -) { - let combinedQuery: QueryDslQueryContainer = getDefaultDSLQuery(); - - // FIXME: Add support for AggregateQuery type #150091 - if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { - const ast = fromKueryExpression(query.query); - if (query.query !== '') { - combinedQuery = toElasticsearchQuery(ast, dataView); - } - if (combinedQuery.bool !== undefined) { - const filterQuery = buildQueryFromFilters(filters, dataView); - - if (!Array.isArray(combinedQuery.bool.filter)) { - combinedQuery.bool.filter = - combinedQuery.bool.filter === undefined ? [] : [combinedQuery.bool.filter]; - } - - if (!Array.isArray(combinedQuery.bool.must_not)) { - combinedQuery.bool.must_not = - combinedQuery.bool.must_not === undefined ? [] : [combinedQuery.bool.must_not]; - } - - combinedQuery.bool.filter = [...combinedQuery.bool.filter, ...filterQuery.filter]; - combinedQuery.bool.must_not = [...combinedQuery.bool.must_not, ...filterQuery.must_not]; - } - } else { - combinedQuery = buildEsQuery( - dataView, - query ? [query] : [], - filters ? filters : [], - uiSettings ? getEsQueryConfig(uiSettings) : undefined - ); - } - return combinedQuery; -} - function getSavedSearchSource(savedSearch: SavedSearch) { return savedSearch && 'searchSource' in savedSearch && @@ -174,11 +120,11 @@ export function getEsQueryFromSavedSearch({ if (!savedSearch && userQuery) { if (filterManager && userFilters) filterManager.addFilters(userFilters); - const combinedQuery = createMergedEsQuery( + const combinedQuery = buildEsQuery( + dataView, userQuery, Array.isArray(userFilters) ? userFilters : [], - dataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); return { @@ -199,11 +145,11 @@ export function getEsQueryFromSavedSearch({ if (filterManager) filterManager.setFilters(currentFilters); if (globalFilters) filterManager?.addFilters(globalFilters); - const combinedQuery = createMergedEsQuery( + const combinedQuery = buildEsQuery( + dataView, currentQuery, filterManager ? filterManager?.getFilters() : currentFilters, - dataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); return { diff --git a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx index 45ef73c5dd7b5..4c429e2fd2976 100644 --- a/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx +++ b/x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx @@ -9,7 +9,7 @@ import type { FC, PropsWithChildren } from 'react'; import React, { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react'; import { type DataViewField } from '@kbn/data-views-plugin/public'; import { startWith } from 'rxjs'; -import type { Filter, Query } from '@kbn/es-query'; +import { buildEsQuery, type Filter, type Query } from '@kbn/es-query'; import { usePageUrlState } from '@kbn/ml-url-state'; import { useTimefilter } from '@kbn/ml-date-picker'; import { ES_FIELD_TYPES } from '@kbn/field-types'; @@ -17,12 +17,10 @@ import { type QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types import type { TimeBuckets, TimeBucketsInterval } from '@kbn/ml-time-buckets'; import { useTimeBuckets } from '@kbn/ml-time-buckets'; import { createDefaultQuery } from '@kbn/aiops-common/create_default_query'; +import { getEsQueryConfig } from '@kbn/data-service'; import { useFilterQueryUpdates } from '../../hooks/use_filters_query'; import { type ChangePointType, DEFAULT_AGG_FUNCTION } from './constants'; -import { - createMergedEsQuery, - getEsQueryFromSavedSearch, -} from '../../application/utils/search_utils'; +import { getEsQueryFromSavedSearch } from '../../application/utils/search_utils'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; import { useDataSource } from '../../hooks/use_data_source'; @@ -261,7 +259,12 @@ export const ChangePointDetectionContextProvider: FC> ); const combinedQuery = useMemo(() => { - const mergedQuery = createMergedEsQuery(resultQuery, resultFilters, dataView, uiSettings); + const mergedQuery = buildEsQuery( + dataView, + resultQuery, + resultFilters, + uiSettings ? getEsQueryConfig(uiSettings) : undefined + ); const to = searchBounds.max?.valueOf(); const from = searchBounds.min?.valueOf(); const timeRange = to !== undefined && from !== undefined ? { from, to } : undefined; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_discover.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_discover.tsx index db2135e94ab74..5bf415c5ccfc9 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_discover.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_discover.tsx @@ -12,7 +12,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiSpacer, useEuiPaddingSize } from '@elasti import type { DataViewField } from '@kbn/data-views-plugin/public'; import { i18n } from '@kbn/i18n'; import type { Filter } from '@kbn/es-query'; -import { buildEmptyFilter } from '@kbn/es-query'; +import { buildEmptyFilter, buildEsQuery } from '@kbn/es-query'; import { usePageUrlState } from '@kbn/ml-url-state'; import type { FieldValidationResults } from '@kbn/ml-category-validator'; @@ -24,11 +24,11 @@ import type { EmbeddablePatternAnalysisInput } from '@kbn/aiops-log-pattern-anal import { css } from '@emotion/react'; import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state'; import useMountedState from 'react-use/lib/useMountedState'; +import { getEsQueryConfig } from '@kbn/data-service'; import { type LogCategorizationPageUrlState, getDefaultLogCategorizationAppState, } from '../../../application/url_state/log_pattern_analysis'; -import { createMergedEsQuery } from '../../../application/utils/search_utils'; import { useData } from '../../../hooks/use_data'; import { useSearch } from '../../../hooks/use_search'; import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context'; @@ -90,7 +90,12 @@ export const LogCategorizationDiscover: FC = ( const [stateFromUrl] = usePageUrlState( 'logCategorization', getDefaultLogCategorizationAppState({ - searchQuery: createMergedEsQuery(query, filters, dataView, uiSettings), + searchQuery: buildEsQuery( + dataView, + query ?? [], + filters ?? [], + uiSettings ? getEsQueryConfig(uiSettings) : undefined + ), }) ); const [selectedCategories, setSelectedCategories] = useState([]); diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx index 9178ac7428664..5ca3cd947f7fe 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx @@ -10,7 +10,7 @@ import React, { useState, useEffect, useCallback } from 'react'; import { i18n } from '@kbn/i18n'; import type { Filter } from '@kbn/es-query'; -import { buildEmptyFilter } from '@kbn/es-query'; +import { buildEmptyFilter, buildEsQuery } from '@kbn/es-query'; import type { FieldValidationResults } from '@kbn/ml-category-validator'; import type { Category } from '@kbn/aiops-log-pattern-analysis/types'; @@ -21,11 +21,11 @@ import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state'; import { AIOPS_ANALYSIS_RUN_ORIGIN } from '@kbn/aiops-common/constants'; import datemath from '@elastic/datemath'; import useMountedState from 'react-use/lib/useMountedState'; +import { getEsQueryConfig } from '@kbn/data-service'; import { useFilterQueryUpdates } from '../../../hooks/use_filters_query'; import type { PatternAnalysisProps } from '../../../shared_components/pattern_analysis'; import { useSearch } from '../../../hooks/use_search'; import { getDefaultLogCategorizationAppState } from '../../../application/url_state/log_pattern_analysis'; -import { createMergedEsQuery } from '../../../application/utils/search_utils'; import { useData } from '../../../hooks/use_data'; import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context'; @@ -86,7 +86,12 @@ export const LogCategorizationEmbeddable: FC = }); const appState = getDefaultLogCategorizationAppState({ - searchQuery: createMergedEsQuery(query, filters, dataView, uiSettings), + searchQuery: buildEsQuery( + dataView, + query ?? [], + filters ?? [], + uiSettings ? getEsQueryConfig(uiSettings) : undefined + ), filters, }); const { searchQuery } = useSearch({ dataView, savedSearch: savedSearch ?? null }, appState, true); diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx index 2036c5d3a24e9..203950cbf3971 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx @@ -33,11 +33,12 @@ import type { CategorizationAdditionalFilter } from '@kbn/aiops-log-pattern-anal import type { Category } from '@kbn/aiops-log-pattern-analysis/types'; import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state'; +import { buildEsQuery } from '@kbn/es-query'; +import { getEsQueryConfig } from '@kbn/data-service'; import { type LogCategorizationPageUrlState, getDefaultLogCategorizationAppState, } from '../../application/url_state/log_pattern_analysis'; -import { createMergedEsQuery } from '../../application/utils/search_utils'; import { useData } from '../../hooks/use_data'; import { useSearch } from '../../hooks/use_search'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; @@ -100,7 +101,12 @@ export const LogCategorizationFlyout: FC = ({ const [stateFromUrl] = usePageUrlState( 'logCategorization', getDefaultLogCategorizationAppState({ - searchQuery: createMergedEsQuery(query, filters, dataView, uiSettings), + searchQuery: buildEsQuery( + dataView, + query ?? [], + filters ?? [], + uiSettings ? getEsQueryConfig(uiSettings) : undefined + ), }) ); const [selectedCategories, setSelectedCategories] = useState([]); diff --git a/x-pack/plugins/aiops/public/components/search_panel/search_panel.tsx b/x-pack/plugins/aiops/public/components/search_panel/search_panel.tsx index e80bfbf31d843..b3c5adc7c4ed9 100644 --- a/x-pack/plugins/aiops/public/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/aiops/public/components/search_panel/search_panel.tsx @@ -9,13 +9,13 @@ import type { FC } from 'react'; import React, { useEffect, useState } from 'react'; import { EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import type { Query, Filter } from '@kbn/es-query'; +import { type Query, type Filter, buildEsQuery } from '@kbn/es-query'; import type { TimeRange } from '@kbn/es-query'; import type { DataViewField } from '@kbn/data-views-plugin/public'; import type { SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { getEsQueryConfig } from '@kbn/data-service'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; import { useDataSource } from '../../hooks/use_data_source'; -import { createMergedEsQuery } from '../../application/utils/search_utils'; interface Props { searchString: Query['query']; searchQuery: Query['query']; @@ -66,11 +66,11 @@ export const SearchPanel: FC = ({ searchString, searchQueryLanguage, setS queryManager.filterManager.setFilters(mergedFilters); } - const combinedQuery = createMergedEsQuery( + const combinedQuery = buildEsQuery( + dataView, mergedQuery, queryManager.filterManager.getFilters() ?? [], - dataView, - uiSettings + uiSettings ? getEsQueryConfig(uiSettings) : undefined ); setSearchParams({ diff --git a/x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx b/x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx index d95fcca4cbbff..69da4be087a14 100644 --- a/x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx +++ b/x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx @@ -9,6 +9,8 @@ import type { FC } from 'react'; import React, { useEffect, useMemo } from 'react'; import { css } from '@emotion/react'; import { CHANGE_POINT_DETECTION_VIEW_TYPE } from '@kbn/aiops-change-point-detection/constants'; +import { getEsQueryConfig } from '@kbn/data-service'; +import { buildEsQuery } from '@kbn/es-query'; import type { ChangePointDetectionProps } from '../../shared_components/change_point_detection'; import { ChangePointsTable } from '../../components/change_point_detection/change_points_table'; import { @@ -18,7 +20,6 @@ import { import { useFilterQueryUpdates } from '../../hooks/use_filters_query'; import { useDataSource } from '../../hooks/use_data_source'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; -import { createMergedEsQuery } from '../../application/utils/search_utils'; import { useChangePointResults } from '../../components/change_point_detection/use_change_point_agg_request'; import { ChartsGrid } from '../../components/change_point_detection/charts_grid'; import { NoChangePointsWarning } from '../../components/change_point_detection/no_change_points_warning'; @@ -62,15 +63,13 @@ export const ChartGridEmbeddableWrapper: FC = ({ const { uiSettings } = useAiopsAppContext(); const combinedQuery = useMemo(() => { - const mergedQuery = createMergedEsQuery(query, filters, dataView, uiSettings); - if (!Array.isArray(mergedQuery.bool?.filter)) { - if (!mergedQuery.bool) { - mergedQuery.bool = {}; - } - mergedQuery.bool.filter = []; - } - - mergedQuery.bool!.filter.push({ + const mergedQuery = buildEsQuery( + dataView, + query, + filters, + uiSettings ? getEsQueryConfig(uiSettings) : undefined + ); + mergedQuery.bool.filter.push({ range: { [dataView.timeFieldName!]: { from: searchBounds.min?.valueOf(), diff --git a/x-pack/plugins/aiops/public/hooks/use_search.ts b/x-pack/plugins/aiops/public/hooks/use_search.ts index 060e87dab59c5..14d8b6124eabb 100644 --- a/x-pack/plugins/aiops/public/hooks/use_search.ts +++ b/x-pack/plugins/aiops/public/hooks/use_search.ts @@ -9,14 +9,14 @@ import { useMemo } from 'react'; import type { DataView } from '@kbn/data-views-plugin/public'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; -import { isQuery } from '@kbn/data-plugin/public'; +import { getEsQueryConfig, isQuery } from '@kbn/data-plugin/public'; +import { buildEsQuery } from '@kbn/es-query'; import { getEsQueryFromSavedSearch } from '../application/utils/search_utils'; import { isDefaultSearchQuery, type AiOpsIndexBasedAppState, } from '../application/url_state/common'; -import { createMergedEsQuery } from '../application/utils/search_utils'; import { useAiopsAppContext } from './use_aiops_app_context'; @@ -66,7 +66,12 @@ export const useSearch = ( (isDefaultSearchQuery(searchQuery) || searchQuery === undefined) && isQuery(query) ) { - searchQuery = createMergedEsQuery(query, aiopsListState.filters, dataView, uiSettings); + searchQuery = buildEsQuery( + dataView, + query, + aiopsListState.filters ?? [], + uiSettings ? getEsQueryConfig(uiSettings) : undefined + ); } return {