From dd6a8ff0bb08d38169d650e20d10ef2cc71095f3 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:58:57 +0200 Subject: [PATCH] [8.15] [ES|QL] Automatically encapsulate index names with special chars with quotes (#187899) (#188052) # Backport This will backport the following commits from `main` to `8.15`: - [[ES|QL] Automatically encapsulate index names with special chars with quotes (#187899)](https://github.com/elastic/kibana/pull/187899) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Quynh Nguyen (Quinn) <43350163+qn895@users.noreply.github.com> --- .../src/autocomplete/autocomplete.test.ts | 32 +++++++++---- .../src/autocomplete/autocomplete.ts | 14 +++++- .../src/autocomplete/factories.ts | 11 ++++- .../src/autocomplete/helper.ts | 11 ++++- .../src/shared/helpers.test.ts | 48 +++++++++++++++++++ .../src/shared/helpers.ts | 4 ++ packages/kbn-monaco/src/esql/language.ts | 2 + 7 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 packages/kbn-esql-validation-autocomplete/src/shared/helpers.test.ts diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts index 6f0562d7f118e..dcaa898b15a5d 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts @@ -53,15 +53,27 @@ const fields: Array<{ name: string; type: string; suggestedAs?: string }> = [ ]; const indexes = ([] as Array<{ name: string; hidden: boolean; suggestedAs?: string }>).concat( - ['a', 'index', 'otherIndex', '.secretIndex', 'my-index'].map((name) => ({ + [ + 'a', + 'index', + 'otherIndex', + '.secretIndex', + 'my-index', + 'my-index$', + 'my_index{}', + 'my-index+1', + 'synthetics-*', + ].map((name) => ({ name, hidden: name.startsWith('.'), })), - ['my-index[quoted]', 'my-index$', 'my_index{}'].map((name) => ({ - name, - hidden: false, - suggestedAs: `\`${name}\``, - })) + ['my-index[quoted]', 'my:index', 'my,index', 'logstash-{now/d{yyyy.MM.dd|+12:00}}'].map( + (name) => ({ + name, + hidden: false, + suggestedAs: `"${name}"`, + }) + ) ); const integrations: Integration[] = ['nginx', 'k8s'].map((name) => ({ @@ -368,8 +380,9 @@ describe('autocomplete', () => { }); describe('from', () => { - const suggestedIndexes = indexes.filter(({ hidden }) => !hidden).map(({ name }) => name); - + const suggestedIndexes = indexes + .filter(({ hidden }) => !hidden) + .map(({ name, suggestedAs }) => suggestedAs || name); // Monaco will filter further down here testSuggestions( 'f', @@ -397,8 +410,7 @@ describe('autocomplete', () => { const dataSources = indexes.concat(integrations); const suggestedDataSources = dataSources .filter(({ hidden }) => !hidden) - .map(({ name }) => name); - + .map(({ name, suggestedAs }) => suggestedAs || name); testSuggestions('from ', suggestedDataSources, '', [undefined, dataSources, undefined]); testSuggestions('from a,', suggestedDataSources, '', [undefined, dataSources, undefined]); testSuggestions('from *,', suggestedDataSources, '', [undefined, dataSources, undefined]); diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts index add47e4a5547b..03504552370b6 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts @@ -86,6 +86,7 @@ import { getQueryForFields, getSourcesFromCommands, isAggFunctionUsedAlready, + removeQuoteForSuggestedSources, } from './helper'; import { FunctionParameter } from '../definitions/types'; @@ -857,19 +858,28 @@ async function getExpressionSuggestionsByType( suggestions.push(...(policies.length ? policies : [buildNoPoliciesAvailableDefinition()])); } else { const index = getSourcesFromCommands(commands, 'index'); + const canRemoveQuote = isNewExpression && innerText.includes('"'); + // This is going to be empty for simple indices, and not empty for integrations if (index && index.text && index.text !== EDITOR_MARKER) { const source = index.text.replace(EDITOR_MARKER, ''); const dataSource = await getDatastreamsForIntegration(source); + const newDefinitions = buildSourcesDefinitions( dataSource?.dataStreams?.map(({ name }) => ({ name, isIntegration: false })) || [] ); - suggestions.push(...newDefinitions); + suggestions.push( + ...(canRemoveQuote ? removeQuoteForSuggestedSources(newDefinitions) : newDefinitions) + ); } else { // FROM // @TODO: filter down the suggestions here based on other existing sources defined const sourcesDefinitions = await getSources(); - suggestions.push(...sourcesDefinitions); + suggestions.push( + ...(canRemoveQuote + ? removeQuoteForSuggestedSources(sourcesDefinitions) + : sourcesDefinitions) + ); } } } diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts index 53e65c2f95aba..ad49303c55def 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts @@ -19,7 +19,7 @@ import { CommandOptionsDefinition, CommandModeDefinition, } from '../definitions/types'; -import { getCommandDefinition, shouldBeQuotedText } from '../shared/helpers'; +import { shouldBeQuotedSource, getCommandDefinition, shouldBeQuotedText } from '../shared/helpers'; import { buildDocumentation, buildFunctionDocumentation } from './documentation_util'; import { DOUBLE_BACKTICK, SINGLE_TICK_REGEX } from '../shared/constants'; @@ -37,6 +37,13 @@ function getSafeInsertText(text: string, options: { dashSupported?: boolean } = ? `\`${text.replace(SINGLE_TICK_REGEX, DOUBLE_BACKTICK)}\`` : text; } +export function getQuotedText(text: string) { + return text.startsWith(`"`) && text.endsWith(`"`) ? text : `"${text}"`; +} + +function getSafeInsertSourceText(text: string) { + return shouldBeQuotedSource(text) ? getQuotedText(text) : text; +} export function getSuggestionFunctionDefinition(fn: FunctionDefinition): SuggestionRawDefinition { const fullSignatures = getFunctionSignatures(fn); @@ -148,7 +155,7 @@ export const buildSourcesDefinitions = ( ): SuggestionRawDefinition[] => sources.map(({ name, isIntegration, title }) => ({ label: title ?? name, - text: name, + text: getSafeInsertSourceText(name), isSnippet: isIntegration, ...(isIntegration && { command: TRIGGER_SUGGESTION_COMMAND }), kind: isIntegration ? 'Class' : 'Issue', diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts index fa6e364c78ca9..f9586670b29e6 100644 --- a/packages/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts +++ b/packages/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts @@ -7,8 +7,9 @@ */ import type { ESQLAstItem, ESQLCommand, ESQLFunction, ESQLSource } from '@kbn/esql-ast'; -import { FunctionDefinition } from '../definitions/types'; +import type { FunctionDefinition } from '../definitions/types'; import { getFunctionDefinition, isAssignment, isFunctionItem } from '../shared/helpers'; +import type { SuggestionRawDefinition } from './types'; function extractFunctionArgs(args: ESQLAstItem[]): ESQLFunction[] { return args.flatMap((arg) => (isAssignment(arg) ? arg.args[1] : arg)).filter(isFunctionItem); @@ -71,3 +72,11 @@ export function getSourcesFromCommands(commands: ESQLCommand[], sourceType: 'ind return sources.length === 1 ? sources[0] : undefined; } + +export function removeQuoteForSuggestedSources(suggestions: SuggestionRawDefinition[]) { + return suggestions.map((d) => ({ + ...d, + // "text" -> text + text: d.text.startsWith('"') && d.text.endsWith('"') ? d.text.slice(1, -1) : d.text, + })); +} diff --git a/packages/kbn-esql-validation-autocomplete/src/shared/helpers.test.ts b/packages/kbn-esql-validation-autocomplete/src/shared/helpers.test.ts new file mode 100644 index 0000000000000..87eb31de4d3c9 --- /dev/null +++ b/packages/kbn-esql-validation-autocomplete/src/shared/helpers.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { shouldBeQuotedSource } from './helpers'; + +describe('shouldBeQuotedSource', () => { + it('does not have to be quoted for sources with acceptable characters @-+$', () => { + expect(shouldBeQuotedSource('foo')).toBe(false); + expect(shouldBeQuotedSource('123-test@foo_bar+baz1')).toBe(false); + expect(shouldBeQuotedSource('my-index*')).toBe(false); + expect(shouldBeQuotedSource('my-index$')).toBe(false); + expect(shouldBeQuotedSource('.my-index$')).toBe(false); + }); + it(`should be quoted if containing any of special characters [:"=|,[\]/ \t\r\n]`, () => { + expect(shouldBeQuotedSource('foo\ttest')).toBe(true); + expect(shouldBeQuotedSource('foo\rtest')).toBe(true); + expect(shouldBeQuotedSource('foo\ntest')).toBe(true); + expect(shouldBeQuotedSource('foo:test=bar')).toBe(true); + expect(shouldBeQuotedSource('foo|test=bar')).toBe(true); + expect(shouldBeQuotedSource('foo[test]=bar')).toBe(true); + expect(shouldBeQuotedSource('foo/test=bar')).toBe(true); + expect(shouldBeQuotedSource('foo test=bar')).toBe(true); + expect(shouldBeQuotedSource('foo,test-*,abc')).toBe(true); + expect(shouldBeQuotedSource('foo, test-*, abc, xyz')).toBe(true); + expect(shouldBeQuotedSource('foo, test-*, abc, xyz,test123')).toBe(true); + expect(shouldBeQuotedSource('foo,test,xyz')).toBe(true); + expect( + shouldBeQuotedSource(',') + ).toBe(true); + expect(shouldBeQuotedSource('`backtick`,``multiple`back``ticks```')).toBe(true); + expect(shouldBeQuotedSource('test,metadata,metaata,.metadata')).toBe(true); + expect(shouldBeQuotedSource('cluster:index')).toBe(true); + expect(shouldBeQuotedSource('cluster:index|pattern')).toBe(true); + expect(shouldBeQuotedSource('cluster:.index')).toBe(true); + expect(shouldBeQuotedSource('cluster*:index*')).toBe(true); + expect(shouldBeQuotedSource('cluster*:*')).toBe(true); + expect(shouldBeQuotedSource('*:index*')).toBe(true); + expect(shouldBeQuotedSource('*:index|pattern')).toBe(true); + expect(shouldBeQuotedSource('*:*')).toBe(true); + expect(shouldBeQuotedSource('*:*,cluster*:index|pattern,i|p')).toBe(true); + expect(shouldBeQuotedSource('index-[dd-mm]')).toBe(true); + }); +}); diff --git a/packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts b/packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts index effd6b1b16ddd..94aae4194d367 100644 --- a/packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts +++ b/packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts @@ -566,6 +566,10 @@ export function isRestartingExpression(text: string) { return getLastCharFromTrimmed(text) === ','; } +export function shouldBeQuotedSource(text: string) { + // Based on lexer `fragment UNQUOTED_SOURCE_PART` + return /[:"=|,[\]\/ \t\r\n]/.test(text); +} export function shouldBeQuotedText( text: string, { dashSupported }: { dashSupported?: boolean } = {} diff --git a/packages/kbn-monaco/src/esql/language.ts b/packages/kbn-monaco/src/esql/language.ts index 4e486a5c24188..7c49da41a996e 100644 --- a/packages/kbn-monaco/src/esql/language.ts +++ b/packages/kbn-monaco/src/esql/language.ts @@ -39,11 +39,13 @@ export const ESQLLang: CustomLangModuleType = { { open: '(', close: ')' }, { open: '[', close: ']' }, { open: `'`, close: `'` }, + { open: '"""', close: '"""' }, { open: '"', close: '"' }, ], surroundingPairs: [ { open: '(', close: ')' }, { open: `'`, close: `'` }, + { open: '"""', close: '"""' }, { open: '"', close: '"' }, ], },