[8.15] [ES|QL] Automatically encapsulate index names with special cha…

…rs with quotes (elastic#187899) (elastic#188052) # Backport This will backport the following commits from `main` to `8.15`: - [[ES|QL] Automatically encapsulate index names with special chars with quotes (elastic#187899)](elastic#187899)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Quynh Nguyen (Quinn) <[email protected]>
kibanamachine · Jul 11, 2024 · dd6a8ff · dd6a8ff
1 parent f405425
commit dd6a8ff
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 15 deletions.
diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts
@@ -53,15 +53,27 @@ const fields: Array<{ name: string; type: string; suggestedAs?: string }> = [
 ];
 
 const indexes = ([] as Array<{ name: string; hidden: boolean; suggestedAs?: string }>).concat(
-  ['a', 'index', 'otherIndex', '.secretIndex', 'my-index'].map((name) => ({
+  [
+    'a',
+    'index',
+    'otherIndex',
+    '.secretIndex',
+    'my-index',
+    'my-index$',
+    'my_index{}',
+    'my-index+1',
+    'synthetics-*',
+  ].map((name) => ({
     name,
     hidden: name.startsWith('.'),
   })),
-  ['my-index[quoted]', 'my-index$', 'my_index{}'].map((name) => ({
-    name,
-    hidden: false,
-    suggestedAs: `\`${name}\``,
-  }))
+  ['my-index[quoted]', 'my:index', 'my,index', 'logstash-{now/d{yyyy.MM.dd|+12:00}}'].map(
+    (name) => ({
+      name,
+      hidden: false,
+      suggestedAs: `"${name}"`,
+    })
+  )
 );
 
 const integrations: Integration[] = ['nginx', 'k8s'].map((name) => ({
@@ -368,8 +380,9 @@ describe('autocomplete', () => {
   });
 
   describe('from', () => {
-    const suggestedIndexes = indexes.filter(({ hidden }) => !hidden).map(({ name }) => name);
-
+    const suggestedIndexes = indexes
+      .filter(({ hidden }) => !hidden)
+      .map(({ name, suggestedAs }) => suggestedAs || name);
     // Monaco will filter further down here
     testSuggestions(
       'f',
@@ -397,8 +410,7 @@ describe('autocomplete', () => {
     const dataSources = indexes.concat(integrations);
     const suggestedDataSources = dataSources
       .filter(({ hidden }) => !hidden)
-      .map(({ name }) => name);
-
+      .map(({ name, suggestedAs }) => suggestedAs || name);
     testSuggestions('from ', suggestedDataSources, '', [undefined, dataSources, undefined]);
     testSuggestions('from a,', suggestedDataSources, '', [undefined, dataSources, undefined]);
     testSuggestions('from *,', suggestedDataSources, '', [undefined, dataSources, undefined]);

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts
@@ -86,6 +86,7 @@ import {
   getQueryForFields,
   getSourcesFromCommands,
   isAggFunctionUsedAlready,
+  removeQuoteForSuggestedSources,
 } from './helper';
 import { FunctionParameter } from '../definitions/types';
 
@@ -857,19 +858,28 @@ async function getExpressionSuggestionsByType(
         suggestions.push(...(policies.length ? policies : [buildNoPoliciesAvailableDefinition()]));
       } else {
         const index = getSourcesFromCommands(commands, 'index');
+        const canRemoveQuote = isNewExpression && innerText.includes('"');
+
         // This is going to be empty for simple indices, and not empty for integrations
         if (index && index.text && index.text !== EDITOR_MARKER) {
           const source = index.text.replace(EDITOR_MARKER, '');
           const dataSource = await getDatastreamsForIntegration(source);
+
           const newDefinitions = buildSourcesDefinitions(
             dataSource?.dataStreams?.map(({ name }) => ({ name, isIntegration: false })) || []
           );
-          suggestions.push(...newDefinitions);
+          suggestions.push(
+            ...(canRemoveQuote ? removeQuoteForSuggestedSources(newDefinitions) : newDefinitions)
+          );
         } else {
           // FROM <suggest>
           // @TODO: filter down the suggestions here based on other existing sources defined
           const sourcesDefinitions = await getSources();
-          suggestions.push(...sourcesDefinitions);
+          suggestions.push(
+            ...(canRemoveQuote
+              ? removeQuoteForSuggestedSources(sourcesDefinitions)
+              : sourcesDefinitions)
+          );
         }
       }
     }

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts
@@ -19,7 +19,7 @@ import {
   CommandOptionsDefinition,
   CommandModeDefinition,
 } from '../definitions/types';
-import { getCommandDefinition, shouldBeQuotedText } from '../shared/helpers';
+import { shouldBeQuotedSource, getCommandDefinition, shouldBeQuotedText } from '../shared/helpers';
 import { buildDocumentation, buildFunctionDocumentation } from './documentation_util';
 import { DOUBLE_BACKTICK, SINGLE_TICK_REGEX } from '../shared/constants';
 
@@ -37,6 +37,13 @@ function getSafeInsertText(text: string, options: { dashSupported?: boolean } =
     ? `\`${text.replace(SINGLE_TICK_REGEX, DOUBLE_BACKTICK)}\``
     : text;
 }
+export function getQuotedText(text: string) {
+  return text.startsWith(`"`) && text.endsWith(`"`) ? text : `"${text}"`;
+}
+
+function getSafeInsertSourceText(text: string) {
+  return shouldBeQuotedSource(text) ? getQuotedText(text) : text;
+}
 
 export function getSuggestionFunctionDefinition(fn: FunctionDefinition): SuggestionRawDefinition {
   const fullSignatures = getFunctionSignatures(fn);
@@ -148,7 +155,7 @@ export const buildSourcesDefinitions = (
 ): SuggestionRawDefinition[] =>
   sources.map(({ name, isIntegration, title }) => ({
     label: title ?? name,
-    text: name,
+    text: getSafeInsertSourceText(name),
     isSnippet: isIntegration,
     ...(isIntegration && { command: TRIGGER_SUGGESTION_COMMAND }),
     kind: isIntegration ? 'Class' : 'Issue',

diff --git a/packages/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts b/packages/kbn-esql-validation-autocomplete/src/autocomplete/helper.ts
@@ -7,8 +7,9 @@
  */
 
 import type { ESQLAstItem, ESQLCommand, ESQLFunction, ESQLSource } from '@kbn/esql-ast';
-import { FunctionDefinition } from '../definitions/types';
+import type { FunctionDefinition } from '../definitions/types';
 import { getFunctionDefinition, isAssignment, isFunctionItem } from '../shared/helpers';
+import type { SuggestionRawDefinition } from './types';
 
 function extractFunctionArgs(args: ESQLAstItem[]): ESQLFunction[] {
   return args.flatMap((arg) => (isAssignment(arg) ? arg.args[1] : arg)).filter(isFunctionItem);
@@ -71,3 +72,11 @@ export function getSourcesFromCommands(commands: ESQLCommand[], sourceType: 'ind
 
   return sources.length === 1 ? sources[0] : undefined;
 }
+
+export function removeQuoteForSuggestedSources(suggestions: SuggestionRawDefinition[]) {
+  return suggestions.map((d) => ({
+    ...d,
+    // "text" -> text
+    text: d.text.startsWith('"') && d.text.endsWith('"') ? d.text.slice(1, -1) : d.text,
+  }));
+}
diff --git a/packages/kbn-esql-validation-autocomplete/src/shared/helpers.test.ts b/packages/kbn-esql-validation-autocomplete/src/shared/helpers.test.ts
@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { shouldBeQuotedSource } from './helpers';
+
+describe('shouldBeQuotedSource', () => {
+  it('does not have to be quoted for sources with acceptable characters @-+$', () => {
+    expect(shouldBeQuotedSource('foo')).toBe(false);
+    expect(shouldBeQuotedSource('123-test@foo_bar+baz1')).toBe(false);
+    expect(shouldBeQuotedSource('my-index*')).toBe(false);
+    expect(shouldBeQuotedSource('my-index$')).toBe(false);
+    expect(shouldBeQuotedSource('.my-index$')).toBe(false);
+  });
+  it(`should be quoted if containing any of special characters [:"=|,[\]/ \t\r\n]`, () => {
+    expect(shouldBeQuotedSource('foo\ttest')).toBe(true);
+    expect(shouldBeQuotedSource('foo\rtest')).toBe(true);
+    expect(shouldBeQuotedSource('foo\ntest')).toBe(true);
+    expect(shouldBeQuotedSource('foo:test=bar')).toBe(true);
+    expect(shouldBeQuotedSource('foo|test=bar')).toBe(true);
+    expect(shouldBeQuotedSource('foo[test]=bar')).toBe(true);
+    expect(shouldBeQuotedSource('foo/test=bar')).toBe(true);
+    expect(shouldBeQuotedSource('foo test=bar')).toBe(true);
+    expect(shouldBeQuotedSource('foo,test-*,abc')).toBe(true);
+    expect(shouldBeQuotedSource('foo, test-*, abc, xyz')).toBe(true);
+    expect(shouldBeQuotedSource('foo, test-*, abc, xyz,test123')).toBe(true);
+    expect(shouldBeQuotedSource('foo,test,xyz')).toBe(true);
+    expect(
+      shouldBeQuotedSource('<logstash-{now/M{yyyy.MM}}>,<logstash-{now/d{yyyy.MM.dd|+12:00}}>')
+    ).toBe(true);
+    expect(shouldBeQuotedSource('`backtick`,``multiple`back``ticks```')).toBe(true);
+    expect(shouldBeQuotedSource('test,metadata,metaata,.metadata')).toBe(true);
+    expect(shouldBeQuotedSource('cluster:index')).toBe(true);
+    expect(shouldBeQuotedSource('cluster:index|pattern')).toBe(true);
+    expect(shouldBeQuotedSource('cluster:.index')).toBe(true);
+    expect(shouldBeQuotedSource('cluster*:index*')).toBe(true);
+    expect(shouldBeQuotedSource('cluster*:*')).toBe(true);
+    expect(shouldBeQuotedSource('*:index*')).toBe(true);
+    expect(shouldBeQuotedSource('*:index|pattern')).toBe(true);
+    expect(shouldBeQuotedSource('*:*')).toBe(true);
+    expect(shouldBeQuotedSource('*:*,cluster*:index|pattern,i|p')).toBe(true);
+    expect(shouldBeQuotedSource('index-[dd-mm]')).toBe(true);
+  });
+});
diff --git a/packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts b/packages/kbn-esql-validation-autocomplete/src/shared/helpers.ts
@@ -566,6 +566,10 @@ export function isRestartingExpression(text: string) {
   return getLastCharFromTrimmed(text) === ',';
 }
 
+export function shouldBeQuotedSource(text: string) {
+  // Based on lexer `fragment UNQUOTED_SOURCE_PART`
+  return /[:"=|,[\]\/ \t\r\n]/.test(text);
+}
 export function shouldBeQuotedText(
   text: string,
   { dashSupported }: { dashSupported?: boolean } = {}

diff --git a/packages/kbn-monaco/src/esql/language.ts b/packages/kbn-monaco/src/esql/language.ts
@@ -39,11 +39,13 @@ export const ESQLLang: CustomLangModuleType<ESQLCallbacks> = {
       { open: '(', close: ')' },
       { open: '[', close: ']' },
       { open: `'`, close: `'` },
+      { open: '"""', close: '"""' },
       { open: '"', close: '"' },
     ],
     surroundingPairs: [
       { open: '(', close: ')' },
       { open: `'`, close: `'` },
+      { open: '"""', close: '"""' },
       { open: '"', close: '"' },
     ],
   },