[Discover] Fix Field Statistics when discover:searchFieldsFromSource …

…is enabled (#187250)

- Closes #187241

## Summary

This PR excludes `_source` when processing current `columns` in UI
- For Field Statistics table
- For ES|QL fields callout

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Matthias Wilhelm <[email protected]>

src/plugins/discover/public/application/main/components/field_stats_table/field_stats_table.tsx

@@ -56,7 +56,12 @@ export const FieldStatisticsTable = React.memo((props: FieldStatisticsTableProps
   } = props;
 
   const visibleFields = useMemo(
-    () => convertFieldsToFallbackFields({ fields: columns, additionalFieldGroups }),
+    () =>
+      convertFieldsToFallbackFields({
+        // `discover:searchFieldsFromSource` adds `_source` to the columns, but we should exclude it for Field Statistics
+        fields: columns.filter((col) => col !== '_source'),
+        additionalFieldGroups,
+      }),
     [additionalFieldGroups, columns]
   );
   const allFallbackFields = useMemo(

src/plugins/discover/public/application/main/components/layout/discover_documents.tsx

@@ -277,7 +277,8 @@ function DiscoverDocumentsComponent({
       <>
         <SelectedVSAvailableCallout
           esqlQueryColumns={documents?.esqlQueryColumns}
-          selectedColumns={currentColumns}
+          // `discover:searchFieldsFromSource` adds `_source` to the columns, but we should exclude it from the callout
+          selectedColumns={currentColumns.filter((col) => col !== '_source')}
         />
         <SearchResponseWarningsCallout warnings={documentState.interceptedWarnings ?? []} />
       </>

test/functional/apps/discover/group6/_field_stats_table.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'header']);
+  const esArchiver = getService('esArchiver');
+  const testSubjects = getService('testSubjects');
+  const kibanaServer = getService('kibanaServer');
+  const security = getService('security');
+  const defaultSettings = {
+    defaultIndex: 'logstash-*',
+  };
+
+  describe('discover field statistics table', function () {
+    before(async () => {
+      await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
+      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover');
+    });
+
+    after(async () => {
+      await kibanaServer.importExport.unload('test/functional/fixtures/kbn_archiver/discover');
+      await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
+      await kibanaServer.savedObjects.cleanStandardList();
+    });
+
+    [true, false].forEach((shouldSearchFieldsFromSource) => {
+      describe(`discover:searchFieldsFromSource: ${shouldSearchFieldsFromSource}`, function () {
+        before(async function () {
+          await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings();
+          await kibanaServer.uiSettings.update({
+            ...defaultSettings,
+            'discover:searchFieldsFromSource': shouldSearchFieldsFromSource,
+          });
+          await PageObjects.common.navigateToApp('discover');
+          await PageObjects.header.waitUntilLoadingHasFinished();
+          await PageObjects.discover.waitUntilSearchingHasFinished();
+        });
+
+        after(async () => {
+          await kibanaServer.uiSettings.replace({});
+        });
+
+        it('should show Field Statistics data in data view mode', async () => {
+          await testSubjects.click('dscViewModeFieldStatsButton');
+          await PageObjects.header.waitUntilLoadingHasFinished();
+          await testSubjects.existOrFail('dataVisualizerTableContainer');
+
+          await testSubjects.click('dscViewModeDocumentButton');
+          await PageObjects.header.waitUntilLoadingHasFinished();
+          await testSubjects.existOrFail('discoverDocTable');
+        });
+
+        it('should show Field Statistics data in ES|QL mode', async () => {
+          await PageObjects.discover.selectTextBaseLang();
+          await PageObjects.discover.waitUntilSearchingHasFinished();
+
+          await testSubjects.click('dscViewModeFieldStatsButton');
+          await PageObjects.header.waitUntilLoadingHasFinished();
+          await testSubjects.existOrFail('dataVisualizerTableContainer');
+        });
+      });
+    });
+  });
+}

test/functional/apps/discover/group6/index.ts

@@ -25,5 +25,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./_time_field_column'));
     loadTestFile(require.resolve('./_unsaved_changes_badge'));
     loadTestFile(require.resolve('./_view_mode_toggle'));
+    loadTestFile(require.resolve('./_field_stats_table'));
   });
 }