[Semantic text] Make semantic text work with non-root level fields (#…

…187154) ## Summary This makes semantic text work with non-root level reference fields. It also correctly adds copy_to to existing copy_to fields instead of replacing them, and streamlines a lot of the code. To test these changes: - Create an index - Go to the index mappings page at `app/management/data/index_management/indices/index_details?{yourIndexName}=blah&tab=mappings` - Add an object field with a text field inside - Add a semantic text field referencing that text field - If you're on a Macbook, create a new inference endpoint with the model `.elser_model_2` instead of using the default inference endpoint. - Add a second semantic text field referencing that text field - Save your mappings - Use JSON view to verify that the newly created text field contains a `copy_to` field referencing both newly created semantic text fields - Verify that the newly created semantic text fields are also in the JSON view ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [x] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers)
elastic · Jul 11, 2024 · 460b520 · 460b520
1 parent 879b7b7
commit 460b520
Show file tree

Hide file tree

Showing 37 changed files with 1,524 additions and 1,033 deletions.
diff --git a/x-pack/packages/ml/inference_integration_flyout/components/elasticsearch_models.tsx b/x-pack/packages/ml/inference_integration_flyout/components/elasticsearch_models.tsx
@@ -70,10 +70,7 @@ export const ElasticsearchModels: React.FC<ElasticsearchModelsProps> = ({
     }
   }, [numberOfAllocations, numberOfThreads, serviceType]);
 
-  const elasticSearchModelTypesDescriptions: Record<
-    ElasticsearchModelDefaultOptions | string,
-    ElasticsearchModelDescriptions
-  > = {
+  const elasticSearchModelTypesDescriptions: Record<string, ElasticsearchModelDescriptions> = {
     [ElasticsearchModelDefaultOptions.elser]: {
       description: i18n.translate(
         'xpack.ml.addInferenceEndpoint.elasticsearchModels.elser.description',

diff --git a/x-pack/packages/ml/trained_models_utils/index.ts b/x-pack/packages/ml/trained_models_utils/index.ts
@@ -27,4 +27,13 @@ export {
   ELASTIC_MODEL_TYPE,
   MODEL_STATE,
   type ModelState,
+  ELSER_LINUX_OPTIMIZED_MODEL_ID,
+  ELSER_MODEL_ID,
+  E5_LINUX_OPTIMIZED_MODEL_ID,
+  E5_MODEL_ID,
+  LANG_IDENT_MODEL_ID,
+  LATEST_ELSER_VERSION,
+  LATEST_ELSER_MODEL_ID,
+  LATEST_E5_MODEL_ID,
+  ElserModels,
 } from './src/constants/trained_models';
diff --git a/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts b/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts
@@ -7,6 +7,18 @@
 
 import { i18n } from '@kbn/i18n';
 
+export const ELSER_MODEL_ID = '.elser_model_2';
+export const ELSER_LINUX_OPTIMIZED_MODEL_ID = '.elser_model_2_linux-x86_64';
+export const E5_MODEL_ID = '.multilingual-e5-small';
+export const E5_LINUX_OPTIMIZED_MODEL_ID = '.multilingual-e5-small_linux-x86_64';
+export const LANG_IDENT_MODEL_ID = 'lang_ident_model_1';
+export const ELSER_ID_V1 = '.elser_model_1' as const;
+export const LATEST_ELSER_VERSION: ElserVersion = 2;
+export const LATEST_ELSER_MODEL_ID = ELSER_LINUX_OPTIMIZED_MODEL_ID;
+export const LATEST_E5_MODEL_ID = E5_LINUX_OPTIMIZED_MODEL_ID;
+
+export const ElserModels = [ELSER_MODEL_ID, ELSER_LINUX_OPTIMIZED_MODEL_ID, ELSER_ID_V1];
+
 export const DEPLOYMENT_STATE = {
   STARTED: 'started',
   STARTING: 'starting',
@@ -46,10 +58,8 @@ export const BUILT_IN_MODEL_TAG = 'prepackaged';
 
 export const ELASTIC_MODEL_TAG = 'elastic';
 
-export const ELSER_ID_V1 = '.elser_model_1' as const;
-
 export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object.freeze({
-  '.elser_model_1': {
+  [ELSER_ID_V1]: {
     modelName: 'elser',
     hidden: true,
     version: 1,
@@ -63,7 +73,7 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
     }),
     type: ['elastic', 'pytorch', 'text_expansion'],
   },
-  '.elser_model_2': {
+  [ELSER_MODEL_ID]: {
     modelName: 'elser',
     version: 2,
     default: true,
@@ -77,7 +87,7 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
     }),
     type: ['elastic', 'pytorch', 'text_expansion'],
   },
-  '.elser_model_2_linux-x86_64': {
+  [ELSER_LINUX_OPTIMIZED_MODEL_ID]: {
     modelName: 'elser',
     version: 2,
     os: 'Linux',
@@ -92,7 +102,7 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
     }),
     type: ['elastic', 'pytorch', 'text_expansion'],
   },
-  '.multilingual-e5-small': {
+  [E5_MODEL_ID]: {
     modelName: 'e5',
     version: 1,
     default: true,
@@ -108,7 +118,7 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
     licenseUrl: 'https://huggingface.co/elastic/multilingual-e5-small',
     type: ['pytorch', 'text_embedding'],
   },
-  '.multilingual-e5-small_linux-x86_64': {
+  [E5_LINUX_OPTIMIZED_MODEL_ID]: {
     modelName: 'e5',
     version: 1,
     os: 'Linux',
@@ -178,23 +188,17 @@ export interface GetModelDownloadConfigOptions {
   version?: ElserVersion;
 }
 
+export interface LocalInferenceServiceSettings {
+  service: 'elser' | 'elasticsearch';
+  service_settings: {
+    num_allocations: number;
+    num_threads: number;
+    model_id: string;
+  };
+}
+
 export type InferenceServiceSettings =
-  | {
-      service: 'elser';
-      service_settings: {
-        num_allocations: number;
-        num_threads: number;
-        model_id: string;
-      };
-    }
-  | {
-      service: 'elasticsearch';
-      service_settings: {
-        num_allocations: number;
-        num_threads: number;
-        model_id: string;
-      };
-    }
+  | LocalInferenceServiceSettings
   | {
       service: 'openai';
       service_settings: {

diff --git a/x-pack/plugins/enterprise_search/server/lib/ml/fetch_ml_models.test.ts b/x-pack/plugins/enterprise_search/server/lib/ml/fetch_ml_models.test.ts
@@ -9,15 +9,16 @@ import { mockLogger } from '../../__mocks__';
 
 import { MlTrainedModels } from '@kbn/ml-plugin/server';
 
-import { MlModelDeploymentState } from '../../../common/types/ml';
-
-import { fetchMlModels } from './fetch_ml_models';
 import {
   E5_LINUX_OPTIMIZED_MODEL_ID,
   E5_MODEL_ID,
   ELSER_LINUX_OPTIMIZED_MODEL_ID,
   ELSER_MODEL_ID,
-} from './utils';
+} from '@kbn/ml-trained-models-utils';
+
+import { MlModelDeploymentState } from '../../../common/types/ml';
+
+import { fetchMlModels } from './fetch_ml_models';
 
 describe('fetchMlModels', () => {
   const mockTrainedModelsProvider = {

diff --git a/x-pack/plugins/enterprise_search/server/lib/ml/fetch_ml_models.ts b/x-pack/plugins/enterprise_search/server/lib/ml/fetch_ml_models.ts
@@ -11,22 +11,25 @@ import { Logger } from '@kbn/core/server';
 import { i18n } from '@kbn/i18n';
 import { MlTrainedModels } from '@kbn/ml-plugin/server';
 
+import {
+  E5_LINUX_OPTIMIZED_MODEL_ID,
+  E5_MODEL_ID,
+  ELSER_LINUX_OPTIMIZED_MODEL_ID,
+  ELSER_MODEL_ID,
+  LANG_IDENT_MODEL_ID,
+} from '@kbn/ml-trained-models-utils';
+
 import { getMlModelTypesForModelConfig } from '../../../common/ml_inference_pipeline';
 
 import { MlModelDeploymentState, MlModel } from '../../../common/types/ml';
 
 import {
   BASE_MODEL,
   ELSER_LINUX_OPTIMIZED_MODEL_PLACEHOLDER,
-  ELSER_MODEL_ID,
   ELSER_MODEL_PLACEHOLDER,
   E5_LINUX_OPTIMIZED_MODEL_PLACEHOLDER,
-  E5_MODEL_ID,
   E5_MODEL_PLACEHOLDER,
-  LANG_IDENT_MODEL_ID,
   MODEL_TITLES_BY_TYPE,
-  E5_LINUX_OPTIMIZED_MODEL_ID,
-  ELSER_LINUX_OPTIMIZED_MODEL_ID,
 } from './utils';
 
 let compatibleElserModelId = ELSER_MODEL_ID;

diff --git a/x-pack/plugins/enterprise_search/server/lib/ml/utils.ts b/x-pack/plugins/enterprise_search/server/lib/ml/utils.ts
@@ -8,13 +8,14 @@
 import { i18n } from '@kbn/i18n';
 import { SUPPORTED_PYTORCH_TASKS } from '@kbn/ml-trained-models-utils';
 
-import { MlModelDeploymentState, MlModel } from '../../../common/types/ml';
+import {
+  E5_LINUX_OPTIMIZED_MODEL_ID,
+  E5_MODEL_ID,
+  ELSER_LINUX_OPTIMIZED_MODEL_ID,
+  ELSER_MODEL_ID,
+} from '@kbn/ml-trained-models-utils';
 
-export const ELSER_MODEL_ID = '.elser_model_2';
-export const ELSER_LINUX_OPTIMIZED_MODEL_ID = '.elser_model_2_linux-x86_64';
-export const E5_MODEL_ID = '.multilingual-e5-small';
-export const E5_LINUX_OPTIMIZED_MODEL_ID = '.multilingual-e5-small_linux-x86_64';
-export const LANG_IDENT_MODEL_ID = 'lang_ident_model_1';
+import { MlModelDeploymentState, MlModel } from '../../../common/types/ml';
 
 export const MODEL_TITLES_BY_TYPE: Record<string, string | undefined> = {
   fill_mask: i18n.translate('xpack.enterpriseSearch.content.ml_inference.fill_mask', {

diff --git a/...x_management/__jest__/client_integration/index_details_page/index_details_page.helpers.ts b/...x_management/__jest__/client_integration/index_details_page/index_details_page.helpers.ts
@@ -61,7 +61,7 @@ export interface IndexDetailsPageTestBed extends TestBed {
       selectInferenceIdButtonExists: () => void;
       openSelectInferencePopover: () => void;
       expectDefaultInferenceModelToExists: () => void;
-      expectCustomInferenceModelToExists: (customInference: string) => Promise<void>;
+      expectCustomInferenceModelToExists: (customInference: string) => void;
     };
     settings: {
       getCodeBlockContent: () => string;
@@ -317,23 +317,23 @@ export const setup = async ({
         expect(exists('fieldTypesOptions-semantic_text')).toBe(false);
       });
     },
-    isReferenceFieldVisible: async () => {
-      expect(exists('referenceField.select')).toBe(true);
+    isReferenceFieldVisible: () => {
+      expect(exists('referenceFieldSelect')).toBe(true);
     },
-    selectInferenceIdButtonExists: async () => {
+    selectInferenceIdButtonExists: () => {
       expect(exists('selectInferenceId')).toBe(true);
       expect(exists('inferenceIdButton')).toBe(true);
       find('inferenceIdButton').simulate('click');
     },
-    openSelectInferencePopover: async () => {
+    openSelectInferencePopover: () => {
       expect(exists('addInferenceEndpointButton')).toBe(true);
       expect(exists('manageInferenceEndpointButton')).toBe(true);
     },
-    expectDefaultInferenceModelToExists: async () => {
-      expect(exists('default-inference_elser_model_2')).toBe(true);
-      expect(exists('default-inference_e5')).toBe(true);
+    expectDefaultInferenceModelToExists: () => {
+      expect(exists('custom-inference_elser_model_2')).toBe(true);
+      expect(exists('custom-inference_e5')).toBe(true);
     },
-    expectCustomInferenceModelToExists: async (customInference: string) => {
+    expectCustomInferenceModelToExists: (customInference: string) => {
       expect(exists(customInference)).toBe(true);
     },
   };

diff --git a/...dex_management/__jest__/client_integration/index_details_page/index_details_page.test.tsx b/...dex_management/__jest__/client_integration/index_details_page/index_details_page.test.tsx
@@ -69,6 +69,7 @@ describe('<IndexDetailsPage />', () => {
     httpRequestsMockHelpers.setLoadIndexStatsResponse(testIndexName, testIndexStats);
     httpRequestsMockHelpers.setLoadIndexMappingResponse(testIndexName, testIndexMappings);
     httpRequestsMockHelpers.setLoadIndexSettingsResponse(testIndexName, testIndexSettings);
+    httpRequestsMockHelpers.setInferenceModels([]);
 
     await act(async () => {
       testBed = await setup({
@@ -692,6 +693,7 @@ describe('<IndexDetailsPage />', () => {
                   ml: {
                     mlApi: {
                       trainedModels: {
+                        getModelsDownloadStatus: jest.fn().mockResolvedValue({}),
                         getTrainedModels: jest.fn().mockResolvedValue([
                           {
                             model_id: '.elser_model_2',

diff --git a/...ex_management/__jest__/client_integration/index_details_page/select_inference_id.test.tsx b/...ex_management/__jest__/client_integration/index_details_page/select_inference_id.test.tsx
@@ -5,13 +5,20 @@
  * 2.0.
  */
 
+import {
+  Form,
+  useForm,
+} from '../../../public/application/components/mappings_editor/shared_imports';
 import { registerTestBed } from '@kbn/test-jest-helpers';
 import { act } from 'react-dom/test-utils';
-import { SelectInferenceId } from '../../../public/application/components/mappings_editor/components/document_fields/field_parameters/select_inference_id';
+import {
+  SelectInferenceId,
+  SelectInferenceIdProps,
+} from '../../../public/application/components/mappings_editor/components/document_fields/field_parameters/select_inference_id';
+import React from 'react';
 
-const onChangeMock = jest.fn();
-const setValueMock = jest.fn();
-const setNewInferenceEndpointMock = jest.fn();
+const createInferenceEndpointMock = jest.fn();
+const mockDispatch = jest.fn();
 
 jest.mock('../../../public/application/app_context', () => ({
   useAppContext: jest.fn().mockReturnValue({
@@ -41,23 +48,40 @@ jest.mock(
     }),
   })
 );
+
+jest.mock('../../../public/application/components/mappings_editor/mappings_state_context', () => ({
+  useMappingsState: () => ({ inferenceToModelIdMap: {} }),
+  useDispatch: () => mockDispatch,
+}));
+
+function getTestForm(Component: React.FC<SelectInferenceIdProps>) {
+  return (defaultProps: SelectInferenceIdProps) => {
+    const { form } = useForm();
+    form.setFieldValue('inference_id', 'elser_model_2');
+    return (
+      <Form form={form}>
+        <Component {...(defaultProps as any)} />
+      </Form>
+    );
+  };
+}
+
 describe('SelectInferenceId', () => {
   let exists: any;
   let find: any;
 
   beforeAll(async () => {
-    const setup = registerTestBed(SelectInferenceId, {
-      defaultProps: {
-        onChange: onChangeMock,
-        'data-test-subj': 'data-inference-endpoint-list',
-        setValue: setValueMock,
-        setNewInferenceEndpoint: setNewInferenceEndpointMock,
-      },
+    const defaultProps: SelectInferenceIdProps = {
+      'data-test-subj': 'data-inference-endpoint-list',
+      createInferenceEndpoint: createInferenceEndpointMock,
+    };
+    const setup = registerTestBed(getTestForm(SelectInferenceId), {
+      defaultProps,
       memoryRouter: { wrapComponent: false },
     });
 
     await act(async () => {
-      const testBed = setup();
+      const testBed = await setup();
       exists = testBed.exists;
       find = testBed.find;
     });