Skip to content

Commit

Permalink
[Semantic text] Make semantic text work with non-root level fields (#…
Browse files Browse the repository at this point in the history
…187154)

## Summary

This makes semantic text work with non-root level reference fields. It
also correctly adds copy_to to existing copy_to fields instead of
replacing them, and streamlines a lot of the code.

To test these changes:

- Create an index
- Go to the index mappings page at
`app/management/data/index_management/indices/index_details?{yourIndexName}=blah&tab=mappings`
- Add an object field with a text field inside
- Add a semantic text field referencing that text field
- If you're on a Macbook, create a new inference endpoint with the model
`.elser_model_2` instead of using the default inference endpoint.
- Add a second semantic text field referencing that text field
- Save your mappings
- Use JSON view to verify that the newly created text field contains a
`copy_to` field referencing both newly created semantic text fields
- Verify that the newly created semantic text fields are also in the
JSON view



### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [x] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [x] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [x] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)
  • Loading branch information
sphilipse authored Jul 11, 2024
1 parent 879b7b7 commit 460b520
Show file tree
Hide file tree
Showing 37 changed files with 1,524 additions and 1,033 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,7 @@ export const ElasticsearchModels: React.FC<ElasticsearchModelsProps> = ({
}
}, [numberOfAllocations, numberOfThreads, serviceType]);

const elasticSearchModelTypesDescriptions: Record<
ElasticsearchModelDefaultOptions | string,
ElasticsearchModelDescriptions
> = {
const elasticSearchModelTypesDescriptions: Record<string, ElasticsearchModelDescriptions> = {
[ElasticsearchModelDefaultOptions.elser]: {
description: i18n.translate(
'xpack.ml.addInferenceEndpoint.elasticsearchModels.elser.description',
Expand Down
9 changes: 9 additions & 0 deletions x-pack/packages/ml/trained_models_utils/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,13 @@ export {
ELASTIC_MODEL_TYPE,
MODEL_STATE,
type ModelState,
ELSER_LINUX_OPTIMIZED_MODEL_ID,
ELSER_MODEL_ID,
E5_LINUX_OPTIMIZED_MODEL_ID,
E5_MODEL_ID,
LANG_IDENT_MODEL_ID,
LATEST_ELSER_VERSION,
LATEST_ELSER_MODEL_ID,
LATEST_E5_MODEL_ID,
ElserModels,
} from './src/constants/trained_models';
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,18 @@

import { i18n } from '@kbn/i18n';

export const ELSER_MODEL_ID = '.elser_model_2';
export const ELSER_LINUX_OPTIMIZED_MODEL_ID = '.elser_model_2_linux-x86_64';
export const E5_MODEL_ID = '.multilingual-e5-small';
export const E5_LINUX_OPTIMIZED_MODEL_ID = '.multilingual-e5-small_linux-x86_64';
export const LANG_IDENT_MODEL_ID = 'lang_ident_model_1';
export const ELSER_ID_V1 = '.elser_model_1' as const;
export const LATEST_ELSER_VERSION: ElserVersion = 2;
export const LATEST_ELSER_MODEL_ID = ELSER_LINUX_OPTIMIZED_MODEL_ID;
export const LATEST_E5_MODEL_ID = E5_LINUX_OPTIMIZED_MODEL_ID;

export const ElserModels = [ELSER_MODEL_ID, ELSER_LINUX_OPTIMIZED_MODEL_ID, ELSER_ID_V1];

export const DEPLOYMENT_STATE = {
STARTED: 'started',
STARTING: 'starting',
Expand Down Expand Up @@ -46,10 +58,8 @@ export const BUILT_IN_MODEL_TAG = 'prepackaged';

export const ELASTIC_MODEL_TAG = 'elastic';

export const ELSER_ID_V1 = '.elser_model_1' as const;

export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object.freeze({
'.elser_model_1': {
[ELSER_ID_V1]: {
modelName: 'elser',
hidden: true,
version: 1,
Expand All @@ -63,7 +73,7 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
}),
type: ['elastic', 'pytorch', 'text_expansion'],
},
'.elser_model_2': {
[ELSER_MODEL_ID]: {
modelName: 'elser',
version: 2,
default: true,
Expand All @@ -77,7 +87,7 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
}),
type: ['elastic', 'pytorch', 'text_expansion'],
},
'.elser_model_2_linux-x86_64': {
[ELSER_LINUX_OPTIMIZED_MODEL_ID]: {
modelName: 'elser',
version: 2,
os: 'Linux',
Expand All @@ -92,7 +102,7 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
}),
type: ['elastic', 'pytorch', 'text_expansion'],
},
'.multilingual-e5-small': {
[E5_MODEL_ID]: {
modelName: 'e5',
version: 1,
default: true,
Expand All @@ -108,7 +118,7 @@ export const ELASTIC_MODEL_DEFINITIONS: Record<string, ModelDefinition> = Object
licenseUrl: 'https://huggingface.co/elastic/multilingual-e5-small',
type: ['pytorch', 'text_embedding'],
},
'.multilingual-e5-small_linux-x86_64': {
[E5_LINUX_OPTIMIZED_MODEL_ID]: {
modelName: 'e5',
version: 1,
os: 'Linux',
Expand Down Expand Up @@ -178,23 +188,17 @@ export interface GetModelDownloadConfigOptions {
version?: ElserVersion;
}

export interface LocalInferenceServiceSettings {
service: 'elser' | 'elasticsearch';
service_settings: {
num_allocations: number;
num_threads: number;
model_id: string;
};
}

export type InferenceServiceSettings =
| {
service: 'elser';
service_settings: {
num_allocations: number;
num_threads: number;
model_id: string;
};
}
| {
service: 'elasticsearch';
service_settings: {
num_allocations: number;
num_threads: number;
model_id: string;
};
}
| LocalInferenceServiceSettings
| {
service: 'openai';
service_settings: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,16 @@ import { mockLogger } from '../../__mocks__';

import { MlTrainedModels } from '@kbn/ml-plugin/server';

import { MlModelDeploymentState } from '../../../common/types/ml';

import { fetchMlModels } from './fetch_ml_models';
import {
E5_LINUX_OPTIMIZED_MODEL_ID,
E5_MODEL_ID,
ELSER_LINUX_OPTIMIZED_MODEL_ID,
ELSER_MODEL_ID,
} from './utils';
} from '@kbn/ml-trained-models-utils';

import { MlModelDeploymentState } from '../../../common/types/ml';

import { fetchMlModels } from './fetch_ml_models';

describe('fetchMlModels', () => {
const mockTrainedModelsProvider = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,25 @@ import { Logger } from '@kbn/core/server';
import { i18n } from '@kbn/i18n';
import { MlTrainedModels } from '@kbn/ml-plugin/server';

import {
E5_LINUX_OPTIMIZED_MODEL_ID,
E5_MODEL_ID,
ELSER_LINUX_OPTIMIZED_MODEL_ID,
ELSER_MODEL_ID,
LANG_IDENT_MODEL_ID,
} from '@kbn/ml-trained-models-utils';

import { getMlModelTypesForModelConfig } from '../../../common/ml_inference_pipeline';

import { MlModelDeploymentState, MlModel } from '../../../common/types/ml';

import {
BASE_MODEL,
ELSER_LINUX_OPTIMIZED_MODEL_PLACEHOLDER,
ELSER_MODEL_ID,
ELSER_MODEL_PLACEHOLDER,
E5_LINUX_OPTIMIZED_MODEL_PLACEHOLDER,
E5_MODEL_ID,
E5_MODEL_PLACEHOLDER,
LANG_IDENT_MODEL_ID,
MODEL_TITLES_BY_TYPE,
E5_LINUX_OPTIMIZED_MODEL_ID,
ELSER_LINUX_OPTIMIZED_MODEL_ID,
} from './utils';

let compatibleElserModelId = ELSER_MODEL_ID;
Expand Down
13 changes: 7 additions & 6 deletions x-pack/plugins/enterprise_search/server/lib/ml/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@
import { i18n } from '@kbn/i18n';
import { SUPPORTED_PYTORCH_TASKS } from '@kbn/ml-trained-models-utils';

import { MlModelDeploymentState, MlModel } from '../../../common/types/ml';
import {
E5_LINUX_OPTIMIZED_MODEL_ID,
E5_MODEL_ID,
ELSER_LINUX_OPTIMIZED_MODEL_ID,
ELSER_MODEL_ID,
} from '@kbn/ml-trained-models-utils';

export const ELSER_MODEL_ID = '.elser_model_2';
export const ELSER_LINUX_OPTIMIZED_MODEL_ID = '.elser_model_2_linux-x86_64';
export const E5_MODEL_ID = '.multilingual-e5-small';
export const E5_LINUX_OPTIMIZED_MODEL_ID = '.multilingual-e5-small_linux-x86_64';
export const LANG_IDENT_MODEL_ID = 'lang_ident_model_1';
import { MlModelDeploymentState, MlModel } from '../../../common/types/ml';

export const MODEL_TITLES_BY_TYPE: Record<string, string | undefined> = {
fill_mask: i18n.translate('xpack.enterpriseSearch.content.ml_inference.fill_mask', {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ export interface IndexDetailsPageTestBed extends TestBed {
selectInferenceIdButtonExists: () => void;
openSelectInferencePopover: () => void;
expectDefaultInferenceModelToExists: () => void;
expectCustomInferenceModelToExists: (customInference: string) => Promise<void>;
expectCustomInferenceModelToExists: (customInference: string) => void;
};
settings: {
getCodeBlockContent: () => string;
Expand Down Expand Up @@ -317,23 +317,23 @@ export const setup = async ({
expect(exists('fieldTypesOptions-semantic_text')).toBe(false);
});
},
isReferenceFieldVisible: async () => {
expect(exists('referenceField.select')).toBe(true);
isReferenceFieldVisible: () => {
expect(exists('referenceFieldSelect')).toBe(true);
},
selectInferenceIdButtonExists: async () => {
selectInferenceIdButtonExists: () => {
expect(exists('selectInferenceId')).toBe(true);
expect(exists('inferenceIdButton')).toBe(true);
find('inferenceIdButton').simulate('click');
},
openSelectInferencePopover: async () => {
openSelectInferencePopover: () => {
expect(exists('addInferenceEndpointButton')).toBe(true);
expect(exists('manageInferenceEndpointButton')).toBe(true);
},
expectDefaultInferenceModelToExists: async () => {
expect(exists('default-inference_elser_model_2')).toBe(true);
expect(exists('default-inference_e5')).toBe(true);
expectDefaultInferenceModelToExists: () => {
expect(exists('custom-inference_elser_model_2')).toBe(true);
expect(exists('custom-inference_e5')).toBe(true);
},
expectCustomInferenceModelToExists: async (customInference: string) => {
expectCustomInferenceModelToExists: (customInference: string) => {
expect(exists(customInference)).toBe(true);
},
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ describe('<IndexDetailsPage />', () => {
httpRequestsMockHelpers.setLoadIndexStatsResponse(testIndexName, testIndexStats);
httpRequestsMockHelpers.setLoadIndexMappingResponse(testIndexName, testIndexMappings);
httpRequestsMockHelpers.setLoadIndexSettingsResponse(testIndexName, testIndexSettings);
httpRequestsMockHelpers.setInferenceModels([]);

await act(async () => {
testBed = await setup({
Expand Down Expand Up @@ -692,6 +693,7 @@ describe('<IndexDetailsPage />', () => {
ml: {
mlApi: {
trainedModels: {
getModelsDownloadStatus: jest.fn().mockResolvedValue({}),
getTrainedModels: jest.fn().mockResolvedValue([
{
model_id: '.elser_model_2',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,20 @@
* 2.0.
*/

import {
Form,
useForm,
} from '../../../public/application/components/mappings_editor/shared_imports';
import { registerTestBed } from '@kbn/test-jest-helpers';
import { act } from 'react-dom/test-utils';
import { SelectInferenceId } from '../../../public/application/components/mappings_editor/components/document_fields/field_parameters/select_inference_id';
import {
SelectInferenceId,
SelectInferenceIdProps,
} from '../../../public/application/components/mappings_editor/components/document_fields/field_parameters/select_inference_id';
import React from 'react';

const onChangeMock = jest.fn();
const setValueMock = jest.fn();
const setNewInferenceEndpointMock = jest.fn();
const createInferenceEndpointMock = jest.fn();
const mockDispatch = jest.fn();

jest.mock('../../../public/application/app_context', () => ({
useAppContext: jest.fn().mockReturnValue({
Expand Down Expand Up @@ -41,23 +48,40 @@ jest.mock(
}),
})
);

jest.mock('../../../public/application/components/mappings_editor/mappings_state_context', () => ({
useMappingsState: () => ({ inferenceToModelIdMap: {} }),
useDispatch: () => mockDispatch,
}));

function getTestForm(Component: React.FC<SelectInferenceIdProps>) {
return (defaultProps: SelectInferenceIdProps) => {
const { form } = useForm();
form.setFieldValue('inference_id', 'elser_model_2');
return (
<Form form={form}>
<Component {...(defaultProps as any)} />
</Form>
);
};
}

describe('SelectInferenceId', () => {
let exists: any;
let find: any;

beforeAll(async () => {
const setup = registerTestBed(SelectInferenceId, {
defaultProps: {
onChange: onChangeMock,
'data-test-subj': 'data-inference-endpoint-list',
setValue: setValueMock,
setNewInferenceEndpoint: setNewInferenceEndpointMock,
},
const defaultProps: SelectInferenceIdProps = {
'data-test-subj': 'data-inference-endpoint-list',
createInferenceEndpoint: createInferenceEndpointMock,
};
const setup = registerTestBed(getTestForm(SelectInferenceId), {
defaultProps,
memoryRouter: { wrapComponent: false },
});

await act(async () => {
const testBed = setup();
const testBed = await setup();
exists = testBed.exists;
find = testBed.find;
});
Expand Down
Loading

0 comments on commit 460b520

Please sign in to comment.