[Auto Import] Ask LLM to map to non-reserved ECS fields (elastic#195168)

## Release Note

Automatic Import does not ask the LLM to map the fields to the reserved 
ECS fields anymore.

## Summary

Previously we have given the LLM the whole list of ECS fields, but later
failed the validation if the LLM's suggested a mapping into one of the
reserved fields (like `event.created`). With these changes, we hide the
reserved fields from the LLM when creating the prompt, so the likelihood
of this happening is reduced.

We test with the Teleport integration (see GitHub).
---------

Co-authored-by: Elastic Machine <[email protected]>

x-pack/plugins/integration_assistant/common/ecs.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-interface EcsFields {
+export interface EcsFields {
   [key: string]: string;
 }
 

x-pack/plugins/integration_assistant/server/graphs/ecs/constants.ts

@@ -5,9 +5,7 @@
  * 2.0.
  */
 
-interface EcsFields {
-  [key: string]: string;
-}
+import { EcsFields } from '../../../common/ecs';
 
 export const ECS_TYPES: EcsFields = {
   '@timestamp': 'date',

x-pack/plugins/integration_assistant/server/graphs/ecs/model.ts

@@ -10,6 +10,7 @@ import { mergeAndChunkSamples } from './chunk';
 import { ECS_EXAMPLE_ANSWER, ECS_FIELDS } from './constants';
 import { createPipeline } from './pipeline';
 import type { EcsBaseNodeParams } from './types';
+import { removeReservedFields } from './validate';
 
 export function modelSubOutput({ state }: EcsBaseNodeParams): Partial<EcsMappingState> {
   return {
@@ -33,7 +34,7 @@ export function modelInput({ state }: EcsBaseNodeParams): Partial<EcsMappingStat
   const sampleChunks = mergeAndChunkSamples(prefixedSamples, state.chunkSize);
   return {
     exAnswer: JSON.stringify(ECS_EXAMPLE_ANSWER, null, 2),
-    ecs: JSON.stringify(ECS_FIELDS, null, 2),
+    ecs: JSON.stringify(removeReservedFields(ECS_FIELDS), null, 2),
     prefixedSamples,
     sampleChunks,
     finalized: false,

x-pack/plugins/integration_assistant/server/graphs/ecs/validate.test.ts

@@ -5,7 +5,14 @@
  * 2.0.
  */
 
-import { findDuplicateFields, findInvalidEcsFields, processMapping } from './validate';
+import { ECS_RESERVED } from './constants';
+
+import {
+  findDuplicateFields,
+  findInvalidEcsFields,
+  processMapping,
+  removeReservedFields,
+} from './validate';
 
 describe('Testing ecs handler', () => {
   it('processMapping()', async () => {
@@ -122,3 +129,56 @@ describe('findDuplicateFields', () => {
     ]);
   });
 });
+
+describe('removeReservedFields', () => {
+  it('should remove reserved fields from the mapping', () => {
+    const ecsMapping = {
+      'ecs.version': 'Version',
+      'event.category': 'Category',
+      'source.ip': 'IP',
+    };
+
+    const expectedMapping = {
+      'source.ip': 'IP',
+    };
+
+    const result = removeReservedFields(ecsMapping);
+    expect(result).toEqual(expectedMapping);
+  });
+
+  it('should remove all fields if all are reserved', () => {
+    const ecsMapping = Object.fromEntries(ECS_RESERVED.map((key) => [key, key]));
+    const result = removeReservedFields(ecsMapping);
+    expect(result).toEqual({});
+  });
+
+  it('should return the same mapping if there are no reserved fields', () => {
+    const ecsMapping = {
+      'source.ip': 'Some IP',
+      'destination.ip': 'Another IP',
+    };
+
+    const result = removeReservedFields(ecsMapping);
+    expect(result).toEqual(ecsMapping);
+  });
+
+  it('should handle an empty mapping', () => {
+    const ecsMapping = {};
+
+    const result = removeReservedFields(ecsMapping);
+    expect(result).toEqual({});
+  });
+
+  it('should not modify the original mapping object', () => {
+    const ecsMapping = {
+      'ecs.version': 'Version',
+      'source.ip': 'IP',
+    };
+
+    const ecsMappingCopy = { ...ecsMapping };
+
+    const result = removeReservedFields(ecsMapping);
+    expect(ecsMapping).toEqual(ecsMappingCopy);
+    expect(ecsMapping).not.toEqual(result);
+  });
+});

x-pack/plugins/integration_assistant/server/graphs/ecs/validate.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { ECS_FULL } from '../../../common/ecs';
+import { EcsFields, ECS_FULL } from '../../../common/ecs';
 import { mergeSamples } from '../../util/samples';
 import { ECS_RESERVED } from './constants';
 import type { EcsBaseNodeParams } from './types';
@@ -122,6 +122,15 @@ export function findDuplicateFields(prefixedSamples: string[], ecsMapping: AnyOb
   return results;
 }
 
+// Produces a version of ECS mapping without reserved fields.
+export function removeReservedFields(mapping: EcsFields): EcsFields {
+  const mappingCopy = { ...mapping };
+  for (const field of ECS_RESERVED) {
+    delete mappingCopy[field];
+  }
+  return mappingCopy;
+}
+
 // Function to find invalid ECS fields
 export function findInvalidEcsFields(currentMapping: AnyObject): string[] {
   const results: string[] = [];