[8.16] [SecuritySolution] Update Entity Store transform to read frequ…

…ency and delay from config (#197992) (#198317)

# Backport

This will backport the following commits from `main` to `8.16`:
- [SecuritySolution] Update Entity Store transform to read frequency and
delay from config (#197992) (4d4de51)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Pablo
Machado","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-30T10:26:12Z","message":"[SecuritySolution]
Update Entity Store transform to read frequency and delay from config
(#197992)\n\n## Summary\r\n\r\nUpdate Entity Store transform to read
frequency and delay from config.\r\n\r\nNew
Config:\r\n```\r\nxpack.securitySolution.entityAnalytics.entityStore.frequency:
'60s'\r\nxpack.securitySolution.entityAnalytics.entityStore.syncDelay:
'60s'\r\n```\r\n\r\n\r\n### How to test it?\r\n* Update Kibana
config\r\n* Start the entity store\r\n\r\n*** If you update the config
after the entity store is installed it has\r\nno
effect","sha":"4d4de51af979acb79e807408393ce89bdc24d0bc"},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[]}]
BACKPORT-->

x-pack/plugins/security_solution/server/config.mock.ts

@@ -10,6 +10,7 @@ import type { ExperimentalFeatures } from '../common/experimental_features';
 import { parseExperimentalConfigValue } from '../common/experimental_features';
 import { getDefaultConfigSettings } from '../common/config_settings';
 import type { ConfigType } from './config';
+import { duration } from 'moment';
 
 export const createMockConfig = (): ConfigType => {
   const enableExperimental: Array<keyof ExperimentalFeatures> = ['responseActionUploadEnabled'];
@@ -45,6 +46,8 @@ export const createMockConfig = (): ConfigType => {
         },
       },
       entityStore: {
+        frequency: duration('1m'),
+        syncDelay: duration('5m'),
         developer: {
           pipelineDebugMode: false,
         },

x-pack/plugins/security_solution/server/config.ts

@@ -176,6 +176,8 @@ export const configSchema = schema.object({
       }),
     }),
     entityStore: schema.object({
+      syncDelay: schema.duration({ defaultValue: '60s' }),
+      frequency: schema.duration({ defaultValue: '60s' }),
       developer: schema.object({
         pipelineDebugMode: schema.boolean({ defaultValue: false }),
       }),

x-pack/plugins/security_solution/server/endpoint/mocks/mocks.ts

@@ -75,6 +75,7 @@ import type { EndpointAuthz } from '../../../common/endpoint/types/authz';
 import { createLicenseServiceMock } from '../../../common/license/mocks';
 import { createFeatureUsageServiceMock } from '../services/feature_usage/mocks';
 import { createProductFeaturesServiceMock } from '../../lib/product_features_service/mocks';
+import type { ConfigType } from '../../config';
 
 /**
  * Creates a mocked EndpointAppContext.
@@ -158,11 +159,15 @@ export const createMockEndpointAppContextServiceSetupContract =
     };
   };
 
+type CreateMockEndpointAppContextServiceStartContractType = Omit<
+  DeeplyMockedKeys<EndpointAppContextServiceStartContract>,
+  'config'
+> & { config: ConfigType }; // DeeplyMockedKeys doesn't support moment.Duration
 /**
  * Creates a mocked input contract for the `EndpointAppContextService#start()` method
  */
 export const createMockEndpointAppContextServiceStartContract =
-  (): DeeplyMockedKeys<EndpointAppContextServiceStartContract> => {
+  (): CreateMockEndpointAppContextServiceStartContractType => {
     const config = createMockConfig();
 
     const logger = loggingSystemMock.create().get('mock_endpoint_app_context');
@@ -184,7 +189,7 @@ export const createMockEndpointAppContextServiceStartContract =
       securityMock.createMockAuthenticatedUser({ roles: ['superuser'] })
     );
 
-    const startContract: DeeplyMockedKeys<EndpointAppContextServiceStartContract> = {
+    const startContract: CreateMockEndpointAppContextServiceStartContractType = {
       security,
       config,
       productFeaturesService: createProductFeaturesServiceMock(

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts

@@ -9,8 +9,6 @@ import type { EngineStatus } from '../../../../common/api/entity_analytics';
 
 export const DEFAULT_LOOKBACK_PERIOD = '24h';
 
-export const DEFAULT_INTERVAL = '30s';
-
 export const ENGINE_STATUS: Record<Uppercase<EngineStatus>, EngineStatus> = {
   INSTALLING: 'installing',
   STARTED: 'started',

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -15,6 +15,7 @@ import type { SortOrder } from '@elastic/elasticsearch/lib/api/types';
 import type { EntityType } from '../../../../common/api/entity_analytics/entity_store/common.gen';
 import type { DataViewsService } from '@kbn/data-views-plugin/common';
 import type { AppClient } from '../../..';
+import type { EntityStoreConfig } from './types';
 
 describe('EntityStoreDataClient', () => {
   const mockSavedObjectClient = savedObjectsClientMock.create();
@@ -28,6 +29,7 @@ describe('EntityStoreDataClient', () => {
     kibanaVersion: '9.0.0',
     dataViewsService: {} as DataViewsService,
     appClient: {} as AppClient,
+    config: {} as EntityStoreConfig,
   });
 
   const defaultSearchParams = {

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -52,7 +52,7 @@ import {
   isPromiseFulfilled,
   isPromiseRejected,
 } from './utils';
-import type { EntityRecord } from './types';
+import type { EntityRecord, EntityStoreConfig } from './types';
 import { CRITICALITY_VALUES } from '../asset_criticality/constants';
 
 interface EntityStoreClientOpts {
@@ -65,6 +65,7 @@ interface EntityStoreClientOpts {
   kibanaVersion: string;
   dataViewsService: DataViewsService;
   appClient: AppClient;
+  config: EntityStoreConfig;
 }
 
 interface SearchEntitiesParams {
@@ -120,7 +121,7 @@ export class EntityStoreDataClient {
       throw new Error('Task Manager is not available');
     }
 
-    const { logger } = this.options;
+    const { logger, config } = this.options;
 
     await this.riskScoreDataClient.createRiskScoreLatestIndex();
 
@@ -151,9 +152,10 @@ export class EntityStoreDataClient {
       this.options.taskManager,
       indexPattern,
       filter,
+      config,
       pipelineDebugMode
     ).catch((error) => {
-      logger.error(`There was an error during async setup of the Entity Store: ${error}`);
+      logger.error(`There was an error during async setup of the Entity Store: ${error.message}`);
     });
 
     return descriptor;
@@ -165,6 +167,7 @@ export class EntityStoreDataClient {
     taskManager: TaskManagerStartContract,
     indexPattern: string,
     filter: string,
+    config: EntityStoreConfig,
     pipelineDebugMode: boolean
   ) {
     const { esClient, logger, namespace, appClient, dataViewsService } = this.options;
@@ -175,6 +178,8 @@ export class EntityStoreDataClient {
       entityType,
       namespace,
       fieldHistoryLength,
+      syncDelay: `${config.syncDelay.asSeconds()}s`,
+      frequency: `${config.frequency.asSeconds()}s`,
     });
     const { entityManagerDefinition } = unitedDefinition;
 
@@ -327,15 +332,19 @@ export class EntityStoreDataClient {
     taskManager: TaskManagerStartContract,
     options = { deleteData: false, deleteEngine: true }
   ) {
-    const { namespace, logger, esClient, appClient, dataViewsService } = this.options;
+    const { namespace, logger, esClient, appClient, dataViewsService, config } = this.options;
     const { deleteData, deleteEngine } = options;
     const descriptor = await this.engineClient.maybeGet(entityType);
     const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
+
+    // TODO delete unitedDefinition from this method. we only need the id for deletion
     const unitedDefinition = getUnitedEntityDefinition({
       indexPatterns,
       entityType,
       namespace: this.options.namespace,
       fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10,
+      syncDelay: `${config.syncDelay.asSeconds()}s`,
+      frequency: `${config.frequency.asSeconds()}s`,
     });
     const { entityManagerDefinition } = unitedDefinition;
     logger.info(`In namespace ${namespace}: Deleting entity store for ${entityType}`);

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/stop.ts

@@ -47,7 +47,7 @@ export const stopEntityEngineRoute = (
 
           return response.ok({ body: { stopped: engine.status === ENGINE_STATUS.STOPPED } });
         } catch (e) {
-          logger.error('Error in StopEntityEngine:', e);
+          logger.error(`Error in StopEntityEngine: ${e.message}`);
           const error = transformError(e);
           return siemResponse.error({
             statusCode: error.statusCode,

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/types.ts

@@ -7,6 +7,7 @@
 
 import type { HostEntity, UserEntity } from '../../../../common/api/entity_analytics';
 import type { CriticalityValues } from '../asset_criticality/constants';
+import type { EntityAnalyticsConfig } from '../types';
 
 export interface HostEntityRecord extends Omit<HostEntity, 'asset'> {
   asset?: {
@@ -24,3 +25,5 @@ export interface UserEntityRecord extends Omit<UserEntity, 'asset'> {
  * It represents the data stored in the entity store index.
  */
 export type EntityRecord = HostEntityRecord | UserEntityRecord;
+
+export type EntityStoreConfig = EntityAnalyticsConfig['entityStore'];

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts

@@ -15,6 +15,8 @@ describe('getUnitedEntityDefinition', () => {
       namespace: 'test',
       fieldHistoryLength: 10,
       indexPatterns,
+      syncDelay: '1m',
+      frequency: '1m',
     });
 
     it('mapping', () => {
@@ -172,6 +174,10 @@ describe('getUnitedEntityDefinition', () => {
           ],
           "latest": Object {
             "lookbackPeriod": "24h",
+            "settings": Object {
+              "frequency": "1m",
+              "syncDelay": "1m",
+            },
             "timestampField": "@timestamp",
           },
           "managed": true,
@@ -312,6 +318,8 @@ describe('getUnitedEntityDefinition', () => {
       namespace: 'test',
       fieldHistoryLength: 10,
       indexPatterns,
+      syncDelay: '1m',
+      frequency: '1m',
     });
 
     it('mapping', () => {
@@ -445,6 +453,10 @@ describe('getUnitedEntityDefinition', () => {
           ],
           "latest": Object {
             "lookbackPeriod": "24h",
+            "settings": Object {
+              "frequency": "1m",
+              "syncDelay": "1m",
+            },
             "timestampField": "@timestamp",
           },
           "managed": true,

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts

@@ -25,6 +25,8 @@ interface Options {
   namespace: string;
   fieldHistoryLength: number;
   indexPatterns: string[];
+  syncDelay: string;
+  frequency: string;
 }
 
 export const getUnitedEntityDefinition = memoize(
@@ -33,6 +35,8 @@ export const getUnitedEntityDefinition = memoize(
     namespace,
     fieldHistoryLength,
     indexPatterns,
+    syncDelay,
+    frequency,
   }: Options): UnitedEntityDefinition => {
     const unitedDefinition = unitedDefinitionBuilders[entityType](fieldHistoryLength);
 
@@ -47,6 +51,8 @@ export const getUnitedEntityDefinition = memoize(
       ...unitedDefinition,
       namespace,
       indexPatterns,
+      syncDelay,
+      frequency,
     });
   }
 );

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts

@@ -7,7 +7,7 @@
 import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
 import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
 import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
-import { DEFAULT_INTERVAL, DEFAULT_LOOKBACK_PERIOD } from '../constants';
+import { DEFAULT_LOOKBACK_PERIOD } from '../constants';
 import { buildEntityDefinitionId, getIdentityFieldForEntityType } from '../utils';
 import type {
   FieldRetentionDefinition,
@@ -25,26 +25,32 @@ export class UnitedEntityDefinition {
   entityManagerDefinition: EntityDefinition;
   fieldRetentionDefinition: FieldRetentionDefinition;
   indexMappings: MappingTypeMapping;
+  syncDelay: string;
+  frequency: string;
 
   constructor(opts: {
     version: string;
     entityType: EntityType;
     indexPatterns: string[];
     fields: UnitedDefinitionField[];
     namespace: string;
+    syncDelay: string;
+    frequency: string;
   }) {
     this.version = opts.version;
     this.entityType = opts.entityType;
     this.indexPatterns = opts.indexPatterns;
     this.fields = opts.fields;
+    this.frequency = opts.frequency;
+    this.syncDelay = opts.syncDelay;
     this.namespace = opts.namespace;
     this.entityManagerDefinition = this.toEntityManagerDefinition();
     this.fieldRetentionDefinition = this.toFieldRetentionDefinition();
     this.indexMappings = this.toIndexMappings();
   }
 
   private toEntityManagerDefinition(): EntityDefinition {
-    const { entityType, namespace, indexPatterns } = this;
+    const { entityType, namespace, indexPatterns, syncDelay, frequency } = this;
     const identityField = getIdentityFieldForEntityType(this.entityType);
     const metadata = this.fields
       .filter((field) => field.definition)
@@ -61,7 +67,10 @@ export class UnitedEntityDefinition {
       latest: {
         timestampField: '@timestamp',
         lookbackPeriod: DEFAULT_LOOKBACK_PERIOD,
-        interval: DEFAULT_INTERVAL,
+        settings: {
+          syncDelay,
+          frequency,
+        },
       },
       version: this.version,
       managed: true,

x-pack/plugins/security_solution/server/request_context_factory.ts

@@ -212,6 +212,7 @@ export class RequestContextFactory implements IRequestContextFactory {
           taskManager: startPlugins.taskManager,
           auditLogger: getAuditLogger(),
           kibanaVersion: options.kibanaVersion,
+          config: config.entityAnalytics.entityStore,
         });
       }),
     };