Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.16] [SecuritySolution] Update Entity Store transform to read frequency and delay from config (#197992) #198317

Merged
merged 1 commit into from
Oct 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions x-pack/plugins/security_solution/server/config.mock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import type { ExperimentalFeatures } from '../common/experimental_features';
import { parseExperimentalConfigValue } from '../common/experimental_features';
import { getDefaultConfigSettings } from '../common/config_settings';
import type { ConfigType } from './config';
import { duration } from 'moment';

export const createMockConfig = (): ConfigType => {
const enableExperimental: Array<keyof ExperimentalFeatures> = ['responseActionUploadEnabled'];
Expand Down Expand Up @@ -45,6 +46,8 @@ export const createMockConfig = (): ConfigType => {
},
},
entityStore: {
frequency: duration('1m'),
syncDelay: duration('5m'),
developer: {
pipelineDebugMode: false,
},
Expand Down
2 changes: 2 additions & 0 deletions x-pack/plugins/security_solution/server/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,8 @@ export const configSchema = schema.object({
}),
}),
entityStore: schema.object({
syncDelay: schema.duration({ defaultValue: '60s' }),
frequency: schema.duration({ defaultValue: '60s' }),
developer: schema.object({
pipelineDebugMode: schema.boolean({ defaultValue: false }),
}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ import type { EndpointAuthz } from '../../../common/endpoint/types/authz';
import { createLicenseServiceMock } from '../../../common/license/mocks';
import { createFeatureUsageServiceMock } from '../services/feature_usage/mocks';
import { createProductFeaturesServiceMock } from '../../lib/product_features_service/mocks';
import type { ConfigType } from '../../config';

/**
* Creates a mocked EndpointAppContext.
Expand Down Expand Up @@ -158,11 +159,15 @@ export const createMockEndpointAppContextServiceSetupContract =
};
};

type CreateMockEndpointAppContextServiceStartContractType = Omit<
DeeplyMockedKeys<EndpointAppContextServiceStartContract>,
'config'
> & { config: ConfigType }; // DeeplyMockedKeys doesn't support moment.Duration
/**
* Creates a mocked input contract for the `EndpointAppContextService#start()` method
*/
export const createMockEndpointAppContextServiceStartContract =
(): DeeplyMockedKeys<EndpointAppContextServiceStartContract> => {
(): CreateMockEndpointAppContextServiceStartContractType => {
const config = createMockConfig();

const logger = loggingSystemMock.create().get('mock_endpoint_app_context');
Expand All @@ -184,7 +189,7 @@ export const createMockEndpointAppContextServiceStartContract =
securityMock.createMockAuthenticatedUser({ roles: ['superuser'] })
);

const startContract: DeeplyMockedKeys<EndpointAppContextServiceStartContract> = {
const startContract: CreateMockEndpointAppContextServiceStartContractType = {
security,
config,
productFeaturesService: createProductFeaturesServiceMock(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@ import type { EngineStatus } from '../../../../common/api/entity_analytics';

export const DEFAULT_LOOKBACK_PERIOD = '24h';

export const DEFAULT_INTERVAL = '30s';

export const ENGINE_STATUS: Record<Uppercase<EngineStatus>, EngineStatus> = {
INSTALLING: 'installing',
STARTED: 'started',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import type { SortOrder } from '@elastic/elasticsearch/lib/api/types';
import type { EntityType } from '../../../../common/api/entity_analytics/entity_store/common.gen';
import type { DataViewsService } from '@kbn/data-views-plugin/common';
import type { AppClient } from '../../..';
import type { EntityStoreConfig } from './types';

describe('EntityStoreDataClient', () => {
const mockSavedObjectClient = savedObjectsClientMock.create();
Expand All @@ -28,6 +29,7 @@ describe('EntityStoreDataClient', () => {
kibanaVersion: '9.0.0',
dataViewsService: {} as DataViewsService,
appClient: {} as AppClient,
config: {} as EntityStoreConfig,
});

const defaultSearchParams = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ import {
isPromiseFulfilled,
isPromiseRejected,
} from './utils';
import type { EntityRecord } from './types';
import type { EntityRecord, EntityStoreConfig } from './types';
import { CRITICALITY_VALUES } from '../asset_criticality/constants';

interface EntityStoreClientOpts {
Expand All @@ -65,6 +65,7 @@ interface EntityStoreClientOpts {
kibanaVersion: string;
dataViewsService: DataViewsService;
appClient: AppClient;
config: EntityStoreConfig;
}

interface SearchEntitiesParams {
Expand Down Expand Up @@ -120,7 +121,7 @@ export class EntityStoreDataClient {
throw new Error('Task Manager is not available');
}

const { logger } = this.options;
const { logger, config } = this.options;

await this.riskScoreDataClient.createRiskScoreLatestIndex();

Expand Down Expand Up @@ -151,9 +152,10 @@ export class EntityStoreDataClient {
this.options.taskManager,
indexPattern,
filter,
config,
pipelineDebugMode
).catch((error) => {
logger.error(`There was an error during async setup of the Entity Store: ${error}`);
logger.error(`There was an error during async setup of the Entity Store: ${error.message}`);
});

return descriptor;
Expand All @@ -165,6 +167,7 @@ export class EntityStoreDataClient {
taskManager: TaskManagerStartContract,
indexPattern: string,
filter: string,
config: EntityStoreConfig,
pipelineDebugMode: boolean
) {
const { esClient, logger, namespace, appClient, dataViewsService } = this.options;
Expand All @@ -175,6 +178,8 @@ export class EntityStoreDataClient {
entityType,
namespace,
fieldHistoryLength,
syncDelay: `${config.syncDelay.asSeconds()}s`,
frequency: `${config.frequency.asSeconds()}s`,
});
const { entityManagerDefinition } = unitedDefinition;

Expand Down Expand Up @@ -327,15 +332,19 @@ export class EntityStoreDataClient {
taskManager: TaskManagerStartContract,
options = { deleteData: false, deleteEngine: true }
) {
const { namespace, logger, esClient, appClient, dataViewsService } = this.options;
const { namespace, logger, esClient, appClient, dataViewsService, config } = this.options;
const { deleteData, deleteEngine } = options;
const descriptor = await this.engineClient.maybeGet(entityType);
const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);

// TODO delete unitedDefinition from this method. we only need the id for deletion
const unitedDefinition = getUnitedEntityDefinition({
indexPatterns,
entityType,
namespace: this.options.namespace,
fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10,
syncDelay: `${config.syncDelay.asSeconds()}s`,
frequency: `${config.frequency.asSeconds()}s`,
});
const { entityManagerDefinition } = unitedDefinition;
logger.info(`In namespace ${namespace}: Deleting entity store for ${entityType}`);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ export const stopEntityEngineRoute = (

return response.ok({ body: { stopped: engine.status === ENGINE_STATUS.STOPPED } });
} catch (e) {
logger.error('Error in StopEntityEngine:', e);
logger.error(`Error in StopEntityEngine: ${e.message}`);
const error = transformError(e);
return siemResponse.error({
statusCode: error.statusCode,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import type { HostEntity, UserEntity } from '../../../../common/api/entity_analytics';
import type { CriticalityValues } from '../asset_criticality/constants';
import type { EntityAnalyticsConfig } from '../types';

export interface HostEntityRecord extends Omit<HostEntity, 'asset'> {
asset?: {
Expand All @@ -24,3 +25,5 @@ export interface UserEntityRecord extends Omit<UserEntity, 'asset'> {
* It represents the data stored in the entity store index.
*/
export type EntityRecord = HostEntityRecord | UserEntityRecord;

export type EntityStoreConfig = EntityAnalyticsConfig['entityStore'];
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ describe('getUnitedEntityDefinition', () => {
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns,
syncDelay: '1m',
frequency: '1m',
});

it('mapping', () => {
Expand Down Expand Up @@ -172,6 +174,10 @@ describe('getUnitedEntityDefinition', () => {
],
"latest": Object {
"lookbackPeriod": "24h",
"settings": Object {
"frequency": "1m",
"syncDelay": "1m",
},
"timestampField": "@timestamp",
},
"managed": true,
Expand Down Expand Up @@ -312,6 +318,8 @@ describe('getUnitedEntityDefinition', () => {
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns,
syncDelay: '1m',
frequency: '1m',
});

it('mapping', () => {
Expand Down Expand Up @@ -445,6 +453,10 @@ describe('getUnitedEntityDefinition', () => {
],
"latest": Object {
"lookbackPeriod": "24h",
"settings": Object {
"frequency": "1m",
"syncDelay": "1m",
},
"timestampField": "@timestamp",
},
"managed": true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ interface Options {
namespace: string;
fieldHistoryLength: number;
indexPatterns: string[];
syncDelay: string;
frequency: string;
}

export const getUnitedEntityDefinition = memoize(
Expand All @@ -33,6 +35,8 @@ export const getUnitedEntityDefinition = memoize(
namespace,
fieldHistoryLength,
indexPatterns,
syncDelay,
frequency,
}: Options): UnitedEntityDefinition => {
const unitedDefinition = unitedDefinitionBuilders[entityType](fieldHistoryLength);

Expand All @@ -47,6 +51,8 @@ export const getUnitedEntityDefinition = memoize(
...unitedDefinition,
namespace,
indexPatterns,
syncDelay,
frequency,
});
}
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import { DEFAULT_INTERVAL, DEFAULT_LOOKBACK_PERIOD } from '../constants';
import { DEFAULT_LOOKBACK_PERIOD } from '../constants';
import { buildEntityDefinitionId, getIdentityFieldForEntityType } from '../utils';
import type {
FieldRetentionDefinition,
Expand All @@ -25,26 +25,32 @@ export class UnitedEntityDefinition {
entityManagerDefinition: EntityDefinition;
fieldRetentionDefinition: FieldRetentionDefinition;
indexMappings: MappingTypeMapping;
syncDelay: string;
frequency: string;

constructor(opts: {
version: string;
entityType: EntityType;
indexPatterns: string[];
fields: UnitedDefinitionField[];
namespace: string;
syncDelay: string;
frequency: string;
}) {
this.version = opts.version;
this.entityType = opts.entityType;
this.indexPatterns = opts.indexPatterns;
this.fields = opts.fields;
this.frequency = opts.frequency;
this.syncDelay = opts.syncDelay;
this.namespace = opts.namespace;
this.entityManagerDefinition = this.toEntityManagerDefinition();
this.fieldRetentionDefinition = this.toFieldRetentionDefinition();
this.indexMappings = this.toIndexMappings();
}

private toEntityManagerDefinition(): EntityDefinition {
const { entityType, namespace, indexPatterns } = this;
const { entityType, namespace, indexPatterns, syncDelay, frequency } = this;
const identityField = getIdentityFieldForEntityType(this.entityType);
const metadata = this.fields
.filter((field) => field.definition)
Expand All @@ -61,7 +67,10 @@ export class UnitedEntityDefinition {
latest: {
timestampField: '@timestamp',
lookbackPeriod: DEFAULT_LOOKBACK_PERIOD,
interval: DEFAULT_INTERVAL,
settings: {
syncDelay,
frequency,
},
},
version: this.version,
managed: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ export class RequestContextFactory implements IRequestContextFactory {
taskManager: startPlugins.taskManager,
auditLogger: getAuditLogger(),
kibanaVersion: options.kibanaVersion,
config: config.entityAnalytics.entityStore,
});
}),
};
Expand Down