github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[SecuritySolution] Load entity store indices from security solution data view (#195862)

Browse source 
## Summary

* Update the Entity Store to retrieve indices from the security solution
data view.
* Create a new API that updates all installed entity engine indices
(`api/entity_store/engines/apply_dataview_indices`)


### How to test it?
* Install the entity store
* Check if the transform index has the security solutions data view
indices
* Call `apply_dataview_indices` API; it should not return changes
* Update the security solution data view indices
* Call `apply_dataview_indices` API and if the API response contains the
updated indices
* Check if the transform index also got updated

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Pablo Machado 2024-10-14 22:56:58 +02:00 • committed by GitHub
parent 10622964ef
commit 489c0901ff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 954 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

65
oas_docs/output/kibana.serverless.staging.yaml
View file

@ -8421,6 +8421,56 @@ paths:
summary: Stop an Entity Engine
tags:
- Security Entity Analytics API
/api/entity_store/engines/apply_dataview_indices:
post:
operationId: ApplyEntityEngineDataviewIndices
responses:
'200':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
result:
items:
$ref: >-
#/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult
type: array
success:
type: boolean
description: Successful response
'207':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
errors:
items:
type: string
type: array
result:
items:
$ref: >-
#/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult
type: array
success:
type: boolean
description: Partial successful response
'500':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
body:
type: string
statusCode:
type: number
description: Error response
summary: Apply DataView indices to all installed engines
tags:
- Security Entity Analytics API
/api/entity_store/entities/list:
get:
description: List entities records, paging, sorting and filtering as needed.
@ -47909,6 +47959,20 @@ components:
#/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel
required:
- criticality_level
Security_Entity_Analytics_API_EngineDataviewUpdateResult:
type: object
properties:
changes:
type: object
properties:
indexPatterns:
items:
type: string
type: array
type:
type: string
required:
- type
Security_Entity_Analytics_API_EngineDescriptor:
type: object
properties:
@ -47932,6 +47996,7 @@ components:
- installing
- started
- stopped
- updating
type: string
Security_Entity_Analytics_API_Entity:
oneOf:

65
oas_docs/output/kibana.serverless.yaml
View file

@ -8421,6 +8421,56 @@ paths:
summary: Stop an Entity Engine
tags:
- Security Entity Analytics API
/api/entity_store/engines/apply_dataview_indices:
post:
operationId: ApplyEntityEngineDataviewIndices
responses:
'200':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
result:
items:
$ref: >-
#/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult
type: array
success:
type: boolean
description: Successful response
'207':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
errors:
items:
type: string
type: array
result:
items:
$ref: >-
#/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult
type: array
success:
type: boolean
description: Partial successful response
'500':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
body:
type: string
statusCode:
type: number
description: Error response
summary: Apply DataView indices to all installed engines
tags:
- Security Entity Analytics API
/api/entity_store/entities/list:
get:
description: List entities records, paging, sorting and filtering as needed.
@ -47909,6 +47959,20 @@ components:
#/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel
required:
- criticality_level
Security_Entity_Analytics_API_EngineDataviewUpdateResult:
type: object
properties:
changes:
type: object
properties:
indexPatterns:
items:
type: string
type: array
type:
type: string
required:
- type
Security_Entity_Analytics_API_EngineDescriptor:
type: object
properties:
@ -47932,6 +47996,7 @@ components:
- installing
- started
- stopped
- updating
type: string
Security_Entity_Analytics_API_Entity:
oneOf:

65
oas_docs/output/kibana.staging.yaml
View file

@ -11850,6 +11850,56 @@ paths:
summary: Stop an Entity Engine
tags:
- Security Entity Analytics API
/api/entity_store/engines/apply_dataview_indices:
post:
operationId: ApplyEntityEngineDataviewIndices
responses:
'200':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
result:
items:
$ref: >-
#/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult
type: array
success:
type: boolean
description: Successful response
'207':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
errors:
items:
type: string
type: array
result:
items:
$ref: >-
#/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult
type: array
success:
type: boolean
description: Partial successful response
'500':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
body:
type: string
statusCode:
type: number
description: Error response
summary: Apply DataView indices to all installed engines
tags:
- Security Entity Analytics API
/api/entity_store/entities/list:
get:
description: List entities records, paging, sorting and filtering as needed.
@ -56675,6 +56725,20 @@ components:
#/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel
required:
- criticality_level
Security_Entity_Analytics_API_EngineDataviewUpdateResult:
type: object
properties:
changes:
type: object
properties:
indexPatterns:
items:
type: string
type: array
type:
type: string
required:
- type
Security_Entity_Analytics_API_EngineDescriptor:
type: object
properties:
@ -56698,6 +56762,7 @@ components:
- installing
- started
- stopped
- updating
type: string
Security_Entity_Analytics_API_Entity:
oneOf:

65
oas_docs/output/kibana.yaml
View file

@ -11850,6 +11850,56 @@ paths:
summary: Stop an Entity Engine
tags:
- Security Entity Analytics API
/api/entity_store/engines/apply_dataview_indices:
post:
operationId: ApplyEntityEngineDataviewIndices
responses:
'200':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
result:
items:
$ref: >-
#/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult
type: array
success:
type: boolean
description: Successful response
'207':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
errors:
items:
type: string
type: array
result:
items:
$ref: >-
#/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult
type: array
success:
type: boolean
description: Partial successful response
'500':
content:
application/json; Elastic-Api-Version=2023-10-31:
schema:
type: object
properties:
body:
type: string
statusCode:
type: number
description: Error response
summary: Apply DataView indices to all installed engines
tags:
- Security Entity Analytics API
/api/entity_store/entities/list:
get:
description: List entities records, paging, sorting and filtering as needed.
@ -56675,6 +56725,20 @@ components:
#/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel
required:
- criticality_level
Security_Entity_Analytics_API_EngineDataviewUpdateResult:
type: object
properties:
changes:
type: object
properties:
indexPatterns:
items:
type: string
type: array
type:
type: string
required:
- type
Security_Entity_Analytics_API_EngineDescriptor:
type: object
properties:
@ -56698,6 +56762,7 @@ components:
- installing
- started
- stopped
- updating
type: string
Security_Entity_Analytics_API_Entity:
oneOf:

13
x-pack/plugins/entity_manager/server/lib/entities/errors/entity_definition_update_conflict.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export class EntityDefinitionUpdateConflict extends Error {
constructor(message: string) {
super(message);
this.name = 'EntityDefinitionUpdateConflict';
}
}

56
x-pack/plugins/entity_manager/server/lib/entity_client.ts
View file

@ -5,17 +5,23 @@
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { Logger } from '@kbn/logging';
import { installEntityDefinition } from './entities/install_entity_definition';
import {
installEntityDefinition,
installationInProgress,
reinstallEntityDefinition,
} from './entities/install_entity_definition';
import { startTransforms } from './entities/start_transforms';
import { findEntityDefinitions } from './entities/find_entity_definition';
import { findEntityDefinitionById, findEntityDefinitions } from './entities/find_entity_definition';
import { uninstallEntityDefinition } from './entities/uninstall_entity_definition';
import { EntityDefinitionNotFound } from './entities/errors/entity_not_found';
import { stopTransforms } from './entities/stop_transforms';
import { EntityDefinitionWithState } from './entities/types';
import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
export class EntityClient {
constructor(
@ -47,6 +53,50 @@ export class EntityClient {
return installedDefinition;
}
async updateEntityDefinition({
id,
definitionUpdate,
}: {
id: string;
definitionUpdate: EntityDefinitionUpdate;
}) {
const definition = await findEntityDefinitionById({
id,
soClient: this.options.soClient,
esClient: this.options.esClient,
includeState: true,
});
if (!definition) {
const message = `Unable to find entity definition with [${id}]`;
this.options.logger.error(message);
throw new EntityDefinitionNotFound(message);
}
if (installationInProgress(definition)) {
const message = `Entity definition [${definition.id}] has changes in progress`;
this.options.logger.error(message);
throw new EntityDefinitionUpdateConflict(message);
}
const shouldRestartTransforms = (
definition as EntityDefinitionWithState
).state.components.transforms.some((transform) => transform.running);
const updatedDefinition = await reinstallEntityDefinition({
definition,
definitionUpdate,
soClient: this.options.soClient,
esClient: this.options.esClient,
logger: this.options.logger,
});
if (shouldRestartTransforms) {
await startTransforms(this.options.esClient, updatedDefinition, this.options.logger);
}
return updatedDefinition;
}
async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) {
const [definition] = await findEntityDefinitions({
id,

2
x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
View file

@ -25,7 +25,7 @@ export type IndexPattern = z.infer<typeof IndexPattern>;
export const IndexPattern = z.string();
export type EngineStatus = z.infer<typeof EngineStatus>;
export const EngineStatus = z.enum(['installing', 'started', 'stopped']);
export const EngineStatus = z.enum(['installing', 'started', 'stopped', 'updating']);
export type EngineStatusEnum = typeof EngineStatus.enum;
export const EngineStatusEnum = EngineStatus.enum;

1
x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
View file

@ -37,6 +37,7 @@ components:
- installing
- started
- stopped
- updating
IndexPattern:
type: string

35
x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/*
* NOTICE: Do not edit this file manually.
* This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
*
* info:
* title: Apply DataView indices to all installed engines
* version: 2023-10-31
*/
import { z } from '@kbn/zod';
export type EngineDataviewUpdateResult = z.infer<typeof EngineDataviewUpdateResult>;
export const EngineDataviewUpdateResult = z.object({
type: z.string(),
changes: z
.object({
indexPatterns: z.array(z.string()).optional(),
})
.optional(),
});
export type ApplyEntityEngineDataviewIndicesResponse = z.infer<
typeof ApplyEntityEngineDataviewIndicesResponse
>;
export const ApplyEntityEngineDataviewIndicesResponse = z.object({
success: z.boolean().optional(),
result: z.array(EngineDataviewUpdateResult).optional(),
});

71
x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.schema.yaml Normal file
View file

@ -0,0 +1,71 @@
openapi: 3.0.0
info:
title: Apply DataView indices to all installed engines
version: '2023-10-31'
paths:
/api/entity_store/engines/apply_dataview_indices:
post:
x-labels: [ess, serverless]
x-codegen-enabled: true
operationId: ApplyEntityEngineDataviewIndices
summary: Apply DataView indices to all installed engines
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
result:
type: array
items:
$ref: '#/components/schemas/EngineDataviewUpdateResult'
'207':
description: Partial successful response
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
result:
type: array
items:
$ref: '#/components/schemas/EngineDataviewUpdateResult'
errors:
type: array
items:
type: string
'500':
description: Error response
content:
application/json:
schema:
type: object
properties:
body:
type: string
statusCode:
type: number
components:
schemas:
EngineDataviewUpdateResult:
type: object
properties:
type:
type: string
changes:
type: object
properties:
indexPatterns:
type: array
items:
type: string
required:
- type

1
x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/index.ts
View file

@ -12,3 +12,4 @@ export * from './list.gen';
export * from './start.gen';
export * from './stats.gen';
export * from './stop.gen';
export * from './apply_dataview_indices.gen';

13
x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts
View file

@ -243,6 +243,7 @@ import type {
InternalUploadAssetCriticalityRecordsResponse,
UploadAssetCriticalityRecordsResponse,
} from './entity_analytics/asset_criticality/upload_asset_criticality_csv.gen';
import type { ApplyEntityEngineDataviewIndicesResponse } from './entity_analytics/entity_store/engine/apply_dataview_indices.gen';
import type {
DeleteEntityEngineRequestQueryInput,
DeleteEntityEngineRequestParamsInput,
@ -397,6 +398,18 @@ after 30 days. It also deletes other artifacts specific to the migration impleme
})
.catch(catchAxiosErrorFormatAndThrow);
}
async applyEntityEngineDataviewIndices() {
this.log.info(`${new Date().toISOString()} Calling API ApplyEntityEngineDataviewIndices`);
return this.kbnClient
.request<ApplyEntityEngineDataviewIndicesResponse>({
path: '/api/entity_store/engines/apply_dataview_indices',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
})
.catch(catchAxiosErrorFormatAndThrow);
}
async assetCriticalityGetPrivileges() {
this.log.info(`${new Date().toISOString()} Calling API AssetCriticalityGetPrivileges`);
return this.kbnClient

63
x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
View file

@ -452,6 +452,54 @@ paths:
summary: Stop an Entity Engine
tags:
- Security Entity Analytics API
/api/entity_store/engines/apply_dataview_indices:
post:
operationId: ApplyEntityEngineDataviewIndices
responses:
'200':
content:
application/json:
schema:
type: object
properties:
result:
items:
$ref: '#/components/schemas/EngineDataviewUpdateResult'
type: array
success:
type: boolean
description: Successful response
'207':
content:
application/json:
schema:
type: object
properties:
errors:
items:
type: string
type: array
result:
items:
$ref: '#/components/schemas/EngineDataviewUpdateResult'
type: array
success:
type: boolean
description: Partial successful response
'500':
content:
application/json:
schema:
type: object
properties:
body:
type: string
statusCode:
type: number
description: Error response
summary: Apply DataView indices to all installed engines
tags:
- Security Entity Analytics API
/api/entity_store/entities/list:
get:
description: List entities records, paging, sorting and filtering as needed.
@ -720,6 +768,20 @@ components:
$ref: '#/components/schemas/AssetCriticalityLevel'
required:
- criticality_level
EngineDataviewUpdateResult:
type: object
properties:
changes:
type: object
properties:
indexPatterns:
items:
type: string
type: array
type:
type: string
required:
- type
EngineDescriptor:
type: object
properties:
@ -743,6 +805,7 @@ components:
- installing
- started
- stopped
- updating
type: string
Entity:
oneOf:

63
x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
View file

@ -452,6 +452,54 @@ paths:
summary: Stop an Entity Engine
tags:
- Security Entity Analytics API
/api/entity_store/engines/apply_dataview_indices:
post:
operationId: ApplyEntityEngineDataviewIndices
responses:
'200':
content:
application/json:
schema:
type: object
properties:
result:
items:
$ref: '#/components/schemas/EngineDataviewUpdateResult'
type: array
success:
type: boolean
description: Successful response
'207':
content:
application/json:
schema:
type: object
properties:
errors:
items:
type: string
type: array
result:
items:
$ref: '#/components/schemas/EngineDataviewUpdateResult'
type: array
success:
type: boolean
description: Partial successful response
'500':
content:
application/json:
schema:
type: object
properties:
body:
type: string
statusCode:
type: number
description: Error response
summary: Apply DataView indices to all installed engines
tags:
- Security Entity Analytics API
/api/entity_store/entities/list:
get:
description: List entities records, paging, sorting and filtering as needed.
@ -720,6 +768,20 @@ components:
$ref: '#/components/schemas/AssetCriticalityLevel'
required:
- criticality_level
EngineDataviewUpdateResult:
type: object
properties:
changes:
type: object
properties:
indexPatterns:
items:
type: string
type: array
type:
type: string
required:
- type
EngineDescriptor:
type: object
properties:
@ -743,6 +805,7 @@ components:
- installing
- started
- stopped
- updating
type: string
Entity:
oneOf:

3
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts
View file

@ -6,13 +6,11 @@
*/
import type { EngineStatus } from '../../../../common/api/entity_analytics/entity_store/common.gen';
import { DEFAULT_INDEX_PATTERN } from '../../../../common/constants';
/**
* Default index pattern for entity store
* This is the same as the default index pattern for the SIEM app but might diverge in the future
*/
export const ENTITY_STORE_DEFAULT_SOURCE_INDICES = DEFAULT_INDEX_PATTERN;
export const DEFAULT_LOOKBACK_PERIOD = '24h';
@ -22,6 +20,7 @@ export const ENGINE_STATUS: Record<Uppercase<EngineStatus>, EngineStatus> = {
INSTALLING: 'installing',
STARTED: 'started',
STOPPED: 'stopped',
UPDATING: 'updating',
};
export const MAX_SEARCH_RESPONSE_SIZE = 10_000;

4
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts
View file

@ -13,6 +13,8 @@ import {
import { EntityStoreDataClient } from './entity_store_data_client';
import type { SortOrder } from '@elastic/elasticsearch/lib/api/types';
import type { EntityType } from '../../../../common/api/entity_analytics/entity_store/common.gen';
import type { DataViewsService } from '@kbn/data-views-plugin/common';
import type { AppClient } from '../../..';
describe('EntityStoreDataClient', () => {
const mockSavedObjectClient = savedObjectsClientMock.create();
@ -24,6 +26,8 @@ describe('EntityStoreDataClient', () => {
namespace: 'default',
soClient: mockSavedObjectClient,
kibanaVersion: '9.0.0',
dataViewsService: {} as DataViewsService,
appClient: {} as AppClient,
});
const defaultSearchParams = {

102
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
View file

@ -14,6 +14,10 @@ import type {
import { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client';
import type { SortOrder } from '@elastic/elasticsearch/lib/api/types';
import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
import type { DataViewsService } from '@kbn/data-views-plugin/common';
import { isEqual } from 'lodash/fp';
import type { EngineDataviewUpdateResult } from '../../../../common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen';
import type { AppClient } from '../../..';
import type { Entity } from '../../../../common/api/entity_analytics/entity_store/entities/common.gen';
import type {
InitEntityEngineRequestBody,
@ -24,7 +28,6 @@ import type {
InspectQuery,
} from '../../../../common/api/entity_analytics/entity_store/common.gen';
import { EngineDescriptorClient } from './saved_object/engine_descriptor';
import { buildEntityDefinitionId, getEntitiesIndexName } from './utils';
import { ENGINE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
import { AssetCriticalityEcsMigrationClient } from '../asset_criticality/asset_criticality_migration_client';
import { getUnitedEntityDefinition } from './united_entity_definitions';
@ -44,6 +47,13 @@ import {
deleteFieldRetentionEnrichPolicy,
} from './elasticsearch_assets';
import { RiskScoreDataClient } from '../risk_score/risk_score_data_client';
import {
buildEntityDefinitionId,
buildIndexPatterns,
getEntitiesIndexName,
isPromiseFulfilled,
isPromiseRejected,
} from './utils';
interface EntityStoreClientOpts {
logger: Logger;
@ -53,6 +63,8 @@ interface EntityStoreClientOpts {
taskManager?: TaskManagerStartContract;
auditLogger?: AuditLogger;
kibanaVersion: string;
dataViewsService: DataViewsService;
appClient: AppClient;
}
interface SearchEntitiesParams {
@ -108,7 +120,7 @@ export class EntityStoreDataClient {
throw new Error('Task Manager is not available');
}
const { logger, esClient, namespace, taskManager } = this.options;
const { logger, esClient, namespace, taskManager, appClient, dataViewsService } = this.options;
await this.riskScoreDataClient.createRiskScoreLatestIndex();
@ -132,9 +144,11 @@ export class EntityStoreDataClient {
indexPattern,
});
logger.debug(`Initialized engine for ${entityType}`);
const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
// first create the entity definition without starting it
// so that the index template is created which we can add a component template to
const unitedDefinition = getUnitedEntityDefinition({
indexPatterns,
entityType,
namespace,
fieldHistoryLength,
@ -221,7 +235,6 @@ export class EntityStoreDataClient {
public async start(entityType: EntityType, options?: { force: boolean }) {
const descriptor = await this.engineClient.get(entityType);
if (!options?.force && descriptor.status !== ENGINE_STATUS.STOPPED) {
throw new Error(
`In namespace ${this.options.namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}`
@ -273,9 +286,11 @@ export class EntityStoreDataClient {
taskManager: TaskManagerStartContract,
deleteData: boolean
) {
const { namespace, logger, esClient } = this.options;
const { namespace, logger, esClient, appClient, dataViewsService } = this.options;
const descriptor = await this.engineClient.maybeGet(entityType);
const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService);
const unitedDefinition = getUnitedEntityDefinition({
indexPatterns,
entityType,
namespace: this.options.namespace,
fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10,
@ -368,4 +383,83 @@ export class EntityStoreDataClient {
return { records, total, inspect };
}
public async applyDataViewIndices(): Promise<{
successes: EngineDataviewUpdateResult[];
errors: Error[];
}> {
const { logger } = this.options;
logger.info(
`In namespace ${this.options.namespace}: Applying data view indices to the entity store`
);
const { engines } = await this.engineClient.list();
const updateDefinitionPromises: Array<Promise<EngineDataviewUpdateResult>> = await engines.map(
async (engine) => {
const originalStatus = engine.status;
const id = buildEntityDefinitionId(engine.type, this.options.namespace);
const definition = await this.getExistingEntityDefinition(engine.type);
if (
originalStatus === ENGINE_STATUS.INSTALLING ||
originalStatus === ENGINE_STATUS.UPDATING
) {
throw new Error(
`Error updating entity store: There is an changes already in progress for engine ${id}`
);
}
const indexPatterns = await buildIndexPatterns(
this.options.namespace,
this.options.appClient,
this.options.dataViewsService
);
// Skip update if index patterns are the same
if (isEqual(definition.indexPatterns, indexPatterns)) {
return { type: engine.type, changes: {} };
}
// Update savedObject status
await this.engineClient.update(engine.type, ENGINE_STATUS.UPDATING);
try {
// Update entity manager definition
await this.entityClient.updateEntityDefinition({
id,
definitionUpdate: {
...definition,
indexPatterns,
},
});
// Restore the savedObject status and set the new index pattern
await this.engineClient.update(engine.type, originalStatus);
return { type: engine.type, changes: { indexPatterns } };
} catch (error) {
// Rollback the engine initial status when the update fails
await this.engineClient.update(engine.type, originalStatus);
throw error;
}
}
);
const updatedDefinitions = await Promise.allSettled(updateDefinitionPromises);
const updateErrors = updatedDefinitions
.filter(isPromiseRejected)
.map((result) => result.reason);
const updateSuccesses = updatedDefinitions
.filter(isPromiseFulfilled<EngineDataviewUpdateResult>)
.map((result) => result.value);
return {
successes: updateSuccesses,
errors: updateErrors,
};
}
}

85
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/apply_dataview_indices.ts Normal file
View file

@ -0,0 +1,85 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IKibanaResponse, Logger } from '@kbn/core/server';
import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import type { ApplyEntityEngineDataviewIndicesResponse } from '../../../../../common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen';
import { API_VERSIONS, APP_ID } from '../../../../../common/constants';
import type { EntityAnalyticsRoutesDeps } from '../../types';
export const applyDataViewIndicesEntityEngineRoute = (
router: EntityAnalyticsRoutesDeps['router'],
logger: Logger
) => {
router.versioned
.post({
access: 'public',
path: '/api/entity_store/engines/apply_dataview_indices',
options: {
tags: ['access:securitySolution', `access:${APP_ID}-entity-analytics`],
},
})
.addVersion(
{
version: API_VERSIONS.public.v1,
validate: {
request: {},
},
},
async (
context,
_,
response
): Promise<IKibanaResponse<ApplyEntityEngineDataviewIndicesResponse>> => {
const siemResponse = buildSiemResponse(response);
try {
const secSol = await context.securitySolution;
const { errors, successes } = await secSol
.getEntityStoreDataClient()
.applyDataViewIndices();
const errorMessages = errors.map((e) => e.message);
if (successes.length === 0 && errors.length > 0) {
return siemResponse.error({
statusCode: 500,
body: `Error in ApplyEntityEngineDataViewIndices. Errors: [${errorMessages.join(
', '
)}]`,
});
}
if (errors.length === 0) {
return response.ok({
body: {
success: true,
result: successes,
},
});
} else {
return response.multiStatus({
body: {
success: false,
errors: errorMessages,
result: successes,
},
});
}
} catch (e) {
logger.error('Error in ApplyEntityEngineDataViewIndices:', e);
const error = transformError(e);
return siemResponse.error({
statusCode: error.statusCode,
body: error.message,
});
}
}
);
};

2
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/register_entity_store_routes.ts
View file

@ -6,6 +6,7 @@
*/
import type { EntityAnalyticsRoutesDeps } from '../../types';
import { applyDataViewIndicesEntityEngineRoute } from './apply_dataview_indices';
import { deleteEntityEngineRoute } from './delete';
import { listEntitiesRoute } from './entities/list';
import { getEntityEngineRoute } from './get';
@ -27,4 +28,5 @@ export const registerEntityStoreRoutes = ({
getEntityEngineRoute(router, logger);
listEntityEnginesRoute(router, logger);
listEntitiesRoute(router, logger);
applyDataViewIndicesEntityEngineRoute(router, logger);
};

27
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts
View file

@ -8,11 +8,13 @@
import { getUnitedEntityDefinition } from './get_united_definition';
describe('getUnitedEntityDefinition', () => {
const indexPatterns = ['test*'];
describe('host', () => {
const unitedDefinition = getUnitedEntityDefinition({
entityType: 'host',
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns,
});
it('mapping', () => {
@ -151,17 +153,7 @@ describe('getUnitedEntityDefinition', () => {
},
],
"indexPatterns": Array [
"apm-*-transaction*",
"auditbeat-*",
"endgame-*",
"filebeat-*",
"logs-*",
"packetbeat-*",
"traces-apm*",
"winlogbeat-*",
"-*elastic-cloud-logs-*",
".asset-criticality.asset-criticality-test",
"risk-score.risk-score-latest-test",
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",
@ -286,6 +278,7 @@ describe('getUnitedEntityDefinition', () => {
entityType: 'user',
namespace: 'test',
fieldHistoryLength: 10,
indexPatterns,
});
it('mapping', () => {
@ -416,17 +409,7 @@ describe('getUnitedEntityDefinition', () => {
},
],
"indexPatterns": Array [
"apm-*-transaction*",
"auditbeat-*",
"endgame-*",
"filebeat-*",
"logs-*",
"packetbeat-*",
"traces-apm*",
"winlogbeat-*",
"-*elastic-cloud-logs-*",
".asset-criticality.asset-criticality-test",
"risk-score.risk-score-latest-test",
"test*",
],
"latest": Object {
"lookbackPeriod": "24h",

9
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts
View file

@ -24,10 +24,16 @@ interface Options {
entityType: EntityType;
namespace: string;
fieldHistoryLength: number;
indexPatterns: string[];
}
export const getUnitedEntityDefinition = memoize(
({ entityType, namespace, fieldHistoryLength }: Options): UnitedEntityDefinition => {
({
entityType,
namespace,
fieldHistoryLength,
indexPatterns,
}: Options): UnitedEntityDefinition => {
const unitedDefinition = unitedDefinitionBuilders[entityType](fieldHistoryLength);
unitedDefinition.fields.push(
@ -40,6 +46,7 @@ export const getUnitedEntityDefinition = memoize(
return new UnitedEntityDefinition({
...unitedDefinition,
namespace,
indexPatterns,
});
},
({ entityType, namespace, fieldHistoryLength }: Options) =>

19
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts
View file

@ -7,13 +7,7 @@
import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema';
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
import {
DEFAULT_INTERVAL,
DEFAULT_LOOKBACK_PERIOD,
ENTITY_STORE_DEFAULT_SOURCE_INDICES,
} from '../constants';
import { DEFAULT_INTERVAL, DEFAULT_LOOKBACK_PERIOD } from '../constants';
import { buildEntityDefinitionId, getIdentityFieldForEntityType } from '../utils';
import type {
FieldRetentionDefinition,
@ -25,6 +19,7 @@ import { BASE_ENTITY_INDEX_MAPPING } from './constants';
export class UnitedEntityDefinition {
version: string;
entityType: EntityType;
indexPatterns: string[];
fields: UnitedDefinitionField[];
namespace: string;
entityManagerDefinition: EntityDefinition;
@ -34,11 +29,13 @@ export class UnitedEntityDefinition {
constructor(opts: {
version: string;
entityType: EntityType;
indexPatterns: string[];
fields: UnitedDefinitionField[];
namespace: string;
}) {
this.version = opts.version;
this.entityType = opts.entityType;
this.indexPatterns = opts.indexPatterns;
this.fields = opts.fields;
this.namespace = opts.namespace;
this.entityManagerDefinition = this.toEntityManagerDefinition();
@ -47,7 +44,7 @@ export class UnitedEntityDefinition {
}
private toEntityManagerDefinition(): EntityDefinition {
const { entityType, namespace } = this;
const { entityType, namespace, indexPatterns } = this;
const identityField = getIdentityFieldForEntityType(this.entityType);
const metadata = this.fields
.filter((field) => field.definition)
@ -57,11 +54,7 @@ export class UnitedEntityDefinition {
id: buildEntityDefinitionId(entityType, namespace),
name: `Security '${entityType}' Entity Store Definition`,
type: entityType,
indexPatterns: [
...ENTITY_STORE_DEFAULT_SOURCE_INDICES,
getAssetCriticalityIndex(namespace),
getRiskScoreLatestIndex(namespace),
],
indexPatterns,
identityFields: [identityField],
displayNameTemplate: `{{${identityField}}}`,
metadata,

50
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts
View file

@ -10,6 +10,10 @@ import {
ENTITY_SCHEMA_VERSION_V1,
entitiesIndexPattern,
} from '@kbn/entities-schema';
import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common';
import type { AppClient } from '../../../../types';
import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine';
import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality';
import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen';
import { entityEngineDescriptorTypeName } from '../saved_object';
@ -19,6 +23,44 @@ export const getIdentityFieldForEntityType = (entityType: EntityType) => {
return 'user.name';
};
export const buildIndexPatterns = async (
space: string,
appClient: AppClient,
dataViewsService: DataViewsService
) => {
const { alertsIndex, securitySolutionDataViewIndices } = await getSecuritySolutionIndices(
appClient,
dataViewsService
);
return [
...securitySolutionDataViewIndices.filter((item) => item !== alertsIndex),
getAssetCriticalityIndex(space),
getRiskScoreLatestIndex(space),
];
};
const getSecuritySolutionIndices = async (
appClient: AppClient,
dataViewsService: DataViewsService
) => {
const securitySolutionDataViewId = appClient.getSourcererDataViewId();
let dataView: DataView;
try {
dataView = await dataViewsService.get(securitySolutionDataViewId);
} catch (e) {
if (e.isBoom && e.output.statusCode === 404) {
throw new Error(`Data view not found '${securitySolutionDataViewId}'`);
}
throw e;
}
const dataViewIndexPattern = dataView.getIndexPattern();
return {
securitySolutionDataViewIndices: dataViewIndexPattern.split(','),
alertsIndex: appClient.getAlertsIndex(),
};
};
export const getByEntityTypeQuery = (entityType: EntityType) => {
return `${entityEngineDescriptorTypeName}.attributes.type: ${entityType}`;
};
@ -33,3 +75,11 @@ export const getEntitiesIndexName = (entityType: EntityType, namespace: string)
export const buildEntityDefinitionId = (entityType: EntityType, space: string) => {
return `security_${entityType}_${space}`;
};
export const isPromiseFulfilled = <T>(
result: PromiseSettledResult<T>
): result is PromiseFulfilledResult<T> => result.status === 'fulfilled';
export const isPromiseRejected = <T>(
result: PromiseSettledResult<T>
): result is PromiseRejectedResult => result.status === 'rejected';

11
x-pack/plugins/security_solution/server/request_context_factory.ts
View file

@ -74,6 +74,12 @@ export class RequestContextFactory implements IRequestContextFactory {
const licensing = await context.licensing;
const actionsClient = await startPlugins.actions.getActionsClientWithRequest(request);
const dataViewsService = await startPlugins.dataViews.dataViewsServiceFactory(
coreContext.savedObjects.client,
coreContext.elasticsearch.client.asInternalUser,
request
);
const getSpaceId = (): string =>
startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID;
@ -84,6 +90,7 @@ export class RequestContextFactory implements IRequestContextFactory {
kibanaBranch: options.kibanaBranch,
buildFlavor: options.buildFlavor,
});
const getAppClient = () => appClientFactory.create(request);
const getAuditLogger = () => security?.audit.asScoped(request);
@ -109,7 +116,7 @@ export class RequestContextFactory implements IRequestContextFactory {
getFrameworkRequest: () => frameworkRequest,
getAppClient: () => appClientFactory.create(request),
getAppClient,
getSpaceId,
@ -197,6 +204,8 @@ export class RequestContextFactory implements IRequestContextFactory {
const soClient = coreContext.savedObjects.client;
return new EntityStoreDataClient({
namespace: getSpaceId(),
dataViewsService,
appClient: getAppClient(),
esClient,
logger,
soClient,

7
x-pack/test/api_integration/services/security_solution_api.gen.ts
View file

@ -155,6 +155,13 @@ after 30 days. It also deletes other artifacts specific to the migration impleme
.set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
.send(props.body as object);
},
applyEntityEngineDataviewIndices(kibanaSpace: string = 'default') {
return supertest
.post(routeWithNamespace('/api/entity_store/engines/apply_dataview_indices', kibanaSpace))
.set('kbn-xsrf', 'true')
.set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31')
.set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana');
},
assetCriticalityGetPrivileges(kibanaSpace: string = 'default') {
return supertest
.get(routeWithNamespace('/internal/asset_criticality/privileges', kibanaSpace))

67
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts
View file

@ -8,8 +8,10 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../ftr_provider_context';
import { EntityStoreUtils, elasticAssetCheckerFactory } from '../../utils';
import { dataViewRouteHelpersFactory } from '../../utils/data_view';
export default ({ getService }: FtrProviderContext) => {
const api = getService('securitySolutionApi');
const supertest = getService('supertest');
const {
expectTransformExists,
expectTransformNotFound,
@ -24,8 +26,15 @@ export default ({ getService }: FtrProviderContext) => {
const utils = EntityStoreUtils(getService);
// TODO: unskip once permissions issue is resolved
describe.skip('@ess @serverless @skipInServerlessMKI Entity Store Engine APIs', () => {
const dataView = dataViewRouteHelpersFactory(supertest);
before(async () => {
await utils.cleanEngines();
await dataView.create('security-solution');
});
after(async () => {
await dataView.delete('security-solution');
});
describe('init', () => {
@ -75,9 +84,9 @@ export default ({ getService }: FtrProviderContext) => {
expect(getResponse.body).to.eql({
status: 'started',
type: 'host',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
indexPattern: '',
filter: '',
fieldHistoryLength: 10,
});
});
@ -91,9 +100,9 @@ export default ({ getService }: FtrProviderContext) => {
expect(getResponse.body).to.eql({
status: 'started',
type: 'user',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
indexPattern: '',
filter: '',
fieldHistoryLength: 10,
});
});
});
@ -109,16 +118,16 @@ export default ({ getService }: FtrProviderContext) => {
{
status: 'started',
type: 'host',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
indexPattern: '',
filter: '',
fieldHistoryLength: 10,
},
{
status: 'started',
type: 'user',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
indexPattern: '',
filter: '',
fieldHistoryLength: 10,
},
]);
});
@ -200,5 +209,47 @@ export default ({ getService }: FtrProviderContext) => {
await expectIngestPipelineNotFound(`ea_default_user_entity_store-latest@platform`);
});
});
describe('apply_dataview_indices', () => {
before(async () => {
await utils.initEntityEngineForEntityType('host');
});
after(async () => {
await utils.cleanEngines();
});
afterEach(async () => {
await dataView.delete('security-solution');
await dataView.create('security-solution');
});
it("should not update the index patten when it didn't change", async () => {
const response = await api.applyEntityEngineDataviewIndices();
expect(response.body).to.eql({ success: true, result: [{ type: 'host', changes: {} }] });
});
it('should update the index pattern when the data view changes', async () => {
await dataView.updateIndexPattern('security-solution', 'test-*');
const response = await api.applyEntityEngineDataviewIndices();
expect(response.body).to.eql({
success: true,
result: [
{
type: 'host',
changes: {
indexPatterns: [
'test-*',
'.asset-criticality.asset-criticality-default',
'risk-score.risk-score-latest-default',
],
},
},
],
});
});
});
});
};

44
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type SuperTest from 'supertest';
export const dataViewRouteHelpersFactory = (
supertest: SuperTest.Agent,
namespace: string = 'default'
) => ({
create: (name: string) => {
return supertest
.post(`/api/data_views/data_view`)
.set('kbn-xsrf', 'foo')
.send({
data_view: {
title: `logs-*`,
timeFieldName: '@timestamp',
name: `${name}-${namespace}`,
id: `${name}-${namespace}`,
},
})
.expect(200);
},
delete: (name: string) => {
return supertest
.delete(`/api/data_views/data_view/${name}-${namespace}`)
.set('kbn-xsrf', 'foo')
.expect(200);
},
updateIndexPattern: (name: string, indexPattern: string) => {
return supertest
.post(`/api/data_views/data_view/${name}-${namespace}`)
.set('kbn-xsrf', 'foo')
.send({
data_view: {
title: indexPattern,
},
})
.expect(200);
},
});

2
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts
View file

@ -37,7 +37,7 @@ export const EntityStoreUtils = (
}
};
const initEntityEngineForEntityType = async (entityType: EntityType) => {
const initEntityEngineForEntityType = (entityType: EntityType) => {
log.info(`Initializing engine for entity type ${entityType} in namespace ${namespace}`);
return api
.initEntityEngine(