mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
[Security Solution][Endpoint] adds new alert loading utility and un-skip FTR test for endpoint (#144133)
* FTR generator for endpoint rule alerts + mappings for the security alerts index * new method to load endpoint alerts directly to the index (bypass the rule) * Modify endpoint FTR test to use new alert loading tool * Fix Bulk Delete body payload format * adjustments from code review feedback
This commit is contained in:
Paul Tavares
GitHub
parent
5ca5ef3d00
commit
e5179d39fb
4 changed files with 5752 additions and 8 deletions
|
|
@ -9,6 +9,7 @@ import { IndexedHostsAndAlertsResponse } from '@kbn/security-solution-plugin/com
|
|||
import { TimelineResponse } from '@kbn/security-solution-plugin/common/types';
|
||||
import { kibanaPackageJson } from '@kbn/utils';
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
import { IndexedEndpointRuleAlerts } from '../../../security_solution_ftr/services/detections';
|
||||
|
||||
/**
|
||||
* Test suite is meant to cover usages of endpoint functionality or access to endpoint
|
||||
|
|
@ -22,9 +23,9 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
|||
const testSubjects = getService('testSubjects');
|
||||
const pageObjects = getPageObjects(['common', 'timeline']);
|
||||
|
||||
// FLAKY: https://github.com/elastic/kibana/issues/140701
|
||||
describe.skip('App level Endpoint functionality', () => {
|
||||
describe('App level Endpoint functionality', () => {
|
||||
let indexedData: IndexedHostsAndAlertsResponse;
|
||||
let indexedAlerts: IndexedEndpointRuleAlerts;
|
||||
let endpointAgentId: string;
|
||||
|
||||
before(async () => {
|
||||
|
|
@ -37,14 +38,13 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
|||
|
||||
await endpointService.waitForUnitedEndpoints([endpointAgentId]);
|
||||
|
||||
// Ensure our Endpoint is for v8.0 (or whatever is running in kibana now)
|
||||
// Ensure our Endpoint is for current version of kibana
|
||||
await endpointService.sendEndpointMetadataUpdate(endpointAgentId, {
|
||||
agent: { version: kibanaPackageJson.version },
|
||||
});
|
||||
|
||||
// start/stop the endpoint rule. This should cause the rule to run immediately
|
||||
// and avoid us having to wait for the interval (of 5 minutes)
|
||||
await detectionsTestService.stopStartEndpointRule();
|
||||
// Load alerts for our endpoint (so that we don't have to wait for the rule to run)
|
||||
indexedAlerts = await detectionsTestService.loadEndpointRuleAlerts(endpointAgentId);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
|
|
@ -52,6 +52,10 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
|||
log.info('Cleaning up loaded endpoint data');
|
||||
await endpointService.unloadEndpointData(indexedData);
|
||||
}
|
||||
|
||||
if (indexedAlerts) {
|
||||
await indexedAlerts.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
describe('from Timeline', () => {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,274 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { BaseDataGenerator } from '@kbn/security-solution-plugin/common/endpoint/data_generators/base_data_generator';
|
||||
import endpointPrePackagedRule from '@kbn/security-solution-plugin/server/lib/detection_engine/prebuilt_rules/content/prepackaged_rules/elastic_endpoint_security.json';
|
||||
import { kibanaPackageJson } from '@kbn/utils';
|
||||
import { mergeWith } from 'lodash';
|
||||
import { EndpointMetadataGenerator } from '@kbn/security-solution-plugin/common/endpoint/data_generators/endpoint_metadata_generator';
|
||||
import { HostMetadata } from '@kbn/security-solution-plugin/common/endpoint/types';
|
||||
import { DeepPartial } from 'utility-types';
|
||||
|
||||
const mergeAndReplaceArrays = <T, S>(destinationObj: T, srcObj: S): T => {
|
||||
const customizer = (objValue: T[keyof T], srcValue: S[keyof S]) => {
|
||||
if (Array.isArray(objValue)) {
|
||||
return srcValue;
|
||||
}
|
||||
};
|
||||
|
||||
return mergeWith(destinationObj, srcObj, customizer);
|
||||
};
|
||||
|
||||
type EndpointRuleAlert = Pick<
|
||||
HostMetadata,
|
||||
'Endpoint' | 'agent' | 'elastic' | 'host' | 'data_stream'
|
||||
> & {
|
||||
[key: string]: any;
|
||||
};
|
||||
|
||||
export class EndpointRuleAlertGenerator extends BaseDataGenerator {
|
||||
/** Generates an Endpoint Rule Alert document */
|
||||
generate(overrides: DeepPartial<EndpointRuleAlert> = {}): EndpointRuleAlert {
|
||||
const endpointMetadataGenerator = new EndpointMetadataGenerator();
|
||||
const endpointMetadata = endpointMetadataGenerator.generate({
|
||||
agent: { version: kibanaPackageJson.version },
|
||||
});
|
||||
const now = overrides['@timestamp'] ?? new Date().toISOString();
|
||||
const endpointAgentId = overrides?.agent?.id ?? this.seededUUIDv4();
|
||||
|
||||
return mergeAndReplaceArrays(
|
||||
{
|
||||
'@timestamp': now,
|
||||
Endpoint: endpointMetadata.Endpoint,
|
||||
agent: {
|
||||
id: endpointAgentId,
|
||||
type: 'endpoint',
|
||||
version: kibanaPackageJson.version,
|
||||
},
|
||||
elastic: endpointMetadata.elastic,
|
||||
host: endpointMetadata.host,
|
||||
data_stream: {
|
||||
dataset: 'endpoint.alerts',
|
||||
namespace: 'default',
|
||||
type: 'logs',
|
||||
},
|
||||
ecs: {
|
||||
version: '1.4.0',
|
||||
},
|
||||
file: {
|
||||
Ext: {
|
||||
code_signature: [
|
||||
{
|
||||
subject_name: 'bad signer',
|
||||
trusted: false,
|
||||
},
|
||||
],
|
||||
malware_classification: {
|
||||
identifier: 'endpointpe',
|
||||
score: 1,
|
||||
threshold: 0.66,
|
||||
version: '3.0.33',
|
||||
},
|
||||
quarantine_message: 'fake quarantine message',
|
||||
quarantine_result: true,
|
||||
temp_file_path: 'C:/temp/fake_malware.exe',
|
||||
},
|
||||
accessed: 1666818167432,
|
||||
created: 1666818167432,
|
||||
hash: {
|
||||
md5: 'fake file md5',
|
||||
sha1: 'fake file sha1',
|
||||
sha256: 'fake file sha256',
|
||||
},
|
||||
mtime: 1666818167432,
|
||||
name: 'fake_malware.exe',
|
||||
owner: 'SYSTEM',
|
||||
path: 'C:/fake_malware.exe',
|
||||
size: 3456,
|
||||
},
|
||||
dll: [
|
||||
{
|
||||
Ext: {
|
||||
compile_time: 1534424710,
|
||||
malware_classification: {
|
||||
identifier: 'Whitelisted',
|
||||
score: 0,
|
||||
threshold: 0,
|
||||
version: '3.0.0',
|
||||
},
|
||||
mapped_address: 5362483200,
|
||||
mapped_size: 0,
|
||||
},
|
||||
code_signature: {
|
||||
subject_name: 'Cybereason Inc',
|
||||
trusted: true,
|
||||
},
|
||||
hash: {
|
||||
md5: '1f2d082566b0fc5f2c238a5180db7451',
|
||||
sha1: 'ca85243c0af6a6471bdaa560685c51eefd6dbc0d',
|
||||
sha256: '8ad40c90a611d36eb8f9eb24fa04f7dbca713db383ff55a03aa0f382e92061a2',
|
||||
},
|
||||
path: 'C:\\Program Files\\Cybereason ActiveProbe\\AmSvc.exe',
|
||||
pe: {
|
||||
architecture: 'x64',
|
||||
},
|
||||
},
|
||||
],
|
||||
process: {
|
||||
Ext: {
|
||||
ancestry: ['epyg8z2d21', '26qhqfy8a1'],
|
||||
code_signature: [
|
||||
{
|
||||
subject_name: 'bad signer',
|
||||
trusted: false,
|
||||
},
|
||||
],
|
||||
token: {
|
||||
domain: 'NT AUTHORITY',
|
||||
integrity_level: 16384,
|
||||
integrity_level_name: 'system',
|
||||
privileges: [
|
||||
{
|
||||
description: 'Replace a process level token',
|
||||
enabled: false,
|
||||
name: 'SeAssignPrimaryTokenPrivilege',
|
||||
},
|
||||
],
|
||||
sid: 'S-1-5-18',
|
||||
type: 'tokenPrimary',
|
||||
user: 'SYSTEM',
|
||||
},
|
||||
user: 'SYSTEM',
|
||||
},
|
||||
entity_id: '0gwuy9lpud',
|
||||
entry_leader: {
|
||||
entity_id: '8kfl83q6vl',
|
||||
name: 'fake entry',
|
||||
pid: 945,
|
||||
},
|
||||
executable: 'C:/malware.exe',
|
||||
group_leader: {
|
||||
entity_id: '8kfl83q6vl',
|
||||
name: 'fake leader',
|
||||
pid: 120,
|
||||
},
|
||||
hash: {
|
||||
md5: 'fake md5',
|
||||
sha1: 'fake sha1',
|
||||
sha256: 'fake sha256',
|
||||
},
|
||||
name: 'malware writer',
|
||||
parent: {
|
||||
entity_id: 'epyg8z2d21',
|
||||
pid: 1,
|
||||
},
|
||||
pid: 2,
|
||||
session_leader: {
|
||||
entity_id: '8kfl83q6vl',
|
||||
name: 'fake session',
|
||||
pid: 279,
|
||||
},
|
||||
start: 1666818167432,
|
||||
uptime: 0,
|
||||
},
|
||||
'event.action': 'creation',
|
||||
'event.agent_id_status': 'auth_metadata_missing',
|
||||
'event.category': 'malware',
|
||||
'event.code': 'malicious_file',
|
||||
'event.dataset': 'endpoint',
|
||||
'event.id': this.seededUUIDv4(),
|
||||
'event.ingested': now,
|
||||
'event.kind': 'signal',
|
||||
'event.module': 'endpoint',
|
||||
'event.sequence': 5,
|
||||
'event.type': 'creation',
|
||||
'kibana.alert.ancestors': [
|
||||
{
|
||||
depth: 0,
|
||||
id: 'QBUaFoQBGSAAfHJkxoRQ',
|
||||
index: '.ds-logs-endpoint.alerts-default-2022.10.26-000001',
|
||||
type: 'event',
|
||||
},
|
||||
],
|
||||
'kibana.alert.depth': 1,
|
||||
'kibana.alert.original_event.action': 'creation',
|
||||
'kibana.alert.original_event.agent_id_status': 'auth_metadata_missing',
|
||||
'kibana.alert.original_event.category': 'malware',
|
||||
'kibana.alert.original_event.code': 'malicious_file',
|
||||
'kibana.alert.original_event.dataset': 'endpoint',
|
||||
'kibana.alert.original_event.id': this.seededUUIDv4(),
|
||||
'kibana.alert.original_event.ingested': now,
|
||||
'kibana.alert.original_event.kind': 'alert',
|
||||
'kibana.alert.original_event.module': 'endpoint',
|
||||
'kibana.alert.original_event.sequence': 5,
|
||||
'kibana.alert.original_event.type': 'creation',
|
||||
'kibana.alert.original_time': this.randomPastDate(),
|
||||
'kibana.alert.reason':
|
||||
'malware event with process malware writer, file fake_malware.exe, on Host-4xu9tiwmfp created medium alert Endpoint Security.',
|
||||
'kibana.alert.risk_score': 47,
|
||||
'kibana.alert.rule.actions': [],
|
||||
'kibana.alert.rule.author': ['Elastic'],
|
||||
'kibana.alert.rule.category': 'Custom Query Rule',
|
||||
'kibana.alert.rule.consumer': 'siem',
|
||||
'kibana.alert.rule.created_at': '2022-10-26T21:02:00.237Z',
|
||||
'kibana.alert.rule.created_by': 'some_user',
|
||||
'kibana.alert.rule.description':
|
||||
'Generates a detection alert each time an Elastic Endpoint Security alert is received. Enabling this rule allows you to immediately begin investigating your Endpoint alerts.',
|
||||
'kibana.alert.rule.enabled': true,
|
||||
'kibana.alert.rule.exceptions_list': [
|
||||
{
|
||||
id: 'endpoint_list',
|
||||
list_id: 'endpoint_list',
|
||||
namespace_type: 'agnostic',
|
||||
type: 'endpoint',
|
||||
},
|
||||
],
|
||||
'kibana.alert.rule.execution.uuid': this.seededUUIDv4(),
|
||||
'kibana.alert.rule.false_positives': [],
|
||||
'kibana.alert.rule.from': endpointPrePackagedRule.from,
|
||||
'kibana.alert.rule.immutable': true,
|
||||
'kibana.alert.rule.indices': endpointPrePackagedRule.index,
|
||||
'kibana.alert.rule.interval': '5m',
|
||||
'kibana.alert.rule.license': 'Elastic License v2',
|
||||
'kibana.alert.rule.max_signals': 10000,
|
||||
'kibana.alert.rule.name': endpointPrePackagedRule.name,
|
||||
'kibana.alert.rule.parameters': endpointPrePackagedRule,
|
||||
'kibana.alert.rule.producer': 'siem',
|
||||
'kibana.alert.rule.references': [],
|
||||
'kibana.alert.rule.risk_score': endpointPrePackagedRule.risk_score,
|
||||
'kibana.alert.rule.risk_score_mapping': [
|
||||
{
|
||||
field: 'event.risk_score',
|
||||
operator: 'equals',
|
||||
value: '',
|
||||
},
|
||||
],
|
||||
'kibana.alert.rule.rule_id': endpointPrePackagedRule.rule_id,
|
||||
'kibana.alert.rule.rule_name_override': 'message',
|
||||
'kibana.alert.rule.rule_type_id': 'siem.queryRule',
|
||||
'kibana.alert.rule.severity': 'medium',
|
||||
'kibana.alert.rule.severity_mapping': endpointPrePackagedRule.severity_mapping,
|
||||
'kibana.alert.rule.tags': endpointPrePackagedRule.tags,
|
||||
'kibana.alert.rule.threat': [],
|
||||
'kibana.alert.rule.timestamp_override': endpointPrePackagedRule.timestamp_override,
|
||||
'kibana.alert.rule.to': 'now',
|
||||
'kibana.alert.rule.type': 'query',
|
||||
'kibana.alert.rule.updated_at': '2022-10-26T21:02:00.237Z',
|
||||
'kibana.alert.rule.updated_by': 'some_user',
|
||||
'kibana.alert.rule.uuid': '6eae8572-5571-11ed-a602-953b659b2e32',
|
||||
'kibana.alert.rule.version': 100,
|
||||
'kibana.alert.severity': 'medium',
|
||||
'kibana.alert.status': 'active',
|
||||
'kibana.alert.uuid': 'e25f166b83234cbcfc41600a0191ee6a0efec0f959c6899a325d8026711e6c02',
|
||||
'kibana.alert.workflow_status': 'open',
|
||||
'kibana.space_ids': ['default'],
|
||||
'kibana.version': kibanaPackageJson.version,
|
||||
},
|
||||
overrides
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@
|
|||
import { Response } from 'superagent';
|
||||
import { EndpointError } from '@kbn/security-solution-plugin/common/endpoint/errors';
|
||||
import {
|
||||
DEFAULT_ALERTS_INDEX,
|
||||
DETECTION_ENGINE_QUERY_SIGNALS_URL,
|
||||
DETECTION_ENGINE_RULES_BULK_ACTION,
|
||||
DETECTION_ENGINE_RULES_URL,
|
||||
|
|
@ -15,13 +16,23 @@ import {
|
|||
import { estypes } from '@elastic/elasticsearch';
|
||||
import endpointPrePackagedRule from '@kbn/security-solution-plugin/server/lib/detection_engine/prebuilt_rules/content/prepackaged_rules/elastic_endpoint_security.json';
|
||||
import { Rule } from '@kbn/security-solution-plugin/public/detection_engine/rule_management/logic/types';
|
||||
import { kibanaPackageJson } from '@kbn/utils';
|
||||
import { wrapErrorIfNeeded } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/utils';
|
||||
import { FtrService } from '../../../functional/ftr_provider_context';
|
||||
import { EndpointRuleAlertGenerator } from './endpoint_rule_alert_generator';
|
||||
import { getAlertsIndexMappings } from './alerts_security_index_mappings';
|
||||
|
||||
export interface IndexedEndpointRuleAlerts {
|
||||
alerts: estypes.WriteResponseBase[];
|
||||
cleanup: () => Promise<void>;
|
||||
}
|
||||
|
||||
export class DetectionsTestService extends FtrService {
|
||||
private readonly supertest = this.ctx.getService('supertest');
|
||||
private readonly log = this.ctx.getService('log');
|
||||
private readonly retry = this.ctx.getService('retry');
|
||||
private readonly config = this.ctx.getService('config');
|
||||
private readonly esClient = this.ctx.getService('es');
|
||||
private readonly defaultTimeout = this.config.get('timeouts.waitFor');
|
||||
|
||||
/**
|
||||
|
|
@ -51,6 +62,36 @@ export class DetectionsTestService extends FtrService {
|
|||
};
|
||||
}
|
||||
|
||||
private async ensureEndpointRuleAlertsIndexExists(): Promise<void> {
|
||||
const indexMappings = getAlertsIndexMappings().value;
|
||||
|
||||
if (indexMappings.mappings?._meta?.kibana.version) {
|
||||
indexMappings.mappings._meta.kibana.version = kibanaPackageJson.version;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.esClient.indices.create({
|
||||
index: indexMappings.index,
|
||||
body: {
|
||||
settings: indexMappings.settings,
|
||||
mappings: indexMappings.mappings,
|
||||
aliases: indexMappings.aliases,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
// ignore error that indicate index is already created
|
||||
if (
|
||||
['resource_already_exists_exception', 'invalid_alias_name_exception'].includes(
|
||||
error?.body?.error?.type
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
throw wrapErrorIfNeeded(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the endpoint security rule using the pre-packaged `rule_id`
|
||||
*/
|
||||
|
|
@ -99,10 +140,11 @@ export class DetectionsTestService extends FtrService {
|
|||
}
|
||||
|
||||
/**
|
||||
* Waits for alerts to have been loaded into `.alerts-security.alerts-default` index
|
||||
* Waits for alerts to have been loaded by continuously calling the alerts api until data shows up
|
||||
* @param query
|
||||
* @param timeoutMs
|
||||
*/
|
||||
async waitForAlerts(query: object = { match_all: {} }, timeoutMs?: number) {
|
||||
async waitForAlerts(query: object = { match_all: {} }, timeoutMs?: number): Promise<void> {
|
||||
await this.retry.waitForWithTimeout(
|
||||
'Checking alerts index for data',
|
||||
timeoutMs ?? this.defaultTimeout,
|
||||
|
|
@ -128,4 +170,62 @@ export class DetectionsTestService extends FtrService {
|
|||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads alerts for Endpoint directly into the internal index that the Endpoint Rule
|
||||
* would have written them to for a given endpoint
|
||||
* @param endpointAgentId
|
||||
* @param count
|
||||
*/
|
||||
async loadEndpointRuleAlerts(
|
||||
endpointAgentId: string,
|
||||
count: number = 2
|
||||
): Promise<IndexedEndpointRuleAlerts> {
|
||||
this.log.info(`Loading ${count} endpoint rule alerts`);
|
||||
|
||||
await this.ensureEndpointRuleAlertsIndexExists();
|
||||
|
||||
const alertsGenerator = new EndpointRuleAlertGenerator();
|
||||
const esClient = this.esClient;
|
||||
const indexedAlerts: estypes.IndexResponse[] = [];
|
||||
|
||||
for (let n = 0; n < count; n++) {
|
||||
const alert = alertsGenerator.generate({ agent: { id: endpointAgentId } });
|
||||
const indexedAlert = await esClient.index({
|
||||
index: `${DEFAULT_ALERTS_INDEX}-default`,
|
||||
refresh: 'wait_for',
|
||||
body: alert,
|
||||
});
|
||||
|
||||
indexedAlerts.push(indexedAlert);
|
||||
}
|
||||
|
||||
this.log.info(`Endpoint rule alerts created:`, indexedAlerts);
|
||||
|
||||
return {
|
||||
alerts: indexedAlerts,
|
||||
cleanup: async (): Promise<void> => {
|
||||
if (indexedAlerts.length) {
|
||||
this.log.info('cleaning up loaded endpoint rule alerts');
|
||||
|
||||
await esClient.bulk({
|
||||
body: indexedAlerts.map((indexedDoc) => {
|
||||
return {
|
||||
delete: {
|
||||
_index: indexedDoc._index,
|
||||
_id: indexedDoc._id,
|
||||
},
|
||||
};
|
||||
}),
|
||||
});
|
||||
|
||||
this.log.info(
|
||||
`Deleted ${indexedAlerts.length} endpoint rule alerts. Ids: [${indexedAlerts
|
||||
.map((alert) => alert._id)
|
||||
.join()}]`
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue