github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Detection Engine] fixes ES|QL alert on alert (#208894)

Browse source 
## Summary

- addresses https://github.com/elastic/kibana/issues/205419: 
  - rule does not fail anymore and ancestors array is built correctly
- partly addresses https://github.com/elastic/security-team/issues/11116
by using [drop_null_columns
parameter](https://www.elastic.co/guide/en/elasticsearch/reference/8.15/esql-query-api.html#esql-query-api-query-params
)

### To reproduce

1. Create ES|QL rule alert on alert.
2. Use 2 queries:
3. `from .alerts-security* metadata _id` - rule generates alert and
ancestors array has only 1 item
4. `from .alerts-security* metadata _id | keep _id` - rule fails with
error "existingAncestors is not iterable"
This commit is contained in:
Vitalii Dmyterko 2025-02-05 18:39:26 +00:00 • committed by GitHub
parent 896ba294cc
commit 04102c4141
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 360 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

18
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts
View file

@ -24,7 +24,7 @@ import { wrapEsqlAlerts } from './wrap_esql_alerts';
import { wrapSuppressedEsqlAlerts } from './wrap_suppressed_esql_alerts';
import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory';
import { createEnrichEventsFunction } from '../utils/enrichments';
import { rowToDocument } from './utils';
import { rowToDocument, mergeEsqlResultInSource } from './utils';
import { fetchSourceDocuments } from './fetch_source_documents';
import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters';
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
@ -110,10 +110,11 @@ export const esqlExecutor = async ({
secondaryTimestamp,
exceptionFilter,
});
const esqlQueryString = { drop_null_columns: true };
if (isLoggedRequestsEnabled) {
loggedRequests.push({
request: logEsqlRequest(esqlRequest),
request: logEsqlRequest(esqlRequest, esqlQueryString),
description: i18n.ESQL_SEARCH_REQUEST_DESCRIPTION,
});
}
@ -128,7 +129,8 @@ export const esqlExecutor = async ({
const response = await performEsqlRequest({
esClient: services.scopedClusterClient.asCurrentUser,
requestParams: esqlRequest,
requestBody: esqlRequest,
requestQueryParams: esqlQueryString,
});
const esqlSearchDuration = performance.now() - esqlSignalSearchStart;
@ -177,13 +179,15 @@ export const esqlExecutor = async ({
});
const syntheticHits: Array<estypes.SearchHit<SignalSource>> = results.map((document) => {
const { _id, _version, _index, ...source } = document;
const { _id, _version, _index, ...esqlResult } = document;
const sourceDocument = _id ? sourceDocuments[_id] : undefined;
return {
_source: source as SignalSource,
fields: _id ? sourceDocuments[_id]?.fields : {},
_source: mergeEsqlResultInSource(sourceDocument?._source, esqlResult),
fields: sourceDocument?.fields,
_id: _id ?? '',
_index: _index ?? '',
_index: _index || sourceDocument?._index || '',
_version: sourceDocument?._version,
};
});

11
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql_request.ts
View file

@ -22,20 +22,21 @@ export interface EsqlTable {
export const performEsqlRequest = async ({
esClient,
requestParams,
requestBody,
requestQueryParams,
}: {
logger?: Logger;
esClient: ElasticsearchClient;
requestParams: Record<string, unknown>;
requestBody: Record<string, unknown>;
requestQueryParams?: { drop_null_columns?: boolean };
}): Promise<EsqlTable> => {
const search = async () => {
try {
const rawResponse = await esClient.transport.request<EsqlTable>({
method: 'POST',
path: '/_query',
body: {
...requestParams,
},
body: requestBody,
querystring: requestQueryParams,
});
return {
rawResponse,

33
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/fetch_source_documents.ts
View file

@ -10,6 +10,14 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/types';
import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
import { logQueryRequest } from '../utils/logged_requests';
import * as i18n from '../translations';
import type { SignalSource } from '../types';
interface FetchedDocument {
fields: estypes.SearchHit['fields'];
_source?: SignalSource;
_index: estypes.SearchHit['_index'];
_version: estypes.SearchHit['_version'];
}
interface FetchSourceDocumentsArgs {
isRuleAggregating: boolean;
@ -29,7 +37,7 @@ export const fetchSourceDocuments = async ({
esClient,
index,
loggedRequests,
}: FetchSourceDocumentsArgs): Promise<Record<string, { fields: estypes.SearchHit['fields'] }>> => {
}: FetchSourceDocumentsArgs): Promise<Record<string, FetchedDocument>> => {
const ids = results.reduce<string[]>((acc, doc) => {
if (doc._id) {
acc.push(doc._id);
@ -54,7 +62,7 @@ export const fetchSourceDocuments = async ({
const searchBody = {
query: idsQuery.query,
_source: false,
_source: true,
fields: ['*'],
};
const ignoreUnavailable = true;
@ -66,7 +74,7 @@ export const fetchSourceDocuments = async ({
});
}
const response = await esClient.search({
const response = await esClient.search<SignalSource>({
index,
body: searchBody,
ignore_unavailable: ignoreUnavailable,
@ -76,12 +84,15 @@ export const fetchSourceDocuments = async ({
loggedRequests[loggedRequests.length - 1].duration = response.took;
}
return response.hits.hits.reduce<Record<string, { fields: estypes.SearchHit['fields'] }>>(
(acc, hit) => {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
acc[hit._id!] = { fields: hit.fields };
return acc;
},
{}
);
return response.hits.hits.reduce<Record<string, FetchedDocument>>((acc, hit) => {
if (hit._id) {
acc[hit._id] = {
fields: hit.fields,
_source: hit._source,
_index: hit._index,
_version: hit._version,
};
}
return acc;
}, {});
};

1
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts
View file

@ -7,3 +7,4 @@
export * from './row_to_document';
export * from './generate_alert_id';
export * from './merge_esql_result_in_source';

52
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/merge_esql_result_in_source.test.ts Normal file
View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { mergeEsqlResultInSource } from './merge_esql_result_in_source';
describe('mergeEsqlResultInSource', () => {
it('ES|QL field should overwrite nested object field', () => {
const source = {
agent: { name: 'test-1' },
};
const esqlResult = { 'agent.name': 'custom ES|QL' };
expect(mergeEsqlResultInSource(source, esqlResult)).toEqual({ 'agent.name': 'custom ES|QL' });
});
it('ES|QL field should overwrite flattened object field', () => {
const source = { 'agent.name': 'test-1' };
const esqlResult = { 'agent.name': 'custom ES|QL' };
expect(mergeEsqlResultInSource(source, esqlResult)).toEqual({ 'agent.name': 'custom ES|QL' });
});
it('ES|QL field should overwrite mixed notation object field', () => {
const source = { 'log.syslog': { hostname: 'host-1' } };
const esqlResult = { 'log.syslog.hostname': 'esql host' };
expect(mergeEsqlResultInSource(source, esqlResult)).toEqual({
'log.syslog.hostname': 'esql host',
});
});
it('ES|QL field should be merged into source', () => {
const source = { agent: { hostname: 'host-1' } };
const esqlResult = { 'log.syslog.hostname': 'esql host' };
expect(mergeEsqlResultInSource(source, esqlResult)).toEqual({
agent: { hostname: 'host-1' },
'log.syslog.hostname': 'esql host',
});
});
it('ES|QL field should be merged into source without dropping any existing fields', () => {
const source = { 'log.syslog': { hostname: 'host-1', other: 'other' } };
const esqlResult = { 'log.syslog.hostname': 'esql host' };
expect(mergeEsqlResultInSource(source, esqlResult)).toEqual({
'log.syslog': { other: 'other' },
'log.syslog.hostname': 'esql host',
});
});
});

27
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/merge_esql_result_in_source.ts Normal file
View file

@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { SignalSource } from '../../types';
import {
robustGet,
robustUnset,
} from '../../utils/source_fields_merging/utils/robust_field_access';
export const mergeEsqlResultInSource = (
source: SignalSource | undefined,
esqlResult: Record<string, string>
): SignalSource => {
const document = source ?? {};
Object.keys(esqlResult).forEach((field) => {
if (robustGet({ key: field, document })) {
robustUnset({ key: field, document });
}
document[field] = esqlResult[field];
});
return document;
};

9
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/row_to_document.ts
View file

@ -16,11 +16,12 @@ import type { EsqlResultRow, EsqlResultColumn } from '../esql_request';
export const rowToDocument = (
columns: EsqlResultColumn[],
row: EsqlResultRow
): Record<string, string | null> => {
return columns.reduce<Record<string, string | null>>((acc, column, i) => {
): Record<string, string> => {
return columns.reduce<Record<string, string>>((acc, column, i) => {
const cell = row[i];
// skips nulls, as ES|QL return null for each existing mapping field
if (row[i] !== null) {
acc[column.name] = row[i];
if (cell !== null) {
acc[column.name] = cell;
}
return acc;
}, {});

4
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts
View file

@ -152,8 +152,8 @@ export const buildParent = (doc: SimpleHit): AncestorLatest => {
*/
export const buildAncestors = (doc: SimpleHit): AncestorLatest[] => {
const newAncestor = buildParent(doc);
const existingAncestors: AncestorLatest[] =
(getField(doc, ALERT_ANCESTORS) as AncestorLatest[] | undefined) ?? [];
const ancestorsField = getField(doc, ALERT_ANCESTORS);
const existingAncestors: AncestorLatest[] = Array.isArray(ancestorsField) ? ancestorsField : [];
return [...existingAncestors, newAncestor];
};

23
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/logged_requests/log_esql.ts
View file

@ -7,9 +7,22 @@
import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
export const logEsqlRequest = (esqlRequest: {
query: string;
filter: QueryDslQueryContainer;
}): string => {
return `POST _query\n${JSON.stringify(esqlRequest, null, 2)}`;
export const logEsqlRequest = (
requestBody: {
query: string;
filter: QueryDslQueryContainer;
},
requestQueryParams?: { drop_null_columns?: boolean }
): string => {
const urlParams = Object.entries(requestQueryParams ?? {})
.reduce<string[]>((acc, [key, value]) => {
if (value != null) {
acc.push(`${key}=${value}`);
}
return acc;
}, [])
.join('&');
return `POST _query${urlParams ? `?${urlParams}` : ''}\n${JSON.stringify(requestBody, null, 2)}`;
};

59
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { robustGet, robustSet } from './robust_field_access';
import { robustGet, robustSet, robustUnset } from './robust_field_access';
describe('robust field access', () => {
describe('get', () => {
@ -117,4 +117,61 @@ describe('robust field access', () => {
).toEqual({ 'a.b': 'test-new' });
});
});
describe('unset', () => {
it('unsets a value with a basic key', () => {
const document = { a: { b: { c: 'test-value', d: 'x' } } };
robustUnset({ key: 'a.b.c', document });
expect(document).toEqual({
a: { b: { d: 'x' } },
});
});
it('unsets a value with a basic key and remove empty objects', () => {
const document = { a: { b: { c: 'test-value' } } };
robustUnset({ key: 'a.b.c', document });
expect(document).toEqual({});
});
it('unsets a value inside an object at a dot notation path', () => {
const document = { 'a.b': { c: 'test-value', d: 'x' } };
robustUnset({ key: 'a.b.c', document });
expect(document).toEqual({
'a.b': { d: 'x' },
});
});
it('unsets a value inside an object at a dot notation path and removed empty object', () => {
const document = { 'a.b': { c: 'test-value' } };
robustUnset({ key: 'a.b.c', document });
expect(document).toEqual({});
});
it('unsets a value with dot notation key', () => {
const document = { 'a.b.c': 'test-value' };
robustUnset({ key: 'a.b.c', document });
expect(document).toEqual({});
});
it('ignores non-object values on the path', () => {
const document = { 'a.b': 'test-ignore' };
robustUnset({ key: 'a.b.c', document });
expect(document).toEqual({
'a.b': 'test-ignore',
});
});
it('unsets object value', () => {
const document = { 'a.b': { c: 1 } };
robustUnset({ key: 'a.b', document });
expect(document).toEqual({});
});
});
});

39
x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.ts
View file

@ -6,6 +6,7 @@
*/
import { set } from '@kbn/safer-lodash-set';
import { unset } from 'lodash';
import type { SearchTypes } from '../../../../../../../common/detection_engine/types';
import { isObjectTypeGuard } from './is_objectlike_or_array_of_objectlikes';
@ -80,3 +81,41 @@ export const robustSet = <T extends Record<string, unknown>>({
}
return set(document, key, valueToSet);
};
/**
* Similar to lodash unset, but instead of handling only pure dot or nested notation this function handles any mix of dot and nested notation
* @param key Path to field, in dot notation
* @param document Object to insert value into
* @returns updated document
*/
export const robustUnset = <T extends Record<string, unknown>>({
key,
document,
}: {
key: string;
document: T;
}) => {
const splitKey = key.split('.');
let tempKey = splitKey[0];
for (let i = 0; i < splitKey.length - 1; i++) {
if (i > 0) {
tempKey += `.${splitKey[i]}`;
}
const value = document[tempKey];
if (value != null) {
if (isObjectTypeGuard(value)) {
if (Object.keys(value).length !== 0) {
robustUnset({ key: splitKey.slice(i + 1).join('.'), document: value });
// check if field was removed from object, if so, we remove empty parent too
if (Object.keys(value).length === 0) {
unset(document, tempKey);
}
}
return document;
}
}
}
unset(document, key);
return document;
};

120
x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql.ts
View file

@ -8,10 +8,15 @@
import expect from 'expect';
import { v4 as uuidv4 } from 'uuid';
import moment from 'moment';
import { ALERT_RULE_EXECUTION_TYPE, ALERT_SUPPRESSION_DOCS_COUNT } from '@kbn/rule-data-utils';
import {
ALERT_RULE_EXECUTION_TYPE,
ALERT_SUPPRESSION_DOCS_COUNT,
ALERT_RULE_UUID,
} from '@kbn/rule-data-utils';
import { EsqlRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema';
import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks';
import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring';
import { ALERT_ANCESTORS } from '@kbn/security-solution-plugin/common/field_maps/field_names';
import { getMaxSignalsWarning as getMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils';
import { EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION } from '@kbn/security-solution-plugin/common/constants';
@ -27,6 +32,7 @@ import {
stopAllManualRuns,
waitForBackfillExecuted,
setAdvancedSettings,
getOpenAlerts,
} from '../../../../utils';
import {
deleteAllRules,
@ -1463,5 +1469,117 @@ export default ({ getService }: FtrProviderContext) => {
);
});
});
describe('alerts on alerts', () => {
let id: string;
let ruleId: string;
beforeEach(async () => {
id = uuidv4();
const doc1 = { id, agent: { name: 'test-1' }, '@timestamp': '2020-10-28T06:05:00.000Z' };
const ruleQuery = `from ecs_compliant metadata _id ${internalIdPipe(
id
)} | where agent.name=="test-1"`;
const rule: EsqlRuleCreateProps = {
...getCreateEsqlRulesSchemaMock('rule-1', true),
query: ruleQuery,
from: '2020-10-28T06:00:00.000Z',
interval: '1h',
};
await indexListOfDocuments([doc1]);
const createdRule = await createRule(supertest, log, rule);
await getOpenAlerts(supertest, log, es, createdRule);
ruleId = createdRule.id;
});
it('should create alert on alert with correct ancestors', async () => {
const ruleOnAlert: EsqlRuleCreateProps = {
...getCreateEsqlRulesSchemaMock(),
query: `from .alerts-security* metadata _id | where ${ALERT_RULE_UUID}=="${ruleId}"`,
from: 'now-1h',
interval: '1h',
};
const { previewId } = await previewRule({
supertest,
rule: ruleOnAlert,
timeframeEnd: new Date(),
});
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts[0]?._source?.[ALERT_ANCESTORS]).toHaveLength(2);
expect(previewAlerts[0]?._source?.[ALERT_ANCESTORS]).toEqual([
{
depth: 0,
id: expect.any(String),
index: 'ecs_compliant',
type: 'event',
},
{
depth: 1,
id: expect.any(String),
index: expect.stringContaining('alerts'),
rule: ruleId,
type: 'signal',
},
]);
});
it('should create alert on alert when properties dropped in ES|QL query', async () => {
const ruleOnAlert: EsqlRuleCreateProps = {
...getCreateEsqlRulesSchemaMock(),
query: `from .alerts-security* metadata _id | where ${ALERT_RULE_UUID}=="${ruleId}" | keep _id`,
from: 'now-1h',
interval: '1h',
};
const { previewId } = await previewRule({
supertest,
rule: ruleOnAlert,
timeframeEnd: new Date(),
});
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts[0]?._source?.[ALERT_ANCESTORS]).toHaveLength(2);
expect(previewAlerts[0]?._source?.[ALERT_ANCESTORS]).toEqual([
{
depth: 0,
id: expect.any(String),
index: 'ecs_compliant',
type: 'event',
},
{
depth: 1,
id: expect.any(String),
index: expect.stringContaining('alerts'),
rule: ruleId,
type: 'signal',
},
]);
});
it('should create alert on alert for aggregating query', async () => {
const ruleOnAlert: EsqlRuleCreateProps = {
...getCreateEsqlRulesSchemaMock(),
query: `from .alerts-security* | where ${ALERT_RULE_UUID}=="${ruleId}" | stats _count=count(agent.name) `,
from: 'now-1h',
interval: '1h',
};
const { previewId } = await previewRule({
supertest,
rule: ruleOnAlert,
timeframeEnd: new Date(),
});
const previewAlerts = await getPreviewAlerts({ es, previewId });
// since we don't fetch source document when using aggregating query, only one ancestors item is present
expect(previewAlerts[0]?._source?.[ALERT_ANCESTORS]).toHaveLength(1);
expect(previewAlerts[0]?._source?.[ALERT_ANCESTORS]).toEqual([
{ depth: 0, id: '', index: '', type: 'event' },
]);
});
});
});
};