github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Detection Engine] adds data tier filters to Kibana advanced settings (#186908)

Browse source 
## Summary

- addresses https://github.com/elastic/security-team/issues/9228
- introduces new Kibana advanced settings option
`securitySolution:excludedDataTiersForRuleExecution`, that allows to
exclude cold and frozen data tiers from search during rule execution
  - users would be able to add `data_cold` or/and `data_frozen` tiers
- **ES|QL** rule does not support this feature:
https://github.com/elastic/elasticsearch/issues/108264
- **Machine learning** rule does not support this feature
- Advanced setting available only for ESS

### UI

<img width="2300" alt="Screenshot 2024-07-04 at 17 31 34"
src="39beeda3-8030-4943-959c-53eb064fe5ae">


### Demo

1. Checking there are 3M+ documents in cold data tier of `test-frozen`
index
2. When rule executes, it generates alerts.
3. Checking kibana ancestor index of generated alert - it's
`restored-test-frozen-000001`, which confirms alert was created from a
document in cold tier
4. In advanced settings exlcude `data_cold` tier
5. Execute rule again, observe no alerts were created


c8b2f612-628a-452d-98e5-555c2e89d957

### How to test

Create a deployment with cold and frozen data tiers and use following
commands to create index and ILM

<details>
<summary>Data tiers commands</summary>

```JSON

PUT /_cluster/settings
{
  "persistent": {
    "indices.lifecycle.poll_interval": "1m"
  }
}


PUT /_ilm/policy/filtering_data_tiers
{
  "policy": {
    "phases": {
        "frozen": {
          "min_age": "10m",
          "actions": {
            "searchable_snapshot": {
              "snapshot_repository": "found-snapshots",
              "force_merge_index": true
            }
          }
        },
        "cold": {
          "min_age": "1m",
          "actions": {
            "searchable_snapshot": {
              "snapshot_repository": "found-snapshots",
              "force_merge_index": true
            },
            "set_priority": {
              "priority": 0
            }
          }
        },
        "hot": {
          "min_age": "0ms",
          "actions": {
            "set_priority": {
              "priority": 100
            }
          }
        }
    }
  }
}


PUT /_index_template/filtering_data_tiers_template
{
  "index_patterns": [
    "filtering_data_tiers*"
  ],
  "template": {
    "settings": {
      "index.lifecycle.name": "filtering_data_tiers",
      "index.lifecycle.rollover_alias": "test-filtering_data_tiers"
    },
    "mappings": {
      "_meta": {
        "version": "1.6.0"
      },
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "host": {
          "properties": {
            "name": {
              "type": "keyword",
              "ignore_above": 1024
            }
          }
        }
      }
    }
  }
}

PUT /filtering_data_tiers-000001
{
  "aliases": {
    "filtering_data_tiers": {
      "is_write_index": true
    }
  }
}


POST filtering_data_tiers/_doc
{
  "@timestamp": "2024-07-08T17:00:01.000Z",
  "host.name": "test-0"
}


```

</details>

**OR**
reach out to @vitaliidm to get access to already existing deployment/pR
deployment, where `test-frozen` index has cold and frozen nodes and ILM
policy that move any data to a tier according to config.

Check number of documents in tier by

```JSON
GET test-frozen/_count
{
    "query": {
     "bool": {
        "must": {
          "terms": {
            "_tier": ["data_cold"]
          }
        }
     }
   }
}
```

Create rule of supported type and query that index


### Checklist

- [x] Functional changes are covered with a test plan and automated
tests.
  - https://github.com/elastic/security-team/pull/9896

- [x] Comprehensive manual testing is done by two engineers: the PR
author and one of the PR reviewers. Changes are tested in both ESS and
Serverless.

- [x] Functional changes are communicated to the Docs team. A ticket or
PR is opened in https://github.com/elastic/security-docs. The following
information is included: any feature flags used, affected environments
(Serverless, ESS, or both).
  - https://github.com/elastic/security-docs/issues/5483
This commit is contained in:
Vitalii Dmyterko 2024-07-24 17:09:20 +01:00 • committed by GitHub
parent 58c4be1d2e
commit 4d5de12e9e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 375 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/plugins/kibana_usage_collection/server/collectors/management/schema.ts
View file

@ -14,6 +14,13 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
type: 'keyword',
_meta: { description: 'Default value of the setting was changed.' },
},
'securitySolution:excludedDataTiersForRuleExecution': {
type: 'array',
items: {
type: 'keyword',
_meta: { description: 'Non-default value of setting.' },
},
},
'securitySolution:defaultThreatIndex': {
type: 'keyword',
_meta: { description: 'Default value of the setting was changed.' },

1
src/plugins/kibana_usage_collection/server/collectors/management/types.ts
View file

@ -180,4 +180,5 @@ export interface UsageStats {
'devTools:enablePersistentConsole': boolean;
'aiAssistant:preferredAIAssistantType': string;
'observability:profilingFetchTopNFunctionsFromStacktraces': boolean;
'securitySolution:excludedDataTiersForRuleExecution': string[];
}

9
src/plugins/telemetry/schema/oss_plugins.json
View file

@ -9879,6 +9879,15 @@
"description": "Default value of the setting was changed."
}
},
"securitySolution:excludedDataTiersForRuleExecution": {
"type": "array",
"items": {
"type": "keyword",
"_meta": {
"description": "Non-default value of setting."
}
}
},
"securitySolution:defaultThreatIndex": {
"type": "keyword",
"_meta": {

4
x-pack/plugins/security_solution/common/constants.ts
View file

@ -194,6 +194,10 @@ export const EXTENDED_RULE_EXECUTION_LOGGING_MIN_LEVEL_SETTING =
/** This Kibana Advanced Setting allows users to enable/disable the Asset Criticality feature */
export const ENABLE_ASSET_CRITICALITY_SETTING = 'securitySolution:enableAssetCriticality' as const;
/** This Kibana Advanced Setting allows users to exclude selected data tiers from search during rule execution */
export const EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION =
'securitySolution:excludedDataTiersForRuleExecution' as const;
/**
* Id for the notifications alerting type
* @deprecated Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function

1
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts
View file

@ -90,6 +90,7 @@ export const createRuleTypeMocks = (
} as SavedObject<AlertAttributes>);
const services = {
uiSettingsClient: { get: jest.fn().mockResolvedValue([]) },
savedObjectsClient: mockSavedObjectsClient,
scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(),
alertFactory: {

56
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts
View file

@ -17,8 +17,12 @@ import type { EqlRuleParams } from '../../rule_schema';
import { getCompleteRuleMock, getEqlRuleParams } from '../../rule_schema/mocks';
import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
import { eqlExecutor } from './eql';
import { getDataTierFilter } from '../utils/get_data_tier_filter';
jest.mock('../../routes/index/get_index_version');
jest.mock('../utils/get_data_tier_filter', () => ({ getDataTierFilter: jest.fn() }));
const getDataTierFilterMock = getDataTierFilter as jest.Mock;
describe('eql_executor', () => {
const version = '8.0.0';
@ -43,6 +47,7 @@ describe('eql_executor', () => {
events: [],
},
});
getDataTierFilterMock.mockResolvedValue([]);
});
describe('eqlExecutor', () => {
@ -152,5 +157,56 @@ describe('eql_executor', () => {
});
expect(result.userError).toEqual(true);
});
it('should pass frozen tier filters in eql search request', async () => {
getDataTierFilterMock.mockResolvedValue([
{
meta: { negate: true },
query: {
terms: {
_tier: ['data_cold'],
},
},
},
]);
await eqlExecutor({
inputIndex: DEFAULT_INDEX_PATTERN,
runtimeMappings: {},
completeRule: eqlCompleteRule,
tuple,
ruleExecutionLogger,
services: alertServices,
version,
bulkCreate: jest.fn(),
wrapHits: jest.fn(),
wrapSequences: jest.fn(),
primaryTimestamp: '@timestamp',
exceptionFilter: undefined,
unprocessedExceptions: [],
wrapSuppressedHits: jest.fn(),
alertTimestampOverride: undefined,
alertWithSuppression: jest.fn(),
isAlertSuppressionActive: true,
experimentalFeatures: mockExperimentalFeatures,
});
const searchArgs =
alertServices.scopedClusterClient.asCurrentUser.eql.search.mock.calls[0][0];
expect(searchArgs).toHaveProperty(
'body.filter.bool.filter',
expect.arrayContaining([
{
bool: {
filter: [],
must: [],
must_not: [{ terms: { _tier: ['data_cold'] } }],
should: [],
},
},
])
);
});
});
});

7
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts
View file

@ -44,6 +44,7 @@ import type {
} from '../../../../../common/api/detection_engine/model/alerts';
import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory';
import { getDataTierFilter } from '../utils/get_data_tier_filter';
interface EqlExecutorParams {
inputIndex: string[];
@ -93,13 +94,17 @@ export const eqlExecutor = async ({
return withSecuritySpan('eqlExecutor', async () => {
const result = createSearchAfterReturnType();
const dataTiersFilters = await getDataTierFilter({
uiSettingsClient: services.uiSettingsClient,
});
const request = buildEqlSearchRequest({
query: ruleParams.query,
index: inputIndex,
from: tuple.from.toISOString(),
to: tuple.to.toISOString(),
size: ruleParams.maxSignals,
filters: ruleParams.filters,
filters: [...(ruleParams.filters || []), ...dataTiersFilters],
primaryTimestamp,
secondaryTimestamp,
runtimeMappings,

9
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts
View file

@ -37,6 +37,7 @@ import { getMappingFilters } from './get_mapping_filters';
import { THREAT_PIT_KEEP_ALIVE } from '../../../../../../common/cti/constants';
import { getMaxSignalsWarning, getSafeSortIds } from '../../utils/utils';
import { getFieldsForWildcard } from '../../utils/get_fields_for_wildcard';
import { getDataTierFilter } from '../../utils/get_data_tier_filter';
export const createThreatSignals = async ({
alertId,
@ -106,9 +107,13 @@ export const createThreatSignals = async ({
warningMessages: [],
};
const dataTiersFilters = await getDataTierFilter({
uiSettingsClient: services.uiSettingsClient,
});
const { eventMappingFilter, indicatorMappingFilter } = getMappingFilters(threatMapping);
const allEventFilters = [...filters, eventMappingFilter];
const allThreatFilters = [...threatFilters, indicatorMappingFilter];
const allEventFilters = [...filters, eventMappingFilter, ...dataTiersFilters];
const allThreatFilters = [...threatFilters, indicatorMappingFilter, ...dataTiersFilters];
const eventCount = await getEventCount({
esClient: services.scopedClusterClient.asCurrentUser,

53
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_data_tier_filter.test.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getDataTierFilter } from './get_data_tier_filter';
import type { IUiSettingsClient } from '@kbn/core/server';
const uiSettingsClientMock = {
get: jest.fn(),
} as unknown as IUiSettingsClient;
describe('getDataTierFilter', () => {
it('should return empty array if ui settings empty', async () => {
(uiSettingsClientMock.get as jest.Mock).mockResolvedValueOnce([]);
const filters = await getDataTierFilter({ uiSettingsClient: uiSettingsClientMock });
expect(filters).toEqual([]);
});
it('should return filters array if ui settings populated with single value', async () => {
(uiSettingsClientMock.get as jest.Mock).mockResolvedValueOnce(['data_cold']);
const filters = await getDataTierFilter({ uiSettingsClient: uiSettingsClientMock });
expect(filters).toEqual([
{
meta: { negate: true },
query: {
terms: {
_tier: ['data_cold'],
},
},
},
]);
});
it('should return filters array if ui settings populated with multiple values', async () => {
(uiSettingsClientMock.get as jest.Mock).mockResolvedValueOnce(['data_cold', 'data_frozen']);
const filters = await getDataTierFilter({ uiSettingsClient: uiSettingsClientMock });
expect(filters).toEqual([
{
meta: { negate: true },
query: {
terms: {
_tier: ['data_cold', 'data_frozen'],
},
},
},
]);
});
});

40
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_data_tier_filter.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IUiSettingsClient } from '@kbn/core/server';
import type { Filter } from '@kbn/es-query';
import { EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION } from '../../../../../common/constants';
/**
* reads Kibana advanced settings for filtering data tiers during rule executions
* returns {@link Filter} array
*/
export const getDataTierFilter = async ({
uiSettingsClient,
}: {
uiSettingsClient: IUiSettingsClient;
}): Promise<Filter[]> => {
const excludedTiers = await uiSettingsClient.get<Array<'data_cold' | 'data_frozen'>>(
EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION
);
if (!excludedTiers?.length) {
return [];
}
return [
{
meta: { negate: true },
query: {
terms: {
_tier: excludedTiers,
},
},
},
];
};

144
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.test.ts
View file

@ -6,11 +6,16 @@
*/
import { getFilter } from './get_filter';
import type { GetFilterArgs } from './get_filter';
import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks';
import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock';
import { buildExceptionFilter } from '@kbn/lists-plugin/server/services/exception_lists';
import { getDataTierFilter } from './get_data_tier_filter';
jest.mock('./get_data_tier_filter', () => ({ getDataTierFilter: jest.fn() }));
const getDataTierFilterMock = getDataTierFilter as jest.Mock;
describe('get_filter', () => {
let servicesMock: RuleExecutorServicesMock;
@ -31,6 +36,7 @@ describe('get_filter', () => {
filters: [],
},
}));
getDataTierFilterMock.mockResolvedValue([]);
});
afterEach(() => {
@ -318,5 +324,143 @@ describe('get_filter', () => {
}
`);
});
describe('data tiers filters', () => {
let defaultFilterProps: Omit<GetFilterArgs, 'type'>;
beforeEach(() => {
getDataTierFilterMock.mockResolvedValue([
{
meta: { negate: true },
query: {
terms: {
_tier: ['data_cold', 'data_frozen'],
},
},
},
]);
defaultFilterProps = {
filters: [
{
query: {
match_phrase: {
'event.module': 'system',
},
},
},
],
language: 'kuery',
query: 'host.name: siem',
savedId: undefined,
services: servicesMock,
index: ['auditbeat-*'],
exceptionFilter: undefined,
};
});
it('adds data tier clause for query rule type', async () => {
const esFilter = await getFilter({
...defaultFilterProps,
type: 'query',
});
expect(esFilter.bool).toHaveProperty('must_not', [
{
terms: {
_tier: ['data_cold', 'data_frozen'],
},
},
]);
});
it('adds data tier clause for new terms rule type', async () => {
const esFilter = await getFilter({
...defaultFilterProps,
type: 'new_terms',
});
expect(esFilter.bool).toHaveProperty('must_not', [
{
terms: {
_tier: ['data_cold', 'data_frozen'],
},
},
]);
});
it('adds data tier clause for indicator match rule type', async () => {
const esFilter = await getFilter({
...defaultFilterProps,
type: 'threshold',
});
expect(esFilter.bool).toHaveProperty('must_not', [
{
terms: {
_tier: ['data_cold', 'data_frozen'],
},
},
]);
});
it('adds data tier clause for saved_query rule type', async () => {
const esFilter = await getFilter({
...defaultFilterProps,
type: 'saved_query',
savedId: 'mock-saved-id',
});
expect(esFilter.bool).toHaveProperty('must_not', [
{
terms: {
_tier: ['data_cold', 'data_frozen'],
},
},
]);
});
it('does not adds data tier clause for threat_match rule type', async () => {
const esFilter = await getFilter({
...defaultFilterProps,
type: 'threat_match',
});
expect(esFilter.bool).toHaveProperty('must_not', []);
});
it('should not call getDataTierFilterMock for eql rule type', async () => {
await expect(
getFilter({
...defaultFilterProps,
type: 'eql',
})
).rejects.toThrow();
expect(getDataTierFilterMock).not.toHaveBeenCalled();
});
it('should not call getDataTierFilterMock for esql rule type', async () => {
await expect(
getFilter({
...defaultFilterProps,
type: 'esql',
})
).rejects.toThrow();
expect(getDataTierFilterMock).not.toHaveBeenCalled();
});
it('should not call getDataTierFilterMock for machine_learning rule type', async () => {
await expect(
getFilter({
...defaultFilterProps,
type: 'machine_learning',
})
).rejects.toThrow();
expect(getDataTierFilterMock).not.toHaveBeenCalled();
});
});
});
});

26
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts
View file

@ -28,10 +28,21 @@ import { withSecuritySpan } from '../../../../utils/with_security_span';
import type { ESBoolQuery } from '../../../../../common/typed_json';
import { getQueryFilter as getQueryFilterNoLoadFields } from './get_query_filter';
import { getQueryFilterLoadFields } from './get_query_filter_load_fields';
import { getDataTierFilter } from './get_data_tier_filter';
/**
* EQL and threat_match rules support tier filtering too, but it is implemented inside rule executors
*/
const ruleTypesSupportingTierFilters = new Set<Type>([
'threshold',
'new_terms',
'query',
'saved_query',
]);
export interface GetFilterArgs {
type: Type;
filters: unknown | undefined;
filters: unknown[] | undefined;
language: LanguageOrUndefined;
query: RuleQuery | undefined;
savedId: SavedIdOrUndefined;
@ -66,12 +77,19 @@ export const getFilter = async ({
const getQueryFilter = loadFields
? getQueryFilterLoadFields(services.dataViews)
: getQueryFilterNoLoadFields;
const dataTiersFilters = ruleTypesSupportingTierFilters.has(type)
? await getDataTierFilter({ uiSettingsClient: services.uiSettingsClient })
: [];
const mergedFilters = [...(filters ? filters : []), ...dataTiersFilters];
const queryFilter = () => {
if (query != null && language != null && index != null) {
return getQueryFilter({
query,
language,
filters: filters || [],
filters: mergedFilters,
index,
exceptionFilter,
fields,
@ -91,7 +109,7 @@ export const getFilter = async ({
return getQueryFilter({
query: savedObject.attributes.query.query,
language: savedObject.attributes.query.language,
filters: savedObject.attributes.filters,
filters: [...savedObject.attributes.filters, ...dataTiersFilters],
index,
exceptionFilter,
fields,
@ -103,7 +121,7 @@ export const getFilter = async ({
return getQueryFilter({
query,
language,
filters: filters || [],
filters: mergedFilters,
index,
exceptionFilter,
fields,

25
x-pack/plugins/security_solution/server/ui_settings.ts
View file

@ -37,6 +37,7 @@ import {
DEFAULT_ALERT_TAGS_KEY,
DEFAULT_ALERT_TAGS_VALUE,
EXCLUDE_COLD_AND_FROZEN_TIERS_IN_ANALYZER,
EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION,
ENABLE_ASSET_CRITICALITY_SETTING,
} from '../common/constants';
import type { ExperimentalFeatures } from '../common/experimental_features';
@ -322,6 +323,30 @@ export const initUiSettings = (
requiresPageReload: true,
schema: schema.arrayOf(schema.string()),
},
[EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION]: {
name: i18n.translate(
'xpack.securitySolution.uiSettings.excludedDataTiersForRuleExecutionLabel',
{
defaultMessage: 'Exclude cold or frozen data tier from rule execution',
}
),
description: i18n.translate(
'xpack.securitySolution.uiSettings.excludedDataTiersForRuleExecutionDescription',
{
defaultMessage: `
When configured, events from the specified data tiers are not searched during rules executions.
<br/>This might help to improve rule performance or reduce execution time.
<br/>If you specify multiple data tiers, separate values with commas. For example: data_frozen,data_cold`,
}
),
type: 'array',
schema: schema.arrayOf(
schema.oneOf([schema.literal('data_cold'), schema.literal('data_frozen')])
),
value: [],
category: [APP_ID],
requiresPageReload: false,
},
...(experimentalFeatures.extendedRuleExecutionLoggingEnabled
? {
[EXTENDED_RULE_EXECUTION_LOGGING_ENABLED_SETTING]: {