github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solutions] Fixes exception lists to be able to filter on os type (#106494)

Browse source 
## Summary

Fixes https://github.com/elastic/kibana/issues/102613, and targets `7.14.0` as a blocker/critical

Previously we never fully finished the plumbing for using the `os_types` (operating system type) in the exception lists to be able to filter out values based on this type. With the endpoint exceptions now having specific selections for os_type we have to filter it with exceptions and basically make it work.

Some caveats is that the endpoints utilize `host.os.name.casless` for filtering against os_type, while agents such as auditbeat, winlogbeat, etc... use `host.os.type`. Really `host.os.type` is the correct ECS field to use, but to retain compatibility with the current version of endpoint agents I support both in one query to where if either of these two matches, then that will trigger the exceptions.

* Adds e2e tests
* Enhances the e2e tooling to do endpoint exception testing with `os_types`.
* Adds the logic to handle os_type
* Updates the unit tests

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
This commit is contained in:
Frank Hassanabad 2021-07-22 12:54:40 -06:00 • committed by GitHub
parent cd667d06bc
commit 0a5c96b117
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 1429 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

80
packages/kbn-securitysolution-list-utils/src/build_exception_filter/index.ts
View file

@ -19,6 +19,7 @@ import {
entriesMatch,
entriesMatchAny,
entriesNested,
OsTypeArray,
} from '@kbn/securitysolution-io-ts-list-types';
import { hasLargeValueList } from '../has_large_value_list';
@ -69,26 +70,87 @@ export const chunkExceptions = (
return chunk(chunkSize, exceptions);
};
export const buildExceptionItemFilter = (
exceptionItem: ExceptionItemSansLargeValueLists
): BooleanFilter | NestedFilter => {
const { entries } = exceptionItem;
/**
* Transforms the os_type into a regular filter as if the user had created it
* from the fields for the next state of transforms which will create the elastic filters
* from it.
*
* Note: We use two types of fields, the "host.os.type" and "host.os.name.caseless"
* The endpoint/endgame agent has been using "host.os.name.caseless" as the same value as the ECS
* value of "host.os.type" where the auditbeat, winlogbeat, etc... (other agents) are all using
* "host.os.type". In order to be compatible with both, I create an "OR" between these two data types
* where if either has a match then we will exclude it as part of the match. This should also be
* forwards compatible for endpoints/endgame agents when/if they upgrade to using "host.os.type"
* rather than using "host.os.name.caseless" values.
*
* Also we create another "OR" from the osType names so that if there are multiples such as ['windows', 'linux']
* this will exclude anything with either 'windows' or with 'linux'
* @param osTypes The os_type array from the REST interface that is an array such as ['windows', 'linux']
* @param entries The entries to join the OR's with before the elastic filter change out
*/
export const transformOsType = (
osTypes: OsTypeArray,
entries: NonListEntry[]
): NonListEntry[][] => {
const hostTypeTransformed = osTypes.map<NonListEntry[]>((osType) => {
return [
{ field: 'host.os.type', operator: 'included', type: 'match', value: osType },
...entries,
];
});
const caseLessTransformed = osTypes.map<NonListEntry[]>((osType) => {
return [
{ field: 'host.os.name.caseless', operator: 'included', type: 'match', value: osType },
...entries,
];
});
return [...hostTypeTransformed, ...caseLessTransformed];
};
if (entries.length === 1) {
return createInnerAndClauses(entries[0]);
} else {
/**
* This builds an exception item filter with the os type
* @param osTypes The os_type array from the REST interface that is an array such as ['windows', 'linux']
* @param entries The entries to join the OR's with before the elastic filter change out
*/
export const buildExceptionItemFilterWithOsType = (
osTypes: OsTypeArray,
entries: NonListEntry[]
): BooleanFilter[] => {
const entriesWithOsTypes = transformOsType(osTypes, entries);
return entriesWithOsTypes.map((entryWithOsType) => {
return {
bool: {
filter: entries.map((entry) => createInnerAndClauses(entry)),
filter: entryWithOsType.map((entry) => createInnerAndClauses(entry)),
},
};
});
};
export const buildExceptionItemFilter = (
exceptionItem: ExceptionItemSansLargeValueLists
): Array<BooleanFilter | NestedFilter> => {
const { entries, os_types: osTypes } = exceptionItem;
if (osTypes != null && osTypes.length > 0) {
return buildExceptionItemFilterWithOsType(osTypes, entries);
} else {
if (entries.length === 1) {
return [createInnerAndClauses(entries[0])];
} else {
return [
{
bool: {
filter: entries.map((entry) => createInnerAndClauses(entry)),
},
},
];
}
}
};
export const createOrClauses = (
exceptionItems: ExceptionItemSansLargeValueLists[]
): Array<BooleanFilter | NestedFilter> => {
return exceptionItems.map((exceptionItem) => buildExceptionItemFilter(exceptionItem));
return exceptionItems.flatMap((exceptionItem) => buildExceptionItemFilter(exceptionItem));
};
export const buildExceptionFilter = ({

173
x-pack/plugins/lists/common/exceptions/build_exceptions_filter.test.ts
View file

@ -611,114 +611,115 @@ describe('build_exceptions_filter', () => {
getEntryExistsExcludedMock(),
],
});
expect(exceptionItemFilter).toEqual({
bool: {
filter: [
{
nested: {
path: 'parent.field',
query: {
bool: {
filter: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'parent.field.host.name': 'some host name',
expect(exceptionItemFilter).toEqual([
{
bool: {
filter: [
{
nested: {
path: 'parent.field',
query: {
bool: {
filter: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'parent.field.host.name': 'some host name',
},
},
},
],
],
},
},
},
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'parent.field.host.name': 'some host name',
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'parent.field.host.name': 'some host name',
},
},
},
],
],
},
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'parent.field.host.name': 'some other host name',
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'parent.field.host.name': 'some other host name',
},
},
},
],
],
},
},
},
],
],
},
},
},
},
},
{
bool: {
minimum_should_match: 1,
should: [{ exists: { field: 'parent.field.host.name' } }],
{
bool: {
minimum_should_match: 1,
should: [{ exists: { field: 'parent.field.host.name' } }],
},
},
],
},
},
score_mode: 'none',
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
bool: {
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some "host" name' } }],
},
],
},
},
{
bool: {
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
},
},
],
},
score_mode: 'none',
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some "host" name' } }],
should: [{ match_phrase: { 'host.name': 'some host name' } }],
},
},
{
bool: {
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
},
},
],
},
},
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some host name' } }],
},
},
},
},
{
bool: {
must_not: {
bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] },
{
bool: {
must_not: {
bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] },
},
},
},
},
],
],
},
},
});
]);
});
});

2
x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts
View file

@ -39,7 +39,7 @@ export const getExceptionListItemSchemaMock = (
meta: META,
name: NAME,
namespace_type: NAMESPACE_TYPE,
os_types: ['linux'],
os_types: [],
tags: ['user added string for a tag', 'malware'],
tie_breaker_id: TIE_BREAKER,
type: ITEM_TYPE,

2
x-pack/plugins/security_solution/public/common/components/exceptions/helpers.test.tsx
View file

@ -185,7 +185,7 @@ describe('Exception helpers', () => {
meta: {},
name: 'some name',
namespace_type: 'single',
os_types: ['linux'],
os_types: [],
tags: ['user added string for a tag', 'malware'],
type: 'simple',
};

10
x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/exception_details.test.tsx
View file

@ -157,7 +157,7 @@ describe('ExceptionDetails', () => {
});
test('it renders the operating system if one is specified in the exception item', () => {
const exceptionItem = getExceptionListItemSchemaMock();
const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] });
const wrapper = mount(
<ThemeProvider theme={mockTheme}>
<ExceptionDetails
@ -173,7 +173,7 @@ describe('ExceptionDetails', () => {
});
test('it renders the exception item creator', () => {
const exceptionItem = getExceptionListItemSchemaMock();
const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] });
const wrapper = mount(
<ThemeProvider theme={mockTheme}>
<ExceptionDetails
@ -191,7 +191,7 @@ describe('ExceptionDetails', () => {
});
test('it renders the exception item creation timestamp', () => {
const exceptionItem = getExceptionListItemSchemaMock();
const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] });
const wrapper = mount(
<ThemeProvider theme={mockTheme}>
<ExceptionDetails
@ -207,7 +207,7 @@ describe('ExceptionDetails', () => {
});
test('it renders the description if one is included on the exception item', () => {
const exceptionItem = getExceptionListItemSchemaMock();
const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] });
const wrapper = mount(
<ThemeProvider theme={mockTheme}>
<ExceptionDetails
@ -223,7 +223,7 @@ describe('ExceptionDetails', () => {
});
test('it renders with Name and Modified info when showName and showModified props are true', () => {
const exceptionItem = getExceptionListItemSchemaMock();
const exceptionItem = getExceptionListItemSchemaMock({ os_types: ['linux'] });
exceptionItem.comments = [];
const wrapper = mount(

17
x-pack/plugins/security_solution/public/common/components/exceptions/viewer/helpers.test.tsx
View file

@ -154,7 +154,7 @@ describe('Exception viewer helpers', () => {
describe('#getDescriptionListContent', () => {
test('it returns formatted description list with os if one is specified', () => {
const payload = getExceptionListItemSchemaMock();
const payload = getExceptionListItemSchemaMock({ os_types: ['linux'] });
payload.description = '';
const result = getDescriptionListContent(payload);
const expected: DescriptionListItem[] = [
@ -176,7 +176,7 @@ describe('Exception viewer helpers', () => {
});
test('it returns formatted description list with a description if one specified', () => {
const payload = getExceptionListItemSchemaMock();
const payload = getExceptionListItemSchemaMock({ os_types: ['linux'] });
payload.description = 'Im a description';
const result = getDescriptionListContent(payload);
const expected: DescriptionListItem[] = [
@ -202,7 +202,7 @@ describe('Exception viewer helpers', () => {
});
test('it returns just user and date created if no other fields specified', () => {
const payload = getExceptionListItemSchemaMock();
const payload = getExceptionListItemSchemaMock({ os_types: ['linux'] });
payload.description = '';
const result = getDescriptionListContent(payload);
const expected: DescriptionListItem[] = [
@ -224,7 +224,10 @@ describe('Exception viewer helpers', () => {
});
test('it returns Modified By/On info. when `includeModified` is true', () => {
const result = getDescriptionListContent(getExceptionListItemSchemaMock(), true);
const result = getDescriptionListContent(
getExceptionListItemSchemaMock({ os_types: ['linux'] }),
true
);
expect(result).toEqual([
{
description: 'Linux',
@ -254,7 +257,11 @@ describe('Exception viewer helpers', () => {
});
test('it returns Name when `includeName` is true', () => {
const result = getDescriptionListContent(getExceptionListItemSchemaMock(), false, true);
const result = getDescriptionListContent(
getExceptionListItemSchemaMock({ os_types: ['linux'] }),
false,
true
);
expect(result).toEqual([
{
description: 'some name',

1
x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.test.tsx
View file

@ -115,7 +115,6 @@ describe('When on the Event Filters List Page', () => {
expect(eventMeta).toEqual([
'some name',
'Linux',
'April 20th 2020 @ 11:25:31',
'some user',
'April 20th 2020 @ 11:25:31',

862
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts Normal file
View file

@ -0,0 +1,862 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import {
createListsIndex,
deleteAllExceptions,
deleteListsIndex,
} from '../../../lists_api_integration/utils';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import {
createRule,
createRuleWithExceptionEntries,
createSignalsIndex,
deleteAllAlerts,
deleteSignalsIndex,
getRuleForSignalTesting,
getSignalsById,
waitForRuleSuccessOrStatus,
waitForSignalsToBePresent,
} from '../../utils';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const es = getService('es');
describe('Rule exception operators for endpoints', () => {
beforeEach(async () => {
await createSignalsIndex(supertest);
await createListsIndex(supertest);
await esArchiver.load(
'x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type'
);
await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/agent');
});
afterEach(async () => {
await deleteSignalsIndex(supertest);
await deleteAllAlerts(supertest);
await deleteAllExceptions(es);
await deleteListsIndex(supertest);
await esArchiver.unload(
'x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type'
);
await esArchiver.unload('x-pack/test/functional/es_archives/rule_exceptions/agent');
});
describe('no exceptions set', () => {
it('should find all the "hosts" from a "agent" index when no exceptions are set on the rule', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRule(supertest, rule);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 4, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host).sort();
expect(hits).to.eql([
{
os: { type: 'linux' },
},
{
os: { type: 'windows' },
},
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
it('should find all the "hosts" from a "endpoint_without_host_type" index when no exceptions are set on the rule', async () => {
const rule = getRuleForSignalTesting(['endpoint_without_host_type']);
const { id } = await createRule(supertest, rule);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 4, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host).sort();
expect(hits).to.eql([
{
os: { name: 'Linux' },
},
{
os: { name: 'Windows' },
},
{
os: { name: 'Macos' },
},
{
os: { name: 'Linux' },
},
]);
});
});
describe('operating system types (os_types)', () => {
describe('endpoints', () => {
it('should filter 1 operating system types (os_type) if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['endpoint_without_host_type']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 3, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { name: 'Windows' },
},
{
os: { name: 'Macos' },
},
{
os: { name: 'Linux' },
},
]);
});
it('should filter 2 operating system types as an "OR" (os_type) if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['endpoint_without_host_type']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux', 'macos'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 3, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { name: 'Windows' },
},
{
os: { name: 'Macos' },
},
{
os: { name: 'Linux' },
},
]);
});
it('should filter multiple operating system types if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['endpoint_without_host_type']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
{
osTypes: ['windows'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '2',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 2, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { name: 'Macos' },
},
{
os: { name: 'Linux' },
},
]);
});
it('should filter multiple operating system types (os_type) with multiple filter items for an endpoint', async () => {
const rule = getRuleForSignalTesting(['endpoint_without_host_type']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['macos', 'linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
{
osTypes: ['windows', 'linux', 'macos'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '2',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 2, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { name: 'Macos' },
},
{
os: { name: 'Linux' },
},
]);
});
});
describe('agent', () => {
it('should filter 1 operating system types (os_type) if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 3, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'windows' },
},
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
it('should filter 1 operating system type as an "OR" (os_type) if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux', 'macos'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 3, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'windows' },
},
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
it('should filter multiple operating system types if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
{
osTypes: ['windows'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '2',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 2, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
it('should filter multiple operating system types (os_type) with multiple filter items for an endpoint', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['macos', 'linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
{
osTypes: ['windows', 'linux', 'macos'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '2',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 2, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
});
describe('agent and endpoint', () => {
it('should filter 2 operating system types (os_type) if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['agent', 'endpoint_without_host_type']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 6, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'windows' },
},
{
os: { name: 'Windows' },
},
{
os: { type: 'macos' },
},
{
os: { name: 'Macos' },
},
{
os: { type: 'linux' },
},
{
os: { name: 'Linux' },
},
]);
});
it('should filter 2 operating system types as an "OR" (os_type) if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['agent', 'endpoint_without_host_type']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux', 'macos'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 6, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'windows' },
},
{
os: { name: 'Windows' },
},
{
os: { type: 'macos' },
},
{
os: { name: 'Macos' },
},
{
os: { type: 'linux' },
},
{
os: { name: 'Linux' },
},
]);
});
it('should filter multiple operating system types if it is set as part of an endpoint exception', async () => {
const rule = getRuleForSignalTesting(['agent', 'endpoint_without_host_type']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
{
osTypes: ['windows'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '2',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 4, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'macos' },
},
{
os: { name: 'Macos' },
},
{
os: { type: 'linux' },
},
{
os: { name: 'Linux' },
},
]);
});
it('should filter multiple operating system types (os_type) with multiple filter items for an endpoint', async () => {
const rule = getRuleForSignalTesting(['agent', 'endpoint_without_host_type']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['macos', 'linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '1',
},
],
},
{
osTypes: ['windows', 'linux', 'macos'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match',
value: '2',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 4, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'macos' },
},
{
os: { name: 'Macos' },
},
{
os: { type: 'linux' },
},
{
os: { name: 'Linux' },
},
]);
});
});
});
describe('"is" operator', () => {
it('should filter 1 value set as an endpoint exception and 1 value set as a normal rule exception ', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[
[
{
field: 'host.os.type',
operator: 'included',
type: 'match',
value: 'linux',
},
],
],
[
{
osTypes: undefined, // This "undefined" is not possible through the user interface but is possible in the REST API
entries: [
{
field: 'host.os.type',
operator: 'included',
type: 'match',
value: 'windows',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 1, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'macos' },
},
]);
});
it('should filter 1 value set as an endpoint exception and 1 value set as a normal rule exception with os_type set', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[
[
{
field: 'host.os.type',
operator: 'included',
type: 'match',
value: 'linux',
},
],
],
[
{
osTypes: ['windows'],
entries: [
{
field: 'host.os.type',
operator: 'included',
type: 'match',
value: 'windows',
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 1, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'macos' },
},
]);
});
});
describe('"is one of" operator', () => {
it('should filter 1 single value if it is set as an exception and the os_type is set to only 1 value', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['windows'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match_any',
value: ['1', '2'],
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 3, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'linux' },
},
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
it('should filter 2 values if it is set as an exception and the os_type is set to 2 values', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['windows', 'linux'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match_any',
value: ['1', '2'],
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 2, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
it('should filter 2 values if it is set as an exception and the os_type is set to undefined', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: undefined, // This is only possible through the REST API
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match_any',
value: ['1', '2'],
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 2, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
it('should filter no values if they are set as an exception but the os_type is set to something not within the documents', async () => {
const rule = getRuleForSignalTesting(['agent']);
const { id } = await createRuleWithExceptionEntries(
supertest,
rule,
[],
[
{
osTypes: ['macos'],
entries: [
{
field: 'event.code',
operator: 'included',
type: 'match_any',
value: ['1', '2'],
},
],
},
]
);
await waitForRuleSuccessOrStatus(supertest, id);
await waitForSignalsToBePresent(supertest, 4, [id]);
const signalsOpen = await getSignalsById(supertest, id);
const hits = signalsOpen.hits.hits.map((hit) => hit._source.host);
expect(hits).to.eql([
{
os: { type: 'linux' },
},
{
os: { type: 'windows' },
},
{
os: { type: 'macos' },
},
{
os: { type: 'linux' },
},
]);
});
});
});
};

1
x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts
View file

@ -14,6 +14,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
this.tags('ciGroup11');
loadTestFile(require.resolve('./aliases'));
loadTestFile(require.resolve('./create_endpoint_exceptions'));
loadTestFile(require.resolve('./add_actions'));
loadTestFile(require.resolve('./update_actions'));
loadTestFile(require.resolve('./add_prepackaged_rules'));

155
x-pack/test/detection_engine_api_integration/utils.ts
View file

@ -12,7 +12,11 @@ import { SuperTest } from 'supertest';
import supertestAsPromised from 'supertest-as-promised';
import { Context } from '@elastic/elasticsearch/lib/Transport';
import { SearchResponse } from 'elasticsearch';
import type { NonEmptyEntriesArray } from '@kbn/securitysolution-io-ts-list-types';
import type {
ListArray,
NonEmptyEntriesArray,
OsTypeArray,
} from '@kbn/securitysolution-io-ts-list-types';
import type {
CreateExceptionListItemSchema,
CreateExceptionListSchema,
@ -21,7 +25,6 @@ import type {
} from '@kbn/securitysolution-io-ts-list-types';
import { EXCEPTION_LIST_ITEM_URL, EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants';
import { PrePackagedRulesAndTimelinesStatusSchema } from '../../plugins/security_solution/common/detection_engine/schemas/response';
import { getCreateExceptionListDetectionSchemaMock } from '../../plugins/lists/common/schemas/request/create_exception_list_schema.mock';
import {
CreateRulesSchema,
UpdateRulesSchema,
@ -45,7 +48,6 @@ import {
INTERNAL_IMMUTABLE_KEY,
INTERNAL_RULE_ID_KEY,
} from '../../plugins/security_solution/common/constants';
import { getCreateExceptionListItemMinimalSchemaMockWithoutId } from '../../plugins/lists/common/schemas/request/create_exception_list_item_schema.mock';
/**
* This will remove server generated properties such as date times, etc...
@ -1149,28 +1151,97 @@ export const installPrePackagedRules = async (
};
/**
* Convenience testing function where you can pass in just the entries and you will
* get a rule created with the entries added to an exception list and exception list item
* all auto-created at once.
* Convenience testing function where you can pass in just the endpoint entries and you will
* get a container created with the entries.
* @param supertest super test agent
* @param rule The rule to create and attach an exception list to
* @param entries The entries to create the rule and exception list from
* @param endpointEntries The endpoint entries to create the rule and exception list from
* @param osTypes The os types to optionally add or not to add to the container
*/
export const createRuleWithExceptionEntries = async (
export const createContainerWithEndpointEntries = async (
supertest: SuperTest<supertestAsPromised.Test>,
rule: CreateRulesSchema,
entries: NonEmptyEntriesArray[]
): Promise<FullResponseSchema> => {
endpointEntries: Array<{
entries: NonEmptyEntriesArray;
osTypes: OsTypeArray | undefined;
}>
): Promise<ListArray> => {
// If not given any endpoint entries, return without any
if (endpointEntries.length === 0) {
return [];
}
// create the endpoint exception list container
// eslint-disable-next-line @typescript-eslint/naming-convention
const { id, list_id, namespace_type, type } = await createExceptionList(
supertest,
getCreateExceptionListDetectionSchemaMock()
const { id, list_id, namespace_type, type } = await createExceptionList(supertest, {
description: 'endpoint description',
list_id: 'endpoint_list',
name: 'endpoint_list',
type: 'endpoint',
});
// Add the endpoint exception list container to the backend
await Promise.all(
endpointEntries.map((endpointEntry) => {
const exceptionListItem: CreateExceptionListItemSchema = {
description: 'endpoint description',
entries: endpointEntry.entries,
list_id: 'endpoint_list',
name: 'endpoint_list',
os_types: endpointEntry.osTypes,
type: 'simple',
};
return createExceptionListItem(supertest, exceptionListItem);
})
);
// To reduce the odds of in-determinism and/or bugs we ensure we have
// the same length of entries before continuing.
await waitFor(async () => {
const { body } = await supertest.get(`${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${list_id}`);
return body.data.length === endpointEntries.length;
}, `within createContainerWithEndpointEntries ${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${list_id}`);
return [
{
id,
list_id,
namespace_type,
type,
},
];
};
/**
* Convenience testing function where you can pass in just the endpoint entries and you will
* get a container created with the entries.
* @param supertest super test agent
* @param entries The entries to create the rule and exception list from
* @param osTypes The os types to optionally add or not to add to the container
*/
export const createContainerWithEntries = async (
supertest: SuperTest<supertestAsPromised.Test>,
entries: NonEmptyEntriesArray[]
): Promise<ListArray> => {
// If not given any endpoint entries, return without any
if (entries.length === 0) {
return [];
}
// Create the rule exception list container
// eslint-disable-next-line @typescript-eslint/naming-convention
const { id, list_id, namespace_type, type } = await createExceptionList(supertest, {
description: 'some description',
list_id: 'some-list-id',
name: 'some name',
type: 'detection',
});
// Add the rule exception list container to the backend
await Promise.all(
entries.map((entry) => {
const exceptionListItem: CreateExceptionListItemSchema = {
...getCreateExceptionListItemMinimalSchemaMockWithoutId(),
description: 'some description',
list_id: 'some-list-id',
name: 'some name',
type: 'simple',
entries: entry,
};
return createExceptionListItem(supertest, exceptionListItem);
@ -1180,13 +1251,44 @@ export const createRuleWithExceptionEntries = async (
// To reduce the odds of in-determinism and/or bugs we ensure we have
// the same length of entries before continuing.
await waitFor(async () => {
const { body } = await supertest.get(
`${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${
getCreateExceptionListDetectionSchemaMock().list_id
}`
);
const { body } = await supertest.get(`${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${list_id}`);
return body.data.length === entries.length;
}, `within createRuleWithExceptionEntries ${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${getCreateExceptionListDetectionSchemaMock().list_id}`);
}, `within createContainerWithEntries ${EXCEPTION_LIST_ITEM_URL}/_find?list_id=${list_id}`);
return [
{
id,
list_id,
namespace_type,
type,
},
];
};
/**
* Convenience testing function where you can pass in just the entries and you will
* get a rule created with the entries added to an exception list and exception list item
* all auto-created at once.
* @param supertest super test agent
* @param rule The rule to create and attach an exception list to
* @param entries The entries to create the rule and exception list from
* @param endpointEntries The endpoint entries to create the rule and exception list from
* @param osTypes The os types to optionally add or not to add to the container
*/
export const createRuleWithExceptionEntries = async (
supertest: SuperTest<supertestAsPromised.Test>,
rule: CreateRulesSchema,
entries: NonEmptyEntriesArray[],
endpointEntries?: Array<{
entries: NonEmptyEntriesArray;
osTypes: OsTypeArray | undefined;
}>
): Promise<FullResponseSchema> => {
const maybeExceptionList = await createContainerWithEntries(supertest, entries);
const maybeEndpointList = await createContainerWithEndpointEntries(
supertest,
endpointEntries ?? []
);
// create the rule but don't run it immediately as running it immediately can cause
// the rule to sometimes not filter correctly the first time with an exception list
@ -1195,14 +1297,7 @@ export const createRuleWithExceptionEntries = async (
const ruleWithException: CreateRulesSchema = {
...rule,
enabled: false,
exceptions_list: [
{
id,
list_id,
namespace_type,
type,
},
],
exceptions_list: [...maybeExceptionList, ...maybeEndpointList],
};
const ruleResponse = await createRule(supertest, ruleWithException);
await supertest

3
x-pack/test/functional/es_archives/rule_exceptions/README.md
View file

@ -1,7 +1,8 @@
Within this folder is input test data for tests such as:
```ts
security_and_spaces/tests/rule_exceptions.ts
security_and_spaces/tests/operator_data_types
security_and_spaces/tests/create_endpoint_exceptions.ts
```
where these are small ECS compliant input indexes that try to express tests that exercise different parts of

79
x-pack/test/functional/es_archives/rule_exceptions/agent/data.json Normal file
View file

@ -0,0 +1,79 @@
{
"type": "doc",
"value": {
"id": "1",
"index": "agent",
"source": {
"@timestamp": "2020-10-28T05:00:53.000Z",
"host": {
"os": {
"type": "linux"
}
},
"event": {
"code": 1
}
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "2",
"index": "agent",
"source": {
"@timestamp": "2020-10-28T05:01:53.000Z",
"host": {
"os": {
"type": "windows"
}
},
"event": {
"code": 2
}
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "3",
"index": "agent",
"source": {
"@timestamp": "2020-10-28T05:02:53.000Z",
"host": {
"os": {
"type": "macos"
}
},
"event": {
"code": 3
}
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "4",
"index": "agent",
"source": {
"@timestamp": "2020-10-28T05:03:53.000Z",
"host": {
"os": {
"type": "linux"
}
},
"event": {
"code": 4
}
},
"type": "_doc"
}
}

40
x-pack/test/functional/es_archives/rule_exceptions/agent/mappings.json Normal file
View file

@ -0,0 +1,40 @@
{
"type": "index",
"value": {
"index": "agent",
"mappings": {
"dynamic": "strict",
"properties": {
"@timestamp": {
"type": "date"
},
"host": {
"properties": {
"os": {
"properties": {
"type": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
},
"event": {
"properties": {
"code": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
},
"settings": {
"index": {
"number_of_replicas": "1",
"number_of_shards": "1"
}
}
}
}

79
x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type/data.json Normal file
View file

@ -0,0 +1,79 @@
{
"type": "doc",
"value": {
"id": "1",
"index": "endpoint_without_host_type",
"source": {
"@timestamp": "2020-10-28T05:00:53.000Z",
"host": {
"os": {
"name": "Linux"
}
},
"event": {
"code": 1
}
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "2",
"index": "endpoint_without_host_type",
"source": {
"@timestamp": "2020-10-28T05:01:53.000Z",
"host": {
"os": {
"name": "Windows"
}
},
"event": {
"code": 2
}
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "3",
"index": "endpoint_without_host_type",
"source": {
"@timestamp": "2020-10-28T05:02:53.000Z",
"host": {
"os": {
"name": "Macos"
}
},
"event": {
"code": 3
}
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"id": "4",
"index": "endpoint_without_host_type",
"source": {
"@timestamp": "2020-10-28T05:03:53.000Z",
"host": {
"os": {
"name": "Linux"
}
},
"event": {
"code": 4
}
},
"type": "_doc"
}
}

64
x-pack/test/functional/es_archives/rule_exceptions/endpoint_without_host_type/mappings.json Normal file
View file

@ -0,0 +1,64 @@
{
"type": "index",
"value": {
"index": "endpoint_without_host_type",
"mappings": {
"dynamic": "strict",
"properties": {
"@timestamp": {
"type": "date"
},
"host": {
"properties": {
"os": {
"properties": {
"name": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "text"
}
}
}
}
},
"name": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "text"
}
}
}
}
},
"event": {
"properties": {
"code": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
},
"settings": {
"index": {
"number_of_replicas": "1",
"number_of_shards": "1"
}
}
}
}