github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Alerts] adds support for multi fields in new terms rule (#143943)

Browse source 
## Summary

- addresses https://github.com/elastic/kibana/issues/142862
   - allows up to 3 fields in `New terms`
   - displays new terms fields in alerts details
- For multiple new terms fields(['source.host', 'source.ip']), in terms aggregation uses a runtime field. Which is created by joining values from new terms fields into one single keyword. Field values encoded in base64 and joined with a configured delimiter symbol, which is not part of base64 symbols(a–Z, 0–9, +, /,  =) to avoid a situation when delimiter can be part of field value. Include parameter consists of encoded in base64 results from Phase 1. 
For single field, implementation remains the same to avoid performance penalties
- Performance measurements:
    - [msearch POC for one field](https://github.com/elastic/kibana/pull/131010#issuecomment-1135280478) 
    - [historical POC of multi fields](https://github.com/elastic/kibana/issues/142862#issuecomment-1290471400)
    - [current implementation](https://github.com/elastic/kibana/issues/142862#issuecomment-1298910940)

## UI

### Alert details
#### Before
<img width="1128" alt="Screenshot 2022-10-26 at 18 29 00" src="https://user-images.githubusercontent.com/92328789/199074598-33483312-a660-49e2-aab6-9aea4f70a23e.png">

#### After
<img width="1129" alt="Screenshot 2022-11-09 at 19 05 10" src="https://user-images.githubusercontent.com/92328789/201698130-b907a200-3cdd-49e6-ae76-a5e0ef9a9450.png">


Delete any items that are not applicable to this PR.

- [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials
- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

### For maintainers

- [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Vitalii Dmyterko 2022-11-14 16:55:20 +00:00 • committed by GitHub
parent 164de3c5b4
commit f1117c8959
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 1116 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/security_solution/common/constants.ts
View file

@ -469,4 +469,6 @@ export const RISKY_HOSTS_DOC_LINK =
export const RISKY_USERS_DOC_LINK =
'https://www.elastic.co/guide/en/security/current/user-risk-score.html';
export const MAX_NUMBER_OF_NEW_TERMS_FIELDS = 3;
export const BULK_ADD_TO_TIMELINE_LIMIT = 2000;

9
x-pack/plugins/security_solution/common/detection_engine/rule_schema/model/specific_attributes/new_terms_attributes.ts
View file

@ -7,14 +7,19 @@
import * as t from 'io-ts';
import { LimitedSizeArray, NonEmptyString } from '@kbn/securitysolution-io-ts-types';
import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../constants';
// Attributes specific to New Terms rules
/**
* New terms rule type currently only supports a single term, but should support more in the future
* New terms rule type supports a limited number of fields. Max number of fields is 3 and defined in common constants as MAX_NUMBER_OF_NEW_TERMS_FIELDS
*/
export type NewTermsFields = t.TypeOf<typeof NewTermsFields>;
export const NewTermsFields = LimitedSizeArray({ codec: t.string, minSize: 1, maxSize: 1 });
export const NewTermsFields = LimitedSizeArray({
codec: t.string,
minSize: 1,
maxSize: MAX_NUMBER_OF_NEW_TERMS_FIELDS,
});
export type HistoryWindowStart = t.TypeOf<typeof HistoryWindowStart>;
export const HistoryWindowStart = NonEmptyString;

3
x-pack/plugins/security_solution/common/field_maps/field_names.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from '@kbn/rule-data-utils';
import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils';
export const ALERT_ANCESTORS = `${ALERT_NAMESPACE}.ancestors` as const;
export const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const;
@ -16,6 +16,7 @@ export const ALERT_ORIGINAL_TIME = `${ALERT_NAMESPACE}.original_time` as const;
export const ALERT_THRESHOLD_RESULT = `${ALERT_NAMESPACE}.threshold_result` as const;
export const ALERT_THRESHOLD_RESULT_COUNT = `${ALERT_THRESHOLD_RESULT}.count` as const;
export const ALERT_NEW_TERMS = `${ALERT_NAMESPACE}.new_terms` as const;
export const ALERT_NEW_TERMS_FIELDS = `${ALERT_RULE_PARAMETERS}.new_terms_fields` as const;
export const ALERT_ORIGINAL_EVENT = `${ALERT_NAMESPACE}.original_event` as const;
export const ALERT_ORIGINAL_EVENT_ACTION = `${ALERT_ORIGINAL_EVENT}.action` as const;

9
x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.test.tsx
View file

@ -703,18 +703,25 @@ describe('AlertSummaryView', () => {
values: ['127.0.0.1'],
originalValue: ['127.0.0.1'],
},
{
category: 'kibana',
field: 'kibana.alert.rule.parameters.new_terms_fields',
values: ['host.ip'],
originalValue: ['host.ip'],
},
] as TimelineEventsDetailsItem[];
const renderProps = {
...props,
data: enhancedData,
};
const { getByText } = render(
<TestProvidersComponent>
<AlertSummaryView {...renderProps} />
</TestProvidersComponent>
);
['New Terms'].forEach((fieldId) => {
['New Terms', '127.0.0.1', 'New Terms fields', 'host.ip'].forEach((fieldId) => {
expect(getByText(fieldId));
});
});

11
x-pack/plugins/security_solution/public/common/components/event_details/get_alert_summary_rows.tsx
View file

@ -16,8 +16,13 @@ import {
ALERTS_HEADERS_THRESHOLD_TERMS,
ALERTS_HEADERS_RULE_DESCRIPTION,
ALERTS_HEADERS_NEW_TERMS,
ALERTS_HEADERS_NEW_TERMS_FIELDS,
} from '../../../detections/components/alerts_table/translations';
import { ALERT_NEW_TERMS, ALERT_THRESHOLD_RESULT } from '../../../../common/field_maps/field_names';
import {
ALERT_NEW_TERMS_FIELDS,
ALERT_NEW_TERMS,
ALERT_THRESHOLD_RESULT,
} from '../../../../common/field_maps/field_names';
import { AGENT_STATUS_FIELD_NAME } from '../../../timelines/components/timeline/body/renderers/constants';
import type { AlertSummaryRow } from './helpers';
import { getEnrichedFieldInfo } from './helpers';
@ -172,6 +177,10 @@ function getFieldsByRuleType(ruleType?: string): EventSummaryField[] {
];
case 'new_terms':
return [
{
id: ALERT_NEW_TERMS_FIELDS,
label: ALERTS_HEADERS_NEW_TERMS_FIELDS,
},
{
id: ALERT_NEW_TERMS,
label: ALERTS_HEADERS_NEW_TERMS,

7
x-pack/plugins/security_solution/public/detections/components/alerts_table/translations.ts
View file

@ -123,6 +123,13 @@ export const ALERTS_HEADERS_NEW_TERMS = i18n.translate(
}
);
export const ALERTS_HEADERS_NEW_TERMS_FIELDS = i18n.translate(
'xpack.securitySolution.eventsViewer.alerts.defaultHeaders.newTermsFields',
{
defaultMessage: 'New Terms fields',
}
);
export const ACTION_INVESTIGATE_IN_TIMELINE = i18n.translate(
'xpack.securitySolution.detectionEngine.alerts.actions.investigateInTimelineTitle',
{

8
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx
View file

@ -59,6 +59,7 @@ import {
useFormData,
} from '../../../../shared_imports';
import { schema } from './schema';
import { getTermsAggregationFields } from './utils';
import * as i18n from './translations';
import {
isEqlRule,
@ -297,6 +298,11 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
setAggregatableFields(aggregatableFields(fields as BrowserField[]));
}, [indexPattern]);
const termsAggregationFields: BrowserField[] = useMemo(
() => getTermsAggregationFields(aggFields),
[aggFields]
);
const [
threatIndexPatternsLoading,
{ browserFields: threatBrowserFields, indexPatterns: threatIndexPatterns },
@ -836,7 +842,7 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
path="newTermsFields"
component={NewTermsFields}
componentProps={{
browserFields: aggFields,
browserFields: termsAggregationFields,
}}
/>
<UseField

7
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx
View file

@ -22,6 +22,7 @@ import {
isThreatMatchRule,
isThresholdRule,
} from '../../../../../common/detection_engine/utils';
import { MAX_NUMBER_OF_NEW_TERMS_FIELDS } from '../../../../../common/constants';
import { isMlRule } from '../../../../../common/machine_learning/helpers';
import type { FieldValueQueryBar } from '../query_bar';
import type { ERROR_CODE, FormSchema, ValidationFunc } from '../../../../shared_imports';
@ -585,7 +586,7 @@ export const schema: FormSchema<DefineStepRule> = {
i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.newTermsFieldsMin',
{
defaultMessage: 'Number of fields must be 1.',
defaultMessage: 'A minimum of one field is required.',
}
)
)(...args);
@ -601,11 +602,11 @@ export const schema: FormSchema<DefineStepRule> = {
return;
}
return fieldValidators.maxLengthField({
length: 1,
length: MAX_NUMBER_OF_NEW_TERMS_FIELDS,
message: i18n.translate(
'xpack.securitySolution.detectionEngine.validations.stepDefineRule.newTermsFieldsMax',
{
defaultMessage: 'Number of fields must be 1.',
defaultMessage: 'Number of fields must be 3 or less.',
}
),
})(...args);

21
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/utils.ts Normal file
View file

@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { BrowserField } from '../../../../common/containers/source';
/**
* Filters out fields, that are not supported in terms aggregation.
* Terms aggregation supports limited number of types:
* Keyword, Numeric, ip, boolean, or binary.
* https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html
*/
export const getTermsAggregationFields = (fields: BrowserField[]): BrowserField[] => {
// binary types is excluded, as binary field has property aggregatable === false
const allowedTypesSet = new Set(['string', 'number', 'ip', 'boolean']);
return fields.filter((field) => field.aggregatable === true && allowedTypesSet.has(field.type));
};

5
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/README.md
View file

@ -2,7 +2,7 @@
The rule accepts 2 new parameters that are unique to the new_terms rule type, in addition to common Security rule parameters such as query, index, and filters, to, from, etc. The new parameters are:
- `new_terms_fields`: an array of field names, currently limited to an array of size 1. In the future we will likely allow multiple field names to be specified here.
- `new_terms_fields`: an array of field names, currently limited to an array of size 3.
Example: ['host.ip']
- `history_window_start`: defines the additional time range to search over when determining if a term is "new". If a term is found between the times `history_window_start` and from then it will not be classified as a new term.
Example: now-30d
@ -12,6 +12,7 @@ Each page is evaluated in 3 phases.
Phase 1: Collect "recent" terms - terms that have appeared in the last rule interval, without regard to whether or not they have appeared in historical data. This is done using a composite aggregation to ensure we can iterate over every term.
Phase 2: Check if the page of terms contains any new terms. This uses a regular terms agg with the include parameter - every term is added to the array of include values, so the terms agg is limited to only aggregating on the terms of interest from phase 1. This avoids issues with the terms agg providing approximate results due to getting different terms from different shards.
For multiple new terms fields(['source.host', 'source.ip']), in terms aggregation uses a runtime field. Which is created by joining values from new terms fields into one single keyword value. Fields values encoded in base64 and joined with configured a delimiter symbol, which is not part of base64 symbols(a–Z, 0–9, +, /, =) to avoid a situation when delimiter can be part of field value. Include parameter consists of encoded in base64 results from Phase 1.
Phase 3: Any new terms from phase 2 are processed and the first document to contain that term is retrieved. The document becomes the basis of the generated alert. This is done with an aggregation query that is very similar to the agg used in phase 2, except it also includes a top_hits agg. top_hits is moved to a separate, later phase for efficiency - top_hits is slow and most terms will not be new in phase 2. This means we only execute the top_hits agg on the terms that are actually new which is faster.
@ -26,4 +27,4 @@ The new terms rule type reuses the singleSearchAfter function which implements t
## Limitations and future enhancements
- Value list exceptions are not supported at the moment. Commit ead04ce removes an experimental method I tried for evaluating value list exceptions.
- In the future we may want to support searching for new sets of terms, e.g. a pair of `host.ip` and `host.id` that has never been seen together before.
- Runtime field supports only 100 emitted values. So for large arrays or combination of values greater than 100, results may not be exhaustive. This applies only to new terms with multiple fields

34
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/__snapshots__/build_new_terms_aggregation.test.ts.snap generated
View file

@ -135,3 +135,37 @@ Object {
},
}
`;
exports[`aggregations buildRecentTermsAgg builds a correct composite aggregation with multiple fields 1`] = `
Object {
"new_terms": Object {
"composite": Object {
"after": undefined,
"size": 10000,
"sources": Array [
Object {
"host.name": Object {
"terms": Object {
"field": "host.name",
},
},
},
Object {
"host.port": Object {
"terms": Object {
"field": "host.port",
},
},
},
Object {
"host.url": Object {
"terms": Object {
"field": "host.url",
},
},
},
],
},
},
}
`;

13
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.test.ts
View file

@ -16,7 +16,7 @@ describe('aggregations', () => {
describe('buildRecentTermsAgg', () => {
test('builds a correct composite agg without `after`', () => {
const aggregation = buildRecentTermsAgg({
field: 'host.name',
fields: ['host.name'],
after: undefined,
});
@ -25,12 +25,21 @@ describe('aggregations', () => {
test('builds a correct composite aggregation with `after`', () => {
const aggregation = buildRecentTermsAgg({
field: 'host.name',
fields: ['host.name'],
after: { 'host.name': 'myHost' },
});
expect(aggregation).toMatchSnapshot();
});
test('builds a correct composite aggregation with multiple fields', () => {
const aggregation = buildRecentTermsAgg({
fields: ['host.name', 'host.port', 'host.url'],
after: undefined,
});
expect(aggregation).toMatchSnapshot();
});
});
describe('buildNewTermsAggregation', () => {

22
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/build_new_terms_aggregation.ts
View file

@ -31,24 +31,24 @@ const PAGE_SIZE = 10000;
* without regard to whether or not they're actually new.
*/
export const buildRecentTermsAgg = ({
field,
fields,
after,
}: {
field: string;
fields: string[];
after: Record<string, string | number | null> | undefined;
}) => {
const sources = fields.map((field) => ({
[field]: {
terms: {
field,
},
},
}));
return {
new_terms: {
composite: {
sources: [
{
[field]: {
terms: {
field,
},
},
},
],
sources,
size: PAGE_SIZE,
after,
},

48
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts
View file

@ -5,7 +5,6 @@
* 2.0.
*/
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
import { NEW_TERMS_RULE_TYPE_ID } from '@kbn/securitysolution-rules';
import { SERVER_APP_ID } from '../../../../../common/constants';
@ -16,6 +15,7 @@ import type { CreateRuleOptions, SecurityAlertType } from '../types';
import { singleSearchAfter } from '../../signals/single_search_after';
import { getFilter } from '../../signals/get_filter';
import { wrapNewTermsAlerts } from '../factories/utils/wrap_new_terms_alerts';
import type { EventsAndTerms } from '../factories/utils/wrap_new_terms_alerts';
import type {
DocFetchAggResult,
RecentTermsAggResult,
@ -26,9 +26,15 @@ import {
buildRecentTermsAgg,
buildNewTermsAgg,
} from './build_new_terms_aggregation';
import type { SignalSource } from '../../signals/types';
import { validateIndexPatterns } from '../utils';
import { parseDateString, validateHistoryWindowStart } from './utils';
import {
parseDateString,
validateHistoryWindowStart,
transformBucketsToValues,
getNewTermsRuntimeMappings,
getAggregationField,
decodeMatchedValues,
} from './utils';
import {
addToSearchAfterReturn,
createSearchAfterReturnType,
@ -154,7 +160,7 @@ export const createNewTermsAlertType = (
// ones are new.
const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
aggregations: buildRecentTermsAgg({
field: params.newTermsFields[0],
fields: params.newTermsFields,
after: afterKey,
}),
searchAfterSortIds: undefined,
@ -187,10 +193,7 @@ export const createNewTermsAlertType = (
break;
}
const bucketsForField = searchResultWithAggs.aggregations.new_terms.buckets;
const includeValues = bucketsForField
.map((bucket) => Object.values(bucket.key)[0])
.filter((value): value is string | number => value != null);
const includeValues = transformBucketsToValues(params.newTermsFields, bucketsForField);
// PHASE 2: Take the page of results from Phase 1 and determine if each term exists in the history window.
// The aggregation filters out buckets for terms that exist prior to `tuple.from`, so the buckets in the
// response correspond to each new term.
@ -202,10 +205,13 @@ export const createNewTermsAlertType = (
aggregations: buildNewTermsAgg({
newValueWindowStart: tuple.from,
timestampField: aggregatableTimestampField,
field: params.newTermsFields[0],
field: getAggregationField(params.newTermsFields),
include: includeValues,
}),
runtimeMappings,
runtimeMappings: {
...runtimeMappings,
...getNewTermsRuntimeMappings(params.newTermsFields),
},
searchAfterSortIds: undefined,
index: inputIndex,
// For Phase 2, we expand the time range to aggregate over the history window
@ -245,10 +251,13 @@ export const createNewTermsAlertType = (
} = await singleSearchAfter({
aggregations: buildDocFetchAgg({
timestampField: aggregatableTimestampField,
field: params.newTermsFields[0],
field: getAggregationField(params.newTermsFields),
include: actualNewTerms,
}),
runtimeMappings,
runtimeMappings: {
...runtimeMappings,
...getNewTermsRuntimeMappings(params.newTermsFields),
},
searchAfterSortIds: undefined,
index: inputIndex,
// For phase 3, we go back to aggregating only over the rule interval - excluding the history window
@ -270,13 +279,14 @@ export const createNewTermsAlertType = (
throw new Error('Aggregations were missing on document fetch search result');
}
const eventsAndTerms: Array<{
event: estypes.SearchHit<SignalSource>;
newTerms: Array<string | number | null>;
}> = docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => ({
event: bucket.docs.hits.hits[0],
newTerms: [bucket.key],
}));
const eventsAndTerms: EventsAndTerms[] =
docFetchResultWithAggs.aggregations.new_terms.buckets.map((bucket) => {
const newTerms = decodeMatchedValues(params.newTermsFields, bucket.key);
return {
event: bucket.docs.hits.hits[0],
newTerms,
};
});
const alertTimestampOverride = isPreview ? startedAt : undefined;
const wrappedAlerts = wrapNewTermsAlerts({

146
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.test.ts
View file

@ -5,7 +5,15 @@
* 2.0.
*/
import { parseDateString, validateHistoryWindowStart } from './utils';
import {
parseDateString,
validateHistoryWindowStart,
transformBucketsToValues,
getAggregationField,
decodeMatchedValues,
getNewTermsRuntimeMappings,
AGG_FIELD_NAME,
} from './utils';
describe('new terms utils', () => {
describe('parseDateString', () => {
@ -64,4 +72,140 @@ describe('new terms utils', () => {
);
});
});
describe('transformBucketsToValues', () => {
it('should return correct value for a single new terms field', () => {
expect(
transformBucketsToValues(
['source.host'],
[
{
key: {
'source.host': 'host-0',
},
doc_count: 1,
},
{
key: {
'source.host': 'host-1',
},
doc_count: 3,
},
]
)
).toEqual(['host-0', 'host-1']);
});
it('should filter null values for a single new terms field', () => {
expect(
transformBucketsToValues(
['source.host'],
[
{
key: {
'source.host': 'host-0',
},
doc_count: 1,
},
{
key: {
'source.host': null,
},
doc_count: 3,
},
]
)
).toEqual(['host-0']);
});
it('should return correct value for multiple new terms fields', () => {
expect(
transformBucketsToValues(
['source.host', 'source.ip'],
[
{
key: {
'source.host': 'host-0',
'source.ip': '127.0.0.1',
},
doc_count: 1,
},
{
key: {
'source.host': 'host-1',
'source.ip': '127.0.0.1',
},
doc_count: 1,
},
]
)
).toEqual(['aG9zdC0w_MTI3LjAuMC4x', 'aG9zdC0x_MTI3LjAuMC4x']);
});
it('should filter null values for multiple new terms fields', () => {
expect(
transformBucketsToValues(
['source.host', 'source.ip'],
[
{
key: {
'source.host': 'host-0',
'source.ip': '127.0.0.1',
},
doc_count: 1,
},
{
key: {
'source.host': 'host-1',
'source.ip': null,
},
doc_count: 1,
},
]
)
).toEqual(['aG9zdC0w_MTI3LjAuMC4x']);
});
});
describe('getAggregationField', () => {
it('should return correct value for a single new terms field', () => {
expect(getAggregationField(['source.ip'])).toBe('source.ip');
});
it('should return correct value for multiple new terms fields', () => {
expect(getAggregationField(['source.host', 'source.ip'])).toBe(AGG_FIELD_NAME);
});
});
describe('decodeMatchedValues', () => {
it('should return correct value for a single new terms field', () => {
expect(decodeMatchedValues(['source.ip'], '127.0.0.1')).toEqual(['127.0.0.1']);
});
it('should return correct value for multiple new terms fields', () => {
expect(decodeMatchedValues(['source.host', 'source.ip'], 'aG9zdC0w_MTI3LjAuMC4x')).toEqual([
'host-0',
'127.0.0.1',
]);
});
});
describe('getNewTermsRuntimeMappings', () => {
it('should not return runtime field if new terms fields is empty', () => {
expect(getNewTermsRuntimeMappings([])).toBeUndefined();
});
it('should not return runtime field if new terms fields has only one field', () => {
expect(getNewTermsRuntimeMappings(['host.name'])).toBeUndefined();
});
it('should return runtime field if new terms fields has more than one field', () => {
const runtimeMappings = getNewTermsRuntimeMappings(['host.name', 'host.ip']);
expect(runtimeMappings?.[AGG_FIELD_NAME]).toMatchObject({
type: 'keyword',
script: {
params: { fields: ['host.name', 'host.ip'] },
source: expect.any(String),
},
});
});
});
});

107
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/utils.ts
View file

@ -7,6 +7,10 @@
import dateMath from '@elastic/datemath';
import moment from 'moment';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
export const AGG_FIELD_NAME = 'new_terms_values';
const DELIMITER = '_';
export const parseDateString = ({
date,
@ -46,3 +50,106 @@ export const validateHistoryWindowStart = ({
);
}
};
/**
* Takes a list of buckets and creates value from them to be used in 'include' clause of terms aggregation.
* For a single new terms field, value equals to bucket name
* For multiple new terms fields and buckets, value equals to concatenated base64 encoded bucket names
* @returns for buckets('host-0', 'test'), resulted value equals to: 'aG9zdC0w_dGVzdA=='
*/
export const transformBucketsToValues = (
newTermsFields: string[],
buckets: estypes.AggregationsCompositeBucket[]
): Array<string | number> => {
// if new terms include only one field we don't use runtime mappings and don't stich fields buckets together
if (newTermsFields.length === 1) {
return buckets
.map((bucket) => Object.values(bucket.key)[0])
.filter((value): value is string | number => value != null);
}
return buckets
.map((bucket) => Object.values(bucket.key))
.filter((values) => !values.some((value) => value == null))
.map((values) =>
values
.map((value) =>
Buffer.from(typeof value !== 'string' ? value.toString() : value).toString('base64')
)
.join(DELIMITER)
);
};
export const getNewTermsRuntimeMappings = (
newTermsFields: string[]
): undefined | { [AGG_FIELD_NAME]: estypes.MappingRuntimeField } => {
// if new terms include only one field we don't use runtime mappings and don't stich fields buckets together
if (newTermsFields.length <= 1) {
return undefined;
}
return {
[AGG_FIELD_NAME]: {
type: 'keyword',
script: {
params: { fields: newTermsFields },
source: `
def stack = new Stack();
// ES has limit in 100 values for runtime field, after this query will fail
int emitLimit = 100;
stack.add([0, '']);
while (stack.length > 0) {
if (emitLimit == 0) {
break;
}
def tuple = stack.pop();
def index = tuple[0];
def line = tuple[1];
if (index === params['fields'].length) {
emit(line);
emitLimit = emitLimit - 1;
} else {
for (field in doc[params['fields'][index]]) {
def delimiter = index === 0 ? '' : '${DELIMITER}';
def nextLine = line + delimiter + String.valueOf(field).encodeBase64();
stack.add([index + 1, nextLine])
}
}
}
`,
},
},
};
};
/**
* For a single new terms field, aggregation field equals to new terms field
* For multiple new terms fields, aggregation field equals to defined AGG_FIELD_NAME, which is runtime field
*/
export const getAggregationField = (newTermsFields: string[]): string => {
// if new terms include only one field we don't use runtime mappings and don't stich fields buckets together
if (newTermsFields.length === 1) {
return newTermsFields[0];
}
return AGG_FIELD_NAME;
};
const decodeBucketKey = (bucketKey: string): string[] => {
return bucketKey
.split(DELIMITER)
.map((encodedValue) => Buffer.from(encodedValue, 'base64').toString());
};
/**
* decodes matched values(bucket keys) from terms aggregation and returns fields as array
* @returns 'aG9zdC0w_dGVzdA==' bucket key will result in ['host-0', 'test']
*/
export const decodeMatchedValues = (newTermsFields: string[], bucketKey: string | number) => {
// if newTermsFields has length greater than 1, bucketKey can't be number, so casting is safe here
const values = newTermsFields.length === 1 ? [bucketKey] : decodeBucketKey(bucketKey as string);
return values;
};

17
x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts
View file

@ -40,5 +40,22 @@ export default ({ getService }: FtrProviderContext) => {
"params invalid: History window size is smaller than rule interval + additional lookback, 'historyWindowStart' must be earlier than 'from'"
);
});
it('should not be able to create a new terms rule with fields number greater than 3', async () => {
const rule = {
...getCreateNewTermsRulesSchemaMock('rule-1'),
history_window_start: 'now-5m',
new_terms_fields: ['field1', 'field2', 'field3', 'field4'],
};
const response = await supertest
.post(DETECTION_ENGINE_RULES_URL)
.set('kbn-xsrf', 'true')
.send(rule);
expect(response.status).to.equal(400);
expect(response.body.message).to.be(
'[request body]: Array size (4) is out of bounds: min: 1, max: 3'
);
});
});
};

356
x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/new_terms.ts
View file

@ -11,6 +11,10 @@ import { NewTermsRuleCreateProps } from '@kbn/security-solution-plugin/common/de
import { orderBy } from 'lodash';
import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/detection_engine/rule_schema/mocks';
import { DetectionAlert } from '@kbn/security-solution-plugin/common/detection_engine/schemas/alerts';
import {
getNewTermsRuntimeMappings,
AGG_FIELD_NAME,
} from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/new_terms/utils';
import {
createRule,
deleteAllAlerts,
@ -18,6 +22,7 @@ import {
getOpenSignals,
getPreviewAlerts,
previewRule,
performSearchQuery,
} from '../../utils';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { previewRuleWithExceptionEntries } from '../../utils/preview_rule_with_exception_entries';
@ -50,10 +55,12 @@ export default ({ getService }: FtrProviderContext) => {
describe('New terms type rules', () => {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/auditbeat/hosts');
await esArchiver.load('x-pack/test/functional/es_archives/security_solution/new_terms');
});
after(async () => {
await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts');
await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/new_terms');
await deleteSignalsIndex(supertest, log);
await deleteAllAlerts(supertest, log);
});
@ -228,6 +235,130 @@ export default ({ getService }: FtrProviderContext) => {
]);
});
it('should generate 3 alerts when 1 document has 3 new values for multiple fields', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
new_terms_fields: ['host.name', 'host.ip'],
from: '2019-02-19T20:42:00.000Z',
history_window_start: '2019-01-19T20:42:00.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts.length).eql(3);
const newTerms = orderBy(
previewAlerts.map((item) => item._source?.['kibana.alert.new_terms']),
['0', '1']
);
expect(newTerms).eql([
['zeek-newyork-sha-aa8df15', '10.10.0.6'],
['zeek-newyork-sha-aa8df15', '157.230.208.30'],
['zeek-newyork-sha-aa8df15', 'fe80::24ce:f7ff:fede:a571'],
]);
});
it('should generate 1 alert for unique combination of existing terms', async () => {
// ensure there are no alerts for single new terms fields, it means values are not new
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['host.name', 'host.ip'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
// shouldn't be terms for 'host.ip'
const hostIpPreview = await previewRule({
supertest,
rule: { ...rule, new_terms_fields: ['host.ip'] },
});
const hostIpPreviewAlerts = await getPreviewAlerts({
es,
previewId: hostIpPreview.previewId,
});
expect(hostIpPreviewAlerts.length).eql(0);
// shouldn't be terms for 'host.name'
const hostNamePreview = await previewRule({
supertest,
rule: { ...rule, new_terms_fields: ['host.name'] },
});
const hostNamePreviewAlerts = await getPreviewAlerts({
es,
previewId: hostNamePreview.previewId,
});
expect(hostNamePreviewAlerts.length).eql(0);
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts.length).eql(1);
expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['host-0', '127.0.0.2']);
});
it('should generate 5 alerts, 1 for each new unique combination in 2 fields', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['source.ip', 'tags'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts.length).eql(5);
const newTerms = orderBy(
previewAlerts.map((item) => item._source?.['kibana.alert.new_terms']),
['0', '1']
);
expect(newTerms).eql([
['192.168.1.1', 'tag-new-1'],
['192.168.1.1', 'tag-new-3'],
['192.168.1.2', 'tag-2'],
['192.168.1.2', 'tag-new-1'],
['192.168.1.2', 'tag-new-3'],
]);
});
it('should generate 1 alert for unique combination of terms, one of which is a number', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['user.name', 'user.id'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts.length).eql(1);
expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']);
});
it('should generate 1 alert for unique combination of terms, one of which is a boolean', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['user.name', 'user.enabled'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts.length).eql(1);
expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', 'false']);
});
it('should generate alerts for every term when history window is small', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
@ -251,6 +382,92 @@ export default ({ getService }: FtrProviderContext) => {
expect(hostNames[4]).eql(['zeek-sensor-san-francisco']);
});
describe('null values', () => {
it('should not generate alerts with null values for single field', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['possibly_null_field'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts.length).eql(0);
});
it('should not generate alerts with null values for multiple fields', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['possibly_null_field', 'host.name'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId });
expect(previewAlerts.length).eql(0);
});
});
describe('large arrays values', () => {
it('should generate alerts for unique values in large array for single field from a single document', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['large_array_20'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId, size: 100 });
expect(previewAlerts.length).eql(20);
});
// There is a limit in ES for a number of emitted values in runtime field (100)
// This test ensures rule run doesn't fail if processed fields in runtime script generates 100 values, hard limit for ES
// For this test case: large_array_10 & large_array_5 have 100 unique combination in total
it('should generate alerts for array fields that have 100 unique combination of values in runtime field', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['large_array_10', 'large_array_5'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 });
expect(previewAlerts.length).eql(100);
});
// There is a limit in ES for a number of emitted values in runtime field (100)
// This test ensures rule run doesn't fail if processed fields in runtime script generates 200 values
// In case of this test case: large_array_10 & large_array_20 have 200 unique combination in total
// Rule run should not fail and should generate alerts
it('should generate alert for array fields that have more than 200 unique combination of values in runtime field', async () => {
const rule: NewTermsRuleCreateProps = {
...getCreateNewTermsRulesSchemaMock('rule-1', true),
index: ['new_terms'],
new_terms_fields: ['large_array_10', 'large_array_20'],
from: '2020-10-19T05:00:04.000Z',
history_window_start: '2020-10-13T05:00:04.000Z',
};
const { previewId } = await previewRule({ supertest, rule });
const previewAlerts = await getPreviewAlerts({ es, previewId, size: 200 });
expect(previewAlerts.length).eql(100);
});
});
describe('timestamp override and fallback', () => {
before(async () => {
await esArchiver.load(
@ -381,5 +598,144 @@ export default ({ getService }: FtrProviderContext) => {
expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).to.eql(23);
});
});
describe('runtime field', () => {
it('should return runtime field created from 2 single values', async () => {
// encoded base64 values of "host-0" and "127.0.0.1" joined with underscore
const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x'];
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'first_doc' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME],
runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip']),
});
expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues);
});
it('should return runtime field created from 2 single values, including number value', async () => {
// encoded base64 values of "user-0" and 0 joined with underscore
const expectedEncodedValues = ['dXNlci0w_MA=='];
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'first_doc' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME],
runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.id']),
});
expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues);
});
it('should return runtime field created from 2 single values, including boolean value', async () => {
// encoded base64 values of "user-0" and true joined with underscore
const expectedEncodedValues = ['dXNlci0w_dHJ1ZQ=='];
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'first_doc' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME],
runtimeMappings: getNewTermsRuntimeMappings(['user.name', 'user.enabled']),
});
expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues);
});
it('should return runtime field created from 3 single values', async () => {
// encoded base64 values of "host-0" and "127.0.0.1" and "user-0" joined with underscore
const expectedEncodedValues = ['aG9zdC0w_MTI3LjAuMC4x_dXNlci0w'];
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'first_doc' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME],
runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'host.ip', 'user.name']),
});
expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues);
});
it('should return runtime field created from fields of arrays', async () => {
// encoded base64 values of all combinations of ["192.168.1.1", "192.168.1.2"]
// and ["tag-new-1", "tag-2", "tag-new-3"] joined with underscore
const expectedEncodedValues = [
'MTkyLjE2OC4xLjE=_dGFnLTI=',
'MTkyLjE2OC4xLjE=_dGFnLW5ldy0x',
'MTkyLjE2OC4xLjE=_dGFnLW5ldy0z',
'MTkyLjE2OC4xLjI=_dGFnLTI=',
'MTkyLjE2OC4xLjI=_dGFnLW5ldy0x',
'MTkyLjE2OC4xLjI=_dGFnLW5ldy0z',
];
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'doc_with_source_ip_as_array' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME],
runtimeMappings: getNewTermsRuntimeMappings(['source.ip', 'tags']),
});
expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues);
});
it('should return runtime field without duplicated values', async () => {
// encoded base64 values of "host-0" and ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"]
// joined with underscore, without duplicates in tags
const expectedEncodedValues = ['aG9zdC0w_dGFnLTE=', 'aG9zdC0w_dGFnLTI='];
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'doc_with_duplicated_tags' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME],
runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'tags']),
});
expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.eql(expectedEncodedValues);
});
it('should not return runtime field if one of fields is null', async () => {
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'doc_with_null_field' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME, 'possibly_null_field', 'host.name'],
runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'possibly_null_field']),
});
expect(hits.hits.length).to.be(1);
expect(hits.hits[0].fields?.[AGG_FIELD_NAME]).to.be(undefined);
expect(hits.hits[0].fields?.possibly_null_field).to.be(undefined);
expect(hits.hits[0].fields?.['host.name']).to.eql(['host-0']);
});
it('should not return runtime field if one of fields is not defined', async () => {
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'doc_without_large_arrays' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME],
runtimeMappings: getNewTermsRuntimeMappings(['host.name', 'large_array_5']),
});
expect(hits.hits.length).to.be(1);
expect(hits.hits[0].fields).to.be(undefined);
});
// There is a limit in ES for a number of emitted values in runtime field (100)
// This test makes sure runtime script doesn't cause query failure and returns first 100 results
it('should return runtime field if number of emitted values greater than 100', async () => {
const { hits } = await performSearchQuery({
es,
query: { match: { id: 'first_doc' } },
index: 'new_terms',
fields: [AGG_FIELD_NAME],
runtimeMappings: getNewTermsRuntimeMappings(['large_array_20', 'large_array_10']),
});
// runtime field should have 100 values, as large_array_20 and large_array_10
// give in total 200 combinations
expect(hits.hits[0].fields?.[AGG_FIELD_NAME].length).to.be(100);
});
});
});
};

1
x-pack/test/detection_engine_api_integration/utils/index.ts
View file

@ -81,6 +81,7 @@ export * from './get_web_hook_action';
export * from './index_event_log_execution_events';
export * from './install_prepackaged_rules';
export * from './machine_learning_setup';
export * from './perform_search_query';
export * from './preview_rule_with_exception_entries';
export * from './preview_rule';
export * from './refresh_index';

44
x-pack/test/detection_engine_api_integration/utils/perform_search_query.ts Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { Client } from '@elastic/elasticsearch';
import type {
QueryDslQueryContainer,
MappingRuntimeFields,
IndexName,
Field,
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
interface PerformSearchQueryArgs {
es: Client;
query: QueryDslQueryContainer;
index: IndexName;
size?: number;
runtimeMappings?: MappingRuntimeFields;
fields?: Field[];
}
/**
* run ES search query
*/
export const performSearchQuery = async ({
es,
query,
index,
size = 10,
runtimeMappings,
fields,
}: PerformSearchQueryArgs) => {
return es.search({
index,
size,
fields,
query,
runtime_mappings: runtimeMappings,
});
};

220
x-pack/test/functional/es_archives/security_solution/new_terms/data.json Normal file
View file

@ -0,0 +1,220 @@
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-14T05:00:01.000Z",
"id": "first_doc",
"possibly_null_field": "test-value",
"host": {
"name": "host-0",
"ip": "127.0.0.1"
},
"user.name": "user-0",
"user.id": 0,
"user.enabled": true,
"source.ip": "192.168.1.1",
"tags": ["tag-1", "tag-2"],
"large_array_10": ["value-of-10-0","value-of-10-1","value-of-10-2","value-of-10-3","value-of-10-4","value-of-10-5","value-of-10-6","value-of-10-7","value-of-10-8","value-of-10-9"],
"large_array_5": ["value-of-5-0","value-of-5-1","value-of-5-2","value-of-5-3","value-of-5-4"],
"large_array_20": ["value-of-20-0","value-of-20-1","value-of-20-2","value-of-20-3","value-of-20-4","value-of-20-5","value-of-20-6","value-of-20-7","value-of-20-8","value-of-20-9","value-of-20-10","value-of-20-11","value-of-20-12","value-of-20-13","value-of-20-14","value-of-20-15","value-of-20-16","value-of-20-17","value-of-20-18","value-of-20-19"]
}
,
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-14T05:00:01.000Z",
"id": "doc_without_large_arrays",
"possibly_null_field": "test-value",
"host": {
"name": "host-0",
"ip": "127.0.0.1"
},
"user.name": "user-1",
"user.id": 1,
"user.enabled": false,
"source.ip": "192.168.1.1",
"tags": ["tag-1", "tag-2"]
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-14T05:00:01.000Z",
"id": "doc_with_duplicated_tags",
"possibly_null_field": "test-value",
"host": {
"name": "host-0",
"ip": "127.0.0.1"
},
"user.name": "user-1",
"user.id": 1,
"user.enabled": false,
"source.ip": "192.168.1.1",
"tags": ["tag-1", "tag-2", "tag-2", "tag-1", "tag-1"]
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-14T05:00:02.000Z",
"possibly_null_field": "test-value",
"host": {
"name": "host-1",
"ip": "127.0.0.1"
},
"user.name": "user-0",
"user.id": 0,
"user.enabled": true,
"source.ip": "192.168.1.1"
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-14T05:00:02.000Z",
"possibly_null_field": "test-value",
"host": {
"name": "host-1"
},
"user.name": "user-0",
"user.id": 0,
"user.enabled": true,
"source.ip": "192.168.1.1",
"tags": ["tag-1", "tag-2"]
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-14T05:00:03.000Z",
"possibly_null_field": "test-value",
"host": {
"name": "host-1",
"ip": "127.0.0.2"
},
"user.name": "user-0",
"user.id": 0,
"user.enabled": true,
"source.ip": "192.168.1.1",
"tags": ["tag-1", "tag-2"]
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-14T05:00:04.000Z",
"possibly_null_field": "test-value",
"host": {
"name": "host-0",
"ip": "127.0.0.1"
},
"user.name": "user-0",
"user.id": 0,
"user.enabled": true,
"source.ip": "192.168.1.1",
"tags": ["tag-1", "tag-2"]
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-20T05:00:04.000Z",
"possibly_null_field": "test-value",
"host": {
"name": "host-0",
"ip": "127.0.0.2"
},
"user.name": "user-0",
"user.id": 0,
"user.enabled": true,
"source.ip": "192.168.1.1",
"tags": ["tag-1", "tag-2"]
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-20T05:00:04.000Z",
"id": "doc_with_source_ip_as_array",
"possibly_null_field": "test-value",
"host": {
"name": "host-0",
"ip": "127.0.0.2"
},
"user.name": "user-0",
"user.id": 0,
"user.enabled": true,
"source.ip": ["192.168.1.1", "192.168.1.2"],
"tags": ["tag-new-1", "tag-2", "tag-new-3"]
},
"type": "_doc"
}
}
{
"type": "doc",
"value": {
"index": "new_terms",
"source": {
"@timestamp": "2020-10-20T05:00:04.000Z",
"id": "doc_with_null_field",
"possibly_null_field": null,
"host": {
"name": "host-0",
"ip": "127.0.0.2"
},
"user.name": "user-0",
"user.id": 1,
"user.enabled": false,
"source.ip": "192.168.1.1",
"tags": ["tag-1", "tag-2"],
"large_array_10": ["a-new-value-of-10-0","a-new-value-of-10-1","a-new-value-of-10-2","a-new-value-of-10-3","a-new-value-of-10-4","a-new-value-of-10-5","a-new-value-of-10-6","a-new-value-of-10-7","a-new-value-of-10-8","a-new-value-of-10-9"],
"large_array_5": ["another-new-value-of-10-0","another-new-value-of-10-1","another-new-value-of-10-2","another-new-value-of-10-3","another-new-value-of-10-4","another-new-value-of-10-5","another-new-value-of-10-6","another-new-value-of-10-7","another-new-value-of-10-8","another-new-value-of-10-9"],
"large_array_20": ["a-new-value-of-20-0","a-new-value-of-20-1","a-new-value-of-20-2","a-new-value-of-20-3","a-new-value-of-20-4","a-new-value-of-20-5","a-new-value-of-20-6","a-new-value-of-20-7","a-new-value-of-20-8","a-new-value-of-20-9","a-new-value-of-20-10","a-new-value-of-20-11","a-new-value-of-20-12","a-new-value-of-20-13","a-new-value-of-20-14","a-new-value-of-20-15","a-new-value-of-20-16","a-new-value-of-20-17","a-new-value-of-20-18","a-new-value-of-20-19"]
},
"type": "_doc"
}
}

70
x-pack/test/functional/es_archives/security_solution/new_terms/mappings.json Normal file
View file

@ -0,0 +1,70 @@
{
"type": "index",
"value": {
"index": "new_terms",
"mappings": {
"properties": {
"id": {
"type": "keyword"
},
"@timestamp": {
"type": "date"
},
"host": {
"properties": {
"name": {
"type": "keyword"
},
"ip": {
"type": "ip"
}
}
},
"user": {
"properties": {
"name": {
"type": "keyword"
},
"id": {
"type": "integer"
},
"enabled": {
"type": "boolean"
}
}
},
"source": {
"properties": {
"ip": {
"type": "ip"
}
}
},
"tags": {
"type": "keyword"
},
"blob": {
"type": "binary"
},
"possibly_null_field": {
"type": "keyword"
},
"large_array_10": {
"type": "keyword"
},
"large_array_20": {
"type": "keyword"
},
"large_array_5": {
"type": "keyword"
}
}
},
"settings": {
"index": {
"number_of_replicas": "1",
"number_of_shards": "1"
}
}
}
}