github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 18:51:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Response Ops] add ignore_malformed to alerts mappings AGAIN (#165781)

Browse source 
Resolves https://github.com/elastic/kibana/issues/161465

This is a re-do of https://github.com/elastic/kibana/pull/163414, which
we had to revert since data streams do not support `ignore_malformed` on
the `@timestamp` field. We now specifically add `ignore_malformed:
false` for that field, and then use `ignore_malformed: true` at the
index level.

This ignores malformed content globally across all allowed mapping
types. For existing alerts as data indices, the new setting is not
applied directly to the existing concrete indices but will be applied
whenever the alias rolls over and a new concrete index is created.
This commit is contained in:
Patrick Mueller 2023-09-13 23:53:15 -04:00 • committed by GitHub
parent 21034c12e7
commit f638a38c64
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 21 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts
View file

@ -188,6 +188,7 @@ describe('mappingFromFieldMap', () => {
dynamic: 'strict',
properties: {
'@timestamp': {
ignore_malformed: false,
type: 'date',
},
event: {

4
x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts
View file

@ -43,6 +43,10 @@ export function mappingFromFieldMap(
: rest;
set(mappings.properties, field.name.split('.').join('.properties.'), mapped);
if (name === '@timestamp') {
set(mappings.properties, `${name}.ignore_malformed`, false);
}
});
return mappings;

2
x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts
View file

@ -141,6 +141,7 @@ const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => {
rollover_alias: `.alerts-${context ? context : 'test'}.alerts-${namespace}`,
},
}),
'index.mapping.ignore_malformed': true,
'index.mapping.total_fields.limit': 2500,
},
mappings: {
@ -808,6 +809,7 @@ describe('Alerts Service', () => {
rollover_alias: `.alerts-empty.alerts-default`,
},
}),
'index.mapping.ignore_malformed': true,
'index.mapping.total_fields.limit': 2500,
},
mappings: {

1
x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.test.ts
View file

@ -48,6 +48,7 @@ const IndexTemplate = (namespace: string = 'default', useDataStream: boolean = f
rollover_alias: `.alerts-test.alerts-${namespace}`,
},
}),
'index.mapping.ignore_malformed': true,
'index.mapping.total_fields.limit': 2500,
},
},

1
x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.ts
View file

@ -68,6 +68,7 @@ export const getIndexTemplate = ({
: {
'index.lifecycle': indexLifecycle,
}),
'index.mapping.ignore_malformed': true,
'index.mapping.total_fields.limit': totalFieldsLimit,
},
mappings: {

2
x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts
View file

@ -175,6 +175,7 @@ describe('RiskEngineDataClient', () => {
"dynamic": "strict",
"properties": Object {
"@timestamp": Object {
"ignore_malformed": false,
"type": "date",
},
"host": Object {
@ -360,6 +361,7 @@ describe('RiskEngineDataClient', () => {
dynamic: 'strict',
properties: {
'@timestamp': {
ignore_malformed: false,
type: 'date',
},
host: {

2
x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/install_resources.ts
View file

@ -163,6 +163,7 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
rollover_alias: '.alerts-test.patternfiring.alerts-default',
},
mapping: {
ignore_malformed: 'true',
total_fields: {
limit: '2500',
},
@ -196,6 +197,7 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
});
expect(contextIndex[indexName].settings?.index?.mapping).to.eql({
ignore_malformed: 'true',
total_fields: {
limit: '2500',
},

1
x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine/init_and_status_apis.ts
View file

@ -104,6 +104,7 @@ export default ({ getService }: FtrProviderContext) => {
dynamic: 'strict',
properties: {
'@timestamp': {
ignore_malformed: false,
type: 'date',
},
host: {

16
x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts
View file

@ -56,7 +56,6 @@ export default ({ getService }: FtrProviderContext) => {
};
};
// FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/154277
describe('Non ECS fields in alert document source', () => {
before(async () => {
await esArchiver.load(
@ -257,9 +256,10 @@ export default ({ getService }: FtrProviderContext) => {
expect(alertSource).toHaveProperty('client.nat.port', '3000');
});
// we don't validate it because geo_point is very complex type with many various representations: array, different object, string with few valid patterns
// more on geo_point type https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
it('should fail creating alert when ECS field mapping is geo_point', async () => {
// We don't validate it because geo_point is very complex type with many various representations: array,
// different object, string with few valid patterns.
// More on geo_point type https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
it('should not fail creating alert when ECS field mapping is geo_point', async () => {
const document = {
client: {
geo: {
@ -269,12 +269,10 @@ export default ({ getService }: FtrProviderContext) => {
},
};
const { errors } = await indexAndCreatePreviewAlert(document);
const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
expect(errors[0]).toContain('Bulk Indexing of signals failed');
expect(errors[0]).toContain(
'failed to parse field [client.geo.location] of type [geo_point]'
);
expect(errors).toEqual([]);
expect(alertSource).toHaveProperty('client.geo.location', 'test test');
});
it('should strip invalid boolean values and left valid ones', async () => {