mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[Automatic Import] Fix missing ecs mappings (#209057)
## Summary Fix missing ecs mappings. ## Details If the input is of a multilevel json where there were no ecs mappings identified on first level, For ex: ``` { okta_test: { audit: { actor: [Object], authenticationContext: [Object], client: [Object], debugContext: [Object], displayMessage: [Object], eventType: [Object], legacyEventType: null, outcome: [Object], published: [Object], request: [Object], securityContext: [Object], severity: [Object], target: null, transaction: [Object], uuid: [Object], version: null } } } ``` Then there is no `target` field identified and an iteration is triggered to find the ecs mappings in the next level. Since there is no `target` identified the logic exited and further levels were skipped in identifying the mappings. This PR fixes it and skips handling of Arrays since `rename` processor cannot flatten arrays on its own to map searchable fields. This will handled separately.
This commit is contained in:
parent
a724cdf141
commit
2fe39767ee
2 changed files with 118 additions and 5 deletions
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { ecsPipelineState } from '../../../__jest__/fixtures/ecs_mapping';
|
||||
import type { EcsMappingState } from '../../types';
|
||||
import { createPipeline } from './pipeline';
|
||||
import { createPipeline, generateProcessors } from './pipeline';
|
||||
|
||||
const state: EcsMappingState = ecsPipelineState;
|
||||
|
||||
|
@ -303,4 +303,118 @@ describe('Testing pipeline templates', () => {
|
|||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('should generate processors for empty mapping', () => {
|
||||
const processors = generateProcessors({}, {});
|
||||
expect(processors).toEqual([]);
|
||||
});
|
||||
|
||||
it('should generate processors for nested fields', () => {
|
||||
const mapping = {
|
||||
nested: {
|
||||
field: {
|
||||
target: 'target.field',
|
||||
confidence: 0.8,
|
||||
type: 'keyword',
|
||||
date_formats: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
const samples = {
|
||||
nested: {
|
||||
field: 'test value',
|
||||
},
|
||||
};
|
||||
|
||||
const processors = generateProcessors(mapping, samples);
|
||||
|
||||
expect(processors).toHaveLength(1);
|
||||
expect(processors[0]).toEqual({
|
||||
rename: {
|
||||
field: 'nested.field',
|
||||
target_field: 'target.field',
|
||||
ignore_missing: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
it('should handle nested fields with mixed target mappings', () => {
|
||||
const mapping = {
|
||||
level1: {
|
||||
no_target: {
|
||||
field: {
|
||||
target: 'target.field',
|
||||
confidence: 0.8,
|
||||
type: 'keyword',
|
||||
date_formats: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const samples = {
|
||||
level1: {
|
||||
no_target: {
|
||||
field: 'test value',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const processors = generateProcessors(mapping, samples);
|
||||
|
||||
expect(processors).toHaveLength(1);
|
||||
expect(processors[0]).toEqual({
|
||||
rename: {
|
||||
field: 'level1.no_target.field',
|
||||
target_field: 'target.field',
|
||||
ignore_missing: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
it('should handle multiple nested fields', () => {
|
||||
const mapping = {
|
||||
level1: {
|
||||
level2: {
|
||||
level3: {
|
||||
field1: {
|
||||
target: 'target.field1',
|
||||
confidence: 0.8,
|
||||
type: 'keyword',
|
||||
date_formats: [],
|
||||
},
|
||||
},
|
||||
field2: {
|
||||
target: 'target.field2',
|
||||
confidence: 0.9,
|
||||
type: 'long',
|
||||
date_formats: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const samples = {
|
||||
level1: {
|
||||
level2: {
|
||||
field1: 'string value',
|
||||
field2: '123',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const processors = generateProcessors(mapping, samples);
|
||||
|
||||
expect(processors).toHaveLength(2);
|
||||
expect(processors[0]).toEqual({
|
||||
rename: {
|
||||
field: 'level1.level2.level3.field1',
|
||||
target_field: 'target.field1',
|
||||
ignore_missing: true,
|
||||
},
|
||||
});
|
||||
expect(processors[1]).toEqual({
|
||||
rename: {
|
||||
field: 'level1.level2.field2',
|
||||
target_field: 'target.field2',
|
||||
ignore_missing: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -153,7 +153,7 @@ function needsTypeConversion(sample: unknown, expected: KnownESType): boolean {
|
|||
return false;
|
||||
}
|
||||
|
||||
function generateProcessors(
|
||||
export function generateProcessors(
|
||||
ecsMapping: object,
|
||||
samples: object,
|
||||
basePath: FieldPath = []
|
||||
|
@ -167,10 +167,9 @@ function generateProcessors(
|
|||
|
||||
for (const [key, value] of Object.entries(ecsMapping)) {
|
||||
const currentPath = [...basePath, key];
|
||||
|
||||
if (value !== null && typeof value === 'object' && value?.target !== null) {
|
||||
if (value !== null && !Array.isArray(value) && typeof value === 'object') {
|
||||
const valueKeys = new Set(Object.keys(value));
|
||||
if ([...valueFieldKeys].every((k) => valueKeys.has(k))) {
|
||||
if (value?.target != null && [...valueFieldKeys].every((k) => valueKeys.has(k))) {
|
||||
const processor = generateProcessor(
|
||||
currentPath,
|
||||
value as ECSField,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue