mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[Fleet] Create intermediate objects when using dynamic mappings (#169981)
For `dynamic: "runtime"`, and possibly other configurations different to `dynamic: true`, intermediate objects need to exist for dynamic mappings, otherwise ingestion can fail with `Missing intermediate object` errors. This pull request includes these intermediate objects, and avoids the creation of static properties with wildcards, that is probably not what is expected for these mappings. Intermediate objects are included as static properties for parts of the name before the wildcard, and as dynamic templates when the full path has wildcards. In the modified test files are examples of all of these. This is more an issue since the following recent fixes, that create the actual dynamic mappings for some cases where only empty objects were created before. * https://github.com/elastic/elastic-package/pull/1492 * https://github.com/elastic/kibana/pull/168842
This commit is contained in:
Jaime Soriano Pastor
GitHub
parent
09ff2c462c
commit
d398606134
3 changed files with 207 additions and 55 deletions
|
|
@ -58,7 +58,22 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = `
|
|||
|
||||
exports[`EPM template tests loading cockroachdb_dynamic_templates.yml: cockroachdb_dynamic_templates.yml 1`] = `
|
||||
{
|
||||
"properties": {},
|
||||
"properties": {
|
||||
"cockroachdb": {
|
||||
"properties": {
|
||||
"status": {
|
||||
"properties": {
|
||||
"labels": {
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dynamic_templates": [
|
||||
{
|
||||
"cockroachdb.status.labels": {
|
||||
|
|
@ -78,6 +93,16 @@ exports[`EPM template tests loading cockroachdb_dynamic_templates.yml: cockroach
|
|||
"path_match": "cockroachdb.status.*.value"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cockroachdb.status.*": {
|
||||
"mapping": {
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"match_mapping_type": "object",
|
||||
"path_match": "cockroachdb.status.*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cockroachdb.status.*.counter": {
|
||||
"mapping": {
|
||||
|
|
@ -837,39 +862,24 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = `
|
|||
"network_summary": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"*": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"tcp": {
|
||||
"properties": {
|
||||
"*": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"udp": {
|
||||
"properties": {
|
||||
"*": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"udp_lite": {
|
||||
"properties": {
|
||||
"*": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"icmp": {
|
||||
"properties": {
|
||||
"*": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -883,6 +893,10 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = `
|
|||
"type": "keyword",
|
||||
"ignore_above": 2048
|
||||
},
|
||||
"env": {
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"cpu": {
|
||||
"properties": {
|
||||
"user": {
|
||||
|
|
@ -1076,8 +1090,14 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = `
|
|||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"percpu": {
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"memory": {
|
||||
"properties": {
|
||||
|
|
@ -1355,7 +1375,9 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = `
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"raid": {
|
||||
"properties": {
|
||||
|
|
@ -1388,8 +1410,14 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = `
|
|||
},
|
||||
"failed": {
|
||||
"type": "long"
|
||||
},
|
||||
"states": {
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"dynamic": true
|
||||
},
|
||||
"blocks": {
|
||||
"properties": {
|
||||
|
|
|
|||
|
|
@ -1151,7 +1151,12 @@ describe('EPM template', () => {
|
|||
runtime: true
|
||||
`;
|
||||
const runtimeFieldMapping = {
|
||||
properties: {},
|
||||
properties: {
|
||||
labels: {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
},
|
||||
dynamic_templates: [
|
||||
{
|
||||
'labels.*': {
|
||||
|
|
@ -1177,7 +1182,12 @@ describe('EPM template', () => {
|
|||
object_type: scaled_float
|
||||
`;
|
||||
const runtimeFieldMapping = {
|
||||
properties: {},
|
||||
properties: {
|
||||
numeric_labels: {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
},
|
||||
dynamic_templates: [
|
||||
{
|
||||
numeric_labels: {
|
||||
|
|
@ -1205,7 +1215,12 @@ describe('EPM template', () => {
|
|||
default_metric: "max"
|
||||
`;
|
||||
const runtimeFieldMapping = {
|
||||
properties: {},
|
||||
properties: {
|
||||
aggregate: {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
},
|
||||
dynamic_templates: [
|
||||
{
|
||||
'aggregate.*': {
|
||||
|
|
@ -1226,7 +1241,7 @@ describe('EPM template', () => {
|
|||
expect(mappings).toEqual(runtimeFieldMapping);
|
||||
});
|
||||
|
||||
it('tests processing groub sub fields in a dynamic template', () => {
|
||||
it('tests processing group sub fields in a dynamic template', () => {
|
||||
const textWithRuntimeFieldsLiteralYml = `
|
||||
- name: group.*.network
|
||||
type: group
|
||||
|
|
@ -1236,7 +1251,12 @@ describe('EPM template', () => {
|
|||
metric_type: counter
|
||||
`;
|
||||
const runtimeFieldMapping = {
|
||||
properties: {},
|
||||
properties: {
|
||||
group: {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
},
|
||||
dynamic_templates: [
|
||||
{
|
||||
'group.*.network.bytes': {
|
||||
|
|
@ -1248,6 +1268,26 @@ describe('EPM template', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
'group.*.network': {
|
||||
path_match: 'group.*.network',
|
||||
match_mapping_type: 'object',
|
||||
mapping: {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
'group.*': {
|
||||
path_match: 'group.*',
|
||||
match_mapping_type: 'object',
|
||||
mapping: {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
|
||||
|
|
|
|||
|
|
@ -151,8 +151,8 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
|
|||
path: string;
|
||||
matchingType: string;
|
||||
pathMatch: string;
|
||||
properties: string;
|
||||
runtimeProperties?: string;
|
||||
properties: Properties;
|
||||
runtimeProperties?: Properties;
|
||||
}) => {
|
||||
const name = dynamicMapping.path;
|
||||
if (dynamicTemplateNames.has(name)) {
|
||||
|
|
@ -210,9 +210,64 @@ function _generateMappings(
|
|||
): {
|
||||
properties: IndexTemplateMappings['properties'];
|
||||
hasNonDynamicTemplateMappings: boolean;
|
||||
hasDynamicTemplateMappings: boolean;
|
||||
} {
|
||||
let hasNonDynamicTemplateMappings = false;
|
||||
let hasDynamicTemplateMappings = false;
|
||||
const props: Properties = {};
|
||||
|
||||
function addParentObjectAsStaticProperty(field: Field) {
|
||||
// Don't add intermediate objects for wildcard names, as it will
|
||||
// be added for its parent object.
|
||||
if (field.name.includes('*')) {
|
||||
return;
|
||||
}
|
||||
|
||||
const fieldProps = {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
};
|
||||
|
||||
props[field.name] = fieldProps;
|
||||
hasNonDynamicTemplateMappings = true;
|
||||
}
|
||||
|
||||
function addDynamicMappingWithIntermediateObjects(
|
||||
path: string,
|
||||
pathMatch: string,
|
||||
matchingType: string,
|
||||
dynProperties: Properties,
|
||||
fieldProps?: Properties
|
||||
) {
|
||||
ctx.addDynamicMapping({
|
||||
path,
|
||||
pathMatch,
|
||||
matchingType,
|
||||
properties: dynProperties,
|
||||
runtimeProperties: fieldProps,
|
||||
});
|
||||
hasDynamicTemplateMappings = true;
|
||||
|
||||
// Add dynamic intermediate objects.
|
||||
const parts = pathMatch.split('.');
|
||||
for (let i = parts.length - 1; i > 0; i--) {
|
||||
const name = parts.slice(0, i).join('.');
|
||||
if (!name.includes('*')) {
|
||||
continue;
|
||||
}
|
||||
const dynProps: Properties = {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
};
|
||||
ctx.addDynamicMapping({
|
||||
path: name,
|
||||
pathMatch: name,
|
||||
matchingType: 'object',
|
||||
properties: dynProps,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: this can happen when the fields property in fields.yml is present but empty
|
||||
// Maybe validation should be moved to fields/field.ts
|
||||
if (fields) {
|
||||
|
|
@ -250,13 +305,17 @@ function _generateMappings(
|
|||
const fieldProps = generateRuntimeFieldProps(_field);
|
||||
|
||||
if (dynProperties && matchingType) {
|
||||
ctx.addDynamicMapping({
|
||||
addDynamicMappingWithIntermediateObjects(
|
||||
path,
|
||||
pathMatch,
|
||||
matchingType,
|
||||
properties: dynProperties,
|
||||
runtimeProperties: fieldProps,
|
||||
});
|
||||
dynProperties,
|
||||
fieldProps
|
||||
);
|
||||
|
||||
// Add the parent object as static property, this is needed for
|
||||
// index templates not using `"dynamic": true`.
|
||||
addParentObjectAsStaticProperty(field);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
|
@ -331,12 +390,15 @@ function _generateMappings(
|
|||
type: 'object',
|
||||
object_type: subField.object_type ?? subField.type,
|
||||
}));
|
||||
_generateMappings(subFields, {
|
||||
const mappings = _generateMappings(subFields, {
|
||||
...ctx,
|
||||
groupFieldName: ctx.groupFieldName
|
||||
? `${ctx.groupFieldName}.${field.name}`
|
||||
: field.name,
|
||||
});
|
||||
if (mappings.hasDynamicTemplateMappings) {
|
||||
hasDynamicTemplateMappings = true;
|
||||
}
|
||||
break;
|
||||
case 'flattened':
|
||||
dynProperties.type = field.object_type;
|
||||
|
|
@ -349,12 +411,11 @@ function _generateMappings(
|
|||
}
|
||||
|
||||
if (dynProperties && matchingType) {
|
||||
ctx.addDynamicMapping({
|
||||
path,
|
||||
pathMatch,
|
||||
matchingType,
|
||||
properties: dynProperties,
|
||||
});
|
||||
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);
|
||||
|
||||
// Add the parent object as static property, this is needed for
|
||||
// index templates not using `"dynamic": true`.
|
||||
addParentObjectAsStaticProperty(field);
|
||||
}
|
||||
} else {
|
||||
let fieldProps = getDefaultProperties(field);
|
||||
|
|
@ -367,14 +428,25 @@ function _generateMappings(
|
|||
? `${ctx.groupFieldName}.${field.name}`
|
||||
: field.name,
|
||||
});
|
||||
if (!mappings.hasNonDynamicTemplateMappings) {
|
||||
if (mappings.hasNonDynamicTemplateMappings) {
|
||||
fieldProps = {
|
||||
properties:
|
||||
Object.keys(mappings.properties).length > 0 ? mappings.properties : undefined,
|
||||
...generateDynamicAndEnabled(field),
|
||||
};
|
||||
if (mappings.hasDynamicTemplateMappings) {
|
||||
fieldProps.type = 'object';
|
||||
fieldProps.dynamic = true;
|
||||
}
|
||||
} else if (mappings.hasDynamicTemplateMappings) {
|
||||
fieldProps = {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
};
|
||||
hasDynamicTemplateMappings = true;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
fieldProps = {
|
||||
properties: mappings.properties,
|
||||
...generateDynamicAndEnabled(field),
|
||||
};
|
||||
break;
|
||||
case 'group-nested':
|
||||
fieldProps = {
|
||||
|
|
@ -478,13 +550,25 @@ function _generateMappings(
|
|||
fieldProps.time_series_dimension = field.dimension;
|
||||
}
|
||||
|
||||
props[field.name] = fieldProps;
|
||||
// Even if we don't add the property because it has a wildcard, notify
|
||||
// the parent that there is some kind of property, so the intermediate object
|
||||
// is still created.
|
||||
// This is done for legacy packages that include ambiguous mappings with objects
|
||||
// without object type. This is not allowed starting on Package Spec v3.
|
||||
hasNonDynamicTemplateMappings = true;
|
||||
|
||||
// Avoid including maps with wildcards, they have generated dynamic mappings.
|
||||
if (field.name.includes('*')) {
|
||||
hasDynamicTemplateMappings = true;
|
||||
return;
|
||||
}
|
||||
|
||||
props[field.name] = fieldProps;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return { properties: props, hasNonDynamicTemplateMappings };
|
||||
return { properties: props, hasNonDynamicTemplateMappings, hasDynamicTemplateMappings };
|
||||
}
|
||||
|
||||
function generateDynamicAndEnabled(field: Field) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue