mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
[8.8] [Fleet] Always generate time series metrics/dimensions regardless of whether index_mode: "time_series" is set (#157047) (#157378)
# Backport This will backport the following commits from `main` to `8.8`: - [[Fleet] Always generate time series metrics/dimensions regardless of whether `index_mode: "time_series"` is set (#157047)](https://github.com/elastic/kibana/pull/157047) <!--- Backport version: 8.9.7 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Kyle Pollich","email":"kyle.pollich@elastic.co"},"sourceCommit":{"committedDate":"2023-05-11T11:41:06Z","message":"[Fleet] Always generate time series metrics/dimensions regardless of whether `index_mode: \"time_series\"` is set (#157047)\n\n## Summary\r\n\r\nBlocked by https://github.com/elastic/integrations/pull/6128\r\n\r\nThis PR changes Fleet's mapping generation logic in `@package` component\r\ntemplates such that `time_series_dimension: true` and `metric_type`\r\nproperties are generated for all mappings regardless of whether\r\n`index_mode: \"time_series\"` is set. Previously, Elasticsearch would\r\nerror if this were the case, however in newer versions of Elasticsearch\r\nit's simply a no-op when `index_mode` is not set to `time_series` and\r\nTSDS fields are declared.\r\n\r\nThis allows us to more easily toggle TSDB on/off via API requests for\r\nuser trial.\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>","sha":"c21f2a978f1a9b849d437ed91b3c7708d87c04b9","branchLabelMapping":{"^v8.9.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","Team:Fleet","backport:prev-minor","v8.9.0"],"number":157047,"url":"https://github.com/elastic/kibana/pull/157047","mergeCommit":{"message":"[Fleet] Always generate time series metrics/dimensions regardless of whether `index_mode: \"time_series\"` is set (#157047)\n\n## Summary\r\n\r\nBlocked by https://github.com/elastic/integrations/pull/6128\r\n\r\nThis PR changes Fleet's mapping generation logic in `@package` component\r\ntemplates such that `time_series_dimension: true` and `metric_type`\r\nproperties are generated for all mappings regardless of whether\r\n`index_mode: \"time_series\"` is set. Previously, Elasticsearch would\r\nerror if this were the case, however in newer versions of Elasticsearch\r\nit's simply a no-op when `index_mode` is not set to `time_series` and\r\nTSDS fields are declared.\r\n\r\nThis allows us to more easily toggle TSDB on/off via API requests for\r\nuser trial.\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>","sha":"c21f2a978f1a9b849d437ed91b3c7708d87c04b9"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.9.0","labelRegex":"^v8.9.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/157047","number":157047,"mergeCommit":{"message":"[Fleet] Always generate time series metrics/dimensions regardless of whether `index_mode: \"time_series\"` is set (#157047)\n\n## Summary\r\n\r\nBlocked by https://github.com/elastic/integrations/pull/6128\r\n\r\nThis PR changes Fleet's mapping generation logic in `@package` component\r\ntemplates such that `time_series_dimension: true` and `metric_type`\r\nproperties are generated for all mappings regardless of whether\r\n`index_mode: \"time_series\"` is set. Previously, Elasticsearch would\r\nerror if this were the case, however in newer versions of Elasticsearch\r\nit's simply a no-op when `index_mode` is not set to `time_series` and\r\nTSDS fields are declared.\r\n\r\nThis allows us to more easily toggle TSDB on/off via API requests for\r\nuser trial.\r\n\r\n---------\r\n\r\nCo-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>","sha":"c21f2a978f1a9b849d437ed91b3c7708d87c04b9"}}]}] BACKPORT--> Co-authored-by: Kyle Pollich <kyle.pollich@elastic.co>
This commit is contained in:
Kibana Machine
GitHub
parent
f9298e2a4b
commit
3bb1fdd20c
3 changed files with 47 additions and 129 deletions
|
|
@ -519,7 +519,7 @@ export function prepareTemplate({
|
|||
dataStream.elasticsearch?.index_mode === 'time_series' ||
|
||||
experimentalDataStreamFeature?.features.tsdb;
|
||||
|
||||
const mappings = generateMappings(validFields, { isIndexModeTimeSeries });
|
||||
const mappings = generateMappings(validFields);
|
||||
const templateName = generateTemplateName(dataStream);
|
||||
const templateIndexPattern = generateTemplateIndexPattern(dataStream);
|
||||
const templatePriority = getTemplatePriority(dataStream);
|
||||
|
|
|
|||
|
|
@ -822,13 +822,11 @@ describe('EPM template', () => {
|
|||
};
|
||||
const fields: Field[] = safeLoad(literalYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields, {
|
||||
isIndexModeTimeSeries: true,
|
||||
});
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
it('tests processing dimension field on an long', () => {
|
||||
it('tests processing dimension field on a long', () => {
|
||||
const literalYml = `
|
||||
- name: example.id
|
||||
type: long
|
||||
|
|
@ -848,38 +846,11 @@ describe('EPM template', () => {
|
|||
};
|
||||
const fields: Field[] = safeLoad(literalYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields, {
|
||||
isIndexModeTimeSeries: true,
|
||||
});
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
it('tests processing dimension field on an long without timeserie enabled', () => {
|
||||
const literalYml = `
|
||||
- name: example.id
|
||||
type: long
|
||||
dimension: true
|
||||
`;
|
||||
const expectedMapping = {
|
||||
properties: {
|
||||
example: {
|
||||
properties: {
|
||||
id: {
|
||||
type: 'long',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const fields: Field[] = safeLoad(literalYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields, {
|
||||
isIndexModeTimeSeries: false,
|
||||
});
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
it('tests processing metric_type field with index mode time series', () => {
|
||||
it('tests processing metric_type field', () => {
|
||||
const literalYml = `
|
||||
- name: total.norm.pct
|
||||
type: scaled_float
|
||||
|
|
@ -909,44 +880,11 @@ describe('EPM template', () => {
|
|||
};
|
||||
const fields: Field[] = safeLoad(literalYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields, { isIndexModeTimeSeries: true });
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
it('tests processing metric_type field with index mode time series disabled', () => {
|
||||
const literalYml = `
|
||||
- name: total.norm.pct
|
||||
type: scaled_float
|
||||
metric_type: gauge
|
||||
unit: percent
|
||||
format: percent
|
||||
`;
|
||||
const expectedMapping = {
|
||||
properties: {
|
||||
total: {
|
||||
properties: {
|
||||
norm: {
|
||||
properties: {
|
||||
pct: {
|
||||
scaling_factor: 1000,
|
||||
type: 'scaled_float',
|
||||
meta: {
|
||||
unit: 'percent',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const fields: Field[] = safeLoad(literalYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields, { isIndexModeTimeSeries: false });
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
it('tests processing metric_type field with long field and index mode timeseries', () => {
|
||||
it('tests processing metric_type field with long field', () => {
|
||||
const literalYml = `
|
||||
- name: total
|
||||
type: long
|
||||
|
|
@ -969,7 +907,7 @@ describe('EPM template', () => {
|
|||
};
|
||||
const fields: Field[] = safeLoad(literalYml);
|
||||
const processedFields = processFields(fields);
|
||||
const mappings = generateMappings(processedFields, { isIndexModeTimeSeries: true });
|
||||
const mappings = generateMappings(processedFields);
|
||||
expect(mappings).toEqual(expectedMapping);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -101,10 +101,6 @@ export function getTemplate({
|
|||
return template;
|
||||
}
|
||||
|
||||
interface GenerateMappingsOptions {
|
||||
isIndexModeTimeSeries?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate mapping takes the given nested fields array and creates the Elasticsearch
|
||||
* mapping properties out of it.
|
||||
|
|
@ -113,44 +109,37 @@ interface GenerateMappingsOptions {
|
|||
*
|
||||
* @param fields
|
||||
*/
|
||||
export function generateMappings(
|
||||
fields: Field[],
|
||||
options?: GenerateMappingsOptions
|
||||
): IndexTemplateMappings {
|
||||
export function generateMappings(fields: Field[]): IndexTemplateMappings {
|
||||
const dynamicTemplates: Array<Record<string, Properties>> = [];
|
||||
const dynamicTemplateNames = new Set<string>();
|
||||
|
||||
const { properties } = _generateMappings(
|
||||
fields,
|
||||
{
|
||||
addDynamicMapping: (dynamicMapping: {
|
||||
path: string;
|
||||
matchingType: string;
|
||||
pathMatch: string;
|
||||
properties: string;
|
||||
}) => {
|
||||
const name = dynamicMapping.path;
|
||||
if (dynamicTemplateNames.has(name)) {
|
||||
return;
|
||||
}
|
||||
const { properties } = _generateMappings(fields, {
|
||||
addDynamicMapping: (dynamicMapping: {
|
||||
path: string;
|
||||
matchingType: string;
|
||||
pathMatch: string;
|
||||
properties: string;
|
||||
}) => {
|
||||
const name = dynamicMapping.path;
|
||||
if (dynamicTemplateNames.has(name)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const dynamicTemplate: Properties = {
|
||||
mapping: dynamicMapping.properties,
|
||||
};
|
||||
const dynamicTemplate: Properties = {
|
||||
mapping: dynamicMapping.properties,
|
||||
};
|
||||
|
||||
if (dynamicMapping.matchingType) {
|
||||
dynamicTemplate.match_mapping_type = dynamicMapping.matchingType;
|
||||
}
|
||||
if (dynamicMapping.matchingType) {
|
||||
dynamicTemplate.match_mapping_type = dynamicMapping.matchingType;
|
||||
}
|
||||
|
||||
if (dynamicMapping.pathMatch) {
|
||||
dynamicTemplate.path_match = dynamicMapping.pathMatch;
|
||||
}
|
||||
dynamicTemplateNames.add(name);
|
||||
dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate });
|
||||
},
|
||||
if (dynamicMapping.pathMatch) {
|
||||
dynamicTemplate.path_match = dynamicMapping.pathMatch;
|
||||
}
|
||||
dynamicTemplateNames.add(name);
|
||||
dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate });
|
||||
},
|
||||
options
|
||||
);
|
||||
});
|
||||
|
||||
return dynamicTemplates.length
|
||||
? {
|
||||
|
|
@ -173,8 +162,7 @@ function _generateMappings(
|
|||
ctx: {
|
||||
addDynamicMapping: any;
|
||||
groupFieldName?: string;
|
||||
},
|
||||
options?: GenerateMappingsOptions
|
||||
}
|
||||
): {
|
||||
properties: IndexTemplateMappings['properties'];
|
||||
hasNonDynamicTemplateMappings: boolean;
|
||||
|
|
@ -234,16 +222,12 @@ function _generateMappings(
|
|||
|
||||
switch (type) {
|
||||
case 'group':
|
||||
const mappings = _generateMappings(
|
||||
field.fields!,
|
||||
{
|
||||
...ctx,
|
||||
groupFieldName: ctx.groupFieldName
|
||||
? `${ctx.groupFieldName}.${field.name}`
|
||||
: field.name,
|
||||
},
|
||||
options
|
||||
);
|
||||
const mappings = _generateMappings(field.fields!, {
|
||||
...ctx,
|
||||
groupFieldName: ctx.groupFieldName
|
||||
? `${ctx.groupFieldName}.${field.name}`
|
||||
: field.name,
|
||||
});
|
||||
if (!mappings.hasNonDynamicTemplateMappings) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -255,16 +239,12 @@ function _generateMappings(
|
|||
break;
|
||||
case 'group-nested':
|
||||
fieldProps = {
|
||||
properties: _generateMappings(
|
||||
field.fields!,
|
||||
{
|
||||
...ctx,
|
||||
groupFieldName: ctx.groupFieldName
|
||||
? `${ctx.groupFieldName}.${field.name}`
|
||||
: field.name,
|
||||
},
|
||||
options
|
||||
).properties,
|
||||
properties: _generateMappings(field.fields!, {
|
||||
...ctx,
|
||||
groupFieldName: ctx.groupFieldName
|
||||
? `${ctx.groupFieldName}.${field.name}`
|
||||
: field.name,
|
||||
}).properties,
|
||||
...generateNestedProps(field),
|
||||
type: 'nested',
|
||||
};
|
||||
|
|
@ -352,10 +332,10 @@ function _generateMappings(
|
|||
}
|
||||
}
|
||||
|
||||
if (options?.isIndexModeTimeSeries && 'metric_type' in field) {
|
||||
if ('metric_type' in field) {
|
||||
fieldProps.time_series_metric = field.metric_type;
|
||||
}
|
||||
if (options?.isIndexModeTimeSeries && field.dimension) {
|
||||
if (field.dimension) {
|
||||
fieldProps.time_series_dimension = field.dimension;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue