mirror of
https://github.com/elastic/kibana.git
synced 2025-04-25 02:09:32 -04:00
[UII] Fill in empty values for constant_keyword fields from existing mappings (#188145)
## Summary Resolves https://github.com/elastic/kibana/issues/178528. Some packages declare `constant_keyword` type fields without an explicit value. This causes ES to fill in the value in the mappings using the first ingested value. When upgrading this type of package & field after the value has already been populated in this way, the mappings update fail due to pushing a `null` value into an existing value, triggering unnecessary rollovers. This PR fixes that by filling in the empty values from the existing mappings. ## Test 1. On an empty cluster, turn on debug logs 2. Set up Fleet Server policy and Fleet Server agent 3. Force install old version of Elastic Agent integration, v1.19.2: ``` POST kbn:/api/fleet/epm/packages/elastic_agent/1.19.2 { "force": true } ``` 4. Create a new empty policy, **deselect system and agent monitoring** (otherwise the integration will be upgraded, we do not want this yet) 5. Manually add Elastic Agent integration v1.19.2 to the new policy 6. Edit the policy to enable logs and metrics monitoring 7. Enroll agent into the policy, confirm that monitoring logs and metrics are being ingested and that a value exists for `event.dataset` mapping for the logs: ``` GET logs-elastic_agent*/_mappings ``` ``` "dataset": { "type": "constant_keyword", "value": "elastic_agent" } ``` 9. Upgrade Elastic Agent integration to v1.20.0 (note we are not upgrading to the newest versions, 2.0+, because these **are** expected to trigger rollovers for some data streams): ``` POST kbn:/api/fleet/epm/packages/elastic_agent/1.20.0 { "force": true } ``` 10. Confirm in Kibana logs that no rollovers triggered during the upgrade 11. Confirm that there is still only 1 backing index for monitoring logs: ``` GET logs-elastic_agent* ``` ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
This commit is contained in:
Jen Huang
GitHub
parent
1f82d5d68c
commit
b7c96f4c09
3 changed files with 270 additions and 6 deletions
|
|
@ -39,7 +39,7 @@ import { retryTransientEsErrors } from '../retry';
|
|||
import { PackageESError, PackageInvalidArchiveError } from '../../../../errors';
|
||||
|
||||
import { getDefaultProperties, histogram, keyword, scaledFloat } from './mappings';
|
||||
import { isUserSettingsTemplate } from './utils';
|
||||
import { isUserSettingsTemplate, fillConstantKeywordValues } from './utils';
|
||||
|
||||
interface Properties {
|
||||
[key: string]: any;
|
||||
|
|
@ -986,7 +986,7 @@ const updateAllDataStreams = async (
|
|||
});
|
||||
},
|
||||
{
|
||||
// Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster
|
||||
// Limit concurrent putMapping/rollover requests to avoid overwhelming ES cluster
|
||||
concurrency: 20,
|
||||
}
|
||||
);
|
||||
|
|
@ -1017,19 +1017,23 @@ const updateExistingDataStream = async ({
|
|||
const currentSourceType = currentBackingIndexConfig.mappings?._source?.mode;
|
||||
|
||||
let settings: IndicesIndexSettings;
|
||||
let mappings: MappingTypeMapping;
|
||||
let mappings: MappingTypeMapping = {};
|
||||
let lifecycle: any;
|
||||
let subobjectsFieldChanged: boolean = false;
|
||||
let simulateResult: any = {};
|
||||
try {
|
||||
const simulateResult = await retryTransientEsErrors(async () =>
|
||||
simulateResult = await retryTransientEsErrors(async () =>
|
||||
esClient.indices.simulateTemplate({
|
||||
name: await getIndexTemplate(esClient, dataStreamName),
|
||||
})
|
||||
);
|
||||
|
||||
settings = simulateResult.template.settings;
|
||||
mappings = simulateResult.template.mappings;
|
||||
// @ts-expect-error template is not yet typed with DLM
|
||||
mappings = fillConstantKeywordValues(
|
||||
currentBackingIndexConfig?.mappings || {},
|
||||
simulateResult.template.mappings
|
||||
);
|
||||
|
||||
lifecycle = simulateResult.template.lifecycle;
|
||||
|
||||
// for now, remove from object so as not to update stream or data stream properties of the index until type and name
|
||||
|
|
@ -1063,6 +1067,7 @@ const updateExistingDataStream = async ({
|
|||
subobjectsFieldChanged
|
||||
) {
|
||||
logger.info(`Mappings update for ${dataStreamName} failed due to ${err}`);
|
||||
logger.trace(`Attempted mappings: ${mappings}`);
|
||||
if (options?.skipDataStreamRollover === true) {
|
||||
logger.info(
|
||||
`Skipping rollover for ${dataStreamName} as "skipDataStreamRollover" is enabled`
|
||||
|
|
@ -1075,6 +1080,7 @@ const updateExistingDataStream = async ({
|
|||
}
|
||||
}
|
||||
logger.error(`Mappings update for ${dataStreamName} failed due to unexpected error: ${err}`);
|
||||
logger.trace(`Attempted mappings: ${mappings}`);
|
||||
if (options?.ignoreMappingUpdateErrors === true) {
|
||||
logger.info(`Ignore mapping update errors as "ignoreMappingUpdateErrors" is enabled`);
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,226 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import { fillConstantKeywordValues } from './utils';
|
||||
|
||||
describe('fillConstantKeywordValues', () => {
|
||||
const oldMappings = {
|
||||
dynamic: false,
|
||||
_meta: {
|
||||
managed_by: 'fleet',
|
||||
managed: true,
|
||||
package: {
|
||||
name: 'elastic_agent',
|
||||
},
|
||||
},
|
||||
dynamic_templates: [
|
||||
{
|
||||
ecs_timestamp: {
|
||||
match: '@timestamp',
|
||||
mapping: {
|
||||
ignore_malformed: false,
|
||||
type: 'date',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
date_detection: false,
|
||||
properties: {
|
||||
'@timestamp': {
|
||||
type: 'date',
|
||||
ignore_malformed: false,
|
||||
},
|
||||
load: {
|
||||
properties: {
|
||||
'1': {
|
||||
type: 'double',
|
||||
},
|
||||
'5': {
|
||||
type: 'double',
|
||||
},
|
||||
'15': {
|
||||
type: 'double',
|
||||
},
|
||||
},
|
||||
},
|
||||
event: {
|
||||
properties: {
|
||||
agent_id_status: {
|
||||
type: 'keyword',
|
||||
ignore_above: 1024,
|
||||
},
|
||||
dataset: {
|
||||
type: 'constant_keyword',
|
||||
value: 'elastic_agent.metricbeat',
|
||||
},
|
||||
ingested: {
|
||||
type: 'date',
|
||||
format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
|
||||
ignore_malformed: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
message: {
|
||||
type: 'match_only_text',
|
||||
},
|
||||
'dot.field': {
|
||||
type: 'keyword',
|
||||
},
|
||||
constant_keyword_without_value: {
|
||||
type: 'constant_keyword',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const newMappings = {
|
||||
dynamic: false,
|
||||
_meta: {
|
||||
managed_by: 'fleet',
|
||||
managed: true,
|
||||
package: {
|
||||
name: 'elastic_agent',
|
||||
},
|
||||
},
|
||||
dynamic_templates: [
|
||||
{
|
||||
ecs_timestamp: {
|
||||
match: '@timestamp',
|
||||
mapping: {
|
||||
ignore_malformed: false,
|
||||
type: 'date',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
date_detection: false,
|
||||
properties: {
|
||||
'@timestamp': {
|
||||
type: 'date',
|
||||
ignore_malformed: false,
|
||||
},
|
||||
load: {
|
||||
properties: {
|
||||
'1': {
|
||||
type: 'double',
|
||||
},
|
||||
'5': {
|
||||
type: 'double',
|
||||
},
|
||||
'15': {
|
||||
type: 'double',
|
||||
},
|
||||
},
|
||||
},
|
||||
event: {
|
||||
properties: {
|
||||
agent_id_status: {
|
||||
type: 'keyword',
|
||||
ignore_above: 1024,
|
||||
},
|
||||
dataset: {
|
||||
type: 'constant_keyword',
|
||||
},
|
||||
ingested: {
|
||||
type: 'date',
|
||||
format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
|
||||
ignore_malformed: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
message: {
|
||||
type: 'match_only_text',
|
||||
},
|
||||
'dot.field': {
|
||||
type: 'keyword',
|
||||
},
|
||||
some_new_field: {
|
||||
type: 'keyword',
|
||||
},
|
||||
constant_keyword_without_value: {
|
||||
type: 'constant_keyword',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
it('should fill in missing constant_keyword values from old mappings correctly', () => {
|
||||
// @ts-ignore
|
||||
expect(fillConstantKeywordValues(oldMappings, newMappings)).toEqual({
|
||||
dynamic: false,
|
||||
_meta: {
|
||||
managed_by: 'fleet',
|
||||
managed: true,
|
||||
package: {
|
||||
name: 'elastic_agent',
|
||||
},
|
||||
},
|
||||
dynamic_templates: [
|
||||
{
|
||||
ecs_timestamp: {
|
||||
match: '@timestamp',
|
||||
mapping: {
|
||||
ignore_malformed: false,
|
||||
type: 'date',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
date_detection: false,
|
||||
properties: {
|
||||
'@timestamp': {
|
||||
type: 'date',
|
||||
ignore_malformed: false,
|
||||
},
|
||||
load: {
|
||||
properties: {
|
||||
'1': {
|
||||
type: 'double',
|
||||
},
|
||||
'5': {
|
||||
type: 'double',
|
||||
},
|
||||
'15': {
|
||||
type: 'double',
|
||||
},
|
||||
},
|
||||
},
|
||||
event: {
|
||||
properties: {
|
||||
agent_id_status: {
|
||||
type: 'keyword',
|
||||
ignore_above: 1024,
|
||||
},
|
||||
dataset: {
|
||||
type: 'constant_keyword',
|
||||
value: 'elastic_agent.metricbeat',
|
||||
},
|
||||
ingested: {
|
||||
type: 'date',
|
||||
format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
|
||||
ignore_malformed: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
message: {
|
||||
type: 'match_only_text',
|
||||
},
|
||||
'dot.field': {
|
||||
type: 'keyword',
|
||||
},
|
||||
some_new_field: {
|
||||
type: 'keyword',
|
||||
},
|
||||
constant_keyword_without_value: {
|
||||
type: 'constant_keyword',
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should return the same mappings if old mappings are not provided', () => {
|
||||
// @ts-ignore
|
||||
expect(fillConstantKeywordValues({}, newMappings)).toMatchObject(newMappings);
|
||||
});
|
||||
});
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
|
||||
import { USER_SETTINGS_TEMPLATE_SUFFIX } from '../../../../constants';
|
||||
|
||||
|
|
@ -12,3 +13,34 @@ type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPL
|
|||
|
||||
export const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName =>
|
||||
name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX);
|
||||
|
||||
// For any `constant_keyword` fields in `newMappings` that don't have a `value`, access the same field in
|
||||
// the `oldMappings` and fill in the value from there
|
||||
export const fillConstantKeywordValues = (
|
||||
oldMappings: MappingTypeMapping,
|
||||
newMappings: MappingTypeMapping
|
||||
) => {
|
||||
const filledMappings = JSON.parse(JSON.stringify(newMappings)) as MappingTypeMapping;
|
||||
const deepGet = (obj: any, keys: string[]) => keys.reduce((xs, x) => xs?.[x] ?? undefined, obj);
|
||||
|
||||
const fillEmptyConstantKeywordFields = (mappings: unknown, currentPath: string[] = []) => {
|
||||
if (!mappings) return;
|
||||
for (const [key, potentialField] of Object.entries(mappings)) {
|
||||
const path = [...currentPath, key];
|
||||
if (typeof potentialField === 'object') {
|
||||
if (potentialField.type === 'constant_keyword' && potentialField.value === undefined) {
|
||||
const valueFromOldMappings = deepGet(oldMappings.properties, [...path, 'value']);
|
||||
if (valueFromOldMappings !== undefined) {
|
||||
potentialField.value = valueFromOldMappings;
|
||||
}
|
||||
} else if (potentialField.properties && typeof potentialField.properties === 'object') {
|
||||
fillEmptyConstantKeywordFields(potentialField.properties, [...path, 'properties']);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
fillEmptyConstantKeywordFields(filledMappings.properties);
|
||||
|
||||
return filledMappings;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue