[8.15] [UII] Fill in empty values for `constant_keyword` fields from existing mappings (#188145) (#188170)

# Backport This will backport the following commits from `main` to `8.15`: - [[UII] Fill in empty values for `constant_keyword` fields from existing mappings (#188145)](https://github.com/elastic/kibana/pull/188145)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Jen Huang <its.jenetic@gmail.com>
2025-04-23 17:28:26 -04:00 · 2024-07-12 06:48:50 +02:00 · 2024-07-12 06:48:50 +02:00 · 6302a65c80
commit 6302a65c80
parent d8611f1b13
3 changed files with 270 additions and 6 deletions
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
@ -39,7 +39,7 @@ import { retryTransientEsErrors } from '../retry';
 import { PackageESError, PackageInvalidArchiveError } from '../../../../errors';

 import { getDefaultProperties, histogram, keyword, scaledFloat } from './mappings';
-import { isUserSettingsTemplate } from './utils';
+import { isUserSettingsTemplate, fillConstantKeywordValues } from './utils';

 interface Properties {
  [key: string]: any;
@ -986,7 +986,7 @@ const updateAllDataStreams = async (
      });
    },
    {
-      // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster
+      // Limit concurrent putMapping/rollover requests to avoid overwhelming ES cluster
      concurrency: 20,
    }
  );
@ -1017,19 +1017,23 @@ const updateExistingDataStream = async ({
  const currentSourceType = currentBackingIndexConfig.mappings?._source?.mode;

  let settings: IndicesIndexSettings;
-  let mappings: MappingTypeMapping;
+  let mappings: MappingTypeMapping = {};
  let lifecycle: any;
  let subobjectsFieldChanged: boolean = false;
+  let simulateResult: any = {};
  try {
-    const simulateResult = await retryTransientEsErrors(async () =>
+    simulateResult = await retryTransientEsErrors(async () =>
      esClient.indices.simulateTemplate({
        name: await getIndexTemplate(esClient, dataStreamName),
      })
    );

    settings = simulateResult.template.settings;
-    mappings = simulateResult.template.mappings;
-    // @ts-expect-error template is not yet typed with DLM
+    mappings = fillConstantKeywordValues(
+      currentBackingIndexConfig?.mappings || {},
+      simulateResult.template.mappings
+    );
+
    lifecycle = simulateResult.template.lifecycle;

    // for now, remove from object so as not to update stream or data stream properties of the index until type and name
@ -1063,6 +1067,7 @@ const updateExistingDataStream = async ({
      subobjectsFieldChanged
    ) {
      logger.info(`Mappings update for ${dataStreamName} failed due to ${err}`);
+      logger.trace(`Attempted mappings: ${mappings}`);
      if (options?.skipDataStreamRollover === true) {
        logger.info(
          `Skipping rollover for ${dataStreamName} as "skipDataStreamRollover" is enabled`
@ -1075,6 +1080,7 @@ const updateExistingDataStream = async ({
      }
    }
    logger.error(`Mappings update for ${dataStreamName} failed due to unexpected error: ${err}`);
+    logger.trace(`Attempted mappings: ${mappings}`);
    if (options?.ignoreMappingUpdateErrors === true) {
      logger.info(`Ignore mapping update errors as "ignoreMappingUpdateErrors" is enabled`);
      return;
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts
@ -0,0 +1,226 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { fillConstantKeywordValues } from './utils';
+
+describe('fillConstantKeywordValues', () => {
+  const oldMappings = {
+    dynamic: false,
+    _meta: {
+      managed_by: 'fleet',
+      managed: true,
+      package: {
+        name: 'elastic_agent',
+      },
+    },
+    dynamic_templates: [
+      {
+        ecs_timestamp: {
+          match: '@timestamp',
+          mapping: {
+            ignore_malformed: false,
+            type: 'date',
+          },
+        },
+      },
+    ],
+    date_detection: false,
+    properties: {
+      '@timestamp': {
+        type: 'date',
+        ignore_malformed: false,
+      },
+      load: {
+        properties: {
+          '1': {
+            type: 'double',
+          },
+          '5': {
+            type: 'double',
+          },
+          '15': {
+            type: 'double',
+          },
+        },
+      },
+      event: {
+        properties: {
+          agent_id_status: {
+            type: 'keyword',
+            ignore_above: 1024,
+          },
+          dataset: {
+            type: 'constant_keyword',
+            value: 'elastic_agent.metricbeat',
+          },
+          ingested: {
+            type: 'date',
+            format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+            ignore_malformed: false,
+          },
+        },
+      },
+      message: {
+        type: 'match_only_text',
+      },
+      'dot.field': {
+        type: 'keyword',
+      },
+      constant_keyword_without_value: {
+        type: 'constant_keyword',
+      },
+    },
+  };
+
+  const newMappings = {
+    dynamic: false,
+    _meta: {
+      managed_by: 'fleet',
+      managed: true,
+      package: {
+        name: 'elastic_agent',
+      },
+    },
+    dynamic_templates: [
+      {
+        ecs_timestamp: {
+          match: '@timestamp',
+          mapping: {
+            ignore_malformed: false,
+            type: 'date',
+          },
+        },
+      },
+    ],
+    date_detection: false,
+    properties: {
+      '@timestamp': {
+        type: 'date',
+        ignore_malformed: false,
+      },
+      load: {
+        properties: {
+          '1': {
+            type: 'double',
+          },
+          '5': {
+            type: 'double',
+          },
+          '15': {
+            type: 'double',
+          },
+        },
+      },
+      event: {
+        properties: {
+          agent_id_status: {
+            type: 'keyword',
+            ignore_above: 1024,
+          },
+          dataset: {
+            type: 'constant_keyword',
+          },
+          ingested: {
+            type: 'date',
+            format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+            ignore_malformed: false,
+          },
+        },
+      },
+      message: {
+        type: 'match_only_text',
+      },
+      'dot.field': {
+        type: 'keyword',
+      },
+      some_new_field: {
+        type: 'keyword',
+      },
+      constant_keyword_without_value: {
+        type: 'constant_keyword',
+      },
+    },
+  };
+
+  it('should fill in missing constant_keyword values from old mappings correctly', () => {
+    // @ts-ignore
+    expect(fillConstantKeywordValues(oldMappings, newMappings)).toEqual({
+      dynamic: false,
+      _meta: {
+        managed_by: 'fleet',
+        managed: true,
+        package: {
+          name: 'elastic_agent',
+        },
+      },
+      dynamic_templates: [
+        {
+          ecs_timestamp: {
+            match: '@timestamp',
+            mapping: {
+              ignore_malformed: false,
+              type: 'date',
+            },
+          },
+        },
+      ],
+      date_detection: false,
+      properties: {
+        '@timestamp': {
+          type: 'date',
+          ignore_malformed: false,
+        },
+        load: {
+          properties: {
+            '1': {
+              type: 'double',
+            },
+            '5': {
+              type: 'double',
+            },
+            '15': {
+              type: 'double',
+            },
+          },
+        },
+        event: {
+          properties: {
+            agent_id_status: {
+              type: 'keyword',
+              ignore_above: 1024,
+            },
+            dataset: {
+              type: 'constant_keyword',
+              value: 'elastic_agent.metricbeat',
+            },
+            ingested: {
+              type: 'date',
+              format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+              ignore_malformed: false,
+            },
+          },
+        },
+        message: {
+          type: 'match_only_text',
+        },
+        'dot.field': {
+          type: 'keyword',
+        },
+        some_new_field: {
+          type: 'keyword',
+        },
+        constant_keyword_without_value: {
+          type: 'constant_keyword',
+        },
+      },
+    });
+  });
+
+  it('should return the same mappings if old mappings are not provided', () => {
+    // @ts-ignore
+    expect(fillConstantKeywordValues({}, newMappings)).toMatchObject(newMappings);
+  });
+});
--- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts
+++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts
@ -4,6 +4,7 @@
 * 2.0; you may not use this file except in compliance with the Elastic License
 * 2.0.
 */
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';

 import { USER_SETTINGS_TEMPLATE_SUFFIX } from '../../../../constants';

@ -12,3 +13,34 @@ type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPL

 export const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName =>
  name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX);
+
+// For any `constant_keyword` fields in `newMappings` that don't have a `value`, access the same field in
+// the `oldMappings` and fill in the value from there
+export const fillConstantKeywordValues = (
+  oldMappings: MappingTypeMapping,
+  newMappings: MappingTypeMapping
+) => {
+  const filledMappings = JSON.parse(JSON.stringify(newMappings)) as MappingTypeMapping;
+  const deepGet = (obj: any, keys: string[]) => keys.reduce((xs, x) => xs?.[x] ?? undefined, obj);
+
+  const fillEmptyConstantKeywordFields = (mappings: unknown, currentPath: string[] = []) => {
+    if (!mappings) return;
+    for (const [key, potentialField] of Object.entries(mappings)) {
+      const path = [...currentPath, key];
+      if (typeof potentialField === 'object') {
+        if (potentialField.type === 'constant_keyword' && potentialField.value === undefined) {
+          const valueFromOldMappings = deepGet(oldMappings.properties, [...path, 'value']);
+          if (valueFromOldMappings !== undefined) {
+            potentialField.value = valueFromOldMappings;
+          }
+        } else if (potentialField.properties && typeof potentialField.properties === 'object') {
+          fillEmptyConstantKeywordFields(potentialField.properties, [...path, 'properties']);
+        }
+      }
+    }
+  };
+
+  fillEmptyConstantKeywordFields(filledMappings.properties);
+
+  return filledMappings;
+};