[UII] Fill in empty values for constant_keyword fields from existing mappings (#188145)

## Summary

Resolves https://github.com/elastic/kibana/issues/178528.

Some packages declare `constant_keyword` type fields without an explicit
value. This causes ES to fill in the value in the mappings using the
first ingested value.

When upgrading this type of package & field after the value has already
been populated in this way, the mappings update fail due to pushing a
`null` value into an existing value, triggering unnecessary rollovers.

This PR fixes that by filling in the empty values from the existing
mappings.

## Test
1. On an empty cluster, turn on debug logs
2. Set up Fleet Server policy and Fleet Server agent
3. Force install old version of Elastic Agent integration, v1.19.2:
```
POST kbn:/api/fleet/epm/packages/elastic_agent/1.19.2
{
  "force": true
}
```
4. Create a new empty policy, **deselect system and agent monitoring**
(otherwise the integration will be upgraded, we do not want this yet)
5. Manually add Elastic Agent integration v1.19.2 to the new policy
6. Edit the policy to enable logs and metrics monitoring
7. Enroll agent into the policy, confirm that monitoring logs and
metrics are being ingested and that a value exists for `event.dataset`
mapping for the logs:
```
GET logs-elastic_agent*/_mappings
```
```
            "dataset": {
              "type": "constant_keyword",
              "value": "elastic_agent"
            }
```
9. Upgrade Elastic Agent integration to v1.20.0 (note we are not
upgrading to the newest versions, 2.0+, because these **are** expected
to trigger rollovers for some data streams):
```
POST kbn:/api/fleet/epm/packages/elastic_agent/1.20.0
{
  "force": true
}
```
10. Confirm in Kibana logs that no rollovers triggered during the
upgrade
11. Confirm that there is still only 1 backing index for monitoring
logs:
```
GET logs-elastic_agent*
```

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -39,7 +39,7 @@ import { retryTransientEsErrors } from '../retry';
 import { PackageESError, PackageInvalidArchiveError } from '../../../../errors';
 
 import { getDefaultProperties, histogram, keyword, scaledFloat } from './mappings';
-import { isUserSettingsTemplate } from './utils';
+import { isUserSettingsTemplate, fillConstantKeywordValues } from './utils';
 
 interface Properties {
   [key: string]: any;
@@ -986,7 +986,7 @@ const updateAllDataStreams = async (
       });
     },
     {
-      // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster
+      // Limit concurrent putMapping/rollover requests to avoid overwhelming ES cluster
       concurrency: 20,
     }
   );
@@ -1017,19 +1017,23 @@ const updateExistingDataStream = async ({
   const currentSourceType = currentBackingIndexConfig.mappings?._source?.mode;
 
   let settings: IndicesIndexSettings;
-  let mappings: MappingTypeMapping;
+  let mappings: MappingTypeMapping = {};
   let lifecycle: any;
   let subobjectsFieldChanged: boolean = false;
+  let simulateResult: any = {};
   try {
-    const simulateResult = await retryTransientEsErrors(async () =>
+    simulateResult = await retryTransientEsErrors(async () =>
       esClient.indices.simulateTemplate({
         name: await getIndexTemplate(esClient, dataStreamName),
       })
     );
 
     settings = simulateResult.template.settings;
-    mappings = simulateResult.template.mappings;
+    mappings = fillConstantKeywordValues(
-    // @ts-expect-error template is not yet typed with DLM
+      currentBackingIndexConfig?.mappings || {},
+      simulateResult.template.mappings
+    );
+
     lifecycle = simulateResult.template.lifecycle;
 
     // for now, remove from object so as not to update stream or data stream properties of the index until type and name
@@ -1063,6 +1067,7 @@ const updateExistingDataStream = async ({
       subobjectsFieldChanged
     ) {
       logger.info(`Mappings update for ${dataStreamName} failed due to ${err}`);
+      logger.trace(`Attempted mappings: ${mappings}`);
       if (options?.skipDataStreamRollover === true) {
         logger.info(
           `Skipping rollover for ${dataStreamName} as "skipDataStreamRollover" is enabled`
@@ -1075,6 +1080,7 @@ const updateExistingDataStream = async ({
       }
     }
     logger.error(`Mappings update for ${dataStreamName} failed due to unexpected error: ${err}`);
+    logger.trace(`Attempted mappings: ${mappings}`);
     if (options?.ignoreMappingUpdateErrors === true) {
       logger.info(`Ignore mapping update errors as "ignoreMappingUpdateErrors" is enabled`);
       return;

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts

@@ -0,0 +1,226 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { fillConstantKeywordValues } from './utils';
+
+describe('fillConstantKeywordValues', () => {
+  const oldMappings = {
+    dynamic: false,
+    _meta: {
+      managed_by: 'fleet',
+      managed: true,
+      package: {
+        name: 'elastic_agent',
+      },
+    },
+    dynamic_templates: [
+      {
+        ecs_timestamp: {
+          match: '@timestamp',
+          mapping: {
+            ignore_malformed: false,
+            type: 'date',
+          },
+        },
+      },
+    ],
+    date_detection: false,
+    properties: {
+      '@timestamp': {
+        type: 'date',
+        ignore_malformed: false,
+      },
+      load: {
+        properties: {
+          '1': {
+            type: 'double',
+          },
+          '5': {
+            type: 'double',
+          },
+          '15': {
+            type: 'double',
+          },
+        },
+      },
+      event: {
+        properties: {
+          agent_id_status: {
+            type: 'keyword',
+            ignore_above: 1024,
+          },
+          dataset: {
+            type: 'constant_keyword',
+            value: 'elastic_agent.metricbeat',
+          },
+          ingested: {
+            type: 'date',
+            format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+            ignore_malformed: false,
+          },
+        },
+      },
+      message: {
+        type: 'match_only_text',
+      },
+      'dot.field': {
+        type: 'keyword',
+      },
+      constant_keyword_without_value: {
+        type: 'constant_keyword',
+      },
+    },
+  };
+
+  const newMappings = {
+    dynamic: false,
+    _meta: {
+      managed_by: 'fleet',
+      managed: true,
+      package: {
+        name: 'elastic_agent',
+      },
+    },
+    dynamic_templates: [
+      {
+        ecs_timestamp: {
+          match: '@timestamp',
+          mapping: {
+            ignore_malformed: false,
+            type: 'date',
+          },
+        },
+      },
+    ],
+    date_detection: false,
+    properties: {
+      '@timestamp': {
+        type: 'date',
+        ignore_malformed: false,
+      },
+      load: {
+        properties: {
+          '1': {
+            type: 'double',
+          },
+          '5': {
+            type: 'double',
+          },
+          '15': {
+            type: 'double',
+          },
+        },
+      },
+      event: {
+        properties: {
+          agent_id_status: {
+            type: 'keyword',
+            ignore_above: 1024,
+          },
+          dataset: {
+            type: 'constant_keyword',
+          },
+          ingested: {
+            type: 'date',
+            format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+            ignore_malformed: false,
+          },
+        },
+      },
+      message: {
+        type: 'match_only_text',
+      },
+      'dot.field': {
+        type: 'keyword',
+      },
+      some_new_field: {
+        type: 'keyword',
+      },
+      constant_keyword_without_value: {
+        type: 'constant_keyword',
+      },
+    },
+  };
+
+  it('should fill in missing constant_keyword values from old mappings correctly', () => {
+    // @ts-ignore
+    expect(fillConstantKeywordValues(oldMappings, newMappings)).toEqual({
+      dynamic: false,
+      _meta: {
+        managed_by: 'fleet',
+        managed: true,
+        package: {
+          name: 'elastic_agent',
+        },
+      },
+      dynamic_templates: [
+        {
+          ecs_timestamp: {
+            match: '@timestamp',
+            mapping: {
+              ignore_malformed: false,
+              type: 'date',
+            },
+          },
+        },
+      ],
+      date_detection: false,
+      properties: {
+        '@timestamp': {
+          type: 'date',
+          ignore_malformed: false,
+        },
+        load: {
+          properties: {
+            '1': {
+              type: 'double',
+            },
+            '5': {
+              type: 'double',
+            },
+            '15': {
+              type: 'double',
+            },
+          },
+        },
+        event: {
+          properties: {
+            agent_id_status: {
+              type: 'keyword',
+              ignore_above: 1024,
+            },
+            dataset: {
+              type: 'constant_keyword',
+              value: 'elastic_agent.metricbeat',
+            },
+            ingested: {
+              type: 'date',
+              format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis',
+              ignore_malformed: false,
+            },
+          },
+        },
+        message: {
+          type: 'match_only_text',
+        },
+        'dot.field': {
+          type: 'keyword',
+        },
+        some_new_field: {
+          type: 'keyword',
+        },
+        constant_keyword_without_value: {
+          type: 'constant_keyword',
+        },
+      },
+    });
+  });
+
+  it('should return the same mappings if old mappings are not provided', () => {
+    // @ts-ignore
+    expect(fillConstantKeywordValues({}, newMappings)).toMatchObject(newMappings);
+  });
+});

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.ts

@@ -4,6 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import { USER_SETTINGS_TEMPLATE_SUFFIX } from '../../../../constants';
 
@@ -12,3 +13,34 @@ type UserSettingsTemplateName = `${TemplateBaseName}${typeof USER_SETTINGS_TEMPL
 
 export const isUserSettingsTemplate = (name: string): name is UserSettingsTemplateName =>
   name.endsWith(USER_SETTINGS_TEMPLATE_SUFFIX);
+
+// For any `constant_keyword` fields in `newMappings` that don't have a `value`, access the same field in
+// the `oldMappings` and fill in the value from there
+export const fillConstantKeywordValues = (
+  oldMappings: MappingTypeMapping,
+  newMappings: MappingTypeMapping
+) => {
+  const filledMappings = JSON.parse(JSON.stringify(newMappings)) as MappingTypeMapping;
+  const deepGet = (obj: any, keys: string[]) => keys.reduce((xs, x) => xs?.[x] ?? undefined, obj);
+
+  const fillEmptyConstantKeywordFields = (mappings: unknown, currentPath: string[] = []) => {
+    if (!mappings) return;
+    for (const [key, potentialField] of Object.entries(mappings)) {
+      const path = [...currentPath, key];
+      if (typeof potentialField === 'object') {
+        if (potentialField.type === 'constant_keyword' && potentialField.value === undefined) {
+          const valueFromOldMappings = deepGet(oldMappings.properties, [...path, 'value']);
+          if (valueFromOldMappings !== undefined) {
+            potentialField.value = valueFromOldMappings;
+          }
+        } else if (potentialField.properties && typeof potentialField.properties === 'object') {
+          fillEmptyConstantKeywordFields(potentialField.properties, [...path, 'properties']);
+        }
+      }
+    }
+  };
+
+  fillEmptyConstantKeywordFields(filledMappings.properties);
+
+  return filledMappings;
+};