[Fleet] experimental toggles for doc-value-only (#149131)

## Summary

Closes https://github.com/elastic/kibana/issues/144357

WIP. Review can be started, but still requires a lot of testing and
fixing the issue below.

How to test locally:
- Turn on `experimentalDataStreamSettings` feature flag
- Go to Add integration, System integration
- On the first data stream, turn on the Doc value only switches, Save
- The mapping changes are visible under Stack Management / Index
Management / Component Templates e.g. `logs-system.auth@package`
- The numeric switch sets `index:false` on all numeric field mappings
(long, double, etc.)
- The other switch sets `index:false` on all other field type mappings
that support it (keyword, ip, date, etc.)
- The new mappings will take effect after rollover

<img width="475" alt="image"
src="https://user-images.githubusercontent.com/90178898/213206641-13ead2fc-f079-407c-9c0e-c58f99dd4903.png">
<img width="1037" alt="image"
src="https://user-images.githubusercontent.com/90178898/213495546-9962c458-590b-4787-bf2d-9f19abea3f67.png">

What works:
- When turning the new doc-value-only numeric and other checkboxes on or
off, the corresponding mapping changes are done in the component
template
- The logic also reads the package spec's template and preserves the
`index:false` values regardless of the switch (tested manually by
setting `@timestamp` field to `index:false` in the template, there is
also the `original` field in `logs-system.auth@package` stream that is
set to `index:false` in the package by default.

```
"original": {
    "index": false,
    "doc_values": false,
    "type": "keyword"
},
```

Pending:
- Add/update unit and integration tests to verify the mapping change
logic - DONE
- Manual testing (turning the switches on/off, create/update package
policy, upgrade package) - DONE
- Clarify TODOs in the code about the supported types - DONE
- Hitting an issue when turning on `doc-value-only` for "other" types
(keyword, date, etc.). Could be that one of the fields doesn't support
`index:false` setting. Didn't experience this when turning on only the
numeric types. - FIXED
```
illegal_argument_exception: [illegal_argument_exception] Reason: updating component template [logs-system.auth@package] results in invalid composable template [logs-system.auth] after templates are merged
```

EDIT: found the root cause of this: `Caused by:
java.lang.IllegalArgumentException: data stream timestamp field
[@timestamp] is not indexed`


### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts

@@ -84,7 +84,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0",
         "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88",
         "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5",
-        "epm-packages": "7d80ba3f1fcd80316aa0b112657272034b66d5a8",
+        "epm-packages": "21e096cf4554abe1652953a6cd2119d68ddc9403",
         "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771",
         "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd",
         "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c",

x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts

@@ -132,6 +132,8 @@ describe('toPackagePolicy', () => {
               features: {
                 synthetic_source: true,
                 tsdb: false,
+                doc_value_only_numeric: false,
+                doc_value_only_other: false,
               },
             },
           ],
@@ -144,6 +146,8 @@ describe('toPackagePolicy', () => {
           features: {
             synthetic_source: true,
             tsdb: false,
+            doc_value_only_numeric: false,
+            doc_value_only_other: false,
           },
         },
       ]);

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -460,7 +460,11 @@ export type PackageInfo =
   | Installable<Merge<ArchivePackage, EpmPackageAdditions>>;
 
 // TODO - Expand this with other experimental indexing types
-export type ExperimentalIndexingFeature = 'synthetic_source' | 'tsdb';
+export type ExperimentalIndexingFeature =
+  | 'synthetic_source'
+  | 'tsdb'
+  | 'doc_value_only_numeric'
+  | 'doc_value_only_other';
 
 export interface ExperimentalDataStreamFeature {
   data_stream: string;

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.test.tsx

@@ -92,6 +92,8 @@ describe('ExperimentDatastreamSettings', () => {
               features: {
                 synthetic_source: false,
                 tsdb: false,
+                doc_value_only_numeric: false,
+                doc_value_only_other: false,
               },
             },
           ]}
@@ -141,6 +143,8 @@ describe('ExperimentDatastreamSettings', () => {
           features: {
             synthetic_source: true,
             tsdb: false,
+            doc_value_only_numeric: false,
+            doc_value_only_other: false,
           },
         },
       ];
@@ -169,6 +173,8 @@ describe('ExperimentDatastreamSettings', () => {
           features: {
             synthetic_source: false,
             tsdb: false,
+            doc_value_only_numeric: false,
+            doc_value_only_other: false,
           },
         },
       ]);
@@ -178,6 +184,8 @@ describe('ExperimentDatastreamSettings', () => {
           features: {
             synthetic_source: true,
             tsdb: false,
+            doc_value_only_numeric: false,
+            doc_value_only_other: false,
           },
         },
       ]);

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/experimental_datastream_settings.tsx

@@ -64,6 +64,18 @@ export const ExperimentDatastreamSettings: React.FunctionComponent<Props> = ({
   const isSyntheticSourceEnabledByDefault =
     registryDataStream.elasticsearch?.source_mode === 'synthetic' || isTimeSeriesEnabledByDefault;
 
+  const docValueOnlyNumericExperimentalValue = getExperimentalFeatureValue(
+    'doc_value_only_numeric',
+    experimentalDataFeatures ?? [],
+    registryDataStream
+  );
+
+  const docValueOnlyOtherExperimentalValue = getExperimentalFeatureValue(
+    'doc_value_only_other',
+    experimentalDataFeatures ?? [],
+    registryDataStream
+  );
+
   const newExperimentalIndexingFeature = {
     synthetic_source:
       typeof syntheticSourceExperimentalValue !== 'undefined'
@@ -73,6 +85,14 @@ export const ExperimentDatastreamSettings: React.FunctionComponent<Props> = ({
       ? isTimeSeriesEnabledByDefault
       : getExperimentalFeatureValue('tsdb', experimentalDataFeatures ?? [], registryDataStream) ??
         false,
+    doc_value_only_numeric:
+      typeof docValueOnlyNumericExperimentalValue !== 'undefined'
+        ? docValueOnlyNumericExperimentalValue
+        : false,
+    doc_value_only_other:
+      typeof docValueOnlyOtherExperimentalValue !== 'undefined'
+        ? docValueOnlyOtherExperimentalValue
+        : false,
   };
 
   const onIndexingSettingChange = (
@@ -178,6 +198,40 @@ export const ExperimentDatastreamSettings: React.FunctionComponent<Props> = ({
             />
           </EuiToolTip>
         </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiSwitch
+            checked={newExperimentalIndexingFeature.doc_value_only_numeric ?? false}
+            data-test-subj="packagePolicyEditor.docValueOnlyNumericExperimentalFeature.switch"
+            label={
+              <FormattedMessage
+                id="xpack.fleet.packagePolicyEditor.experimentalFeatures.docValueOnlyNumericLabel"
+                defaultMessage="Doc value only (numeric types)"
+              />
+            }
+            onChange={(e) => {
+              onIndexingSettingChange({
+                doc_value_only_numeric: e.target.checked,
+              });
+            }}
+          />
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiSwitch
+            checked={newExperimentalIndexingFeature.doc_value_only_other ?? false}
+            data-test-subj="packagePolicyEditor.docValueOnlyOtherExperimentalFeature.switch"
+            label={
+              <FormattedMessage
+                id="xpack.fleet.packagePolicyEditor.experimentalFeatures.docValueOnlyOtherLabel"
+                defaultMessage="Doc value only (other types)"
+              />
+            }
+            onChange={(e) => {
+              onIndexingSettingChange({
+                doc_value_only_other: e.target.checked,
+              });
+            }}
+          />
+        </EuiFlexItem>
       </EuiFlexGroup>
     </EuiFlexItem>
   );

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -289,6 +289,7 @@ const getSavedObjectTypes = (
             data_stream: { type: 'keyword' },
             features: {
               type: 'nested',
+              dynamic: false,
               properties: {
                 synthetic_source: { type: 'boolean' },
                 tsdb: { type: 'boolean' },

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts

@@ -201,6 +201,8 @@ describe('EPM index template install', () => {
         features: {
           synthetic_source: false,
           tsdb: false,
+          doc_value_only_numeric: false,
+          doc_value_only_other: false,
         },
       },
     });
@@ -241,6 +243,8 @@ describe('EPM index template install', () => {
         features: {
           synthetic_source: false,
           tsdb: false,
+          doc_value_only_numeric: false,
+          doc_value_only_other: false,
         },
       },
     });

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -49,6 +49,11 @@ import {
 import { getESAssetMetadata } from '../meta';
 import { retryTransientEsErrors } from '../retry';
 
+import {
+  applyDocOnlyValueToMapping,
+  forEachMappings,
+} from '../../../experimental_datastream_features_helper';
+
 import {
   generateMappings,
   generateTemplateName,
@@ -262,6 +267,7 @@ export function buildComponentTemplates(params: {
     defaultSettings,
     mappings,
     pipelineName,
+    experimentalDataStreamFeature,
   } = params;
   const packageTemplateName = `${templateName}${PACKAGE_TEMPLATE_SUFFIX}`;
   const userSettingsTemplateName = `${templateName}${USER_SETTINGS_TEMPLATE_SUFFIX}`;
@@ -275,6 +281,23 @@ export function buildComponentTemplates(params: {
 
   const indexTemplateMappings = registryElasticsearch?.['index_template.mappings'] ?? {};
 
+  const isDocValueOnlyNumericEnabled =
+    experimentalDataStreamFeature?.features.doc_value_only_numeric === true;
+  const isDocValueOnlyOtherEnabled =
+    experimentalDataStreamFeature?.features.doc_value_only_other === true;
+
+  if (isDocValueOnlyNumericEnabled || isDocValueOnlyOtherEnabled) {
+    forEachMappings(mappings.properties, (mappingProp, name) =>
+      applyDocOnlyValueToMapping(
+        mappingProp,
+        name,
+        experimentalDataStreamFeature,
+        isDocValueOnlyNumericEnabled,
+        isDocValueOnlyOtherEnabled
+      )
+    );
+  }
+
   const mappingsProperties = merge(mappings.properties, indexTemplateMappings.properties ?? {});
 
   const mappingsDynamicTemplates = uniqBy(
@@ -286,8 +309,8 @@ export function buildComponentTemplates(params: {
   const isSyntheticSourceEnabledByDefault = registryElasticsearch?.source_mode === 'synthetic';
 
   const sourceModeSynthetic =
-    params.experimentalDataStreamFeature?.features.synthetic_source !== false &&
+    experimentalDataStreamFeature?.features.synthetic_source !== false &&
-    (params.experimentalDataStreamFeature?.features.synthetic_source === true ||
+    (experimentalDataStreamFeature?.features.synthetic_source === true ||
       isSyntheticSourceEnabledByDefault ||
       isTimeSeriesEnabledByDefault);
 

x-pack/plugins/fleet/server/services/experimental_datastream_features_helper.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  MappingProperty,
+  PropertyName,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+import type { ExperimentalDataStreamFeature } from '../../common/types';
+
+export const forEachMappings = (
+  mappingProperties: Record<PropertyName, MappingProperty>,
+  process: (prop: MappingProperty, name: string) => void
+) => {
+  Object.keys(mappingProperties).forEach((mapping) => {
+    const property = mappingProperties[mapping] as any;
+    if (property.properties) {
+      forEachMappings(property.properties, process);
+    } else {
+      process(property, mapping);
+    }
+  });
+};
+
+export const applyDocOnlyValueToMapping = (
+  mappingProp: MappingProperty,
+  name: string,
+  featureMap: ExperimentalDataStreamFeature,
+  isDocValueOnlyNumericChanged: boolean,
+  isDocValueOnlyOtherChanged: boolean
+) => {
+  const mapping = mappingProp as any;
+
+  const numericTypes = [
+    'long',
+    'integer',
+    'short',
+    'byte',
+    'double',
+    'float',
+    'half_float',
+    'scaled_float',
+    'unsigned_long',
+  ];
+  if (isDocValueOnlyNumericChanged && numericTypes.includes(mapping.type ?? '')) {
+    updateMapping(mapping, featureMap.features.doc_value_only_numeric);
+  }
+
+  const otherSupportedTypes = ['date', 'date_nanos', 'boolean', 'ip', 'geo_point', 'keyword'];
+  if (
+    isDocValueOnlyOtherChanged &&
+    name !== '@timestamp' &&
+    otherSupportedTypes.includes(mapping.type ?? '')
+  ) {
+    updateMapping(mapping, featureMap.features.doc_value_only_other);
+  }
+};
+
+const updateMapping = (mapping: { index?: boolean }, featureValue: boolean) => {
+  if (featureValue === false && mapping.index === false) {
+    delete mapping.index;
+  }
+  if (featureValue && !mapping.index) {
+    mapping.index = false;
+  }
+};

x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts

@@ -9,15 +9,60 @@ import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
 import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
 
 import type { NewPackagePolicy, PackagePolicy } from '../../types';
+import { getInstallation } from '../epm/packages';
 
 import { handleExperimentalDatastreamFeatureOptIn } from './experimental_datastream_features';
 
+jest.mock('../epm/packages', () => {
+  return {
+    getInstallation: jest.fn(),
+    getPackageInfo: jest.fn().mockResolvedValue({
+      data_streams: [
+        {
+          dataset: 'test',
+          type: 'metrics',
+        },
+      ],
+    }),
+  };
+});
+
+const mockGetInstallation = getInstallation as jest.Mock;
+
+jest.mock('../epm/elasticsearch/template/install', () => {
+  return {
+    prepareTemplate: jest.fn().mockReturnValue({
+      componentTemplates: {
+        'metrics-test.test@package': {
+          template: {
+            mappings: {
+              properties: {
+                sequence: {
+                  type: 'long',
+                },
+                name: {
+                  type: 'keyword',
+                  index: false,
+                },
+              },
+            },
+          },
+        },
+      },
+    }),
+  };
+});
+
 function getNewTestPackagePolicy({
   isSyntheticSourceEnabled,
   isTSDBEnabled,
+  isDocValueOnlyNumeric,
+  isDocValueOnlyOther,
 }: {
   isSyntheticSourceEnabled: boolean;
   isTSDBEnabled: boolean;
+  isDocValueOnlyNumeric: boolean;
+  isDocValueOnlyOther: boolean;
 }): NewPackagePolicy {
   const packagePolicy: NewPackagePolicy = {
     name: 'Test policy',
@@ -36,6 +81,8 @@ function getNewTestPackagePolicy({
           features: {
             synthetic_source: isSyntheticSourceEnabled,
             tsdb: isTSDBEnabled,
+            doc_value_only_numeric: isDocValueOnlyNumeric,
+            doc_value_only_other: isDocValueOnlyOther,
           },
         },
       ],
@@ -48,9 +95,13 @@ function getNewTestPackagePolicy({
 function getExistingTestPackagePolicy({
   isSyntheticSourceEnabled,
   isTSDBEnabled,
+  isDocValueOnlyNumeric,
+  isDocValueOnlyOther,
 }: {
   isSyntheticSourceEnabled: boolean;
   isTSDBEnabled: boolean;
+  isDocValueOnlyNumeric: boolean;
+  isDocValueOnlyOther: boolean;
 }): PackagePolicy {
   const packagePolicy: PackagePolicy = {
     id: 'test-policy',
@@ -70,6 +121,8 @@ function getExistingTestPackagePolicy({
           features: {
             synthetic_source: isSyntheticSourceEnabled,
             tsdb: isTSDBEnabled,
+            doc_value_only_numeric: isDocValueOnlyNumeric,
+            doc_value_only_other: isDocValueOnlyOther,
           },
         },
       ],
@@ -104,6 +157,15 @@ describe('experimental_datastream_features', () => {
                     type: 'keyword',
                     time_series_dimension: true,
                   },
+                  sequence: {
+                    type: 'long',
+                  },
+                  name: {
+                    type: 'keyword',
+                  },
+                  '@timestamp': {
+                    type: 'date',
+                  },
                 },
               },
             },
@@ -118,24 +180,26 @@ describe('experimental_datastream_features', () => {
 
   describe('when package policy does not exist (create)', () => {
     beforeEach(() => {
-      soClient.get.mockResolvedValueOnce({
+      mockGetInstallation.mockResolvedValueOnce({
-        attributes: {
         experimental_data_stream_features: [
           {
             data_stream: 'metrics-test.test',
-              features: { synthetic_source: false, tsdb: false },
+            features: {
+              synthetic_source: false,
+              tsdb: false,
+              doc_value_only_numeric: false,
+              doc_value_only_other: false,
+            },
           },
         ],
-        },
-        id: 'mocked',
-        type: 'mocked',
-        references: [],
       });
     });
     it('updates component template', async () => {
       const packagePolicy = getNewTestPackagePolicy({
         isSyntheticSourceEnabled: true,
         isTSDBEnabled: false,
+        isDocValueOnlyNumeric: false,
+        isDocValueOnlyOther: false,
       });
 
       await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
@@ -152,10 +216,98 @@ describe('experimental_datastream_features', () => {
       );
     });
 
+    it('updates component template number fields', async () => {
+      const packagePolicy = getNewTestPackagePolicy({
+        isSyntheticSourceEnabled: false,
+        isTSDBEnabled: false,
+        isDocValueOnlyNumeric: true,
+        isDocValueOnlyOther: false,
+      });
+
+      await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
+
+      expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled();
+      expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith(
+        expect.objectContaining({
+          body: expect.objectContaining({
+            template: expect.objectContaining({
+              mappings: expect.objectContaining({
+                properties: expect.objectContaining({
+                  sequence: {
+                    type: 'long',
+                    index: false,
+                  },
+                }),
+              }),
+            }),
+          }),
+        })
+      );
+    });
+
+    it('updates component template other fields', async () => {
+      const packagePolicy = getNewTestPackagePolicy({
+        isSyntheticSourceEnabled: false,
+        isTSDBEnabled: false,
+        isDocValueOnlyNumeric: false,
+        isDocValueOnlyOther: true,
+      });
+
+      await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
+
+      expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled();
+      expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith(
+        expect.objectContaining({
+          body: expect.objectContaining({
+            template: expect.objectContaining({
+              mappings: expect.objectContaining({
+                properties: expect.objectContaining({
+                  name: {
+                    type: 'keyword',
+                    index: false,
+                  },
+                }),
+              }),
+            }),
+          }),
+        })
+      );
+    });
+
+    it('should not set index:false on @timestamp field', async () => {
+      const packagePolicy = getNewTestPackagePolicy({
+        isSyntheticSourceEnabled: false,
+        isTSDBEnabled: false,
+        isDocValueOnlyNumeric: false,
+        isDocValueOnlyOther: true,
+      });
+
+      await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
+
+      expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled();
+      expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith(
+        expect.objectContaining({
+          body: expect.objectContaining({
+            template: expect.objectContaining({
+              mappings: expect.objectContaining({
+                properties: expect.objectContaining({
+                  '@timestamp': {
+                    type: 'date',
+                  },
+                }),
+              }),
+            }),
+          }),
+        })
+      );
+    });
+
     it('should update index template', async () => {
       const packagePolicy = getNewTestPackagePolicy({
         isSyntheticSourceEnabled: false,
         isTSDBEnabled: true,
+        isDocValueOnlyNumeric: false,
+        isDocValueOnlyOther: false,
       });
 
       esClient.indices.getIndexTemplate.mockResolvedValueOnce({
@@ -197,20 +349,22 @@ describe('experimental_datastream_features', () => {
         const packagePolicy = getExistingTestPackagePolicy({
           isSyntheticSourceEnabled: true,
           isTSDBEnabled: false,
+          isDocValueOnlyNumeric: false,
+          isDocValueOnlyOther: false,
         });
 
-        soClient.get.mockResolvedValueOnce({
+        mockGetInstallation.mockResolvedValueOnce({
-          attributes: {
           experimental_data_stream_features: [
             {
               data_stream: 'metrics-test.test',
-                features: { synthetic_source: true, tsdb: false },
+              features: {
+                synthetic_source: true,
+                tsdb: false,
+                doc_value_only_numeric: false,
+                doc_value_only_other: false,
+              },
             },
           ],
-          },
-          id: 'mocked',
-          type: 'mocked',
-          references: [],
         });
 
         await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
@@ -222,24 +376,26 @@ describe('experimental_datastream_features', () => {
 
     describe('when opt in status is changed', () => {
       beforeEach(() => {
-        soClient.get.mockResolvedValueOnce({
+        mockGetInstallation.mockResolvedValueOnce({
-          attributes: {
           experimental_data_stream_features: [
             {
               data_stream: 'metrics-test.test',
-                features: { synthetic_source: false, tsdb: false },
+              features: {
+                synthetic_source: false,
+                tsdb: false,
+                doc_value_only_numeric: false,
+                doc_value_only_other: true,
+              },
             },
           ],
-          },
-          id: 'mocked',
-          type: 'mocked',
-          references: [],
         });
       });
       it('updates component template', async () => {
         const packagePolicy = getExistingTestPackagePolicy({
           isSyntheticSourceEnabled: true,
           isTSDBEnabled: false,
+          isDocValueOnlyNumeric: false,
+          isDocValueOnlyOther: true,
         });
 
         await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
@@ -256,10 +412,70 @@ describe('experimental_datastream_features', () => {
         );
       });
 
+      it('updates component template number fields', async () => {
+        const packagePolicy = getExistingTestPackagePolicy({
+          isSyntheticSourceEnabled: false,
+          isTSDBEnabled: false,
+          isDocValueOnlyNumeric: true,
+          isDocValueOnlyOther: true,
+        });
+
+        await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
+
+        expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled();
+        expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith(
+          expect.objectContaining({
+            body: expect.objectContaining({
+              template: expect.objectContaining({
+                mappings: expect.objectContaining({
+                  properties: expect.objectContaining({
+                    sequence: {
+                      type: 'long',
+                      index: false,
+                    },
+                  }),
+                }),
+              }),
+            }),
+          })
+        );
+      });
+
+      it('should not remove index:false from a field that has it in package spec', async () => {
+        const packagePolicy = getExistingTestPackagePolicy({
+          isSyntheticSourceEnabled: false,
+          isTSDBEnabled: false,
+          isDocValueOnlyNumeric: false,
+          isDocValueOnlyOther: false,
+        });
+
+        await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy });
+
+        expect(esClient.cluster.getComponentTemplate).toHaveBeenCalled();
+        expect(esClient.cluster.putComponentTemplate).toHaveBeenCalledWith(
+          expect.objectContaining({
+            body: expect.objectContaining({
+              template: expect.objectContaining({
+                mappings: expect.objectContaining({
+                  properties: expect.objectContaining({
+                    name: {
+                      type: 'keyword',
+                      index: false,
+                    },
+                  }),
+                }),
+              }),
+            }),
+          })
+        );
+      });
+
       it('should update index template', async () => {
         const packagePolicy = getExistingTestPackagePolicy({
           isSyntheticSourceEnabled: false,
           isTSDBEnabled: true,
+          isDocValueOnlyNumeric: false,
+          isDocValueOnlyOther: false,
         });
 
         esClient.indices.getIndexTemplate.mockResolvedValueOnce({

x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.ts

@@ -8,9 +8,19 @@
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
 
+import { merge } from 'lodash';
+
+import { getRegistryDataStreamAssetBaseName } from '../../../common/services';
+
+import type { ExperimentalIndexingFeature } from '../../../common/types';
 import type { NewPackagePolicy, PackagePolicy } from '../../types';
-import { getInstallation } from '../epm/packages';
+import { prepareTemplate } from '../epm/elasticsearch/template/install';
+import { getInstallation, getPackageInfo } from '../epm/packages';
 import { updateDatastreamExperimentalFeatures } from '../epm/packages/update';
+import {
+  applyDocOnlyValueToMapping,
+  forEachMappings,
+} from '../experimental_datastream_features_helper';
 
 export async function handleExperimentalDatastreamFeatureOptIn({
   soClient,
@@ -29,12 +39,36 @@ export async function handleExperimentalDatastreamFeatureOptIn({
   // an update to the component templates for the package. So we fetch the saved object
   // for the package policy here to compare later.
   let installation;
+  const templateMappings: { [key: string]: any } = {};
 
   if (packagePolicy.package) {
     installation = await getInstallation({
       savedObjectsClient: soClient,
       pkgName: packagePolicy.package.name,
     });
+
+    const packageInfo = await getPackageInfo({
+      savedObjectsClient: soClient,
+      pkgName: packagePolicy.package.name,
+      pkgVersion: packagePolicy.package.version,
+    });
+
+    // prepare template from package spec to find original index:false values
+    const templates = packageInfo.data_streams?.map((dataStream: any) => {
+      const experimentalDataStreamFeature =
+        packagePolicy.package?.experimental_data_stream_features?.find(
+          (datastreamFeature) =>
+            datastreamFeature.data_stream === getRegistryDataStreamAssetBaseName(dataStream)
+        );
+      return prepareTemplate({ pkg: packageInfo, dataStream, experimentalDataStreamFeature });
+    });
+
+    templates?.forEach((template) => {
+      Object.keys(template.componentTemplates).forEach((templateName) => {
+        templateMappings[templateName] =
+          (template.componentTemplates[templateName].template as any).mappings ?? {};
+      });
+    });
   }
 
   for (const featureMapEntry of packagePolicy.package.experimental_data_stream_features) {
@@ -42,12 +76,25 @@ export async function handleExperimentalDatastreamFeatureOptIn({
       (optIn) => optIn.data_stream === featureMapEntry.data_stream
     );
 
-    const isSyntheticSourceOptInChanged =
+    const hasFeatureChanged = (name: ExperimentalIndexingFeature) =>
-      existingOptIn?.features.synthetic_source !== featureMapEntry.features.synthetic_source;
+      existingOptIn?.features[name] !== featureMapEntry.features[name];
 
-    const isTSDBOptInChanged = existingOptIn?.features.tsdb !== featureMapEntry.features.tsdb;
+    const isSyntheticSourceOptInChanged = hasFeatureChanged('synthetic_source');
 
-    if (!isSyntheticSourceOptInChanged && !isTSDBOptInChanged) continue;
+    const isTSDBOptInChanged = hasFeatureChanged('tsdb');
+
+    const isDocValueOnlyNumericChanged = hasFeatureChanged('doc_value_only_numeric');
+    const isDocValueOnlyOtherChanged = hasFeatureChanged('doc_value_only_other');
+
+    if (
+      [
+        isSyntheticSourceOptInChanged,
+        isTSDBOptInChanged,
+        isDocValueOnlyNumericChanged,
+        isDocValueOnlyOtherChanged,
+      ].every((hasFlagChange) => !hasFlagChange)
+    )
+      continue;
 
     const componentTemplateName = `${featureMapEntry.data_stream}@package`;
     const componentTemplateRes = await esClient.cluster.getComponentTemplate({
@@ -56,22 +103,51 @@ export async function handleExperimentalDatastreamFeatureOptIn({
 
     const componentTemplate = componentTemplateRes.component_templates[0].component_template;
 
+    const mappings = componentTemplate.template.mappings;
+    const componentTemplateChanged =
+      isDocValueOnlyNumericChanged || isDocValueOnlyOtherChanged || isSyntheticSourceOptInChanged;
+
+    let mappingsProperties = componentTemplate.template.mappings?.properties;
+    if (isDocValueOnlyNumericChanged || isDocValueOnlyOtherChanged) {
+      forEachMappings(mappings?.properties ?? {}, (mappingProp, name) =>
+        applyDocOnlyValueToMapping(
+          mappingProp,
+          name,
+          featureMapEntry,
+          isDocValueOnlyNumericChanged,
+          isDocValueOnlyOtherChanged
+        )
+      );
+
+      const templateProperties = (templateMappings[componentTemplateName] ?? {}).properties ?? {};
+      // merge package spec mappings with generated mappings, so that index:false from package spec is not overwritten
+      mappingsProperties = merge(templateProperties, mappings?.properties ?? {});
+    }
+
+    let sourceModeSettings = {};
+
     if (isSyntheticSourceOptInChanged) {
+      sourceModeSettings = {
+        _source: {
+          ...(featureMapEntry.features.synthetic_source ? { mode: 'synthetic' } : {}),
+        },
+      };
+    }
+
+    if (componentTemplateChanged) {
       const body = {
         template: {
           ...componentTemplate.template,
           mappings: {
-            ...componentTemplate.template.mappings,
+            ...mappings,
-            _source: {
+            properties: mappingsProperties ?? {},
-              ...(featureMapEntry.features.synthetic_source ? { mode: 'synthetic' } : {}),
+            ...sourceModeSettings,
-            },
           },
         },
       };
 
       await esClient.cluster.putComponentTemplate({
         name: componentTemplateName,
-        // @ts-expect-error - TODO: Remove when ES client typings include support for synthetic source
         body,
       });
     }

x-pack/plugins/fleet/server/types/models/package_policy.ts

@@ -85,6 +85,8 @@ const ExperimentalDataStreamFeatures = schema.arrayOf(
     features: schema.object({
       synthetic_source: schema.boolean(),
       tsdb: schema.boolean(),
+      doc_value_only_numeric: schema.boolean(),
+      doc_value_only_other: schema.boolean(),
     }),
   })
 );
@@ -125,14 +127,7 @@ const CreatePackagePolicyProps = {
       name: schema.string(),
       title: schema.maybe(schema.string()),
       version: schema.string(),
-      experimental_data_stream_features: schema.maybe(
+      experimental_data_stream_features: schema.maybe(ExperimentalDataStreamFeatures),
-        schema.arrayOf(
-          schema.object({
-            data_stream: schema.string(),
-            features: schema.object({ synthetic_source: schema.boolean(), tsdb: schema.boolean() }),
-          })
-        )
-      ),
     })
   ),
   // Deprecated TODO create remove issue