[Fleet] Add overrides to package policies update endpoint (#181453)

Closes https://github.com/elastic/kibana/issues/177323 ## Summary Allow to override `inputs` in `PUT package_policies/:id` endpoint. This functionality will be useful for support and troubleshooting purposes, but it shouldn't be used in place of normal updates to the policies. - `inputs` parameters are saved in package policy SO `overrides` field and then [merged](https://github.com/elastic/kibana/pull/181453/files#diff-ee6c1fe752205768ab54e6107a869041bcbb6130a0c982fa169bf5aba570a30eR84) to the full agent policy - `compiled_streams` and `compiled_inputs` are not allowed Example with an Nginx policy: ``` PUT kbn:/api/fleet/package_policies/d4fd9578-534f-4e1a-bc75-dfbd4ff0aa14 { "overrides": { "inputs": { "logfile-system-d4fd9578-534f-4e1a-bc75-dfbd4ff0aa14": { "log_level": "debug" } } } } ``` Result in full agent policy: ![Screenshot 2024-05-02 at 10 24 08](3772460b-de29-4679-90e6-d9fdb0d6d67d) ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
2025-04-24 17:59:23 -04:00 · 2024-05-02 13:14:47 +02:00 · 2024-05-02 13:14:47 +02:00 · 2ecda69e64
commit 2ecda69e64
parent 95384b4f9a
17 changed files with 402 additions and 36 deletions
--- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts
+++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts
@ -185,7 +185,7 @@ export const HASH_TO_VERSION_MAP = {
  'ingest-agent-policies|0fd93cd11c019b118e93a9157c22057b': '10.1.0',
  'ingest-download-sources|0b0f6828e59805bd07a650d80817c342': '10.0.0',
  'ingest-outputs|b1237f7fdc0967709e75d65d208ace05': '10.6.0',
-  'ingest-package-policies|a1a074bad36e68d54f98d2158d60f879': '10.0.0',
+  'ingest-package-policies|ca63c4c5a946704f045803a6b975dbc6': '10.9.0',
  'inventory-view|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
  'kql-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
  'legacy-url-alias|0750774cf16475f88f2361e99cc5c8f0': '8.2.0',
--- a/packages/kbn-check-mappings-update-cli/current_fields.json
+++ b/packages/kbn-check-mappings-update-cli/current_fields.json
@ -593,6 +593,7 @@
    "is_managed",
    "name",
    "namespace",
+    "overrides",
    "package",
    "package.name",
    "package.title",
--- a/packages/kbn-check-mappings-update-cli/current_mappings.json
+++ b/packages/kbn-check-mappings-update-cli/current_mappings.json
@ -1991,6 +1991,10 @@
      "namespace": {
        "type": "keyword"
      },
+      "overrides": {
+        "index": false,
+        "type": "flattened"
+      },
      "package": {
        "properties": {
          "name": {
--- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
+++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@ -111,7 +111,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
        "ingest-agent-policies": "d2ee0bf36a512c2ac744b0def1c822b7880f1f83",
        "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
        "ingest-outputs": "daafff49255ab700e07491376fe89f04fc998b91",
-        "ingest-package-policies": "38d7545c1e20f28d35591f97500d2e0c16ba27b0",
+        "ingest-package-policies": "d63e091b2b3cf2eecaa46ae2533bdd5214a983fc",
        "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797",
        "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",
        "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad",
--- a/x-pack/plugins/fleet/common/openapi/bundled.json
+++ b/x-pack/plugins/fleet/common/openapi/bundled.json
@ -7454,6 +7454,9 @@
          },
          "description": {
            "type": "string"
+          },
+          "overrides": {
+            "type": "object"
          }
        },
        "required": [
@ -8164,6 +8167,16 @@
              }
            }
          },
+          "overrides": {
+            "type": "object",
+            "properties": {
+              "inputs": {
+                "type": "object"
+              }
+            },
+            "description": "Override settings that are defined in the package policy. The override option should be used only in unusual circumstances and not as a routine procedure.",
+            "nullable": true
+          },
          "force": {
            "type": "boolean",
            "description": "Force package policy creation even if package is not verified, or if the agent policy is managed."
--- a/x-pack/plugins/fleet/common/openapi/bundled.yaml
+++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml
@ -4783,6 +4783,8 @@ components:
          type: string
        description:
          type: string
+        overrides:
+          type: object
      required:
        - inputs
        - name
@ -5287,6 +5289,16 @@ components:
                      description: >-
                        Stream level variable (see integration documentation for
                        more information)
+        overrides:
+          type: object
+          properties:
+            inputs:
+              type: object
+          description: >-
+            Override settings that are defined in the package policy. The
+            override option should be used only in unusual circumstances and not
+            as a routine procedure.
+          nullable: true
        force:
          type: boolean
          description: >-
--- a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml
+++ b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml
@ -50,6 +50,8 @@ properties:
    type: string
  description:
    type: string
+  overrides:
+    type: object
 required:
  - inputs
  - name
--- a/x-pack/plugins/fleet/common/openapi/components/schemas/package_policy_request.yaml
+++ b/x-pack/plugins/fleet/common/openapi/components/schemas/package_policy_request.yaml
@ -74,6 +74,13 @@ properties:
              vars:
                type: object
                description: Stream level variable (see integration documentation for more information)
+  overrides:
+    type: object
+    properties:
+      inputs:
+        type: object
+    description: Override settings that are defined in the package policy. The override option should be used only in unusual circumstances and not as a routine procedure.
+    nullable: true
  force:
    type: boolean
    description: Force package policy creation even if package is not verified, or if the agent policy is managed.
--- a/x-pack/plugins/fleet/common/types/models/package_policy.ts
+++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts
@ -86,6 +86,7 @@ export interface NewPackagePolicy {
      cluster?: string[];
    };
  };
+  overrides?: { inputs?: { [key: string]: any } } | null;
 }

 export interface UpdatePackagePolicy extends NewPackagePolicy {
--- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts
+++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts
@ -34,6 +34,7 @@ import type {
  PackagePolicy,
  DeleteOnePackagePolicyRequestSchema,
  BulkGetPackagePoliciesRequestSchema,
+  UpdatePackagePolicyRequestBodySchema,
 } from '../../types';
 import type {
  PostDeletePackagePoliciesResponse,
@ -56,6 +57,8 @@ import {

 import type { SimplifiedPackagePolicy } from '../../../common/services/simplified_package_policy_helper';

+import { isSimplifiedCreatePackagePolicyRequest, removeFieldsFromInputSchema } from './utils';
+
 export const isNotNull = <T>(value: T | null): value is T => value !== null;

 export const getPackagePoliciesHandler: FleetRequestHandler<
@ -214,17 +217,6 @@ export const getOrphanedPackagePolicies: RequestHandler<undefined, undefined> =
  }
 };

-function isSimplifiedCreatePackagePolicyRequest(
-  body: Omit<TypeOf<typeof CreatePackagePolicyRequestSchema.body>, 'force' | 'package'>
-): body is SimplifiedPackagePolicy {
-  // If `inputs` is not defined or if it's a non-array, the request body is using the new simplified API
-  if (body.inputs && Array.isArray(body.inputs)) {
-    return false;
-  }
-
-  return true;
-}
-
 export const createPackagePolicyHandler: FleetRequestHandler<
  undefined,
  TypeOf<typeof CreatePackagePolicyRequestSchema.query>,
@ -353,31 +345,30 @@ export const updatePackagePolicyHandler: FleetRequestHandler<
        { experimental_data_stream_features: pkg.experimental_data_stream_features }
      );
    } else {
-      // removed fields not recognized by schema
-      const packagePolicyInputs = packagePolicy.inputs.map((input) => {
-        const newInput = {
-          ...input,
-          streams: input.streams.map((stream) => {
-            const newStream = { ...stream };
-            delete newStream.compiled_stream;
-            return newStream;
-          }),
-        };
-        delete newInput.compiled_input;
-        return newInput;
-      });
+      const { overrides, ...restOfBody } = body as TypeOf<
+        typeof UpdatePackagePolicyRequestBodySchema
+      >;
+      const packagePolicyInputs = removeFieldsFromInputSchema(packagePolicy.inputs);
+
      // listing down accepted properties, because loaded packagePolicy contains some that are not accepted in update
      newData = {
-        ...body,
-        name: body.name ?? packagePolicy.name,
-        description: body.description ?? packagePolicy.description,
-        namespace: body.namespace ?? packagePolicy?.namespace,
-        policy_id: body.policy_id ?? packagePolicy.policy_id,
-        enabled: 'enabled' in body ? body.enabled ?? packagePolicy.enabled : packagePolicy.enabled,
+        ...restOfBody,
+        name: restOfBody.name ?? packagePolicy.name,
+        description: restOfBody.description ?? packagePolicy.description,
+        namespace: restOfBody.namespace ?? packagePolicy?.namespace,
+        policy_id: restOfBody.policy_id ?? packagePolicy.policy_id,
+        enabled:
+          'enabled' in restOfBody
+            ? restOfBody.enabled ?? packagePolicy.enabled
+            : packagePolicy.enabled,
        package: pkg ?? packagePolicy.package,
-        inputs: body.inputs ?? packagePolicyInputs,
-        vars: body.vars ?? packagePolicy.vars,
+        inputs: restOfBody.inputs ?? packagePolicyInputs,
+        vars: restOfBody.vars ?? packagePolicy.vars,
      } as NewPackagePolicy;
+
+      if (overrides) {
+        newData.overrides = overrides;
+      }
    }
    const updatedPackagePolicy = await packagePolicyService.update(
      soClient,
--- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts
+++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts
@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { TypeOf } from '@kbn/config-schema';
+
+import type { CreatePackagePolicyRequestSchema, PackagePolicyInput } from '../../../types';
+
+import type { SimplifiedPackagePolicy } from '../../../../common/services/simplified_package_policy_helper';
+
+export function isSimplifiedCreatePackagePolicyRequest(
+  body: Omit<TypeOf<typeof CreatePackagePolicyRequestSchema.body>, 'force' | 'package'>
+): body is SimplifiedPackagePolicy {
+  // If `inputs` is not defined or if it's a non-array, the request body is using the new simplified API
+  if (body.inputs && Array.isArray(body.inputs)) {
+    return false;
+  }
+
+  return true;
+}
+
+export function removeFieldsFromInputSchema(
+  packagePolicyInputs: PackagePolicyInput[]
+): PackagePolicyInput[] {
+  // removed fields not recognized by schema
+  return packagePolicyInputs.map((input) => {
+    const newInput = {
+      ...input,
+      streams: input.streams.map((stream) => {
+        const newStream = { ...stream };
+        delete newStream.compiled_stream;
+        return newStream;
+      }),
+    };
+    delete newInput.compiled_input;
+    return newInput;
+  });
+}
--- a/x-pack/plugins/fleet/server/saved_objects/index.ts
+++ b/x-pack/plugins/fleet/server/saved_objects/index.ts
@ -428,6 +428,7 @@ export const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
          properties: {},
        },
        secret_references: { properties: { id: { type: 'keyword' } } },
+        overrides: { type: 'flattened', index: false },
        revision: { type: 'integer' },
        updated_at: { type: 'date' },
        updated_by: { type: 'keyword' },
@ -513,6 +514,16 @@ export const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
          },
        ],
      },
+      '9': {
+        changes: [
+          {
+            type: 'mappings_addition',
+            addedMappings: {
+              overrides: { type: 'flattened', index: false },
+            },
+          },
+        ],
+      },
    },
    migrations: {
      '7.10.0': migratePackagePolicyToV7100,
--- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts
+++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts
@ -548,4 +548,193 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => {
      },
    ]);
  });
+
+  it('returns agent inputs merged with overrides from package policies if available for that input', async () => {
+    expect(
+      await storedPackagePoliciesToAgentInputs(
+        [
+          {
+            ...mockPackagePolicy,
+            inputs: [
+              {
+                ...mockInput,
+              },
+            ],
+            namespace: '',
+            overrides: {
+              inputs: {
+                'test-logs-some-uuid': {
+                  log_level: 'debug',
+                },
+              },
+            },
+          },
+        ],
+        packageInfoCache,
+        'default',
+        'agentpolicyspace'
+      )
+    ).toEqual([
+      {
+        id: 'test-logs-some-uuid',
+        name: 'mock_package-policy',
+        package_policy_id: 'some-uuid',
+        revision: 1,
+        type: 'test-logs',
+        data_stream: { namespace: 'agentpolicyspace' },
+        use_output: 'default',
+        log_level: 'debug',
+        streams: [
+          {
+            id: 'test-logs-foo',
+            data_stream: { dataset: 'foo', type: 'logs' },
+            fooKey: 'fooValue1',
+            fooKey2: ['fooValue2'],
+          },
+          {
+            data_stream: {
+              dataset: 'bar',
+              type: 'logs',
+            },
+            id: 'test-logs-bar',
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('returns agent inputs merged with overrides based on passed input id', async () => {
+    expect(
+      await storedPackagePoliciesToAgentInputs(
+        [
+          {
+            ...mockPackagePolicy,
+            package: {
+              name: 'mock_package',
+              title: 'Mock package',
+              version: '0.0.0',
+            },
+            inputs: [mockInput, mockInput2],
+            namespace: '',
+            overrides: {
+              inputs: {
+                'test-logs-some-uuid': {
+                  log_level: 'debug',
+                },
+              },
+            },
+          },
+        ],
+        packageInfoCache
+      )
+    ).toEqual([
+      {
+        id: 'test-logs-some-uuid',
+        log_level: 'debug',
+        data_stream: {
+          namespace: 'default',
+        },
+        meta: {
+          package: {
+            name: 'mock_package',
+            version: '0.0.0',
+          },
+        },
+        name: 'mock_package-policy',
+        package_policy_id: 'some-uuid',
+        revision: 1,
+        streams: [
+          {
+            data_stream: {
+              dataset: 'foo',
+              type: 'logs',
+            },
+            fooKey: 'fooValue1',
+            fooKey2: ['fooValue2'],
+            id: 'test-logs-foo',
+          },
+          {
+            data_stream: {
+              dataset: 'bar',
+              type: 'logs',
+            },
+            id: 'test-logs-bar',
+          },
+        ],
+        type: 'test-logs',
+        use_output: 'default',
+      },
+      {
+        data_stream: {
+          namespace: 'default',
+        },
+        id: 'test-metrics-some-template-some-uuid',
+        meta: {
+          package: {
+            name: 'mock_package',
+            version: '0.0.0',
+          },
+        },
+        name: 'mock_package-policy',
+        package_policy_id: 'some-uuid',
+        revision: 1,
+        streams: [
+          {
+            data_stream: {
+              dataset: 'foo',
+              type: 'metrics',
+            },
+            fooKey: 'fooValue1',
+            fooKey2: ['fooValue2'],
+            id: 'test-metrics-foo',
+          },
+        ],
+        type: 'test-metrics',
+        use_output: 'default',
+      },
+    ]);
+  });
+
+  it('returns unchanged agent inputs if overrides are empty', async () => {
+    expect(
+      await storedPackagePoliciesToAgentInputs(
+        [
+          {
+            ...mockPackagePolicy,
+            inputs: [
+              {
+                ...mockInput,
+                streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }],
+              },
+            ],
+            namespace: '',
+            overrides: {
+              inputs: {},
+            },
+          },
+        ],
+        packageInfoCache,
+        'default',
+        'agentpolicyspace'
+      )
+    ).toEqual([
+      {
+        id: 'test-logs-some-uuid',
+        name: 'mock_package-policy',
+        package_policy_id: 'some-uuid',
+        revision: 1,
+        type: 'test-logs',
+        data_stream: { namespace: 'agentpolicyspace' },
+        use_output: 'default',
+        streams: [
+          {
+            id: 'test-logs-foo',
+            data_stream: { dataset: 'foo', type: 'logs' },
+            fooKey: 'fooValue1',
+            fooKey2: ['fooValue2'],
+          },
+        ],
+      },
+    ]);
+  });
 });
--- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts
+++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts
@ -5,6 +5,7 @@
 * 2.0.
 */
 import { merge } from 'lodash';
+import deepMerge from 'deepmerge';

 import { isPackageLimited } from '../../../common/services';
 import type {
@ -71,7 +72,6 @@ export const storedPackagePolicyToAgentInputs = (
        return acc;
      }, {} as Record<string, unknown>)
    );
-
    if (packagePolicy.package) {
      fullInput.meta = {
        package: {
@ -80,11 +80,29 @@ export const storedPackagePolicyToAgentInputs = (
        },
      };
    }
-    fullInputs.push(fullInput);
+
+    const fullInputWithOverrides = mergeInputsOverrides(packagePolicy, fullInput);
+    fullInputs.push(fullInputWithOverrides);
  });
  return fullInputs;
 };

+export const mergeInputsOverrides = (
+  packagePolicy: PackagePolicy,
+  fullInput: FullAgentPolicyInput
+) => {
+  // check if there are inputs overrides and merge them
+  if (packagePolicy?.overrides?.inputs) {
+    const overrideInputs = packagePolicy.overrides.inputs;
+    const keys = Object.keys(overrideInputs);
+
+    if (keys.length > 0 && fullInput.id === keys[0]) {
+      return deepMerge<FullAgentPolicyInput>(fullInput, overrideInputs[keys[0]]);
+    }
+  }
+  return fullInput;
+};
+
 export const getFullInputStreams = (
  input: PackagePolicyInput,
  allStreamEnabled: boolean = false
--- a/x-pack/plugins/fleet/server/types/models/package_policy.ts
+++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts
@ -111,6 +111,25 @@ const PackagePolicyBaseSchema = {
  output_id: schema.maybe(schema.string()),
  inputs: schema.arrayOf(schema.object(PackagePolicyInputsSchema)),
  vars: schema.maybe(ConfigRecordSchema),
+  overrides: schema.maybe(
+    schema.nullable(
+      schema.object({
+        inputs: schema.maybe(
+          schema.recordOf(schema.string(), schema.any(), {
+            validate: (val) => {
+              if (
+                Object.keys(val).some(
+                  (key) => key.match(/^compiled_inputs(\.)?/) || key.match(/^compiled_stream(\.)?/)
+                )
+              ) {
+                return 'Overrides of compiled_inputs and compiled_stream are not allowed';
+              }
+            },
+          })
+        ),
+      })
+    )
+  ),
 };

 export const NewPackagePolicySchema = schema.object({
--- a/x-pack/plugins/fleet/server/types/so_attributes.ts
+++ b/x-pack/plugins/fleet/server/types/so_attributes.ts
@ -131,6 +131,7 @@ export interface PackagePolicySOAttributes {
    };
  };
  agents?: number;
+  overrides?: any | null;
 }

 interface OutputSoBaseAttributes {
--- a/x-pack/test/fleet_api_integration/apis/package_policy/update.ts
+++ b/x-pack/test/fleet_api_integration/apis/package_policy/update.ts
@ -521,6 +521,62 @@ export default function (providerContext: FtrProviderContext) {
        .expect(400);
    });

+    it('should allow to override inputs', async function () {
+      await supertest
+        .put(`/api/fleet/package_policies/${endpointPackagePolicyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          overrides: {
+            inputs: {
+              'policy-id': {
+                log_level: 'debug',
+              },
+            },
+          },
+        })
+        .expect(200);
+    });
+
+    it('should not allow to override compiled_streams', async function () {
+      await supertest
+        .put(`/api/fleet/package_policies/${endpointPackagePolicyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          overrides: {
+            inputs: {
+              compiled_streams: {},
+            },
+          },
+        })
+        .expect(400);
+    });
+
+    it('should not allow to override compiled_inputs', async function () {
+      await supertest
+        .put(`/api/fleet/package_policies/${endpointPackagePolicyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          overrides: {
+            inputs: {
+              compiled_inputs: {},
+            },
+          },
+        })
+        .expect(400);
+    });
+
+    it('should not allow to override properties other than inputs', async function () {
+      await supertest
+        .put(`/api/fleet/package_policies/${endpointPackagePolicyId}`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({
+          overrides: {
+            name: 'test',
+          },
+        })
+        .expect(400);
+    });
+
    describe('Simplified package policy', async () => {
      it('should work with valid values', async function () {
        await supertest