[Fleet] Add "Keep Policies up to Date" functionality for integrations (#112702)

* Add initial implementation for keep policies up to date functionality

* Upgrade package policies during preconfiguration check

* Only show keep policies up to date switch for default/auto-update packages

* Fix type error

* Fixup setup policy upgrade logic

* Add migration for keep policies up to date flag

* Move setup package policy logic to new module + add tests

* Update snapshots to include keepPoliciesUpToDate field

* Fix type errors

* Fix some CI failures

* Fix more type errors

* Fix type error in isolation test

* Fix package fixtures types

* Fix another type error

* Move policy upgrade error swallowing up a level in setup

* Address PR feedback

- Move keep policies up to date switch to separate component
- Use PACKAGE_POLICY_SAVED_OBJECT_TYPE instead of magic string

* Fix overwriting user values when upgrading

Fixes #113731

* Add test package

* Fix tests for overridePackageVars

* Address PR feedback

- Don't index keep_policies_up_to_date field
- Use SO_SEARCH_LIMIT constant instead of magic number

* Make toast translation usage more consistent

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/fleet/common/services/routes.ts

@@ -59,6 +59,10 @@ export const epmRouteService = {
   getRemovePath: (pkgkey: string) => {
     return EPM_API_ROUTES.DELETE_PATTERN.replace('{pkgkey}', pkgkey).replace(/\/$/, ''); // trim trailing slash
   },
+
+  getUpdatePath: (pkgkey: string) => {
+    return EPM_API_ROUTES.INFO_PATTERN.replace('{pkgkey}', pkgkey);
+  },
 };
 
 export const packagePolicyRouteService = {

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -351,6 +351,7 @@ export interface EpmPackageAdditions {
   assets: AssetsGroupedByServiceByType;
   removable?: boolean;
   notice?: string;
+  keepPoliciesUpToDate?: boolean;
 }
 
 type Merge<FirstType, SecondType> = Omit<FirstType, Extract<keyof FirstType, keyof SecondType>> &
@@ -391,6 +392,7 @@ export interface Installation extends SavedObjectAttributes {
   install_version: string;
   install_started_at: string;
   install_source: InstallSource;
+  keep_policies_up_to_date: boolean;
 }
 
 export interface PackageUsageStats {

x-pack/plugins/fleet/common/types/rest_spec/epm.ts

@@ -57,6 +57,19 @@ export interface GetInfoResponse {
   response: PackageInfo;
 }
 
+export interface UpdatePackageRequest {
+  params: {
+    pkgkey: string;
+  };
+  body: {
+    keepPoliciesUpToDate?: boolean;
+  };
+}
+
+export interface UpdatePackageResponse {
+  response: PackageInfo;
+}
+
 export interface GetStatsRequest {
   params: {
     pkgname: string;

x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_card.stories.tsx

@@ -66,6 +66,7 @@ export const Installed = ({ width, ...props }: Args) => {
       install_status: 'installed',
       install_source: 'registry',
       install_started_at: '2020-01-01T00:00:00.000Z',
+      keep_policies_up_to_date: false,
     },
     references: [],
   };

x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.stories.tsx

@@ -44,6 +44,7 @@ const savedObject: SavedObject<Installation> = {
     install_status: 'installed',
     install_source: 'registry',
     install_started_at: '2020-01-01T00:00:00.000Z',
+    keep_policies_up_to_date: false,
   },
   references: [],
 };

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/index.tsx

@@ -7,3 +7,4 @@
 export { UpdateIcon } from './update_icon';
 export { IntegrationAgentPolicyCount } from './integration_agent_policy_count';
 export { IconPanel, LoadingIconPanel } from './icon_panel';
+export { KeepPoliciesUpToDateSwitch } from './keep_policies_up_to_date_switch';

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/keep_policies_up_to_date_switch.tsx

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { EuiSwitch, EuiSpacer, EuiText, EuiFlexGroup, EuiFlexItem, EuiIcon } from '@elastic/eui';
+
+interface Props {
+  checked: boolean;
+  onChange: () => void;
+}
+
+export const KeepPoliciesUpToDateSwitch: React.FunctionComponent<Props> = ({
+  checked,
+  onChange,
+}) => (
+  <>
+    <EuiSwitch
+      label={i18n.translate(
+        'xpack.fleet.integrations.settings.keepIntegrationPoliciesUpToDateLabel',
+        { defaultMessage: 'Keep integration policies up to date automatically' }
+      )}
+      checked={checked}
+      onChange={onChange}
+    />
+    <EuiSpacer size="s" />
+    <EuiText color="subdued" size="xs">
+      <EuiFlexGroup alignItems="center" gutterSize="none">
+        <EuiFlexItem grow={false}>
+          <EuiIcon type="iInCircle" />
+        </EuiFlexItem>
+        <EuiFlexItem grow={false}>
+          <FormattedMessage
+            id="xpack.fleet.integrations.settings.keepIntegrationPoliciesUpToDateDescription"
+            defaultMessage="When enabled, Fleet will attempt to upgrade and deploy integration policies automatically"
+          />
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    </EuiText>
+  </>
+);

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx

@@ -5,10 +5,11 @@
  * 2.0.
  */
 
-import React, { memo, useEffect, useMemo, useState } from 'react';
+import React, { memo, useCallback, useEffect, useMemo, useState } from 'react';
 import styled from 'styled-components';
 import { FormattedMessage } from '@kbn/i18n/react';
 import semverLt from 'semver/functions/lt';
+import { uniq } from 'lodash';
 
 import {
   EuiCallOut,
@@ -29,8 +30,16 @@ import {
   useGetPackageInstallStatus,
   useLink,
   sendUpgradePackagePolicyDryRun,
+  sendUpdatePackage,
+  useStartServices,
 } from '../../../../../hooks';
-import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../../../constants';
+import {
+  PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+  AUTO_UPDATE_PACKAGES,
+  DEFAULT_PACKAGES,
+} from '../../../../../constants';
+
+import { KeepPoliciesUpToDateSwitch } from '../components';
 
 import { InstallButton } from './install_button';
 import { UpdateButton } from './update_button';
@@ -85,7 +94,7 @@ interface Props {
 }
 
 export const SettingsPage: React.FC<Props> = memo(({ packageInfo }: Props) => {
-  const { name, title, removable, latestVersion, version } = packageInfo;
+  const { name, title, removable, latestVersion, version, keepPoliciesUpToDate } = packageInfo;
   const [dryRunData, setDryRunData] = useState<UpgradePackagePolicyDryRunResponse | null>();
   const [isUpgradingPackagePolicies, setIsUpgradingPackagePolicies] = useState<boolean>(false);
   const getPackageInstallStatus = useGetPackageInstallStatus();
@@ -95,6 +104,67 @@ export const SettingsPage: React.FC<Props> = memo(({ packageInfo }: Props) => {
     kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${name}`,
   });
 
+  const { notifications } = useStartServices();
+
+  const shouldShowKeepPoliciesUpToDateSwitch = useMemo(() => {
+    const packages = [...DEFAULT_PACKAGES, ...AUTO_UPDATE_PACKAGES];
+
+    const packageNames = uniq(packages.map((pkg) => pkg.name));
+
+    return packageNames.includes(name);
+  }, [name]);
+
+  const [keepPoliciesUpToDateSwitchValue, setKeepPoliciesUpToDateSwitchValue] = useState<boolean>(
+    keepPoliciesUpToDate ?? false
+  );
+
+  const handleKeepPoliciesUpToDateSwitchChange = useCallback(() => {
+    const saveKeepPoliciesUpToDate = async () => {
+      try {
+        setKeepPoliciesUpToDateSwitchValue((prev) => !prev);
+
+        await sendUpdatePackage(`${packageInfo.name}-${packageInfo.version}`, {
+          keepPoliciesUpToDate: !keepPoliciesUpToDateSwitchValue,
+        });
+
+        notifications.toasts.addSuccess({
+          title: i18n.translate('xpack.fleet.integrations.integrationSaved', {
+            defaultMessage: 'Integration settings saved',
+          }),
+          text: !keepPoliciesUpToDateSwitchValue
+            ? i18n.translate('xpack.fleet.integrations.keepPoliciesUpToDateEnabledSuccess', {
+                defaultMessage:
+                  'Fleet will automatically keep integration policies up to date for {title}',
+                values: { title },
+              })
+            : i18n.translate('xpack.fleet.integrations.keepPoliciesUpToDateDisabledSuccess', {
+                defaultMessage:
+                  'Fleet will not automatically keep integration policies up to date for {title}',
+                values: { title },
+              }),
+        });
+      } catch (error) {
+        notifications.toasts.addError(error, {
+          title: i18n.translate('xpack.fleet.integrations.integrationSavedError', {
+            defaultMessage: 'Error saving integration settings',
+          }),
+          toastMessage: i18n.translate('xpack.fleet.integrations.keepPoliciesUpToDateError', {
+            defaultMessage: 'Error saving integration settings for {title}',
+            values: { title },
+          }),
+        });
+      }
+    };
+
+    saveKeepPoliciesUpToDate();
+  }, [
+    keepPoliciesUpToDateSwitchValue,
+    notifications.toasts,
+    packageInfo.name,
+    packageInfo.version,
+    title,
+  ]);
+
   const { status: installationStatus, version: installedVersion } = getPackageInstallStatus(name);
   const packageHasUsages = !!packagePoliciesData?.total;
 
@@ -199,6 +269,16 @@ export const SettingsPage: React.FC<Props> = memo(({ packageInfo }: Props) => {
                   </tr>
                 </tbody>
               </table>
+              {shouldShowKeepPoliciesUpToDateSwitch && (
+                <>
+                  <KeepPoliciesUpToDateSwitch
+                    checked={keepPoliciesUpToDateSwitchValue}
+                    onChange={handleKeepPoliciesUpToDateSwitchChange}
+                  />
+                  <EuiSpacer size="l" />
+                </>
+              )}
+
               {(updateAvailable || isUpgradingPackagePolicies) && (
                 <>
                   <UpdatesAvailableMsg latestVersion={latestVersion} />

x-pack/plugins/fleet/public/constants/index.ts

@@ -19,6 +19,9 @@ export {
   // Fleet Server index
   AGENTS_INDEX,
   ENROLLMENT_API_KEYS_INDEX,
+  // Preconfiguration
+  AUTO_UPDATE_PACKAGES,
+  DEFAULT_PACKAGES,
 } from '../../common/constants';
 
 export * from './page_paths';

x-pack/plugins/fleet/public/hooks/use_request/epm.ts

@@ -17,6 +17,8 @@ import type {
   GetInfoResponse,
   InstallPackageResponse,
   DeletePackageResponse,
+  UpdatePackageRequest,
+  UpdatePackageResponse,
 } from '../../types';
 import type { GetStatsResponse } from '../../../common';
 
@@ -113,3 +115,11 @@ export const sendRemovePackage = (pkgkey: string) => {
     method: 'delete',
   });
 };
+
+export const sendUpdatePackage = (pkgkey: string, body: UpdatePackageRequest['body']) => {
+  return sendRequest<UpdatePackageResponse>({
+    path: epmRouteService.getUpdatePath(pkgkey),
+    method: 'put',
+    body,
+  });
+};

x-pack/plugins/fleet/public/types/index.ts

@@ -128,6 +128,8 @@ export {
   Installable,
   RegistryRelease,
   PackageSpecCategory,
+  UpdatePackageRequest,
+  UpdatePackageResponse,
 } from '../../common';
 
 export * from './intra_app_route_state';

x-pack/plugins/fleet/server/constants/index.ts

@@ -58,6 +58,7 @@ export {
   // Preconfiguration
   PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE,
   PRECONFIGURATION_LATEST_KEYWORD,
+  AUTO_UPDATE_PACKAGES,
 } from '../../common';
 
 export {

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -22,6 +22,7 @@ import type {
   BulkInstallPackagesResponse,
   IBulkInstallPackageHTTPError,
   GetStatsResponse,
+  UpdatePackageResponse,
 } from '../../../common';
 import type {
   GetCategoriesRequestSchema,
@@ -33,6 +34,7 @@ import type {
   DeletePackageRequestSchema,
   BulkUpgradePackagesFromRegistryRequestSchema,
   GetStatsRequestSchema,
+  UpdatePackageRequestSchema,
 } from '../../types';
 import {
   bulkInstallPackages,
@@ -53,6 +55,7 @@ import { licenseService } from '../../services';
 import { getArchiveEntry } from '../../services/epm/archive/cache';
 import { getAsset } from '../../services/epm/archive/storage';
 import { getPackageUsageStats } from '../../services/epm/packages/get';
+import { updatePackage } from '../../services/epm/packages/update';
 
 export const getCategoriesHandler: RequestHandler<
   undefined,
@@ -201,6 +204,28 @@ export const getInfoHandler: RequestHandler<TypeOf<typeof GetInfoRequestSchema.p
   }
 };
 
+export const updatePackageHandler: RequestHandler<
+  TypeOf<typeof UpdatePackageRequestSchema.params>,
+  unknown,
+  TypeOf<typeof UpdatePackageRequestSchema.body>
+> = async (context, request, response) => {
+  try {
+    const { pkgkey } = request.params;
+    const savedObjectsClient = context.core.savedObjects.client;
+
+    const { pkgName } = splitPkgKey(pkgkey);
+
+    const res = await updatePackage({ savedObjectsClient, pkgName, ...request.body });
+    const body: UpdatePackageResponse = {
+      response: res,
+    };
+
+    return response.ok({ body });
+  } catch (error) {
+    return defaultIngestErrorHandler({ error, response });
+  }
+};
+
 export const getStatsHandler: RequestHandler<TypeOf<typeof GetStatsRequestSchema.params>> = async (
   context,
   request,

x-pack/plugins/fleet/server/routes/epm/index.ts

@@ -18,6 +18,7 @@ import {
   DeletePackageRequestSchema,
   BulkUpgradePackagesFromRegistryRequestSchema,
   GetStatsRequestSchema,
+  UpdatePackageRequestSchema,
 } from '../../types';
 
 import {
@@ -31,6 +32,7 @@ import {
   deletePackageHandler,
   bulkInstallPackagesFromRegistryHandler,
   getStatsHandler,
+  updatePackageHandler,
 } from './handlers';
 
 const MAX_FILE_SIZE_BYTES = 104857600; // 100MB
@@ -90,6 +92,15 @@ export const registerRoutes = (router: IRouter) => {
     getInfoHandler
   );
 
+  router.put(
+    {
+      path: EPM_API_ROUTES.INFO_PATTERN,
+      validate: UpdatePackageRequestSchema,
+      options: { tags: [`access:${PLUGIN_ID}-all`] },
+    },
+    updatePackageHandler
+  );
+
   router.post(
     {
       path: EPM_API_ROUTES.INSTALL_FROM_REGISTRY_PATTERN,

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -44,6 +44,7 @@ import {
 } from './migrations/to_v7_13_0';
 import { migratePackagePolicyToV7140, migrateInstallationToV7140 } from './migrations/to_v7_14_0';
 import { migratePackagePolicyToV7150 } from './migrations/to_v7_15_0';
+import { migrateInstallationToV7160 } from './migrations/to_v7_16_0';
 
 /*
  * Saved object types and mappings
@@ -298,6 +299,7 @@ const getSavedObjectTypes = (
         version: { type: 'keyword' },
         internal: { type: 'boolean' },
         removable: { type: 'boolean' },
+        keep_policies_up_to_date: { type: 'boolean', index: false },
         es_index_patterns: {
           enabled: false,
           type: 'object',
@@ -332,6 +334,7 @@ const getSavedObjectTypes = (
     migrations: {
       '7.14.0': migrateInstallationToV7140,
       '7.14.1': migrateInstallationToV7140,
+      '7.16.0': migrateInstallationToV7160,
     },
   },
   [ASSETS_SAVED_OBJECT_TYPE]: {

x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_16_0.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectMigrationFn } from 'kibana/server';
+
+import type { Installation } from '../../../common';
+import { AUTO_UPDATE_PACKAGES, DEFAULT_PACKAGES } from '../../../common';
+
+export const migrateInstallationToV7160: SavedObjectMigrationFn<Installation, Installation> = (
+  installationDoc,
+  migrationContext
+) => {
+  const updatedInstallationDoc = installationDoc;
+
+  if (
+    [...AUTO_UPDATE_PACKAGES, ...DEFAULT_PACKAGES].some(
+      (pkg) => pkg.name === updatedInstallationDoc.attributes.name
+    )
+  ) {
+    updatedInstallationDoc.attributes.keep_policies_up_to_date = true;
+  }
+
+  return updatedInstallationDoc;
+};

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -7,7 +7,12 @@
 
 import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server';
 
-import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common';
+import {
+  MAX_TIME_COMPLETE_INSTALL,
+  ASSETS_SAVED_OBJECT_TYPE,
+  PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+  SO_SEARCH_LIMIT,
+} from '../../../../common';
 import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import type { AssetReference, Installation, InstallType } from '../../../types';
@@ -22,6 +27,8 @@ import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install
 import { saveArchiveEntries } from '../archive/storage';
 import { ConcurrentInstallOperationError } from '../../../errors';
 
+import { packagePolicyService } from '../..';
+
 import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install';
 import { deleteKibanaSavedObjectsAssets } from './remove';
 
@@ -192,11 +199,27 @@ export async function _installPackage({
     // update to newly installed version when all assets are successfully installed
     if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
 
-    await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
-      install_version: pkgVersion,
-      install_status: 'installed',
-      package_assets: packageAssetRefs,
-    });
+    const updatedPackage = await savedObjectsClient.update<Installation>(
+      PACKAGES_SAVED_OBJECT_TYPE,
+      pkgName,
+      {
+        install_version: pkgVersion,
+        install_status: 'installed',
+        package_assets: packageAssetRefs,
+      }
+    );
+
+    // If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its
+    // associated package policies after installation
+    if (updatedPackage.attributes.keep_policies_up_to_date) {
+      const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, {
+        page: 1,
+        perPage: SO_SEARCH_LIMIT,
+        kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`,
+      });
+
+      await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items);
+    }
 
     return [
       ...installedKibanaAssetsRefs,

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -137,6 +137,7 @@ export async function getPackageInfo(options: {
     assets: Registry.groupPathsByService(paths || []),
     removable: !isUnremovablePackage(pkgName),
     notice: Registry.getNoticePath(paths || []),
+    keepPoliciesUpToDate: savedObject?.attributes.keep_policies_up_to_date ?? false,
   };
   const updated = { ...packageInfo, ...additions };
 

x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts

@@ -28,6 +28,7 @@ const mockInstallation: SavedObject<Installation> = {
     install_version: '1.0.0',
     install_started_at: new Date().toISOString(),
     install_source: 'registry',
+    keep_policies_up_to_date: false,
   },
 };
 const mockInstallationUpdateFail: SavedObject<Installation> = {
@@ -46,6 +47,7 @@ const mockInstallationUpdateFail: SavedObject<Installation> = {
     install_version: '1.0.1',
     install_started_at: new Date().toISOString(),
     install_source: 'registry',
+    keep_policies_up_to_date: false,
   },
 };
 

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -457,6 +457,7 @@ export async function createInstallation(options: {
       install_status: 'installing',
       install_started_at: new Date().toISOString(),
       install_source: installSource,
+      keep_policies_up_to_date: false,
     },
     { id: pkgName, overwrite: true }
   );

x-pack/plugins/fleet/server/services/epm/packages/update.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectsClientContract } from 'kibana/server';
+import type { TypeOf } from '@kbn/config-schema';
+
+import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
+import type { Installation, UpdatePackageRequestSchema } from '../../../types';
+import { IngestManagerError } from '../../../errors';
+
+import { getInstallationObject, getPackageInfo } from './get';
+
+export async function updatePackage(
+  options: {
+    savedObjectsClient: SavedObjectsClientContract;
+    pkgName: string;
+    keepPoliciesUpToDate?: boolean;
+  } & TypeOf<typeof UpdatePackageRequestSchema.body>
+) {
+  const { savedObjectsClient, pkgName, keepPoliciesUpToDate } = options;
+  const installedPackage = await getInstallationObject({ savedObjectsClient, pkgName });
+
+  if (!installedPackage) {
+    throw new IngestManagerError(`package ${pkgName} is not installed`);
+  }
+
+  await savedObjectsClient.update<Installation>(PACKAGES_SAVED_OBJECT_TYPE, installedPackage.id, {
+    keep_policies_up_to_date: keepPoliciesUpToDate ?? false,
+  });
+
+  const packageInfo = await getPackageInfo({
+    savedObjectsClient,
+    pkgName,
+    pkgVersion: installedPackage.attributes.version,
+  });
+
+  return packageInfo;
+}

x-pack/plugins/fleet/server/services/managed_package_policies.test.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks';
+
+import { upgradeManagedPackagePolicies } from './managed_package_policies';
+import { packagePolicyService } from './package_policy';
+import { getPackageInfo } from './epm/packages';
+
+jest.mock('./package_policy');
+jest.mock('./epm/packages');
+jest.mock('./app_context', () => {
+  return {
+    ...jest.requireActual('./app_context'),
+    appContextService: {
+      getLogger: jest.fn(() => {
+        return { debug: jest.fn() };
+      }),
+    },
+  };
+});
+
+describe('managed package policies', () => {
+  afterEach(() => {
+    (packagePolicyService.get as jest.Mock).mockReset();
+    (getPackageInfo as jest.Mock).mockReset();
+  });
+
+  it('should not upgrade policies for non-managed package', async () => {
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    const soClient = savedObjectsClientMock.create();
+
+    (packagePolicyService.get as jest.Mock).mockImplementationOnce(
+      (savedObjectsClient: any, id: string) => {
+        return {
+          id,
+          inputs: {},
+          version: '',
+          revision: 1,
+          updated_at: '',
+          updated_by: '',
+          created_at: '',
+          created_by: '',
+          package: {
+            name: 'non-managed-package',
+            title: 'Non-Managed Package',
+            version: '0.0.1',
+          },
+        };
+      }
+    );
+
+    (getPackageInfo as jest.Mock).mockImplementationOnce(
+      ({ savedObjectsClient, pkgName, pkgVersion }) => ({
+        name: pkgName,
+        version: pkgVersion,
+        keepPoliciesUpToDate: false,
+      })
+    );
+
+    await upgradeManagedPackagePolicies(soClient, esClient, ['non-managed-package-id']);
+
+    expect(packagePolicyService.upgrade).not.toBeCalled();
+  });
+
+  it('should upgrade policies for managed package', async () => {
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    const soClient = savedObjectsClientMock.create();
+
+    (packagePolicyService.get as jest.Mock).mockImplementationOnce(
+      (savedObjectsClient: any, id: string) => {
+        return {
+          id,
+          inputs: {},
+          version: '',
+          revision: 1,
+          updated_at: '',
+          updated_by: '',
+          created_at: '',
+          created_by: '',
+          package: {
+            name: 'managed-package',
+            title: 'Managed Package',
+            version: '0.0.1',
+          },
+        };
+      }
+    );
+
+    (getPackageInfo as jest.Mock).mockImplementationOnce(
+      ({ savedObjectsClient, pkgName, pkgVersion }) => ({
+        name: pkgName,
+        version: pkgVersion,
+        keepPoliciesUpToDate: true,
+      })
+    );
+
+    await upgradeManagedPackagePolicies(soClient, esClient, ['managed-package-id']);
+
+    expect(packagePolicyService.upgrade).toBeCalledWith(soClient, esClient, ['managed-package-id']);
+  });
+});

x-pack/plugins/fleet/server/services/managed_package_policies.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
+
+import { AUTO_UPDATE_PACKAGES } from '../../common';
+
+import { appContextService } from './app_context';
+import { getPackageInfo } from './epm/packages';
+import { packagePolicyService } from './package_policy';
+
+/**
+ * Upgrade any package policies for packages installed through setup that are denoted as `AUTO_UPGRADE` packages
+ * or have the `keep_policies_up_to_date` flag set to `true`
+ */
+export const upgradeManagedPackagePolicies = async (
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  packagePolicyIds: string[]
+) => {
+  const policyIdsToUpgrade: string[] = [];
+
+  for (const packagePolicyId of packagePolicyIds) {
+    const packagePolicy = await packagePolicyService.get(soClient, packagePolicyId);
+
+    if (!packagePolicy || !packagePolicy.package) {
+      continue;
+    }
+
+    const packageInfo = await getPackageInfo({
+      savedObjectsClient: soClient,
+      pkgName: packagePolicy.package.name,
+      pkgVersion: packagePolicy.package.version,
+    });
+
+    const shouldUpgradePolicies =
+      AUTO_UPDATE_PACKAGES.some((pkg) => pkg.name === packageInfo.name) ||
+      packageInfo.keepPoliciesUpToDate;
+
+    if (shouldUpgradePolicies) {
+      policyIdsToUpgrade.push(packagePolicy.id);
+    }
+  }
+
+  if (policyIdsToUpgrade.length) {
+    appContextService
+      .getLogger()
+      .debug(
+        `Upgrading ${policyIdsToUpgrade.length} package policies: ${policyIdsToUpgrade.join(', ')}`
+      );
+
+    await packagePolicyService.upgrade(soClient, esClient, policyIdsToUpgrade);
+  }
+};

x-pack/plugins/fleet/server/services/package_policy.test.ts

@@ -1085,7 +1085,98 @@ describe('Package policy service', () => {
   });
 
   describe('overridePackageInputs', () => {
-    it('should override variable in base package policy', () => {
+    describe('when variable is already defined', () => {
+      it('preserves original variable value without overwriting', () => {
+        const basePackagePolicy: NewPackagePolicy = {
+          name: 'base-package-policy',
+          description: 'Base Package Policy',
+          namespace: 'default',
+          enabled: true,
+          policy_id: 'xxxx',
+          output_id: 'xxxx',
+          package: {
+            name: 'test-package',
+            title: 'Test Package',
+            version: '0.0.1',
+          },
+          inputs: [
+            {
+              type: 'logs',
+              policy_template: 'template_1',
+              enabled: true,
+              vars: {
+                path: {
+                  type: 'text',
+                  value: ['/var/log/logfile.log'],
+                },
+              },
+              streams: [],
+            },
+          ],
+        };
+
+        const packageInfo: PackageInfo = {
+          name: 'test-package',
+          description: 'Test Package',
+          title: 'Test Package',
+          version: '0.0.1',
+          latestVersion: '0.0.1',
+          release: 'experimental',
+          format_version: '1.0.0',
+          owner: { github: 'elastic/fleet' },
+          policy_templates: [
+            {
+              name: 'template_1',
+              title: 'Template 1',
+              description: 'Template 1',
+              inputs: [
+                {
+                  type: 'logs',
+                  title: 'Log',
+                  description: 'Log Input',
+                  vars: [
+                    {
+                      name: 'path',
+                      type: 'text',
+                    },
+                  ],
+                },
+              ],
+            },
+          ],
+          // @ts-ignore
+          assets: {},
+        };
+
+        const inputsOverride: NewPackagePolicyInput[] = [
+          {
+            type: 'logs',
+            enabled: true,
+            streams: [],
+            vars: {
+              path: {
+                type: 'text',
+                value: '/var/log/new-logfile.log',
+              },
+            },
+          },
+        ];
+
+        const result = overridePackageInputs(
+          basePackagePolicy,
+          packageInfo,
+          // TODO: Update this type assertion when the `InputsOverride` type is updated such
+          // that it no longer causes unresolvable type errors when used directly
+          inputsOverride as InputsOverride[],
+          false
+        );
+        expect(result.inputs[0]?.vars?.path.value).toEqual(['/var/log/logfile.log']);
+      });
+    });
+  });
+
+  describe('when variable is undefined in original object', () => {
+    it('adds the variable definition to the resulting object', () => {
       const basePackagePolicy: NewPackagePolicy = {
         name: 'base-package-policy',
         description: 'Base Package Policy',
@@ -1138,6 +1229,10 @@ describe('Package policy service', () => {
                     name: 'path',
                     type: 'text',
                   },
+                  {
+                    name: 'path_2',
+                    type: 'text',
+                  },
                 ],
               },
             ],
@@ -1157,6 +1252,10 @@ describe('Package policy service', () => {
               type: 'text',
               value: '/var/log/new-logfile.log',
             },
+            path_2: {
+              type: 'text',
+              value: '/var/log/custom.log',
+            },
           },
         },
       ];
@@ -1169,7 +1268,7 @@ describe('Package policy service', () => {
         inputsOverride as InputsOverride[],
         false
       );
-      expect(result.inputs[0]?.vars?.path.value).toBe('/var/log/new-logfile.log');
+      expect(result.inputs[0]?.vars?.path_2.value).toEqual('/var/log/custom.log');
     });
   });
 });

x-pack/plugins/fleet/server/services/package_policy.ts

@@ -980,7 +980,7 @@ export function overridePackageInputs(
         ({ name }) => name === input.policy_template
       );
 
-      // Ignore any policy template removes in the new package version
+      // Ignore any policy templates removed in the new package version
       if (!policyTemplate) {
         return false;
       }
@@ -1000,7 +1000,7 @@ export function overridePackageInputs(
 
     // If there's no corresponding input on the original package policy, just
     // take the override value from the new package as-is. This case typically
-    // occurs when inputs or package policies are added/removed between versions.
+    // occurs when inputs or package policy templates are added/removed between versions.
     if (originalInput === undefined) {
       inputs.push(override as NewPackagePolicyInput);
       continue;
@@ -1092,7 +1092,14 @@ function deepMergeVars(original: any, override: any): any {
 
   for (const { name, ...overrideVal } of overrideVars) {
     const originalVar = original.vars[name];
+
     result.vars[name] = { ...originalVar, ...overrideVal };
+
+    // Ensure that any value from the original object is persisted on the newly merged resulting object,
+    // even if we merge other data about the given variable
+    if (originalVar?.value) {
+      result.vars[name].value = originalVar.value;
+    }
   }
 
   return result;

x-pack/plugins/fleet/server/services/preconfiguration.test.ts

@@ -137,6 +137,7 @@ jest.mock('./package_policy', () => ({
   ...jest.requireActual('./package_policy'),
   packagePolicyService: {
     getByIDs: jest.fn().mockReturnValue([]),
+    listIds: jest.fn().mockReturnValue({ items: [] }),
     create(soClient: any, esClient: any, newPackagePolicy: NewPackagePolicy) {
       return {
         id: 'mocked',
@@ -144,6 +145,12 @@ jest.mock('./package_policy', () => ({
         ...newPackagePolicy,
       };
     },
+    get(soClient: any, id: string) {
+      return {
+        id: 'mocked',
+        version: 'mocked',
+      };
+    },
   },
 }));
 

x-pack/plugins/fleet/server/services/preconfiguration.ts

@@ -35,6 +35,7 @@ import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy';
 import type { InputsOverride } from './package_policy';
 import { overridePackageInputs } from './package_policy';
 import { appContextService } from './app_context';
+import { upgradeManagedPackagePolicies } from './managed_package_policies';
 import { outputService } from './output';
 
 interface PreconfigurationResult {
@@ -313,6 +314,17 @@ export async function ensurePreconfiguredPackagesAndPolicies(
     }
   }
 
+  try {
+    const fulfilledPolicyPackagePolicyIds = fulfilledPolicies.flatMap<string>(
+      ({ policy }) => policy?.package_policies as string[]
+    );
+
+    await upgradeManagedPackagePolicies(soClient, esClient, fulfilledPolicyPackagePolicyIds);
+    // Swallow errors that occur when upgrading
+  } catch (error) {
+    appContextService.getLogger().error(error);
+  }
+
   return {
     policies: fulfilledPolicies.map((p) =>
       p.policy

x-pack/plugins/fleet/server/types/rest_spec/epm.ts

@@ -35,6 +35,15 @@ export const GetInfoRequestSchema = {
   }),
 };
 
+export const UpdatePackageRequestSchema = {
+  params: schema.object({
+    pkgkey: schema.string(),
+  }),
+  body: schema.object({
+    keepPoliciesUpToDate: schema.boolean(),
+  }),
+};
+
 export const GetStatsRequestSchema = {
   params: schema.object({
     pkgName: schema.string(),

x-pack/plugins/fleet/storybook/context/fixtures/packages.ts

@@ -985,6 +985,7 @@ export const response: GetPackagesResponse['response'] = [
         install_status: 'installed',
         install_started_at: '2021-08-25T19:44:41.090Z',
         install_source: 'registry',
+        keep_policies_up_to_date: false,
       },
       references: [],
       coreMigrationVersion: '7.14.0',
@@ -1113,6 +1114,7 @@ export const response: GetPackagesResponse['response'] = [
         install_status: 'installed',
         install_started_at: '2021-08-25T19:44:37.078Z',
         install_source: 'registry',
+        keep_policies_up_to_date: false,
       },
       references: [],
       coreMigrationVersion: '7.14.0',
@@ -4268,6 +4270,7 @@ export const response: GetPackagesResponse['response'] = [
         install_status: 'installed',
         install_started_at: '2021-08-25T19:44:43.380Z',
         install_source: 'registry',
+        keep_policies_up_to_date: false,
       },
       references: [],
       coreMigrationVersion: '7.14.0',

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -1663,6 +1663,7 @@ export class EndpointDocGenerator extends BaseDataGenerator {
           install_status: 'installed',
           install_started_at: '2020-06-24T14:41:23.098Z',
           install_source: 'registry',
+          keep_policies_up_to_date: false,
         },
         references: [],
         updated_at: '2020-06-24T14:41:23.098Z',

x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts

@@ -151,6 +151,7 @@ describe('Host Isolation', () => {
               type: ElasticsearchAssetType.transform,
             },
           ],
+          keep_policies_up_to_date: false,
         })
       );
       licenseEmitter = new Subject();

x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts

@@ -131,6 +131,7 @@ describe('test endpoint route', () => {
               type: ElasticsearchAssetType.transform,
             },
           ],
+          keep_policies_up_to_date: false,
         })
       );
       endpointAppContextService.start({ ...startContract, packageService: mockPackageService });
@@ -390,6 +391,7 @@ describe('test endpoint route', () => {
               type: ElasticsearchAssetType.transform,
             },
           ],
+          keep_policies_up_to_date: false,
         })
       );
       endpointAppContextService.start({ ...startContract, packageService: mockPackageService });

x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap

@@ -276,6 +276,7 @@ Object {
       "type": "image/svg+xml",
     },
   ],
+  "keepPoliciesUpToDate": false,
   "license": "basic",
   "name": "apache",
   "owner": Object {
@@ -449,6 +450,7 @@ Object {
         },
       ],
       "internal": false,
+      "keep_policies_up_to_date": false,
       "name": "apache",
       "package_assets": Array [
         Object {

x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts

@@ -618,6 +618,7 @@ const expectAssetsInstalled = ({
       install_status: 'installed',
       install_started_at: res.attributes.install_started_at,
       install_source: 'registry',
+      keep_policies_up_to_date: false,
     });
   });
 };

x-pack/test/fleet_api_integration/apis/epm/update_assets.ts

@@ -432,6 +432,7 @@ export default function (providerContext: FtrProviderContext) {
         install_status: 'installed',
         install_started_at: res.attributes.install_started_at,
         install_source: 'registry',
+        keep_policies_up_to_date: false,
       });
     });
   });

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/agent/stream/stream.yml.hbs

@@ -0,0 +1 @@
+config.version: "2"

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/manifest.yml

@@ -0,0 +1,15 @@
+title: Test stream
+type: logs
+streams:
+  - input: test_input
+    vars:
+    - name: test_var
+      type: text
+      title: Test Var
+      show_user: true
+      default: Test Value
+    - name: test_var_2
+      type: text
+      title: Test Var 2
+      show_user: true
+      default: Test Value 2

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/docs/README.md

@@ -0,0 +1,3 @@
+# Test package
+
+This is a test package for testing automated upgrades for package policies

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/manifest.yml

@@ -0,0 +1,23 @@
+format_version: 1.0.0
+name: package_policy_upgrade
+title: Tests package policy upgrades
+description: This is a test package for upgrading package policies
+version: 0.2.5-non-breaking-change
+categories: []
+release: beta
+type: integration
+license: basic
+requirement:
+  elasticsearch:
+    versions: '>7.7.0'
+  kibana:
+    versions: '>7.7.0'
+policy_templates:
+  - name: package_policy_upgrade
+    title: Package Policy Upgrade
+    description: Test Package for Upgrading Package Policies
+    inputs:
+      - type: test_input
+        title: Test Input
+        description: Test Input
+        enabled: true

x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts

@@ -162,6 +162,122 @@ export default function (providerContext: FtrProviderContext) {
       });
     });
 
+    describe('when upgrading to a version with no breaking changes', function () {
+      withTestPackageVersion('0.2.5-non-breaking-change');
+
+      beforeEach(async function () {
+        const { body: agentPolicyResponse } = await supertest
+          .post(`/api/fleet/agent_policies`)
+          .set('kbn-xsrf', 'xxxx')
+          .send({
+            name: 'Test policy',
+            namespace: 'default',
+          })
+          .expect(200);
+
+        agentPolicyId = agentPolicyResponse.item.id;
+
+        const { body: packagePolicyResponse } = await supertest
+          .post(`/api/fleet/package_policies`)
+          .set('kbn-xsrf', 'xxxx')
+          .send({
+            name: 'package_policy_upgrade_1',
+            description: '',
+            namespace: 'default',
+            policy_id: agentPolicyId,
+            enabled: true,
+            output_id: '',
+            inputs: [
+              {
+                policy_template: 'package_policy_upgrade',
+                type: 'test_input',
+                enabled: true,
+                streams: [
+                  {
+                    id: 'test-package_policy_upgrade-xxxx',
+                    enabled: true,
+                    data_stream: {
+                      type: 'test_stream',
+                      dataset: 'package_policy_upgrade.test_stream',
+                    },
+                    vars: {
+                      test_var: {
+                        value: 'My custom test value',
+                      },
+                    },
+                  },
+                ],
+              },
+            ],
+            package: {
+              name: 'package_policy_upgrade',
+              title: 'This is a test package for upgrading package policies',
+              version: '0.2.0-add-non-required-test-var',
+            },
+          })
+          .expect(200);
+
+        packagePolicyId = packagePolicyResponse.item.id;
+      });
+
+      afterEach(async function () {
+        await supertest
+          .post(`/api/fleet/package_policies/delete`)
+          .set('kbn-xsrf', 'xxxx')
+          .send({ packagePolicyIds: [packagePolicyId] })
+          .expect(200);
+
+        await supertest
+          .post('/api/fleet/agent_policies/delete')
+          .set('kbn-xsrf', 'xxxx')
+          .send({ agentPolicyId })
+          .expect(200);
+      });
+
+      describe('dry run', function () {
+        it('returns a valid diff', async function () {
+          const { body }: { body: UpgradePackagePolicyDryRunResponse } = await supertest
+            .post(`/api/fleet/package_policies/upgrade`)
+            .set('kbn-xsrf', 'xxxx')
+            .send({
+              packagePolicyIds: [packagePolicyId],
+              dryRun: true,
+            })
+            .expect(200);
+
+          expect(body.length).to.be(1);
+          expect(body[0].diff?.length).to.be(2);
+          expect(body[0].hasErrors).to.be(false);
+
+          const [currentPackagePolicy, proposedPackagePolicy] = body[0].diff ?? [];
+
+          expect(currentPackagePolicy?.package?.version).to.be('0.2.0-add-non-required-test-var');
+          expect(proposedPackagePolicy?.package?.version).to.be('0.2.5-non-breaking-change');
+
+          const testInput = proposedPackagePolicy?.inputs.find(({ type }) => type === 'test_input');
+          const testStream = testInput?.streams[0];
+
+          expect(testStream?.vars?.test_var.value).to.be('My custom test value');
+        });
+      });
+
+      describe('upgrade', function () {
+        it('successfully upgrades package policy', async function () {
+          const { body }: { body: UpgradePackagePolicyResponse } = await supertest
+            .post(`/api/fleet/package_policies/upgrade`)
+            .set('kbn-xsrf', 'xxxx')
+            .send({
+              packagePolicyIds: [packagePolicyId],
+              dryRun: false,
+            })
+            .expect(200);
+
+          expect(body.length).to.be(1);
+          expect(body[0].success).to.be(true);
+        });
+      });
+    });
+
     describe('when upgrading to a version where a non-required variable has been added', function () {
       withTestPackageVersion('0.2.0-add-non-required-test-var');