[Fleet] Save package policy previous revision on package upgrade (#222779)

## Summary

Closes https://github.com/elastic/ingest-dev/issues/5444

* Add `enablePackageRollback` feature flag
* Save package policy previous revision on package upgrade
* Add `latest_revision` boolean property to
`ingest-package-policies/fleet-package-policies` saved object type
   * Package policy SO are created with `latest_revision: true`
* When a package policy is updated with a new package version, the
previous SO is saved to ES with id `{id}:prev` and `latest_revision:
false`
   * Backfill existing SO with `latest_revision: true`
   * GET logic filters for `latest_revision: true`
* Save package previous version
   * Add `previous_version` property to `epm-packages` saved object type
   * When a package is upgraded to a new version, set `previous_version`

### Testing

* Install an integration on an outdated version (edit the version in the
URL and add the integration).
* Check the package policy SO: it should have been created with
`latest_revision: true`.
* Check the package SO: the `previous_version` property should not be
set.
* Upgrade the integration and upgrade package policies.
* Check the package policy SO: there should now be 2 SO for this package
policy:
   * The updated one with `latest_revision: true` and policy id
* The previous one with `latest_revision: false` and `{policy_id}:prev`
* Check the package SO: the `previous_version` property should be set to
the old version

Note: it seems Fleet only allows upgrading packages to the latest
version (please correct me if that's wrong); for testing two consecutive
updates (e.g. check that only the most recent revision is saved), it
might be necessary to run a custom EPR.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

### Identify risks

Risk of bad requests across Fleet wherever packages or package policies
are queried.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -313,12 +313,9 @@
     "enabled",
     "error",
     "filter",
-    "indexPattern",
     "integrationName",
     "managed",
     "matchers",
-    "matchers.fields",
-    "matchers.values",
     "name",
     "type"
   ],
@@ -373,6 +370,7 @@
     "latest_install_failed_attempts",
     "name",
     "package_assets",
+    "previous_version",
     "verification_key_id",
     "verification_status",
     "version"
@@ -550,6 +548,7 @@
     "enabled",
     "inputs",
     "is_managed",
+    "latest_revision",
     "name",
     "namespace",
     "output_id",
@@ -735,6 +734,7 @@
     "enabled",
     "inputs",
     "is_managed",
+    "latest_revision",
     "name",
     "namespace",
     "output_id",
@@ -841,19 +841,6 @@
     "job.job_id",
     "model_id"
   ],
-  "monitoring-entity-source": [
-    "enabled",
-    "error",
-    "filter",
-    "indexPattern",
-    "integrationName",
-    "managed",
-    "matchers",
-    "matchers.fields",
-    "matchers.values",
-    "name",
-    "type"
-  ],
   "monitoring-telemetry": [
     "reportedClusterUuids"
   ],

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1256,6 +1256,9 @@
         "dynamic": false,
         "properties": {}
       },
+      "previous_version": {
+        "type": "keyword"
+      },
       "verification_key_id": {
         "type": "keyword"
       },
@@ -1835,6 +1838,9 @@
       "is_managed": {
         "type": "boolean"
       },
+      "latest_revision": {
+        "type": "boolean"
+      },
       "name": {
         "type": "keyword"
       },
@@ -2438,6 +2444,9 @@
       "is_managed": {
         "type": "boolean"
       },
+      "latest_revision": {
+        "type": "boolean"
+      },
       "name": {
         "type": "keyword"
       },

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -100,7 +100,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "entity-definition": "1c6bff35c423d5dc5650bc806cf2899e4706a0bc",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
         "entity-engine-status": "09f6a617020708e4f638137e5ef35bd9534133be",
-        "epm-packages": "5a9f55e38d424f5b5ebbfeac802788b5b05d867f",
+        "epm-packages": "db1a500677ffca84c2900df498f83626554cb4fb",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
         "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",
         "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88",
@@ -113,7 +113,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "fleet-agent-policies": "f69f7c5639f4cf9e85077c904e161f3574ac3ca2",
         "fleet-fleet-server-host": "232d98738d5321b86edc426e21a9ca2f607da999",
         "fleet-message-signing-keys": "93421f43fed2526b59092a4e3c65d64bc2266c0f",
-        "fleet-package-policies": "b1ded996118af658bc420a737ff3c4d784641fc7",
+        "fleet-package-policies": "efd05a0ed95f387cecf0fad8902c482a5732668b",
         "fleet-preconfiguration-deletion-record": "c52ea1e13c919afe8a5e8e3adbb7080980ecc08e",
         "fleet-proxy": "6cb688f0d2dd856400c1dbc998b28704ff70363d",
         "fleet-setup-lock": "0dc784792c79b5af5a6e6b5dcac06b0dbaa90bde",
@@ -129,7 +129,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ingest-agent-policies": "cfe66f4aeca8f53b26bd4ddb0e956de1637d774e",
         "ingest-download-sources": "5be99940d6b5f9121b2fd279708d14e2bc0bde26",
         "ingest-outputs": "6743521f501bd77b1523dbb1df48d7c47fdad529",
-        "ingest-package-policies": "6a80000fdf2544f2485b0c6a51ecc434b6a12987",
+        "ingest-package-policies": "cb7e5f23e0af62f2d66e194ad2b108c9645e422c",
         "ingest_manager_settings": "111a616eb72627c002029c19feb9e6c439a10505",
         "intercept_interaction_record": "13587751af378409df5cadd08aeb0d3884b1645a",
         "intercept_trigger_record": "9223039379bf9997781ad91df120eb360c3e6b77",

x-pack/platform/plugins/shared/fleet/common/experimental_features.ts

@@ -16,6 +16,7 @@ const _allowedExperimentalValues = {
   installedIntegrationsTabularUI: true,
   enabledUpgradeAgentlessDeploymentsTask: true,
   enableAgentMigrations: false,
+  enablePackageRollback: false,
 };
 
 /**

x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts

@@ -684,6 +684,7 @@ export interface Installation {
   latest_uninstall_failed_attempts?: FailedAttempt[];
   latest_executed_state?: InstallLatestExecutedState;
   latest_custom_asset_install_failed_attempts?: { [asset: string]: CustomAssetFailedAttempt };
+  previous_version?: string;
 }
 
 export interface PackageUsageStats {

x-pack/platform/plugins/shared/fleet/server/integration_tests/fleet_usage_telemetry.test.ts

@@ -367,6 +367,7 @@ describe('fleet usage telemetry', () => {
           },
         },
       ],
+      latest_revision: true,
     });
 
     await soClient.create('ingest-package-policies', {
@@ -381,6 +382,7 @@ describe('fleet usage telemetry', () => {
       policy_id: 'policy2',
       policy_ids: ['policy2', 'policy3'],
       inputs: [],
+      latest_revision: true,
     });
 
     await soClient.create(

x-pack/platform/plugins/shared/fleet/server/saved_objects/index.ts

@@ -103,6 +103,7 @@ import {
 import { backfillAgentPolicyToV4 } from './model_versions/agent_policy_v4';
 import { backfillOutputPolicyToV7 } from './model_versions/outputs';
 import { packagePolicyV17AdvancedFieldsForEndpointV818 } from './model_versions/security_solution/v17_advanced_package_policy_fields';
+import { backfillPackagePolicyLatestRevision } from './model_versions/package_policy_latest_revision_backfill';
 
 /*
  * Saved object types and mappings
@@ -695,6 +696,7 @@ export const getSavedObjectTypes = (
           created_at: { type: 'date' },
           created_by: { type: 'keyword' },
           bump_agent_policy_revision: { type: 'boolean' },
+          latest_revision: { type: 'boolean' },
         },
       },
       modelVersions: {
@@ -875,6 +877,20 @@ export const getSavedObjectTypes = (
             },
           ],
         },
+        '19': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                latest_revision: { type: 'boolean' },
+              },
+            },
+            {
+              type: 'data_backfill',
+              backfillFn: backfillPackagePolicyLatestRevision,
+            },
+          ],
+        },
       },
       migrations: {
         '7.10.0': migratePackagePolicyToV7100,
@@ -938,6 +954,7 @@ export const getSavedObjectTypes = (
           created_at: { type: 'date' },
           created_by: { type: 'keyword' },
           bump_agent_policy_revision: { type: 'boolean' },
+          latest_revision: { type: 'boolean' },
         },
       },
       modelVersions: {
@@ -977,6 +994,20 @@ export const getSavedObjectTypes = (
             },
           ],
         },
+        '5': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                latest_revision: { type: 'boolean' },
+              },
+            },
+            {
+              type: 'data_backfill',
+              backfillFn: backfillPackagePolicyLatestRevision,
+            },
+          ],
+        },
       },
     },
     [PACKAGES_SAVED_OBJECT_TYPE]: {
@@ -1043,6 +1074,7 @@ export const getSavedObjectTypes = (
               },
             },
           },
+          previous_version: { type: 'keyword' },
         },
       },
       modelVersions: {
@@ -1084,6 +1116,16 @@ export const getSavedObjectTypes = (
             },
           ],
         },
+        '5': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                previous_version: { type: 'keyword' },
+              },
+            },
+          ],
+        },
       },
       migrations: {
         '7.14.0': migrateInstallationToV7140,

x-pack/platform/plugins/shared/fleet/server/saved_objects/model_versions/package_policy_latest_revision_backfill.test.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  SavedObject,
+  SavedObjectModelTransformationContext,
+} from '@kbn/core-saved-objects-server';
+
+import type { PackagePolicy } from '../../../common';
+
+import { backfillPackagePolicyLatestRevision } from './package_policy_latest_revision_backfill';
+
+describe('backfillPackagePolicyLatestRevision', () => {
+  const packagePolicyDoc: SavedObject<PackagePolicy & { latest_revision?: boolean }> = {
+    id: 'policy1',
+    type: 'ingest-package-policies',
+    references: [],
+    attributes: {
+      id: 'policy1',
+      name: 'Policy 1',
+      namespace: 'default',
+      description: '',
+      policy_id: 'policy1',
+      policy_ids: ['policy1'],
+      package: { name: 'test-package', version: '1.0.0', title: 'Test Package' },
+      inputs: [],
+      revision: 1,
+      updated_at: '2021-08-17T14:00:00.000Z',
+      updated_by: 'elastic',
+      created_at: '2021-08-17T14:00:00.000Z',
+      created_by: 'elastic',
+      enabled: true,
+    },
+  };
+
+  it('should set latest_revision to true if not defined', () => {
+    const migratedPackagePolicyDoc = backfillPackagePolicyLatestRevision(
+      packagePolicyDoc,
+      {} as SavedObjectModelTransformationContext
+    );
+
+    expect(migratedPackagePolicyDoc.attributes.latest_revision).toBe(true);
+  });
+
+  it('should not change latest_revision if already defined', () => {
+    const migratedPackagePolicyDoc = backfillPackagePolicyLatestRevision(
+      {
+        ...packagePolicyDoc,
+        attributes: {
+          ...packagePolicyDoc.attributes,
+          latest_revision: false,
+        },
+      },
+      {} as SavedObjectModelTransformationContext
+    );
+
+    expect(migratedPackagePolicyDoc.attributes.latest_revision).toBe(false);
+  });
+});

x-pack/platform/plugins/shared/fleet/server/saved_objects/model_versions/package_policy_latest_revision_backfill.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server';
+
+import type { PackagePolicy } from '../../../common';
+
+export const backfillPackagePolicyLatestRevision: SavedObjectModelDataBackfillFn<
+  PackagePolicy & { latest_revision?: boolean },
+  PackagePolicy & { latest_revision?: boolean }
+> = (packagePolicyDoc) => {
+  if (packagePolicyDoc.attributes.latest_revision === undefined) {
+    packagePolicyDoc.attributes.latest_revision = true;
+  }
+
+  return packagePolicyDoc;
+};

x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install.ts

@@ -1129,8 +1129,16 @@ export async function restartInstallation(options: {
   pkgVersion: string;
   installSource: InstallSource;
   verificationResult?: PackageVerificationResult;
+  previousVersion?: string;
 }) {
-  const { savedObjectsClient, pkgVersion, pkgName, installSource, verificationResult } = options;
+  const {
+    savedObjectsClient,
+    pkgVersion,
+    pkgName,
+    installSource,
+    verificationResult,
+    previousVersion,
+  } = options;
 
   let savedObjectUpdate: Partial<Installation> = {
     install_version: pkgVersion,
@@ -1147,6 +1155,10 @@ export async function restartInstallation(options: {
     };
   }
 
+  if (previousVersion) {
+    savedObjectUpdate.previous_version = previousVersion;
+  }
+
   auditLoggingService.writeCustomSoAuditLog({
     action: 'update',
     id: pkgName,

x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import semverGt from 'semver/functions/gt';
+
 import { ConcurrentInstallOperationError } from '../../../../../errors';
 import { MAX_TIME_COMPLETE_INSTALL } from '../../../../../constants';
 
@@ -29,6 +31,9 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
 
   // if some installation already exists
   if (installedPkg) {
+    const previousVersion = semverGt(pkgVersion, installedPkg.attributes.install_version)
+      ? installedPkg.attributes.install_version
+      : undefined;
     const isStatusInstalling = installedPkg.attributes.install_status === 'installing';
     const hasExceededTimeout =
       Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
@@ -50,6 +55,7 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
             pkgVersion,
             installSource,
             verificationResult,
+            previousVersion,
           })
         );
       } else {
@@ -72,6 +78,7 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
           pkgVersion,
           installSource,
           verificationResult,
+          previousVersion,
         })
       );
     }

x-pack/platform/plugins/shared/fleet/server/services/package_policies/utils.ts

@@ -38,7 +38,11 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({
   attributes,
   namespaces,
 }: SavedObject<PackagePolicySOAttributes>): PackagePolicy => {
-  const { bump_agent_policy_revision: bumpAgentPolicyRevision, ...restAttributes } = attributes;
+  const {
+    bump_agent_policy_revision: bumpAgentPolicyRevision,
+    latest_revision: latestRevision,
+    ...restAttributes
+  } = attributes;
   return {
     id,
     version,

x-pack/platform/plugins/shared/fleet/server/services/package_policy.ts

@@ -25,6 +25,7 @@ import { SavedObjectsErrorHelpers } from '@kbn/core/server';
 import { SavedObjectsUtils } from '@kbn/core/server';
 import { v4 as uuidv4 } from 'uuid';
 import { load } from 'js-yaml';
+import semverGt from 'semver/functions/gt';
 
 import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';
 
@@ -453,6 +454,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
             elasticsearch: { privileges: elasticsearchPrivileges },
           }),
           ...(secretReferences?.length && { secret_references: secretReferences }),
+          latest_revision: true,
           revision: 1,
           created_at: isoDate,
           created_by: options?.user?.username ?? 'system',
@@ -653,6 +655,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
             elasticsearch,
             policy_id: agentPolicyIdsOfPackagePolicy[0],
             policy_ids: agentPolicyIdsOfPackagePolicy,
+            latest_revision: true,
             revision: 1,
             created_at: isoDate,
             created_by: options?.user?.username ?? 'system',
@@ -849,7 +852,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
         type: savedObjectType,
         filter: `${savedObjectType}.attributes.policy_ids:${escapeSearchQueryPhrase(
           agentPolicyId
-        )}`,
+        )} AND ${savedObjectType}.attributes.latest_revision:true`,
         perPage: SO_SEARCH_LIMIT,
         namespaces: isSpacesEnabled ? options.spaceIds : undefined,
       })
@@ -969,6 +972,13 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
         )}`
     );
 
+    const filter = _normalizePackagePolicyKuery(
+      savedObjectType,
+      kuery
+        ? `${savedObjectType}.attributes.latest_revision:true AND ${kuery}`
+        : `${savedObjectType}.attributes.latest_revision:true`
+    );
+
     const packagePolicies = await soClient
       .find<PackagePolicySOAttributes>({
         type: savedObjectType,
@@ -977,7 +987,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
         page,
         perPage,
         fields,
-        filter: kuery ? _normalizePackagePolicyKuery(savedObjectType, kuery) : undefined,
+        filter,
         namespaces: isSpacesEnabled && options.spaceId ? [options.spaceId] : undefined,
       })
       .catch(catchAndSetErrorStackTrace.withMessage('failed to find package policies'));
@@ -1013,6 +1023,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
   ): Promise<ListResult<string>> {
     const logger = this.getLogger('listIds');
     const { page = 1, perPage = 20, sortField = 'updated_at', sortOrder = 'desc', kuery } = options;
+    const savedObjectType = await getPackagePolicySavedObjectType();
+    const isSpacesEnabled = await isSpaceAwarenessEnabled();
 
     logger.debug(
       () =>
@@ -1021,8 +1033,13 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
         )}]`
     );
 
-    const savedObjectType = await getPackagePolicySavedObjectType();
+    const filter = _normalizePackagePolicyKuery(
-    const isSpacesEnabled = await isSpaceAwarenessEnabled();
+      savedObjectType,
+      kuery
+        ? `${savedObjectType}.attributes.latest_revision:true AND ${kuery}`
+        : `${savedObjectType}.attributes.latest_revision:true`
+    );
+
     const packagePolicies = await soClient
       .find<{ name: string }>({
         type: savedObjectType,
@@ -1031,7 +1048,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
         page,
         perPage,
         fields: ['name'],
-        filter: kuery ? _normalizePackagePolicyKuery(savedObjectType, kuery) : undefined,
+        filter,
         namespaces: isSpacesEnabled ? options.spaceIds : undefined,
       })
       .catch(catchAndSetErrorStackTrace.withMessage('failed to find package policies IDs'));
@@ -1210,6 +1227,49 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
       packagePolicy: restOfPackagePolicy,
     });
 
+    // If the package version has increased, save the previous package policy revision.
+    if (
+      appContextService.getExperimentalFeatures().enablePackageRollback &&
+      packagePolicy.package &&
+      oldPackagePolicy.package &&
+      semverGt(packagePolicy.package.version, oldPackagePolicy.package.version)
+    ) {
+      logger.debug(
+        `Saving previous revision of package policy ${id} with package version ${oldPackagePolicy.version}`
+      );
+      const currentPackagePolicySO = await soClient.get<PackagePolicySOAttributes>(
+        savedObjectType,
+        id
+      );
+      const previousRevisionSO = {
+        ...currentPackagePolicySO,
+        id: `${id}:prev`,
+        attributes: {
+          ...currentPackagePolicySO.attributes,
+          latest_revision: false,
+        },
+      };
+      try {
+        await soClient.update<PackagePolicySOAttributes>(
+          savedObjectType,
+          `${id}:prev`,
+          previousRevisionSO.attributes
+        );
+      } catch (error) {
+        if (error.output.statusCode === 404) {
+          await soClient.create<PackagePolicySOAttributes>(
+            savedObjectType,
+            previousRevisionSO.attributes,
+            {
+              id: `${id}:prev`,
+            }
+          );
+        } else {
+          throw error;
+        }
+      }
+    }
+
     logger.debug(`Updating SO with revision ${oldPackagePolicy.revision + 1}`);
     await soClient
       .update<PackagePolicySOAttributes>(
@@ -1391,6 +1451,12 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
       packagePolicy: NewPackagePolicyWithId;
       error: Error | SavedObjectError;
     }> = [];
+    const previousPolicyRevisionsToCreate: Array<
+      SavedObjectsBulkCreateObject<PackagePolicySOAttributes>
+    > = [];
+    const previousPolicyRevisionsToUpdate: Array<
+      SavedObjectsBulkUpdateObject<PackagePolicySOAttributes>
+    > = [];
 
     const secretStorageEnabled = await isSecretStorageEnabled(esClient, soClient);
 
@@ -1430,6 +1496,40 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
           this.keepPolicyIdInSync(oldPackagePolicy);
         }
 
+        // If the package version has increased, save the previous package policy revision.
+        if (
+          appContextService.getExperimentalFeatures().enablePackageRollback &&
+          packagePolicy.package &&
+          oldPackagePolicy.package &&
+          semverGt(packagePolicy.package.version, oldPackagePolicy.package.version)
+        ) {
+          logger.debug(
+            `Saving previous revision of package policy ${id} with package version ${oldPackagePolicy.version}`
+          );
+          const currentPackagePolicySO = await soClient.get<PackagePolicySOAttributes>(
+            savedObjectType,
+            id
+          );
+          const previousRevisionSO = {
+            ...currentPackagePolicySO,
+            id: `${id}:prev`,
+            attributes: {
+              ...currentPackagePolicySO.attributes,
+              latest_revision: false,
+            },
+          };
+          try {
+            await soClient.get<PackagePolicySOAttributes>(savedObjectType, `${id}:prev`);
+            previousPolicyRevisionsToUpdate.push(previousRevisionSO);
+          } catch (error) {
+            if (error.output.statusCode === 404) {
+              previousPolicyRevisionsToCreate.push(previousRevisionSO);
+            } else {
+              throw error;
+            }
+          }
+        }
+
         let secretReferences: PolicySecretReference[] | undefined;
 
         const { version } = packagePolicyUpdate;
@@ -1555,6 +1655,29 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
       }
     });
 
+    // Store previous revision.
+    if (appContextService.getExperimentalFeatures().enablePackageRollback) {
+      if (previousPolicyRevisionsToCreate.length > 0) {
+        await soClient
+          .bulkCreate<PackagePolicySOAttributes>(previousPolicyRevisionsToCreate)
+          .catch(
+            catchAndSetErrorStackTrace.withMessage(
+              'Saved objects bulk create of previous package policy revisions failed'
+            )
+          );
+      }
+      if (previousPolicyRevisionsToUpdate.length > 0) {
+        await soClient
+          .bulkUpdate<PackagePolicySOAttributes>(previousPolicyRevisionsToUpdate)
+          .catch(
+            catchAndSetErrorStackTrace.withMessage(
+              'Saved objects bulk update of previous package policy revisions failed'
+            )
+          );
+      }
+    }
+
+    // Update package policies SO.
     const { saved_objects: updatedPolicies } = await soClient
       .bulkUpdate<PackagePolicySOAttributes>(policiesToUpdate)
       .catch(catchAndSetErrorStackTrace.withMessage(`Saved objects bulk update failed]`));

x-pack/platform/plugins/shared/fleet/server/types/so_attributes.ts

@@ -148,6 +148,7 @@ export interface PackagePolicySOAttributes {
   agents?: number;
   overrides?: any | null;
   bump_agent_policy_revision?: boolean;
+  latest_revision?: boolean;
 }
 
 interface OutputSoBaseAttributes {

x-pack/test/fleet_api_integration/apis/agent_policy/agent_policy.ts

@@ -426,6 +426,7 @@ export default function (providerContext: FtrProviderContext) {
                 package: {
                   name: 'system',
                 },
+                latest_revision: true,
               },
             })
           ),
@@ -464,6 +465,7 @@ export default function (providerContext: FtrProviderContext) {
             package: {
               name: 'system',
             },
+            latest_revision: true,
           },
         });
         packagePoliciesToDeleteIds.push('package-policy-1');
@@ -476,6 +478,7 @@ export default function (providerContext: FtrProviderContext) {
             package: {
               name: 'system',
             },
+            latest_revision: true,
           },
         });
         packagePoliciesToDeleteIds.push('package-policy-2');
@@ -880,6 +883,7 @@ export default function (providerContext: FtrProviderContext) {
               package: {
                 name: 'system',
               },
+              latest_revision: true,
             },
           }),
         ]);
@@ -964,6 +968,7 @@ export default function (providerContext: FtrProviderContext) {
               package: {
                 name: 'system',
               },
+              latest_revision: true,
             },
           }),
         ]);

x-pack/test/fleet_api_integration/apis/download_sources/crud.ts

@@ -46,6 +46,7 @@ export default function (providerContext: FtrProviderContext) {
         package: {
           name: 'fleet_server',
         },
+        latest_revision: true,
       },
     });
   };

x-pack/test/fleet_api_integration/apis/epm/update_assets.ts

@@ -622,6 +622,7 @@ export default function (providerContext: FtrProviderContext) {
         latest_install_failed_attempts: [],
         verification_status: 'unknown',
         verification_key_id: null,
+        previous_version: '0.1.0',
       });
     });
   });

x-pack/test/fleet_api_integration/apis/fleet_server_hosts/crud.ts

@@ -76,6 +76,7 @@ export default function (providerContext: FtrProviderContext) {
         package: {
           name: 'fleet_server',
         },
+        latest_revision: true,
       },
     });
   };