[Fleet] Save package policy previous revision on package upgrade (#222779)

## Summary

Closes https://github.com/elastic/ingest-dev/issues/5444

* Add `enablePackageRollback` feature flag
* Save package policy previous revision on package upgrade
* Add `latest_revision` boolean property to
`ingest-package-policies/fleet-package-policies` saved object type
   * Package policy SO are created with `latest_revision: true`
* When a package policy is updated with a new package version, the
previous SO is saved to ES with id `{id}:prev` and `latest_revision:
false`
   * Backfill existing SO with `latest_revision: true`
   * GET logic filters for `latest_revision: true`
* Save package previous version
   * Add `previous_version` property to `epm-packages` saved object type
   * When a package is upgraded to a new version, set `previous_version`

### Testing

* Install an integration on an outdated version (edit the version in the
URL and add the integration).
* Check the package policy SO: it should have been created with
`latest_revision: true`.
* Check the package SO: the `previous_version` property should not be
set.
* Upgrade the integration and upgrade package policies.
* Check the package policy SO: there should now be 2 SO for this package
policy:
   * The updated one with `latest_revision: true` and policy id
* The previous one with `latest_revision: false` and `{policy_id}:prev`
* Check the package SO: the `previous_version` property should be set to
the old version

Note: it seems Fleet only allows upgrading packages to the latest
version (please correct me if that's wrong); for testing two consecutive
updates (e.g. check that only the most recent revision is saved), it
might be necessary to run a custom EPR.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

### Identify risks

Risk of bad requests across Fleet wherever packages or package policies
are queried.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Jill Guyonnet 2025-06-20 10:37:23 +02:00 committed by GitHub
parent 8bd7f0e522
commit c5280d74bd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 307 additions and 26 deletions

View file

@ -313,12 +313,9 @@
"enabled",
"error",
"filter",
"indexPattern",
"integrationName",
"managed",
"matchers",
"matchers.fields",
"matchers.values",
"name",
"type"
],
@ -373,6 +370,7 @@
"latest_install_failed_attempts",
"name",
"package_assets",
"previous_version",
"verification_key_id",
"verification_status",
"version"
@ -550,6 +548,7 @@
"enabled",
"inputs",
"is_managed",
"latest_revision",
"name",
"namespace",
"output_id",
@ -735,6 +734,7 @@
"enabled",
"inputs",
"is_managed",
"latest_revision",
"name",
"namespace",
"output_id",
@ -841,19 +841,6 @@
"job.job_id",
"model_id"
],
"monitoring-entity-source": [
"enabled",
"error",
"filter",
"indexPattern",
"integrationName",
"managed",
"matchers",
"matchers.fields",
"matchers.values",
"name",
"type"
],
"monitoring-telemetry": [
"reportedClusterUuids"
],

View file

@ -1256,6 +1256,9 @@
"dynamic": false,
"properties": {}
},
"previous_version": {
"type": "keyword"
},
"verification_key_id": {
"type": "keyword"
},
@ -1835,6 +1838,9 @@
"is_managed": {
"type": "boolean"
},
"latest_revision": {
"type": "boolean"
},
"name": {
"type": "keyword"
},
@ -2438,6 +2444,9 @@
"is_managed": {
"type": "boolean"
},
"latest_revision": {
"type": "boolean"
},
"name": {
"type": "keyword"
},

View file

@ -100,7 +100,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"entity-definition": "1c6bff35c423d5dc5650bc806cf2899e4706a0bc",
"entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
"entity-engine-status": "09f6a617020708e4f638137e5ef35bd9534133be",
"epm-packages": "5a9f55e38d424f5b5ebbfeac802788b5b05d867f",
"epm-packages": "db1a500677ffca84c2900df498f83626554cb4fb",
"epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
"event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",
"event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88",
@ -113,7 +113,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"fleet-agent-policies": "f69f7c5639f4cf9e85077c904e161f3574ac3ca2",
"fleet-fleet-server-host": "232d98738d5321b86edc426e21a9ca2f607da999",
"fleet-message-signing-keys": "93421f43fed2526b59092a4e3c65d64bc2266c0f",
"fleet-package-policies": "b1ded996118af658bc420a737ff3c4d784641fc7",
"fleet-package-policies": "efd05a0ed95f387cecf0fad8902c482a5732668b",
"fleet-preconfiguration-deletion-record": "c52ea1e13c919afe8a5e8e3adbb7080980ecc08e",
"fleet-proxy": "6cb688f0d2dd856400c1dbc998b28704ff70363d",
"fleet-setup-lock": "0dc784792c79b5af5a6e6b5dcac06b0dbaa90bde",
@ -129,7 +129,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"ingest-agent-policies": "cfe66f4aeca8f53b26bd4ddb0e956de1637d774e",
"ingest-download-sources": "5be99940d6b5f9121b2fd279708d14e2bc0bde26",
"ingest-outputs": "6743521f501bd77b1523dbb1df48d7c47fdad529",
"ingest-package-policies": "6a80000fdf2544f2485b0c6a51ecc434b6a12987",
"ingest-package-policies": "cb7e5f23e0af62f2d66e194ad2b108c9645e422c",
"ingest_manager_settings": "111a616eb72627c002029c19feb9e6c439a10505",
"intercept_interaction_record": "13587751af378409df5cadd08aeb0d3884b1645a",
"intercept_trigger_record": "9223039379bf9997781ad91df120eb360c3e6b77",

View file

@ -16,6 +16,7 @@ const _allowedExperimentalValues = {
installedIntegrationsTabularUI: true,
enabledUpgradeAgentlessDeploymentsTask: true,
enableAgentMigrations: false,
enablePackageRollback: false,
};
/**

View file

@ -684,6 +684,7 @@ export interface Installation {
latest_uninstall_failed_attempts?: FailedAttempt[];
latest_executed_state?: InstallLatestExecutedState;
latest_custom_asset_install_failed_attempts?: { [asset: string]: CustomAssetFailedAttempt };
previous_version?: string;
}
export interface PackageUsageStats {

View file

@ -367,6 +367,7 @@ describe('fleet usage telemetry', () => {
},
},
],
latest_revision: true,
});
await soClient.create('ingest-package-policies', {
@ -381,6 +382,7 @@ describe('fleet usage telemetry', () => {
policy_id: 'policy2',
policy_ids: ['policy2', 'policy3'],
inputs: [],
latest_revision: true,
});
await soClient.create(

View file

@ -103,6 +103,7 @@ import {
import { backfillAgentPolicyToV4 } from './model_versions/agent_policy_v4';
import { backfillOutputPolicyToV7 } from './model_versions/outputs';
import { packagePolicyV17AdvancedFieldsForEndpointV818 } from './model_versions/security_solution/v17_advanced_package_policy_fields';
import { backfillPackagePolicyLatestRevision } from './model_versions/package_policy_latest_revision_backfill';
/*
* Saved object types and mappings
@ -695,6 +696,7 @@ export const getSavedObjectTypes = (
created_at: { type: 'date' },
created_by: { type: 'keyword' },
bump_agent_policy_revision: { type: 'boolean' },
latest_revision: { type: 'boolean' },
},
},
modelVersions: {
@ -875,6 +877,20 @@ export const getSavedObjectTypes = (
},
],
},
'19': {
changes: [
{
type: 'mappings_addition',
addedMappings: {
latest_revision: { type: 'boolean' },
},
},
{
type: 'data_backfill',
backfillFn: backfillPackagePolicyLatestRevision,
},
],
},
},
migrations: {
'7.10.0': migratePackagePolicyToV7100,
@ -938,6 +954,7 @@ export const getSavedObjectTypes = (
created_at: { type: 'date' },
created_by: { type: 'keyword' },
bump_agent_policy_revision: { type: 'boolean' },
latest_revision: { type: 'boolean' },
},
},
modelVersions: {
@ -977,6 +994,20 @@ export const getSavedObjectTypes = (
},
],
},
'5': {
changes: [
{
type: 'mappings_addition',
addedMappings: {
latest_revision: { type: 'boolean' },
},
},
{
type: 'data_backfill',
backfillFn: backfillPackagePolicyLatestRevision,
},
],
},
},
},
[PACKAGES_SAVED_OBJECT_TYPE]: {
@ -1043,6 +1074,7 @@ export const getSavedObjectTypes = (
},
},
},
previous_version: { type: 'keyword' },
},
},
modelVersions: {
@ -1084,6 +1116,16 @@ export const getSavedObjectTypes = (
},
],
},
'5': {
changes: [
{
type: 'mappings_addition',
addedMappings: {
previous_version: { type: 'keyword' },
},
},
],
},
},
migrations: {
'7.14.0': migrateInstallationToV7140,

View file

@ -0,0 +1,63 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type {
SavedObject,
SavedObjectModelTransformationContext,
} from '@kbn/core-saved-objects-server';
import type { PackagePolicy } from '../../../common';
import { backfillPackagePolicyLatestRevision } from './package_policy_latest_revision_backfill';
describe('backfillPackagePolicyLatestRevision', () => {
const packagePolicyDoc: SavedObject<PackagePolicy & { latest_revision?: boolean }> = {
id: 'policy1',
type: 'ingest-package-policies',
references: [],
attributes: {
id: 'policy1',
name: 'Policy 1',
namespace: 'default',
description: '',
policy_id: 'policy1',
policy_ids: ['policy1'],
package: { name: 'test-package', version: '1.0.0', title: 'Test Package' },
inputs: [],
revision: 1,
updated_at: '2021-08-17T14:00:00.000Z',
updated_by: 'elastic',
created_at: '2021-08-17T14:00:00.000Z',
created_by: 'elastic',
enabled: true,
},
};
it('should set latest_revision to true if not defined', () => {
const migratedPackagePolicyDoc = backfillPackagePolicyLatestRevision(
packagePolicyDoc,
{} as SavedObjectModelTransformationContext
);
expect(migratedPackagePolicyDoc.attributes.latest_revision).toBe(true);
});
it('should not change latest_revision if already defined', () => {
const migratedPackagePolicyDoc = backfillPackagePolicyLatestRevision(
{
...packagePolicyDoc,
attributes: {
...packagePolicyDoc.attributes,
latest_revision: false,
},
},
{} as SavedObjectModelTransformationContext
);
expect(migratedPackagePolicyDoc.attributes.latest_revision).toBe(false);
});
});

View file

@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server';
import type { PackagePolicy } from '../../../common';
export const backfillPackagePolicyLatestRevision: SavedObjectModelDataBackfillFn<
PackagePolicy & { latest_revision?: boolean },
PackagePolicy & { latest_revision?: boolean }
> = (packagePolicyDoc) => {
if (packagePolicyDoc.attributes.latest_revision === undefined) {
packagePolicyDoc.attributes.latest_revision = true;
}
return packagePolicyDoc;
};

View file

@ -1129,8 +1129,16 @@ export async function restartInstallation(options: {
pkgVersion: string;
installSource: InstallSource;
verificationResult?: PackageVerificationResult;
previousVersion?: string;
}) {
const { savedObjectsClient, pkgVersion, pkgName, installSource, verificationResult } = options;
const {
savedObjectsClient,
pkgVersion,
pkgName,
installSource,
verificationResult,
previousVersion,
} = options;
let savedObjectUpdate: Partial<Installation> = {
install_version: pkgVersion,
@ -1147,6 +1155,10 @@ export async function restartInstallation(options: {
};
}
if (previousVersion) {
savedObjectUpdate.previous_version = previousVersion;
}
auditLoggingService.writeCustomSoAuditLog({
action: 'update',
id: pkgName,

View file

@ -5,6 +5,8 @@
* 2.0.
*/
import semverGt from 'semver/functions/gt';
import { ConcurrentInstallOperationError } from '../../../../../errors';
import { MAX_TIME_COMPLETE_INSTALL } from '../../../../../constants';
@ -29,6 +31,9 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
// if some installation already exists
if (installedPkg) {
const previousVersion = semverGt(pkgVersion, installedPkg.attributes.install_version)
? installedPkg.attributes.install_version
: undefined;
const isStatusInstalling = installedPkg.attributes.install_status === 'installing';
const hasExceededTimeout =
Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
@ -50,6 +55,7 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
pkgVersion,
installSource,
verificationResult,
previousVersion,
})
);
} else {
@ -72,6 +78,7 @@ export async function stepCreateRestartInstallation(context: InstallContext) {
pkgVersion,
installSource,
verificationResult,
previousVersion,
})
);
}

View file

@ -38,7 +38,11 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({
attributes,
namespaces,
}: SavedObject<PackagePolicySOAttributes>): PackagePolicy => {
const { bump_agent_policy_revision: bumpAgentPolicyRevision, ...restAttributes } = attributes;
const {
bump_agent_policy_revision: bumpAgentPolicyRevision,
latest_revision: latestRevision,
...restAttributes
} = attributes;
return {
id,
version,

View file

@ -25,6 +25,7 @@ import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import { SavedObjectsUtils } from '@kbn/core/server';
import { v4 as uuidv4 } from 'uuid';
import { load } from 'js-yaml';
import semverGt from 'semver/functions/gt';
import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';
@ -453,6 +454,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
elasticsearch: { privileges: elasticsearchPrivileges },
}),
...(secretReferences?.length && { secret_references: secretReferences }),
latest_revision: true,
revision: 1,
created_at: isoDate,
created_by: options?.user?.username ?? 'system',
@ -653,6 +655,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
elasticsearch,
policy_id: agentPolicyIdsOfPackagePolicy[0],
policy_ids: agentPolicyIdsOfPackagePolicy,
latest_revision: true,
revision: 1,
created_at: isoDate,
created_by: options?.user?.username ?? 'system',
@ -849,7 +852,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
type: savedObjectType,
filter: `${savedObjectType}.attributes.policy_ids:${escapeSearchQueryPhrase(
agentPolicyId
)}`,
)} AND ${savedObjectType}.attributes.latest_revision:true`,
perPage: SO_SEARCH_LIMIT,
namespaces: isSpacesEnabled ? options.spaceIds : undefined,
})
@ -969,6 +972,13 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
)}`
);
const filter = _normalizePackagePolicyKuery(
savedObjectType,
kuery
? `${savedObjectType}.attributes.latest_revision:true AND ${kuery}`
: `${savedObjectType}.attributes.latest_revision:true`
);
const packagePolicies = await soClient
.find<PackagePolicySOAttributes>({
type: savedObjectType,
@ -977,7 +987,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
page,
perPage,
fields,
filter: kuery ? _normalizePackagePolicyKuery(savedObjectType, kuery) : undefined,
filter,
namespaces: isSpacesEnabled && options.spaceId ? [options.spaceId] : undefined,
})
.catch(catchAndSetErrorStackTrace.withMessage('failed to find package policies'));
@ -1013,6 +1023,8 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
): Promise<ListResult<string>> {
const logger = this.getLogger('listIds');
const { page = 1, perPage = 20, sortField = 'updated_at', sortOrder = 'desc', kuery } = options;
const savedObjectType = await getPackagePolicySavedObjectType();
const isSpacesEnabled = await isSpaceAwarenessEnabled();
logger.debug(
() =>
@ -1021,8 +1033,13 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
)}]`
);
const savedObjectType = await getPackagePolicySavedObjectType();
const isSpacesEnabled = await isSpaceAwarenessEnabled();
const filter = _normalizePackagePolicyKuery(
savedObjectType,
kuery
? `${savedObjectType}.attributes.latest_revision:true AND ${kuery}`
: `${savedObjectType}.attributes.latest_revision:true`
);
const packagePolicies = await soClient
.find<{ name: string }>({
type: savedObjectType,
@ -1031,7 +1048,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
page,
perPage,
fields: ['name'],
filter: kuery ? _normalizePackagePolicyKuery(savedObjectType, kuery) : undefined,
filter,
namespaces: isSpacesEnabled ? options.spaceIds : undefined,
})
.catch(catchAndSetErrorStackTrace.withMessage('failed to find package policies IDs'));
@ -1210,6 +1227,49 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
packagePolicy: restOfPackagePolicy,
});
// If the package version has increased, save the previous package policy revision.
if (
appContextService.getExperimentalFeatures().enablePackageRollback &&
packagePolicy.package &&
oldPackagePolicy.package &&
semverGt(packagePolicy.package.version, oldPackagePolicy.package.version)
) {
logger.debug(
`Saving previous revision of package policy ${id} with package version ${oldPackagePolicy.version}`
);
const currentPackagePolicySO = await soClient.get<PackagePolicySOAttributes>(
savedObjectType,
id
);
const previousRevisionSO = {
...currentPackagePolicySO,
id: `${id}:prev`,
attributes: {
...currentPackagePolicySO.attributes,
latest_revision: false,
},
};
try {
await soClient.update<PackagePolicySOAttributes>(
savedObjectType,
`${id}:prev`,
previousRevisionSO.attributes
);
} catch (error) {
if (error.output.statusCode === 404) {
await soClient.create<PackagePolicySOAttributes>(
savedObjectType,
previousRevisionSO.attributes,
{
id: `${id}:prev`,
}
);
} else {
throw error;
}
}
}
logger.debug(`Updating SO with revision ${oldPackagePolicy.revision + 1}`);
await soClient
.update<PackagePolicySOAttributes>(
@ -1391,6 +1451,12 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
packagePolicy: NewPackagePolicyWithId;
error: Error | SavedObjectError;
}> = [];
const previousPolicyRevisionsToCreate: Array<
SavedObjectsBulkCreateObject<PackagePolicySOAttributes>
> = [];
const previousPolicyRevisionsToUpdate: Array<
SavedObjectsBulkUpdateObject<PackagePolicySOAttributes>
> = [];
const secretStorageEnabled = await isSecretStorageEnabled(esClient, soClient);
@ -1430,6 +1496,40 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
this.keepPolicyIdInSync(oldPackagePolicy);
}
// If the package version has increased, save the previous package policy revision.
if (
appContextService.getExperimentalFeatures().enablePackageRollback &&
packagePolicy.package &&
oldPackagePolicy.package &&
semverGt(packagePolicy.package.version, oldPackagePolicy.package.version)
) {
logger.debug(
`Saving previous revision of package policy ${id} with package version ${oldPackagePolicy.version}`
);
const currentPackagePolicySO = await soClient.get<PackagePolicySOAttributes>(
savedObjectType,
id
);
const previousRevisionSO = {
...currentPackagePolicySO,
id: `${id}:prev`,
attributes: {
...currentPackagePolicySO.attributes,
latest_revision: false,
},
};
try {
await soClient.get<PackagePolicySOAttributes>(savedObjectType, `${id}:prev`);
previousPolicyRevisionsToUpdate.push(previousRevisionSO);
} catch (error) {
if (error.output.statusCode === 404) {
previousPolicyRevisionsToCreate.push(previousRevisionSO);
} else {
throw error;
}
}
}
let secretReferences: PolicySecretReference[] | undefined;
const { version } = packagePolicyUpdate;
@ -1555,6 +1655,29 @@ class PackagePolicyClientImpl implements PackagePolicyClient {
}
});
// Store previous revision.
if (appContextService.getExperimentalFeatures().enablePackageRollback) {
if (previousPolicyRevisionsToCreate.length > 0) {
await soClient
.bulkCreate<PackagePolicySOAttributes>(previousPolicyRevisionsToCreate)
.catch(
catchAndSetErrorStackTrace.withMessage(
'Saved objects bulk create of previous package policy revisions failed'
)
);
}
if (previousPolicyRevisionsToUpdate.length > 0) {
await soClient
.bulkUpdate<PackagePolicySOAttributes>(previousPolicyRevisionsToUpdate)
.catch(
catchAndSetErrorStackTrace.withMessage(
'Saved objects bulk update of previous package policy revisions failed'
)
);
}
}
// Update package policies SO.
const { saved_objects: updatedPolicies } = await soClient
.bulkUpdate<PackagePolicySOAttributes>(policiesToUpdate)
.catch(catchAndSetErrorStackTrace.withMessage(`Saved objects bulk update failed]`));

View file

@ -148,6 +148,7 @@ export interface PackagePolicySOAttributes {
agents?: number;
overrides?: any | null;
bump_agent_policy_revision?: boolean;
latest_revision?: boolean;
}
interface OutputSoBaseAttributes {

View file

@ -426,6 +426,7 @@ export default function (providerContext: FtrProviderContext) {
package: {
name: 'system',
},
latest_revision: true,
},
})
),
@ -464,6 +465,7 @@ export default function (providerContext: FtrProviderContext) {
package: {
name: 'system',
},
latest_revision: true,
},
});
packagePoliciesToDeleteIds.push('package-policy-1');
@ -476,6 +478,7 @@ export default function (providerContext: FtrProviderContext) {
package: {
name: 'system',
},
latest_revision: true,
},
});
packagePoliciesToDeleteIds.push('package-policy-2');
@ -880,6 +883,7 @@ export default function (providerContext: FtrProviderContext) {
package: {
name: 'system',
},
latest_revision: true,
},
}),
]);
@ -964,6 +968,7 @@ export default function (providerContext: FtrProviderContext) {
package: {
name: 'system',
},
latest_revision: true,
},
}),
]);

View file

@ -46,6 +46,7 @@ export default function (providerContext: FtrProviderContext) {
package: {
name: 'fleet_server',
},
latest_revision: true,
},
});
};

View file

@ -622,6 +622,7 @@ export default function (providerContext: FtrProviderContext) {
latest_install_failed_attempts: [],
verification_status: 'unknown',
verification_key_id: null,
previous_version: '0.1.0',
});
});
});

View file

@ -76,6 +76,7 @@ export default function (providerContext: FtrProviderContext) {
package: {
name: 'fleet_server',
},
latest_revision: true,
},
});
};