github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] Persist package upgrade errors (#171797)

Browse source
This commit is contained in:
Nicolas Chaulet 2023-11-28 10:37:42 -05:00 • committed by GitHub
parent 3b8b829581
commit 348ef4e39c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 350 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

4
packages/kbn-check-mappings-update-cli/current_mappings.json
View file

@ -1967,6 +1967,10 @@
}
}
},
"latest_install_failed_attempts": {
"type": "object",
"enabled": false
},
"installed_kibana": {
"dynamic": false,
"properties": {}

2
src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
View file

@ -84,7 +84,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"dashboard": "0611794ce10d25a36da0770c91376c575e92e8f2",
"endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
"enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
"epm-packages": "2449bb565f987eff70b1b39578bb17e90c404c6e",
"epm-packages": "c23d3d00c051a08817335dba26f542b64b18a56a",
"epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
"event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",
"event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88",

32
x-pack/plugins/fleet/common/openapi/bundled.json
View file

@ -6069,6 +6069,35 @@
"install_format_schema_version": {
"type": "string"
},
"latest_install_failed_attempts": {
"description": "Latest failed install errors",
"type": "array",
"items": {
"type": "object",
"properties": {
"created_at": {
"type": "string"
},
"target_version": {
"type": "string"
},
"error": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"message": {
"type": "string"
},
"stack": {
"type": "string"
}
}
}
}
}
},
"verification_status": {
"type": "string",
"enum": [
@ -6120,7 +6149,8 @@
"install_version",
"install_started_at",
"install_source",
"verification_status"
"verification_status",
"latest_install_failed_attempts"
]
},
"search_result": {

20
x-pack/plugins/fleet/common/openapi/bundled.yaml
View file

@ -3824,6 +3824,25 @@ components:
type: string
install_format_schema_version:
type: string
latest_install_failed_attempts:
description: Latest failed install errors
type: array
items:
type: object
properties:
created_at:
type: string
target_version:
type: string
error:
type: object
properties:
name:
type: string
message:
type: string
stack:
type: string
verification_status:
type: string
enum:
@ -3863,6 +3882,7 @@ components:
- install_started_at
- install_source
- verification_status
- latest_install_failed_attempts
search_result:
title: Search result
type: object

20
x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml
View file

@ -47,6 +47,25 @@ properties:
type: string
install_format_schema_version:
type: string
latest_install_failed_attempts:
description: Latest failed install errors
type: array
items:
type: object
properties:
created_at:
type: string
target_version:
type: string
error:
type: object
properties:
name:
type: string
message:
type: string
stack:
type: string
verification_status:
type: string
enum:
@ -86,3 +105,4 @@ required:
- install_started_at
- install_source
- verification_status
- latest_install_failed_attempts

11
x-pack/plugins/fleet/common/types/models/epm.ts
View file

@ -530,6 +530,16 @@ export interface ExperimentalDataStreamFeature {
features: Partial<Record<ExperimentalIndexingFeature, boolean>>;
}
export interface InstallFailedAttempt {
created_at: string;
target_version: string;
error: {
name: string;
message: string;
stack?: string;
};
}
export interface Installation {
installed_kibana: KibanaAssetReference[];
installed_es: EsAssetReference[];
@ -549,6 +559,7 @@ export interface Installation {
experimental_data_stream_features?: ExperimentalDataStreamFeature[];
internal?: boolean;
removable?: boolean;
latest_install_failed_attempts?: InstallFailedAttempt[];
}
export interface PackageUsageStats {

2
x-pack/plugins/fleet/server/routes/epm/handlers.ts
View file

@ -690,7 +690,9 @@ const soToInstallationInfo = (pkg: PackageListItem | PackageInfo) => {
verification_status: attributes.verification_status,
verification_key_id: attributes.verification_key_id,
experimental_data_stream_features: attributes.experimental_data_stream_features,
latest_install_failed_attempts: attributes.latest_install_failed_attempts,
};
return {
// When savedObject gets removed, replace `pkg` with `...omit(pkg, 'savedObject')`
...pkg,

13
x-pack/plugins/fleet/server/saved_objects/index.ts
View file

@ -457,6 +457,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
deferred: { type: 'boolean' },
},
},
latest_install_failed_attempts: { type: 'object', enabled: false },
installed_kibana: {
dynamic: false,
properties: {},
@ -487,6 +488,18 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
},
},
},
modelVersions: {
'1': {
changes: [
{
type: 'mappings_addition',
addedMappings: {
latest_install_failed_attempts: { type: 'object', enabled: false },
},
},
],
},
},
migrations: {
'7.14.0': migrateInstallationToV7140,
'7.14.1': migrateInstallationToV7140,

9
x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts
View file

@ -57,6 +57,7 @@ import {
installIndexTemplatesAndPipelines,
} from './install';
import { withPackageSpan } from './utils';
import { clearLatestFailedAttempts } from './install_errors_helpers';
// this is only exported for testing
// use a leading underscore to indicate it's not the supported path
@ -103,6 +104,10 @@ export async function _installPackage({
try {
// if some installation already exists
if (installedPkg) {
if (installType === 'update' && pkgVersion === '1.17.0') {
// throw new Error('Test error ');
}
const isStatusInstalling = installedPkg.attributes.install_status === 'installing';
const hasExceededTimeout =
Date.now() - Date.parse(installedPkg.attributes.install_started_at) <
@ -333,6 +338,10 @@ export async function _installPackage({
install_status: 'installed',
package_assets: packageAssetRefs,
install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
latest_install_failed_attempts: clearLatestFailedAttempts(
pkgVersion,
installedPkg?.attributes.latest_install_failed_attempts ?? []
),
})
);

1
x-pack/plugins/fleet/server/services/epm/packages/install.test.ts
View file

@ -243,6 +243,7 @@ describe('install', () => {
it('should send telemetry on install failure, async error', async () => {
jest.mocked(install._installPackage).mockRejectedValue(new Error('error'));
jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true);
await installPackage({
spaceId: DEFAULT_SPACE_ID,
installSource: 'registry',

65
x-pack/plugins/fleet/server/services/epm/packages/install.ts
View file

@ -106,6 +106,7 @@ import { cacheAssets } from './custom_integrations/assets/cache';
import { generateDatastreamEntries } from './custom_integrations/assets/dataset/utils';
import { checkForNamingCollision } from './custom_integrations/validation/check_naming_collision';
import { checkDatasetsNameFormat } from './custom_integrations/validation/check_dataset_name_format';
import { addErrorToLatestFailedAttempts } from './install_errors_helpers';
export async function isPackageInstalled(options: {
savedObjectsClient: SavedObjectsClientContract;
@ -236,6 +237,13 @@ export async function handleInstallPackageFailure({
version: pkgVersion,
});
const latestInstallFailedAttempts = addErrorToLatestFailedAttempts({
error,
targetVersion: pkgVersion,
createdAt: new Date().toISOString(),
latestAttempts: installedPkg?.attributes.latest_install_failed_attempts,
});
// if there is an unknown server error, uninstall any package assets or reinstall the previous version if update
try {
const installType = getInstallType({ pkgVersion, installedPkg });
@ -245,18 +253,18 @@ export async function handleInstallPackageFailure({
return;
}
await updateInstallStatusToFailed({
logger,
savedObjectsClient,
pkgName,
status: 'install_failed',
latestInstallFailedAttempts,
});
if (installType === 'reinstall') {
logger.error(`Failed to reinstall ${pkgkey}: [${error.toString()}]`, { error });
}
await updateInstallStatus({ savedObjectsClient, pkgName, status: 'install_failed' }).catch(
(err) => {
if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {
logger.error(`failed to update package status to: install_failed ${err}`);
}
}
);
if (installType === 'update') {
if (!installedPkg) {
logger.error(
@ -278,13 +286,20 @@ export async function handleInstallPackageFailure({
}
} catch (e) {
// If an error happens while removing the integration or while doing a rollback update the status to failed
await updateInstallStatus({ savedObjectsClient, pkgName, status: 'install_failed' }).catch(
(err) => {
if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {
logger.error(`failed to update package status to: install_failed ${err}`);
}
}
);
await updateInstallStatusToFailed({
logger,
savedObjectsClient,
pkgName,
status: 'install_failed',
latestInstallFailedAttempts: installedPkg
? addErrorToLatestFailedAttempts({
error: e,
targetVersion: installedPkg.attributes.version,
createdAt: installedPkg.attributes.install_started_at,
latestAttempts: latestInstallFailedAttempts,
})
: [],
});
logger.error(`failed to uninstall or rollback package after installation error ${e}`);
}
}
@ -883,24 +898,34 @@ export const updateVersion = async (
});
};
export const updateInstallStatus = async ({
export const updateInstallStatusToFailed = async ({
logger,
savedObjectsClient,
pkgName,
status,
latestInstallFailedAttempts,
}: {
logger: Logger;
savedObjectsClient: SavedObjectsClientContract;
pkgName: string;
status: EpmPackageInstallStatus;
latestInstallFailedAttempts: any;
}) => {
auditLoggingService.writeCustomSoAuditLog({
action: 'update',
id: pkgName,
savedObjectType: PACKAGES_SAVED_OBJECT_TYPE,
});
return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
install_status: status,
});
try {
return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
install_status: status,
latest_install_failed_attempts: latestInstallFailedAttempts,
});
} catch (err) {
if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {
logger.error(`failed to update package status to: install_failed ${err}`);
}
}
};
export async function restartInstallation(options: {

71
x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.test.ts Normal file
View file

@ -0,0 +1,71 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { InstallFailedAttempt } from '../../../types';
import {
clearLatestFailedAttempts,
addErrorToLatestFailedAttempts,
} from './install_errors_helpers';
const generateFailedAttempt = (version: string) => ({
target_version: version,
created_at: new Date().toISOString(),
error: {
name: 'test',
message: 'test',
},
});
const mapFailledAttempsToTargetVersion = (attemps: InstallFailedAttempt[]) =>
attemps.map((attempt) => attempt.target_version);
describe('Install error helpers', () => {
describe('clearLatestFailedAttempts', () => {
const previousFailedAttemps: InstallFailedAttempt[] = [
generateFailedAttempt('0.1.0'),
generateFailedAttempt('0.2.0'),
];
it('should clear previous error on succesfull upgrade', () => {
const currentFailledAttemps = clearLatestFailedAttempts('0.2.0', previousFailedAttemps);
expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual([]);
});
it('should not clear previous upgrade error on succesfull rollback', () => {
const currentFailledAttemps = clearLatestFailedAttempts('0.1.0', previousFailedAttemps);
expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual(['0.2.0']);
});
});
describe('addErrorToLatestFailedAttempts', () => {
it('should only keep 5 errors', () => {
const previousFailedAttemps: InstallFailedAttempt[] = [
generateFailedAttempt('0.2.5'),
generateFailedAttempt('0.2.4'),
generateFailedAttempt('0.2.3'),
generateFailedAttempt('0.2.2'),
generateFailedAttempt('0.2.1'),
];
const currentFailledAttemps = addErrorToLatestFailedAttempts({
targetVersion: '0.2.6',
createdAt: new Date().toISOString(),
error: new Error('new test'),
latestAttempts: previousFailedAttemps,
});
expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual([
'0.2.6',
'0.2.5',
'0.2.4',
'0.2.3',
'0.2.2',
]);
});
});
});

44
x-pack/plugins/fleet/server/services/epm/packages/install_errors_helpers.ts Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { lt } from 'semver';
import type { InstallFailedAttempt } from '../../../types';
const MAX_ATTEMPTS_TO_KEEP = 5;
export function clearLatestFailedAttempts(
installedVersion: string,
latestAttempts: InstallFailedAttempt[] = []
) {
return latestAttempts.filter((attempt) => lt(installedVersion, attempt.target_version));
}
export function addErrorToLatestFailedAttempts({
error,
createdAt,
targetVersion,
latestAttempts = [],
}: {
createdAt: string;
targetVersion: string;
error: Error;
latestAttempts?: InstallFailedAttempt[];
}): InstallFailedAttempt[] {
return [
{
created_at: createdAt,
target_version: targetVersion,
error: {
name: error.name,
message: error.message,
stack: error.stack,
},
},
...latestAttempts,
].slice(0, MAX_ATTEMPTS_TO_KEEP);
}

1
x-pack/plugins/fleet/server/types/index.tsx
View file

@ -41,6 +41,7 @@ export type {
Installation,
EpmPackageInstallStatus,
InstallationStatus,
InstallFailedAttempt,
PackageInfo,
ArchivePackage,
RegistryVarsEntry,

39
x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts
View file

@ -16,10 +16,11 @@ export default function (providerContext: FtrProviderContext) {
const pkgName = 'error_handling';
const goodPackageVersion = '0.1.0';
const badPackageVersion = '0.2.0';
const goodUpgradePackageVersion = '0.3.0';
const kibanaServer = getService('kibanaServer');
const installPackage = async (pkg: string, version: string) => {
await supertest
const installPackage = (pkg: string, version: string) => {
return supertest
.post(`/api/fleet/epm/packages/${pkg}/${version}`)
.set('kbn-xsrf', 'xxxx')
.send({ force: true });
@ -45,28 +46,46 @@ export default function (providerContext: FtrProviderContext) {
afterEach(async () => {
await kibanaServer.savedObjects.cleanStandardList();
await uninstallPackage(pkgName, goodPackageVersion);
await uninstallPackage(pkgName, goodUpgradePackageVersion);
});
it('on a fresh install, it should uninstall a broken package during rollback', async function () {
await supertest
.post(`/api/fleet/epm/packages/${pkgName}/${badPackageVersion}`)
.set('kbn-xsrf', 'xxxx')
.expect(422); // the broken package contains a broken visualization triggering a 422 from Kibana
// the broken package contains a broken visualization triggering a 422 from Kibana
await installPackage(pkgName, badPackageVersion).expect(422);
const pkgInfoResponse = await getPackageInfo(pkgName, badPackageVersion);
expect(JSON.parse(pkgInfoResponse.text).item.status).to.be('not_installed');
expect(pkgInfoResponse.body.item.savedObject).to.be(undefined);
});
it('on an upgrade, it should fall back to the previous good version during rollback', async function () {
await installPackage(pkgName, goodPackageVersion);
await supertest
.post(`/api/fleet/epm/packages/${pkgName}/${badPackageVersion}`)
.set('kbn-xsrf', 'xxxx')
.expect(422); // the broken package contains a broken visualization triggering a 422 from Kibana
// the broken package contains a broken visualization triggering a 422 from Kibana
await installPackage(pkgName, badPackageVersion).expect(422);
const goodPkgInfoResponse = await getPackageInfo(pkgName, goodPackageVersion);
expect(JSON.parse(goodPkgInfoResponse.text).item.status).to.be('installed');
expect(JSON.parse(goodPkgInfoResponse.text).item.version).to.be('0.1.0');
const latestInstallFailedAttempts =
goodPkgInfoResponse.body.item.savedObject.attributes.latest_install_failed_attempts;
expect(latestInstallFailedAttempts).to.have.length(1);
expect(latestInstallFailedAttempts[0].target_version).to.be('0.2.0');
expect(latestInstallFailedAttempts[0].error.message).to.contain(
'Document "sample_visualization" belongs to a more recent version of Kibana [12.7.0]'
);
});
it('on a succesfull upgrade, it should clear previous upgrade errors', async function () {
await installPackage(pkgName, goodPackageVersion);
await installPackage(pkgName, badPackageVersion).expect(422);
await installPackage(pkgName, goodUpgradePackageVersion).expect(200);
const goodPkgInfoResponse = await getPackageInfo(pkgName, goodUpgradePackageVersion);
expect(JSON.parse(goodPkgInfoResponse.text).item.status).to.be('installed');
expect(JSON.parse(goodPkgInfoResponse.text).item.version).to.be('0.3.0');
const latestInstallFailedAttempts =
goodPkgInfoResponse.body.item.savedObject.attributes.latest_install_failed_attempts;
expect(latestInstallFailedAttempts).to.have.length(0);
});
});
}

1
x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts
View file

@ -786,6 +786,7 @@ const expectAssetsInstalled = ({
install_status: 'installed',
install_started_at: res.attributes.install_started_at,
install_source: 'registry',
latest_install_failed_attempts: [],
install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
verification_status: 'unknown',
verification_key_id: null,

1
x-pack/test/fleet_api_integration/apis/epm/update_assets.ts
View file

@ -516,6 +516,7 @@ export default function (providerContext: FtrProviderContext) {
install_started_at: res.attributes.install_started_at,
install_source: 'registry',
install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION,
latest_install_failed_attempts: [],
verification_status: 'unknown',
verification_key_id: null,
});

3
x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/docs/README.md Normal file
View file

@ -0,0 +1,3 @@
This package should install without errors.
Version 0.2.0 of this package should fail during installation. We need this good version to test rollback.

7
x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/img/logo_overrides_64_color.svg Normal file
View file

@ -0,0 +1,7 @@
<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 64 64">
<g fill="none" fill-rule="evenodd">
<path fill="#F04E98" d="M29,32.0001 L15.935,9.4321 C13.48,5.1941 7,6.9351 7,11.8321 L7,52.1681 C7,57.0651 13.48,58.8061 15.935,54.5671 L29,32.0001 Z"/>
<path fill="#FA744E" d="M34.7773,32.0001 L33.3273,34.5051 L20.2613,57.0731 C19.8473,57.7871 19.3533,58.4271 18.8023,59.0001 L34.9273,59.0001 C38.7073,59.0001 42.2213,57.0601 44.2363,53.8611 L58.0003,32.0001 L34.7773,32.0001 Z"/>
<path fill="#343741" d="M44.2363,10.1392 C42.2213,6.9402 38.7073,5.0002 34.9273,5.0002 L18.8023,5.0002 C19.3533,5.5732 19.8473,6.2122 20.2613,6.9272 L33.3273,29.4942 L34.7773,32.0002 L58.0003,32.0002 L44.2363,10.1392 Z"/>
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 751 B

14
x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/kibana/visualization/sample_visualization.json Normal file
View file

@ -0,0 +1,14 @@
{
"attributes": {
"description": "sample visualization",
"title": "sample vis title",
"uiStateJSON": "{}",
"version": 1,
"visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}"
},
"id": "sample_visualization",
"type": "visualization",
"migrationVersion": {
"visualization": "7.7.0"
}
}

22
x-pack/test/fleet_api_integration/apis/fixtures/test_packages/error_handling/0.3.0/manifest.yml Normal file
View file

@ -0,0 +1,22 @@
format_version: 1.0.0
name: error_handling
title: Error handling
description: tests error handling and rollback
version: 0.3.0
categories: []
release: beta
type: integration
license: basic
owner:
github: elastic/fleet
requirement:
elasticsearch:
versions: '>7.7.0'
kibana:
versions: '>7.7.0'
icons:
- src: '/img/logo_overrides_64_color.svg'
size: '16x16'
type: 'image/svg+xml'