github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 18:51:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] Sync Remote integrations: Show warning when package is not found in remote (#223248)

Browse source 
Closes https://github.com/elastic/kibana/issues/220850

## Summary
Add logic that handles the case of custom integrations that cannot be
installed in the remote because of error
` PackageNotFoundError: [pkgName] package not found in registry`

Currently, when the installation is attempted, we just log the error and
exit. I added some logic that creates an "empty" installation SO in
`epm-packages` with status `install_failed` and populates
the`latest_install_failed_attempts` field so we know what happened.
**NOTE**: we could add this logic in the regular install process as
well, it would be useful for debugging purpose

- I also added some logic to `compareSyncIntegrations` to make this
error a warning and because I also made the logic for the warning more
generic

### Testing 
- Install a custom integration in the main cluster, let the
syncIntegrationsTask run and after a while check `GET
.kibana_ingest/_doc/epm-packages:PKGNAME`. You should see the error
saved under `latest_install_failed_attempts`:

<img width="1804" alt="Screenshot 2025-06-10 at 16 20 09"
src="https://github.com/user-attachments/assets/2cffbf01-15f2-4091-bcbd-660fbb390c56"
/>

- In the remote, query `GET
kbn:/api/fleet/remote_synced_integrations/status` and verify that it
returns a warning for the custom integration. Example:
```
 "integrations": [
    {
      "package_name": "agentless_package_links",
      "package_version": "0.0.2",
      "install_status": {
        "main": "installed",
        "remote": "not_installed"
      },
      "updated_at": "2025-05-21T08:55:47.981Z",
      "sync_status": "warning",
      "warning": {
        "title": "agentless_package_links can't be automatically synced",
        "message": "This integration must be manually installed on the remote cluster. Automatic updates and remote installs are not supported."
      }
    },
...
```

- Test the UI - go to Fleet settings and verify that the custom
integration shows a warning:

![Uploading Screenshot 2025-06-11 at 16.26.47.png…]()


### Checklist

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Cristina Amico 2025-06-12 11:53:57 +02:00 • committed by GitHub
parent 1ae725ae01
commit 9cf2b7b799
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 757 additions and 170 deletions
Download patch file Download diff file Expand all files Collapse all files

56
oas_docs/bundle.json
View file

@ -45416,7 +45416,19 @@
"type": "string"
},
"warning": {
"type": "string"
"additionalProperties": false,
"properties": {
"message": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"title"
],
"type": "object"
}
},
"required": [
@ -45428,7 +45440,19 @@
"type": "array"
},
"warning": {
"type": "string"
"additionalProperties": false,
"properties": {
"message": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"title"
],
"type": "object"
}
},
"required": [
@ -45587,7 +45611,19 @@
"type": "string"
},
"warning": {
"type": "string"
"additionalProperties": false,
"properties": {
"message": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"title"
],
"type": "object"
}
},
"required": [
@ -45599,7 +45635,19 @@
"type": "array"
},
"warning": {
"type": "string"
"additionalProperties": false,
"properties": {
"message": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"title"
],
"type": "object"
}
},
"required": [

56
oas_docs/bundle.serverless.json
View file

@ -45416,7 +45416,19 @@
"type": "string"
},
"warning": {
"type": "string"
"additionalProperties": false,
"properties": {
"message": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"title"
],
"type": "object"
}
},
"required": [
@ -45428,7 +45440,19 @@
"type": "array"
},
"warning": {
"type": "string"
"additionalProperties": false,
"properties": {
"message": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"title"
],
"type": "object"
}
},
"required": [
@ -45587,7 +45611,19 @@
"type": "string"
},
"warning": {
"type": "string"
"additionalProperties": false,
"properties": {
"message": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"title"
],
"type": "object"
}
},
"required": [
@ -45599,7 +45635,19 @@
"type": "array"
},
"warning": {
"type": "string"
"additionalProperties": false,
"properties": {
"message": {
"type": "string"
},
"title": {
"type": "string"
}
},
"required": [
"title"
],
"type": "object"
}
},
"required": [

40
oas_docs/output/kibana.serverless.yaml
View file

@ -39855,13 +39855,29 @@ paths:
updated_at:
type: string
warning:
type: string
additionalProperties: false
type: object
properties:
message:
type: string
title:
type: string
required:
- title
required:
- sync_status
- install_status
type: array
warning:
type: string
additionalProperties: false
type: object
properties:
message:
type: string
title:
type: string
required:
- title
required:
- integrations
'400':
@ -39966,13 +39982,29 @@ paths:
updated_at:
type: string
warning:
type: string
additionalProperties: false
type: object
properties:
message:
type: string
title:
type: string
required:
- title
required:
- sync_status
- install_status
type: array
warning:
type: string
additionalProperties: false
type: object
properties:
message:
type: string
title:
type: string
required:
- title
required:
- integrations
'400':

40
oas_docs/output/kibana.yaml
View file

@ -42097,13 +42097,29 @@ paths:
updated_at:
type: string
warning:
type: string
additionalProperties: false
type: object
properties:
message:
type: string
title:
type: string
required:
- title
required:
- sync_status
- install_status
type: array
warning:
type: string
additionalProperties: false
type: object
properties:
message:
type: string
title:
type: string
required:
- title
required:
- integrations
'400':
@ -42208,13 +42224,29 @@ paths:
updated_at:
type: string
warning:
type: string
additionalProperties: false
type: object
properties:
message:
type: string
title:
type: string
required:
- title
required:
- sync_status
- install_status
type: array
warning:
type: string
additionalProperties: false
type: object
properties:
message:
type: string
title:
type: string
required:
- title
required:
- integrations
'400':

5
x-pack/platform/plugins/shared/fleet/common/types/models/remote_synced_integrations.ts
View file

@ -21,7 +21,10 @@ export interface RemoteSyncedIntegrationsBase {
export interface RemoteSyncedIntegrationsStatus extends RemoteSyncedIntegrationsBase {
sync_status: SyncStatus;
error?: string;
warning?: string;
warning?: {
title: string;
message?: string;
};
updated_at?: string;
install_status: {
main: string;

5
x-pack/platform/plugins/shared/fleet/common/types/rest_spec/remote_synced_integrations.ts
View file

@ -10,5 +10,8 @@ export interface GetRemoteSyncedIntegrationsStatusResponse {
integrations: RemoteSyncedIntegrationsStatus[];
custom_assets?: RemoteSyncedCustomAssetsRecord;
error?: string;
warning?: string;
warning?: {
title: string;
message?: string;
};
}

24
x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/outputs_table/integration_status.tsx
View file

@ -202,34 +202,30 @@ export const IntegrationStatus: React.FunctionComponent<{
</>
)}
{integration.sync_status === 'warning' && (
{integration.sync_status === 'warning' && integration?.warning && (
<>
<EuiCallOut
title={
syncUninstalledIntegrations ? (
<FormattedMessage
id="xpack.fleet.integrationSyncStatus.integrationWarningContent"
defaultMessage="Integration was uninstalled, but removal from remote cluster failed."
/>
) : (
<FormattedMessage
id="xpack.fleet.integrationSyncStatus.integrationErrorTitle"
defaultMessage="Warning"
/>
)
<FormattedMessage
id="xpack.fleet.integrationSyncStatus.integrationWarningTitle"
defaultMessage="{Warning}"
values={{
Warning: integration.warning?.title,
}}
/>
}
color="warning"
iconType="warning"
size="s"
data-test-subj="integrationSyncIntegrationWarningCallout"
>
{integration?.warning && (
{integration?.warning?.message && (
<EuiText size="s">
<FormattedMessage
id="xpack.fleet.integrationSyncStatus.integrationWarningContent"
defaultMessage="{uninstallWarning}"
values={{
uninstallWarning: integration?.warning,
uninstallWarning: integration.warning.message,
}}
/>
</EuiText>

2
x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/outputs_table/integration_sync_flyout.test.tsx
View file

@ -64,7 +64,7 @@ describe('IntegrationSyncFlyout', () => {
},
updated_at: '2025-05-19T15:40:26.554Z',
sync_status: SyncStatus.WARNING,
warning: 'Unable to remove package 1password:1.32.0',
warning: { message: 'Unable to remove package 1password:1.32.0', title: 'warning' },
},
{
package_name: 'apache',

5
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts
View file

@ -141,6 +141,11 @@ export async function getPackages(
);
return null;
}
// ignoring errors of type PackageNotFoundError to avoid blocking the UI over a package not found in the registry
if (err instanceof PackageNotFoundError) {
logger.warn(`Package ${pkg.id} ${pkg.attributes.version} not found in registry`);
return null;
}
throw err;
}
} else {

130
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_errors_helpers.test.ts
View file

@ -4,12 +4,19 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { savedObjectsClientMock } from '@kbn/core/server/mocks';
import { loggerMock } from '@kbn/logging-mocks';
import type { Logger } from '@kbn/core/server';
import type { InstallFailedAttempt } from '../../../types';
import { getInstallationObject } from './get';
import {
clearLatestFailedAttempts,
addErrorToLatestFailedAttempts,
createOrUpdateFailedInstallStatus,
} from './install_errors_helpers';
const generateFailedAttempt = (version: string) => ({
@ -20,8 +27,18 @@ const generateFailedAttempt = (version: string) => ({
message: 'test',
},
});
let mockedLogger: jest.Mocked<Logger>;
const mapFailledAttempsToTargetVersion = (attemps: InstallFailedAttempt[]) =>
jest.mock('../../audit_logging');
jest.mock('./get', () => {
return { getInstallationObject: jest.fn() };
});
const getInstallationObjectMock = getInstallationObject as jest.MockedFunction<
typeof getInstallationObject
>;
const mapFailedAttempsToTargetVersion = (attemps: InstallFailedAttempt[]) =>
attemps.map((attempt) => attempt.target_version);
describe('Install error helpers', () => {
@ -30,16 +47,16 @@ describe('Install error helpers', () => {
generateFailedAttempt('0.1.0'),
generateFailedAttempt('0.2.0'),
];
it('should clear previous error on succesfull upgrade', () => {
const currentFailledAttemps = clearLatestFailedAttempts('0.2.0', previousFailedAttemps);
it('should clear previous error on successful upgrade', () => {
const currentFailedAttemps = clearLatestFailedAttempts('0.2.0', previousFailedAttemps);
expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual([]);
expect(mapFailedAttempsToTargetVersion(currentFailedAttemps)).toEqual([]);
});
it('should not clear previous upgrade error on succesfull rollback', () => {
const currentFailledAttemps = clearLatestFailedAttempts('0.1.0', previousFailedAttemps);
it('should not clear previous upgrade error on successful rollback', () => {
const currentFailedAttempts = clearLatestFailedAttempts('0.1.0', previousFailedAttemps);
expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual(['0.2.0']);
expect(mapFailedAttempsToTargetVersion(currentFailedAttempts)).toEqual(['0.2.0']);
});
});
@ -52,14 +69,14 @@ describe('Install error helpers', () => {
generateFailedAttempt('0.2.2'),
generateFailedAttempt('0.2.1'),
];
const currentFailledAttemps = addErrorToLatestFailedAttempts({
const currentFailedAttempts = addErrorToLatestFailedAttempts({
targetVersion: '0.2.6',
createdAt: new Date().toISOString(),
error: new Error('new test'),
latestAttempts: previousFailedAttemps,
});
expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual([
expect(mapFailedAttempsToTargetVersion(currentFailedAttempts)).toEqual([
'0.2.6',
'0.2.5',
'0.2.4',
@ -68,4 +85,99 @@ describe('Install error helpers', () => {
]);
});
});
describe('createOrUpdateFailedInstallStatus', () => {
const soClientMock = savedObjectsClientMock.create();
mockedLogger = loggerMock.create();
it('should create installation object with latest_install_failed_attempts if none was found', async () => {
getInstallationObjectMock.mockResolvedValue(undefined);
await createOrUpdateFailedInstallStatus({
logger: mockedLogger,
savedObjectsClient: soClientMock,
pkgName: 'test-package',
pkgVersion: '0.1.0',
error: new Error('test error'),
installSource: 'registry',
});
expect(soClientMock.create).toHaveBeenCalledWith(
'epm-packages',
{
es_index_patterns: {},
install_source: 'registry',
install_started_at: expect.any(String),
install_status: 'install_failed',
install_version: '0.1.0',
installed_es: [],
installed_kibana: [],
name: 'test-package',
package_assets: [],
verification_status: 'unknown',
version: '0.1.0',
latest_install_failed_attempts: [
{
created_at: expect.any(String),
target_version: '0.1.0',
error: {
message: 'test error',
name: 'Error',
stack: expect.any(String),
},
},
],
},
{ id: 'test-package', overwrite: true }
);
});
it('should update latest_install_failed_attempts in the installation object if it exists', async () => {
getInstallationObjectMock.mockResolvedValue({
es_index_patterns: {},
install_source: 'registry',
install_started_at: '2025-06-11T07:15:06.838Z',
install_status: 'install_failed',
install_version: '0.1.0',
installed_es: [],
installed_kibana: [],
name: 'test-package',
package_assets: [],
verification_status: 'unknown',
version: '0.1.0',
latest_install_failed_attempts: [
{
created_at: '2025-06-11T07:15:06.838Z',
target_version: '0.1.0',
error: {
message: 'test error',
name: 'Error',
stack: 'test error - stacktrace',
},
},
],
} as any);
await createOrUpdateFailedInstallStatus({
logger: mockedLogger,
savedObjectsClient: soClientMock,
pkgName: 'test-package',
pkgVersion: '0.1.0',
error: new Error('test error'),
installSource: 'registry',
});
expect(soClientMock.update).toHaveBeenCalledWith('epm-packages', 'test-package', {
latest_install_failed_attempts: [
{
created_at: expect.any(String),
target_version: '0.1.0',
error: {
message: 'test error',
name: 'Error',
stack: expect.any(String),
},
},
],
});
});
});
});

97
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_errors_helpers.ts
View file

@ -7,7 +7,17 @@
import { lt } from 'semver';
import type { InstallFailedAttempt } from '../../../types';
import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import type { SavedObjectsClientContract } from '@kbn/core/server';
import type { Logger } from '@kbn/core/server';
import type { InstallFailedAttempt, InstallSource, Installation } from '../../../types';
import { auditLoggingService } from '../../audit_logging';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
import { getInstallationObject } from './get';
const MAX_ATTEMPTS_TO_KEEP = 5;
@ -42,3 +52,88 @@ export function addErrorToLatestFailedAttempts({
...latestAttempts,
].slice(0, MAX_ATTEMPTS_TO_KEEP);
}
export const createOrUpdateFailedInstallStatus = async ({
logger,
savedObjectsClient,
pkgName,
pkgVersion,
error,
installSource = 'registry',
}: {
logger: Logger;
savedObjectsClient: SavedObjectsClientContract;
pkgName: string;
pkgVersion: string;
error: Error;
installSource?: InstallSource;
}) => {
const installation = await getInstallationObject({
pkgName,
savedObjectsClient,
});
if (installation) {
auditLoggingService.writeCustomSoAuditLog({
action: 'update',
id: pkgName,
name: pkgName,
savedObjectType: PACKAGES_SAVED_OBJECT_TYPE,
});
const latestInstallFailedAttempts = addErrorToLatestFailedAttempts({
error,
targetVersion: pkgVersion,
createdAt: new Date().toISOString(),
latestAttempts: installation?.attributes?.latest_install_failed_attempts,
});
try {
return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
latest_install_failed_attempts: latestInstallFailedAttempts,
});
} catch (err) {
logger.warn(`Error occurred while updating installation failed attempts: ${err}`);
}
} else {
auditLoggingService.writeCustomSoAuditLog({
action: 'create',
id: pkgName,
name: pkgName,
savedObjectType: PACKAGES_SAVED_OBJECT_TYPE,
});
const installFailedAttempts = addErrorToLatestFailedAttempts({
error,
targetVersion: pkgVersion,
createdAt: new Date().toISOString(),
latestAttempts: [],
});
const savedObject: Installation = {
installed_kibana: [],
installed_es: [],
package_assets: [],
name: pkgName,
version: pkgVersion,
install_version: pkgVersion,
install_status: 'install_failed',
install_started_at: new Date().toISOString(),
verification_status: 'unknown',
latest_install_failed_attempts: installFailedAttempts,
es_index_patterns: {},
install_source: installSource,
};
try {
return await savedObjectsClient.create<Installation>(
PACKAGES_SAVED_OBJECT_TYPE,
savedObject,
{
id: pkgName,
overwrite: true,
}
);
} catch (err) {
if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {
logger.error(`Failed to create package installation: ${err}`);
}
}
}
};

84
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/compare_synced_integrations.test.ts
View file

@ -216,7 +216,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
});
});
it('should return status = synchronizing if there are integrations on sync index but none are installed yet', async () => {
it('should return synchronizing status if there are integrations on sync index but none are installed yet', async () => {
esClientMock.search.mockResolvedValueOnce({
hits: {
hits: [
@ -279,7 +279,76 @@ describe('fetchAndCompareSyncedIntegrations', () => {
],
});
});
it('should compare integrations installed on remote with the ones on sync index', async () => {
it('should return a warning for integrations not found in registry', async () => {
esClientMock.search.mockResolvedValueOnce({
hits: {
hits: [
{
_source: {
integrations: [
{
package_name: 'custom-pkg',
package_version: '1.0.0',
updated_at: '2025-03-20T14:18:40.111Z',
install_status: 'installed',
},
],
},
},
],
},
} as any);
(getPackageSavedObjects as jest.MockedFunction<any>).mockReturnValue({
page: 1,
per_page: 10000,
total: 1,
saved_objects: [
{
type: 'epm-packages',
id: 'custom-pkg',
attributes: {
version: '1.0.0',
install_status: 'install_failed',
latest_install_failed_attempts: [
{
created_at: '2025-06-10T15:30:31.614Z',
target_version: '0.0.2',
error: {
name: 'PackageNotFoundError',
message: '[agentless_package_links] package not found in registry',
},
},
],
},
updated_at: '2025-03-26T14:06:27.611Z',
},
],
});
const res = await fetchAndCompareSyncedIntegrations(
esClientMock,
soClientMock,
'fleet-synced-integrations-ccr-*',
mockedLogger
);
expect(res).toEqual({
integrations: [
{
sync_status: 'warning',
package_name: 'custom-pkg',
package_version: '1.0.0',
install_status: { main: 'installed', remote: 'not_installed' },
updated_at: expect.any(String),
warning: {
message:
'This integration must be manually installed on the remote cluster. Automatic updates and remote installs are not supported.',
title: "Integration can't be automatically synced",
},
},
],
});
});
it('should return completed state when the integrations are correctly synced', async () => {
esClientMock.search.mockResolvedValueOnce({
hits: {
hits: [
@ -461,7 +530,10 @@ describe('fetchAndCompareSyncedIntegrations', () => {
updated_at: expect.any(String),
},
{
warning: 'failed to uninstall at Wed, 26 Mar 2025 14:06:27 GMT',
warning: {
message: 'failed to uninstall at Wed, 26 Mar 2025 14:06:27 GMT',
title: 'Integration was uninstalled, but removal from remote cluster failed.',
},
install_status: {
main: 'not_installed',
remote: 'installed',
@ -645,7 +717,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
error: 'Found incorrect installed version 1.67.2',
},
{
error: `Installation status: install_failed error: installation failure at Tue, 20 Jun 2023 08:47:31 GMT`,
error: `Installation status: install_failed installation failure at Tue, 20 Jun 2023 08:47:31 GMT`,
package_name: 'synthetics',
package_version: '1.4.1',
install_status: { main: 'installed', remote: 'install_failed' },
@ -942,7 +1014,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
});
});
it('should return status = completed if custom assets are equal', async () => {
it('should return completed status if custom assets are equal', async () => {
(getPipelineMock as jest.MockedFunction<any>).mockResolvedValueOnce({
'logs-system.auth@custom': {
processors: [
@ -1042,7 +1114,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
},
});
});
it('should return status = synchronizing if versions do not match', async () => {
it('should return synchronizing status if versions do not match', async () => {
const searchMockWithVersionedPipeline = jest.fn().mockResolvedValue({
hits: {
hits: [

35
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/compare_synced_integrations.ts
View file

@ -180,6 +180,7 @@ const compareIntegrations = (
updated_at: ccrIntegration?.updated_at,
};
}
if (!localIntegrationSO) {
return {
...baseIntegrationData,
@ -216,11 +217,26 @@ const compareIntegrations = (
latestFailedAttemptTime = `at ${new Date(
latestInstallFailedAttempts.created_at
).toUTCString()}`;
latestFailedAttempt = latestInstallFailedAttempts.error?.message
? `error: ${latestInstallFailedAttempts.error?.message}`
: '';
}
// handling special case for those integrations that cannot be found in registry
if (latestInstallFailedAttempts.error?.name === 'PackageNotFoundError') {
return {
...baseIntegrationData,
install_status: {
main: ccrIntegration.install_status,
remote: 'not_installed',
},
updated_at: ccrIntegration.updated_at,
sync_status: SyncStatus.WARNING,
warning: {
title: `Integration can't be automatically synced`,
message: `This integration must be manually installed on the remote cluster. Automatic updates and remote installs are not supported.`,
},
};
} else {
latestFailedAttempt = latestInstallFailedAttempts.error?.message ?? '';
}
}
return {
...baseIntegrationData,
install_status: {
@ -246,9 +262,7 @@ const compareIntegrations = (
latestUninstallFailedAttemptTime = `at ${new Date(
latestInstallFailedAttempts.created_at
).toUTCString()}`;
latestUninstallFailedAttempt = latestInstallFailedAttempts.error?.message
? `${latestInstallFailedAttempts.error?.message}`
: '';
latestUninstallFailedAttempt = latestInstallFailedAttempts.error?.message ?? '';
}
return {
...baseIntegrationData,
@ -259,7 +273,12 @@ const compareIntegrations = (
updated_at: ccrIntegration.updated_at,
sync_status: SyncStatus.WARNING,
...(localIntegrationSO?.attributes.latest_uninstall_failed_attempts !== undefined
? { warning: `${latestUninstallFailedAttempt} ${latestUninstallFailedAttemptTime}` }
? {
warning: {
message: `${latestUninstallFailedAttempt} ${latestUninstallFailedAttemptTime}`,
title: 'Integration was uninstalled, but removal from remote cluster failed.',
},
}
: {}),
};
}

3
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/model.ts
View file

@ -5,11 +5,14 @@
* 2.0.
*/
import type { InstallSource } from '../../types';
export interface IntegrationsData {
package_name: string;
package_version: string;
updated_at: string;
install_status: string;
install_source?: InstallSource;
}
export interface BaseCustomAssetsData {

306
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_on_remote.test.ts
View file

@ -8,14 +8,20 @@
import { PackageNotFoundError } from '../../errors';
import { outputService } from '../../services';
import { createOrUpdateFailedInstallStatus } from '../../services/epm/packages/install_errors_helpers';
import { installCustomAsset } from './custom_assets';
import { syncIntegrationsOnRemote } from './sync_integrations_on_remote';
jest.mock('../../services');
jest.mock('./custom_assets');
jest.mock('../../services/epm/packages/install_errors_helpers');
const outputServiceMock = outputService as jest.Mocked<typeof outputService>;
const createOrUpdateFailedInstallStatusMock = createOrUpdateFailedInstallStatus as jest.Mocked<
typeof createOrUpdateFailedInstallStatus
>;
describe('syncIntegrationsOnRemote', () => {
const abortController = new AbortController();
@ -89,11 +95,19 @@ describe('syncIntegrationsOnRemote', () => {
package_name: 'nginx',
package_version: '2.2.0',
updated_at: '2021-01-01T00:00:00.000Z',
install_source: 'registry',
},
{
package_name: 'system',
package_version: '2.2.0',
updated_at: '2021-01-01T00:00:00.000Z',
install_source: 'registry',
},
{
package_name: 'custom-pkg',
package_version: '1.0.0',
updated_at: '2021-01-01T00:00:00.000Z',
install_source: 'custom',
},
],
custom_assets: {
@ -273,42 +287,26 @@ describe('syncIntegrationsOnRemote', () => {
expect(packageClientMock.installPackage).toHaveBeenCalledTimes(2);
});
it('should not retry if max retry attempts reached', async () => {
it('should call createOrUpdateFailedInstallStatus if installation failed', async () => {
getIndicesMock.mockResolvedValue({
'fleet-synced-integrations-ccr-remote1': {},
});
searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
packageClientMock.getInstallation.mockImplementation((packageName: string) =>
packageName === 'nginx'
? {
install_status: 'install_failed',
version: '2.1.0',
latest_install_failed_attempts: [
{
created_at: new Date().toISOString(),
},
{
created_at: '2025-01-28T08:11:44.395Z',
},
{
created_at: '2025-01-27T08:11:44.395Z',
},
{
created_at: '2025-01-26T08:11:44.395Z',
},
{
created_at: '2025-01-25T08:11:44.395Z',
},
],
}
packageName === 'custom-pkg'
? undefined
: {
install_status: 'installed',
version: '2.2.0',
}
);
packageClientMock.installPackage.mockResolvedValue({
status: 'installed',
packageClientMock.installPackage.mockImplementation(({ pkgName, pkgVersion }: any) => {
if (pkgName === 'custom-pkg') {
throw new PackageNotFoundError('package not found in registry');
}
return {
status: 'installed',
};
});
await syncIntegrationsOnRemote(
@ -319,88 +317,15 @@ describe('syncIntegrationsOnRemote', () => {
loggerMock
);
expect(packageClientMock.installPackage).not.toHaveBeenCalled();
});
it('should not retry if retry time not passed', async () => {
getIndicesMock.mockResolvedValue({
'fleet-synced-integrations-ccr-remote1': {},
expect(packageClientMock.installPackage).toHaveBeenCalledTimes(1);
expect(createOrUpdateFailedInstallStatusMock).toHaveBeenCalledWith({
error: new PackageNotFoundError('package not found in registry'),
installSource: 'custom',
pkgName: 'custom-pkg',
pkgVersion: '1.0.0',
logger: expect.anything(),
savedObjectsClient: expect.anything(),
});
searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
packageClientMock.getInstallation.mockImplementation((packageName: string) =>
packageName === 'nginx'
? {
install_status: 'install_failed',
version: '2.1.0',
latest_install_failed_attempts: [
{
created_at: new Date().toISOString(),
},
{
created_at: '2025-01-28T08:11:44.395Z',
},
{
created_at: '2025-01-27T08:11:44.395Z',
},
{
created_at: '2025-01-26T08:11:44.395Z',
},
],
}
: {
install_status: 'installed',
version: '2.2.0',
}
);
packageClientMock.installPackage.mockResolvedValue({
status: 'installed',
});
await syncIntegrationsOnRemote(
esClientMock,
soClientMock,
packageClientMock,
abortController,
loggerMock
);
expect(packageClientMock.installPackage).not.toHaveBeenCalled();
});
it('should retry if retry time passed', async () => {
getIndicesMock.mockResolvedValue({
'fleet-synced-integrations-ccr-remote1': {},
});
searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
packageClientMock.getInstallation.mockImplementation((packageName: string) =>
packageName === 'nginx'
? {
install_status: 'install_failed',
version: '2.1.0',
latest_install_failed_attempts: [
{
created_at: '2025-02-28T04:11:44.395Z',
},
],
}
: {
install_status: 'installed',
version: '2.2.0',
}
);
packageClientMock.installPackage.mockResolvedValue({
status: 'installed',
});
await syncIntegrationsOnRemote(
esClientMock,
soClientMock,
packageClientMock,
abortController,
loggerMock
);
expect(packageClientMock.installPackage).toHaveBeenCalled();
});
it('should do nothing if sync enabled and the package is installing', async () => {
@ -454,4 +379,171 @@ describe('syncIntegrationsOnRemote', () => {
expect(installCustomAsset).toHaveBeenCalledTimes(2);
});
describe('Retry logic', () => {
it('should not retry if max retry attempts reached', async () => {
getIndicesMock.mockResolvedValue({
'fleet-synced-integrations-ccr-remote1': {},
});
searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
packageClientMock.getInstallation.mockImplementation((packageName: string) =>
packageName === 'nginx'
? {
install_status: 'install_failed',
version: '2.1.0',
latest_install_failed_attempts: [
{
created_at: new Date().toISOString(),
},
{
created_at: '2025-01-28T08:11:44.395Z',
},
{
created_at: '2025-01-27T08:11:44.395Z',
},
{
created_at: '2025-01-26T08:11:44.395Z',
},
{
created_at: '2025-01-25T08:11:44.395Z',
},
],
}
: {
install_status: 'installed',
version: '2.2.0',
}
);
packageClientMock.installPackage.mockResolvedValue({
status: 'installed',
});
await syncIntegrationsOnRemote(
esClientMock,
soClientMock,
packageClientMock,
abortController,
loggerMock
);
expect(packageClientMock.installPackage).not.toHaveBeenCalled();
});
it('should not retry if retry time not passed', async () => {
getIndicesMock.mockResolvedValue({
'fleet-synced-integrations-ccr-remote1': {},
});
searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
packageClientMock.getInstallation.mockImplementation((packageName: string) =>
packageName === 'nginx'
? {
install_status: 'install_failed',
version: '2.1.0',
latest_install_failed_attempts: [
{
created_at: new Date().toISOString(),
},
{
created_at: '2025-01-28T08:11:44.395Z',
},
{
created_at: '2025-01-27T08:11:44.395Z',
},
{
created_at: '2025-01-26T08:11:44.395Z',
},
],
}
: {
install_status: 'installed',
version: '2.2.0',
}
);
packageClientMock.installPackage.mockResolvedValue({
status: 'installed',
});
await syncIntegrationsOnRemote(
esClientMock,
soClientMock,
packageClientMock,
abortController,
loggerMock
);
expect(packageClientMock.installPackage).not.toHaveBeenCalled();
});
it('should retry if retry time passed', async () => {
getIndicesMock.mockResolvedValue({
'fleet-synced-integrations-ccr-remote1': {},
});
searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
packageClientMock.getInstallation.mockImplementation((packageName: string) =>
packageName === 'nginx'
? {
install_status: 'install_failed',
version: '2.1.0',
latest_install_failed_attempts: [
{
created_at: '2025-02-28T04:11:44.395Z',
},
],
}
: {
install_status: 'installed',
version: '2.2.0',
}
);
packageClientMock.installPackage.mockResolvedValue({
status: 'installed',
});
await syncIntegrationsOnRemote(
esClientMock,
soClientMock,
packageClientMock,
abortController,
loggerMock
);
expect(packageClientMock.installPackage).toHaveBeenCalled();
});
it('should not retry if package has install_source custom even if retry time has passed', async () => {
getIndicesMock.mockResolvedValue({
'fleet-synced-integrations-ccr-remote1': {},
});
searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
packageClientMock.getInstallation.mockImplementation((packageName: string) =>
packageName === 'custom'
? {
install_status: 'install_failed',
version: '2.1.0',
latest_install_failed_attempts: [
{
created_at: '2025-02-28T04:11:44.395Z',
},
],
}
: {
install_status: 'installed',
version: '2.2.0',
}
);
packageClientMock.installPackage.mockResolvedValue({
status: 'installed',
});
await syncIntegrationsOnRemote(
esClientMock,
soClientMock,
packageClientMock,
abortController,
loggerMock
);
expect(packageClientMock.installPackage).not.toHaveBeenCalled();
});
});
});

24
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_on_remote.ts
View file

@ -26,6 +26,10 @@ import { getInstallation, removeInstallation } from '../../services/epm/packages
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../constants';
import { createOrUpdateFailedInstallStatus } from '../../services/epm/packages/install_errors_helpers';
import type { InstallSource } from '../../types';
import { installCustomAsset } from './custom_assets';
import type { CustomAssetsData, SyncIntegrationsData } from './model';
@ -93,7 +97,8 @@ async function getSyncIntegrationsEnabled(
}
async function installPackageIfNotInstalled(
pkg: { package_name: string; package_version: string },
savedObjectsClient: SavedObjectsClientContract,
pkg: { package_name: string; package_version: string; install_source?: InstallSource },
packageClient: PackageClient,
logger: Logger,
abortController: AbortController
@ -123,12 +128,13 @@ async function installPackageIfNotInstalled(
}
const lastRetryAttemptTime = installation.latest_install_failed_attempts?.[0].created_at;
// retry install if backoff time has passed since the last attempt
// excluding custom and upload packages from retries
const shouldRetryInstall =
attempt > 0 &&
lastRetryAttemptTime &&
Date.now() - Date.parse(lastRetryAttemptTime) >
RETRY_BACKOFF_MINUTES[attempt - 1] * 60 * 1000;
RETRY_BACKOFF_MINUTES[attempt - 1] * 60 * 1000 &&
(pkg.install_source === 'registry' || pkg.install_source === 'bundled');
if (!shouldRetryInstall) {
return;
}
@ -165,6 +171,16 @@ async function installPackageIfNotInstalled(
logger.error(
`Failed to install package ${pkg.package_name} with version ${pkg.package_version}, error: ${error}`
);
if (error instanceof PackageNotFoundError && error.message.includes('not found in registry')) {
await createOrUpdateFailedInstallStatus({
logger,
savedObjectsClient,
pkgName: pkg.package_name,
pkgVersion: pkg.package_version,
error,
installSource: pkg?.install_source,
});
}
}
}
@ -231,7 +247,7 @@ export const syncIntegrationsOnRemote = async (
if (abortController.signal.aborted) {
throw new Error('Task was aborted');
}
await installPackageIfNotInstalled(pkg, packageClient, logger, abortController);
await installPackageIfNotInstalled(soClient, pkg, packageClient, logger, abortController);
}
const uninstalledIntegrations =

1
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_task.ts
View file

@ -230,6 +230,7 @@ export class SyncIntegrationsTask {
package_version: item.attributes.version,
updated_at: item.updated_at ?? new Date().toISOString(),
install_status: item.attributes.install_status,
install_source: item.attributes.install_source,
};
});

14
x-pack/platform/plugins/shared/fleet/server/types/models/synced_integrations.ts
View file

@ -23,7 +23,12 @@ export const RemoteSyncedIntegrationsStatusSchema = RemoteSyncedIntegrationsBase
schema.literal(SyncStatus.WARNING),
]),
error: schema.maybe(schema.string()),
warning: schema.maybe(schema.string()),
warning: schema.maybe(
schema.object({
title: schema.string(),
message: schema.maybe(schema.string()),
})
),
updated_at: schema.maybe(schema.string()),
install_status: schema.object({
main: schema.string(),
@ -50,5 +55,10 @@ export const GetRemoteSyncedIntegrationsStatusResponseSchema = schema.object({
integrations: schema.arrayOf(RemoteSyncedIntegrationsStatusSchema),
custom_assets: schema.maybe(schema.recordOf(schema.string(), CustomAssetsDataSchema)),
error: schema.maybe(schema.string()),
warning: schema.maybe(schema.string()),
warning: schema.maybe(
schema.object({
title: schema.string(),
message: schema.maybe(schema.string()),
})
),
});