github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 10:40:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] save and read custom asset errors (#218816)

Browse source 
## Summary

Closes https://github.com/elastic/kibana/issues/217154

Improvements to sync integrations status API and error reporting

- Saving custom asset update errors to the package SO in
`latest_custom_asset_install_failed_attempts` field
- Reading these errors in the status API and UI 

- Fix sync status calculation: show `FAILED` if one of integrations or
custom assets are in failed state, `SYNCHRONIZING` if one of
integrations or custom assets are in synchronizing state, otherwise show
`COMPLETED` state.

<img width="608" alt="image"
src="https://github.com/user-attachments/assets/15a17690-443b-4ca1-b705-cc92ec7d3b20"
/>

- Reading the `followStats` API to report on fatal errors, found that
the `followInfo` API doesn't report if the connection to the remote
cluster fails. Reproduced this by updating an active Remote Cluster with
an invalid port. The `followInfo` API still reports `active` status.

<img width="612" alt="image"
src="https://github.com/user-attachments/assets/e95ebc62-4ed9-42c2-9954-93d9438b6ece"
/>


```
GET fleet-synced-integrations-ccr-main/_ccr/stats

{
  "indices": [
    {
      "index": "fleet-synced-integrations-ccr-main",
      "shards": [
        {
          "remote_cluster": "main",
          "leader_index": "fleet-synced-integrations",
          "follower_index": "fleet-synced-integrations-ccr-main",
          ...
          "fatal_exception": {
            "type": "exception",
            "reason": "java.lang.IllegalArgumentException: port out of range:93001",
            "caused_by": {
              "type": "illegal_argument_exception",
              "reason": "port out of range:93001"
            }
          }
        }
      ]
    }
  ]
}
```

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Julia Bardi 2025-04-24 10:01:41 +02:00 • committed by GitHub
parent 29d18cb0d1
commit 0ed82c4d52
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 478 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

36
oas_docs/bundle.json
View file

@ -18202,13 +18202,13 @@
"additionalProperties": false,
"properties": {
"secrets": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"ssl": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"key": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"id": {
"type": "string"
@ -18385,13 +18385,13 @@
"type": "string"
},
"secrets": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"ssl": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"key": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"id": {
"type": "string"
@ -19507,10 +19507,6 @@
"type": "string"
}
},
"required": [
"api_key_id",
"type"
],
"type": "object"
},
"type": "object"
@ -21605,10 +21601,6 @@
"type": "string"
}
},
"required": [
"api_key_id",
"type"
],
"type": "object"
},
"type": "object"
@ -22086,10 +22078,6 @@
"type": "string"
}
},
"required": [
"api_key_id",
"type"
],
"type": "object"
},
"type": "object"
@ -43893,6 +43881,12 @@
"additionalProperties": {
"additionalProperties": false,
"properties": {
"error": {
"type": "string"
},
"is_deleted": {
"type": "boolean"
},
"name": {
"type": "string"
},
@ -44034,6 +44028,12 @@
"additionalProperties": {
"additionalProperties": false,
"properties": {
"error": {
"type": "string"
},
"is_deleted": {
"type": "boolean"
},
"name": {
"type": "string"
},

36
oas_docs/bundle.serverless.json
View file

@ -18202,13 +18202,13 @@
"additionalProperties": false,
"properties": {
"secrets": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"ssl": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"key": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"id": {
"type": "string"
@ -18385,13 +18385,13 @@
"type": "string"
},
"secrets": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"ssl": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"key": {
"additionalProperties": false,
"additionalProperties": true,
"properties": {
"id": {
"type": "string"
@ -19507,10 +19507,6 @@
"type": "string"
}
},
"required": [
"api_key_id",
"type"
],
"type": "object"
},
"type": "object"
@ -21605,10 +21601,6 @@
"type": "string"
}
},
"required": [
"api_key_id",
"type"
],
"type": "object"
},
"type": "object"
@ -22086,10 +22078,6 @@
"type": "string"
}
},
"required": [
"api_key_id",
"type"
],
"type": "object"
},
"type": "object"
@ -43893,6 +43881,12 @@
"additionalProperties": {
"additionalProperties": false,
"properties": {
"error": {
"type": "string"
},
"is_deleted": {
"type": "boolean"
},
"name": {
"type": "string"
},
@ -44034,6 +44028,12 @@
"additionalProperties": {
"additionalProperties": false,
"properties": {
"error": {
"type": "string"
},
"is_deleted": {
"type": "boolean"
},
"name": {
"type": "string"
},

29
oas_docs/output/kibana.serverless.yaml
View file

@ -21380,15 +21380,15 @@ paths:
type: object
properties:
secrets:
additionalProperties: false
additionalProperties: true
type: object
properties:
ssl:
additionalProperties: false
additionalProperties: true
type: object
properties:
key:
additionalProperties: false
additionalProperties: true
type: object
properties:
id:
@ -21503,15 +21503,15 @@ paths:
proxy_url:
type: string
secrets:
additionalProperties: false
additionalProperties: true
type: object
properties:
ssl:
additionalProperties: false
additionalProperties: true
type: object
properties:
key:
additionalProperties: false
additionalProperties: true
type: object
properties:
id:
@ -22431,9 +22431,6 @@ paths:
type: array
type:
type: string
required:
- api_key_id
- type
type: object
packages:
items:
@ -22893,9 +22890,6 @@ paths:
type: array
type:
type: string
required:
- api_key_id
- type
type: object
packages:
items:
@ -23237,9 +23231,6 @@ paths:
type: array
type:
type: string
required:
- api_key_id
- type
type: object
packages:
items:
@ -38636,6 +38627,10 @@ paths:
additionalProperties: false
type: object
properties:
error:
type: string
is_deleted:
type: boolean
name:
type: string
package_name:
@ -38726,6 +38721,10 @@ paths:
additionalProperties: false
type: object
properties:
error:
type: string
is_deleted:
type: boolean
name:
type: string
package_name:

29
oas_docs/output/kibana.yaml
View file

@ -23622,15 +23622,15 @@ paths:
type: object
properties:
secrets:
additionalProperties: false
additionalProperties: true
type: object
properties:
ssl:
additionalProperties: false
additionalProperties: true
type: object
properties:
key:
additionalProperties: false
additionalProperties: true
type: object
properties:
id:
@ -23745,15 +23745,15 @@ paths:
proxy_url:
type: string
secrets:
additionalProperties: false
additionalProperties: true
type: object
properties:
ssl:
additionalProperties: false
additionalProperties: true
type: object
properties:
key:
additionalProperties: false
additionalProperties: true
type: object
properties:
id:
@ -24673,9 +24673,6 @@ paths:
type: array
type:
type: string
required:
- api_key_id
- type
type: object
packages:
items:
@ -25135,9 +25132,6 @@ paths:
type: array
type:
type: string
required:
- api_key_id
- type
type: object
packages:
items:
@ -25479,9 +25473,6 @@ paths:
type: array
type:
type: string
required:
- api_key_id
- type
type: object
packages:
items:
@ -40878,6 +40869,10 @@ paths:
additionalProperties: false
type: object
properties:
error:
type: string
is_deleted:
type: boolean
name:
type: string
package_name:
@ -40968,6 +40963,10 @@ paths:
additionalProperties: false
type: object
properties:
error:
type: string
is_deleted:
type: boolean
name:
type: string
package_name:

20
x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts
View file

@ -612,9 +612,8 @@ export interface ExperimentalDataStreamFeature {
features: Partial<Record<ExperimentalIndexingFeature, boolean>>;
}
export interface InstallFailedAttempt {
export interface FailedAttempt {
created_at: string;
target_version: string;
error: {
name: string;
message: string;
@ -622,13 +621,13 @@ export interface InstallFailedAttempt {
};
}
export interface UninstallFailedAttempt {
created_at: string;
error: {
name: string;
message: string;
stack?: string;
};
export interface InstallFailedAttempt extends FailedAttempt {
target_version: string;
}
export interface CustomAssetFailedAttempt extends FailedAttempt {
type: string;
name: string;
}
export enum INSTALL_STATES {
@ -682,8 +681,9 @@ export interface Installation {
internal?: boolean;
removable?: boolean;
latest_install_failed_attempts?: InstallFailedAttempt[];
latest_uninstall_failed_attempts?: UninstallFailedAttempt[];
latest_uninstall_failed_attempts?: FailedAttempt[];
latest_executed_state?: InstallLatestExecutedState;
latest_custom_asset_install_failed_attempts?: { [asset: string]: CustomAssetFailedAttempt };
}
export interface PackageUsageStats {

25
x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/outputs_table/integration_status.tsx
View file

@ -18,6 +18,7 @@ import {
EuiCallOut,
EuiIcon,
EuiLoadingSpinner,
EuiBadge,
} from '@elastic/eui';
import type { EuiAccordionProps } from '@elastic/eui/src/components/accordion';
@ -36,6 +37,7 @@ import { PackageIcon } from '../../../../../../components';
import { sendGetPackageInfoByKeyForRq } from '../../../../hooks';
import { IntegrationStatusBadge } from './integration_status_badge';
import { getIntegrationStatus } from './integration_sync_status';
const StyledEuiPanel = styled(EuiPanel)`
border: solid 1px ${(props) => props.theme.eui.euiFormBorderColor};
@ -122,6 +124,9 @@ export const IntegrationStatus: React.FunctionComponent<{
});
}, [integration.package_name, integration.package_version]);
const statuses = [integration.sync_status, ...customAssets.map((asset) => asset.sync_status)];
const integrationStatus = getIntegrationStatus(statuses).toUpperCase();
return (
<CollapsablePanel
id={integration.package_name}
@ -148,7 +153,7 @@ export const IntegrationStatus: React.FunctionComponent<{
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<IntegrationStatusBadge status={integration.sync_status.toUpperCase()} />
<IntegrationStatusBadge status={integrationStatus} />
</EuiFlexItem>
</EuiFlexGroup>
</h3>
@ -181,7 +186,23 @@ export const IntegrationStatus: React.FunctionComponent<{
<EuiAccordion
id={`${customAsset.type}:${customAsset.name}`}
key={`${customAsset.type}:${customAsset.name}`}
buttonContent={customAsset.name}
buttonContent={
<EuiFlexGroup alignItems="baseline" gutterSize="xs">
<EuiFlexItem grow={false}>
<EuiText size="s">{customAsset.name}</EuiText>
</EuiFlexItem>
{customAsset.is_deleted && (
<EuiFlexItem grow={false}>
<EuiBadge color="hollow">
<FormattedMessage
id="xpack.fleet.integrationSyncStatus.deletedText"
defaultMessage="Deleted"
/>
</EuiBadge>
</EuiFlexItem>
)}
</EuiFlexGroup>
}
data-test-subj={`${customAsset.type}:${customAsset.name}-accordion`}
extraAction={
customAsset.sync_status === SyncStatus.SYNCHRONIZING ? (

32
x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/outputs_table/integration_sync_status.tsx
View file

@ -7,7 +7,7 @@
import React, { memo, useMemo, useState } from 'react';
import { type Output } from '../../../../../../../common/types';
import { SyncStatus, type Output } from '../../../../../../../common/types';
import { useGetRemoteSyncedIntegrationsStatusQuery } from '../../../../hooks';
@ -18,6 +18,14 @@ interface Props {
output: Output;
}
export function getIntegrationStatus(statuses: SyncStatus[]): SyncStatus {
return statuses.some((current) => current === SyncStatus.FAILED)
? SyncStatus.FAILED
: statuses.some((current) => current === SyncStatus.SYNCHRONIZING)
? SyncStatus.SYNCHRONIZING
: SyncStatus.COMPLETED;
}
export const IntegrationSyncStatus: React.FunctionComponent<Props> = memo(({ output }) => {
const { data: syncedIntegrationsStatus, error } = useGetRemoteSyncedIntegrationsStatusQuery(
output.id,
@ -37,20 +45,18 @@ export const IntegrationSyncStatus: React.FunctionComponent<Props> = memo(({ out
return 'SYNCHRONIZING';
}
const syncCompleted =
syncedIntegrationsStatus?.integrations.every(
(integration) => integration.sync_status === 'completed'
) &&
Object.values(syncedIntegrationsStatus?.custom_assets ?? {}).every(
(asset) => asset.sync_status === 'completed'
);
const statuses = [
...(syncedIntegrationsStatus?.integrations?.map((integration) => integration.sync_status) ||
[]),
...Object.values(syncedIntegrationsStatus?.custom_assets ?? {}).map(
(asset) => asset.sync_status
),
];
const integrationStatus = getIntegrationStatus(statuses).toUpperCase();
const newStatus =
(error as any)?.message || syncedIntegrationsStatus?.error
? 'FAILED'
: syncCompleted
? 'COMPLETED'
: 'SYNCHRONIZING';
(error as any)?.message || syncedIntegrationsStatus?.error ? 'FAILED' : integrationStatus;
return newStatus;
}, [output, syncedIntegrationsStatus, error]);

6
x-pack/platform/plugins/shared/fleet/server/services/epm/packages/uninstall_errors_helpers.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import type { UninstallFailedAttempt } from '../../../types';
import type { FailedAttempt } from '../../../types';
const MAX_ATTEMPTS_TO_KEEP = 5;
@ -16,8 +16,8 @@ export function updateUninstallFailedAttempts({
}: {
error: Error;
createdAt: string;
latestAttempts?: UninstallFailedAttempt[];
}): UninstallFailedAttempt[] {
latestAttempts?: FailedAttempt[];
}): FailedAttempt[] {
return [
{
created_at: createdAt,

4
x-pack/platform/plugins/shared/fleet/server/services/uninstall_errors_helpers.test.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import type { UninstallFailedAttempt } from '../types';
import type { FailedAttempt } from '../types';
import { updateUninstallFailedAttempts } from './epm/packages/uninstall_errors_helpers';
@ -20,7 +20,7 @@ const generateFailedAttempt = () => ({
describe('Uninstall error helpers', () => {
describe('updateUninstallFailedAttempts', () => {
it('should only keep 5 errors', () => {
const previousFailedAttempts: UninstallFailedAttempt[] = Array(5)
const previousFailedAttempts: FailedAttempt[] = Array(5)
.fill(null)
.map((_) => generateFailedAttempt());
const updatedLatestUninstallFailedAttempts = updateUninstallFailedAttempts({

163
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/compare_synced_integrations.test.ts
View file

@ -49,7 +49,7 @@ describe('getFollowerIndexInfo', () => {
get: getIndicesMock,
},
search: searchMock,
ccr: { followInfo: jest.fn() },
ccr: { followInfo: jest.fn(), followStats: jest.fn() },
};
mockedLogger = loggerMock.create();
@ -142,6 +142,35 @@ describe('getFollowerIndexInfo', () => {
error: 'Follower index fleet-synced-integrations-ccr-remote1 paused',
});
});
it('should return error if follow stats have fatal error', async () => {
esClientMock.ccr.followInfo.mockResolvedValue({
follower_indices: [
{
follower_index: 'fleet-synced-integrations-ccr-remote1',
status: 'active',
},
],
} as any);
esClientMock.ccr.followStats.mockResolvedValue({
indices: [
{
shards: [
{
fatal_exception: {
reason: 'java.lang.IllegalArgumentException: port out of range:93001',
},
},
],
},
],
});
expect(await getFollowerIndexInfo(esClientMock, mockedLogger)).toEqual({
error:
'Follower index fleet-synced-integrations-ccr-remote1 fatal exception: java.lang.IllegalArgumentException: port out of range:93001',
});
});
});
describe('fetchAndCompareSyncedIntegrations', () => {
@ -589,6 +618,121 @@ describe('fetchAndCompareSyncedIntegrations', () => {
});
});
it('should return failed state if custom asset failure is found', async () => {
(getPipelineMock as jest.MockedFunction<any>).mockResolvedValue({
'logs-system.auth@custom': {
processors: [
{
user_agent: {
field: 'user_agent',
},
},
],
},
});
(getComponentTemplateMock as jest.MockedFunction<any>).mockResolvedValue({
component_templates: [
{
name: 'logs-system.auth@custom',
component_template: {
template: {
mappings: {
properties: {
new_field: {
type: 'text',
},
},
},
},
},
},
],
});
esClientMock = {
search: searchMockWithCustomAssets,
};
(getPackageSavedObjects as jest.MockedFunction<any>).mockReturnValue({
page: 1,
per_page: 10000,
total: 1,
saved_objects: [
{
type: 'epm-packages',
id: 'system',
attributes: {
version: '1.67.3',
install_status: 'installed',
latest_custom_asset_install_failed_attempts: {
'component_template:logs-system.auth@custom': {
created_at: '2023-06-20T08:47:31.457Z',
error: {
message: 'installation failure',
},
type: 'component_template',
name: 'logs-system.auth@custom',
},
'ingest_pipeline:logs-system.auth@custom': {
created_at: '2023-06-20T08:47:31.457Z',
error: {
message: 'installation failure',
},
type: 'ingest_pipeline',
name: 'logs-system.auth@custom',
},
},
},
updated_at: '2025-03-26T14:06:27.611Z',
},
],
});
const res = await fetchAndCompareSyncedIntegrations(
esClientMock,
soClientMock,
'fleet-synced-integrations-ccr-*',
mockedLogger
);
expect(res).toEqual({
integrations: [
{
package_name: 'system',
package_version: '1.67.3',
sync_status: 'completed',
updated_at: expect.any(String),
},
],
custom_assets: {
'component_template:logs-system.auth@custom': {
name: 'logs-system.auth@custom',
package_name: 'system',
package_version: '1.67.3',
sync_status: 'failed',
type: 'component_template',
error:
'Failed to update component template logs-system.auth@custom - reason: installation failure at Tue, 20 Jun 2023 08:47:31 GMT',
},
'ingest_pipeline:logs-system.auth@custom': {
name: 'logs-system.auth@custom',
package_name: 'system',
package_version: '1.67.3',
sync_status: 'failed',
type: 'ingest_pipeline',
error:
'Failed to update ingest pipeline logs-system.auth@custom - reason: installation failure at Tue, 20 Jun 2023 08:47:31 GMT',
},
'ingest_pipeline:filestream-pipeline1': {
name: 'filestream-pipeline1',
package_name: 'filestream',
package_version: '1.1.0',
sync_status: 'synchronizing',
type: 'ingest_pipeline',
},
},
});
});
it('should return status = completed if custom assets are equal', async () => {
(getPipelineMock as jest.MockedFunction<any>).mockResolvedValueOnce({
'logs-system.auth@custom': {
@ -1061,6 +1205,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
expect(res).toEqual({
custom_assets: {
'component_template:logs-system.auth@custom': {
is_deleted: true,
name: 'logs-system.auth@custom',
package_name: 'system',
package_version: '1.67.3',
@ -1068,6 +1213,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
type: 'component_template',
},
'ingest_pipeline:logs-system.auth@custom': {
is_deleted: true,
name: 'logs-system.auth@custom',
package_name: 'system',
package_version: '1.67.3',
@ -1134,6 +1280,10 @@ describe('fetchAndCompareSyncedIntegrations', () => {
},
],
});
(getPipelineMock as jest.MockedFunction<any>).mockResolvedValue({});
(getComponentTemplateMock as jest.MockedFunction<any>).mockResolvedValue({
component_templates: [],
});
const res = await fetchAndCompareSyncedIntegrations(
esClientMock,
@ -1144,6 +1294,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
expect(res).toEqual({
custom_assets: {
'component_template:logs-system.auth@custom': {
is_deleted: true,
name: 'logs-system.auth@custom',
package_name: 'system',
package_version: '1.67.3',
@ -1151,6 +1302,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
type: 'component_template',
},
'ingest_pipeline:logs-system.auth@custom': {
is_deleted: true,
name: 'logs-system.auth@custom',
package_name: 'system',
package_version: '1.67.3',
@ -1268,7 +1420,7 @@ describe('getRemoteSyncedIntegrationsStatus', () => {
get: getIndicesMock,
},
search: searchMock,
ccr: { followInfo: jest.fn() },
ccr: { followInfo: jest.fn(), followStats: jest.fn() },
};
soClientMock = savedObjectsClientMock.create();
@ -1328,6 +1480,13 @@ describe('getRemoteSyncedIntegrationsStatus', () => {
},
],
}),
followStats: jest.fn().mockResolvedValue({
indices: [
{
shards: [{}],
},
],
}),
},
};
expect(await getRemoteSyncedIntegrationsStatus(esClientMock, soClientMock)).toEqual({

121
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/compare_synced_integrations.ts
View file

@ -56,6 +56,16 @@ export const getFollowerIndexInfo = async (
if (res.follower_indices[0]?.status === 'paused') {
return { error: `Follower index ${index} paused` };
}
const resStats = await esClient.ccr.followStats({
index,
});
if (resStats?.indices[0]?.shards[0]?.fatal_exception) {
return {
error: `Follower index ${index} fatal exception: ${resStats.indices[0].shards[0].fatal_exception?.reason}`,
};
}
return { info: res.follower_indices[0] };
} catch (err) {
if (err?.body?.error?.type === 'index_not_found_exception') {
@ -109,7 +119,12 @@ export const fetchAndCompareSyncedIntegrations = async (
},
{} as Record<string, SavedObjectsFindResult<Installation>>
);
const customAssetsStatus = await fetchAndCompareCustomAssets(esClient, logger, ccrCustomAssets);
const customAssetsStatus = await fetchAndCompareCustomAssets(
esClient,
logger,
ccrCustomAssets,
installedIntegrationsByName
);
const integrationsStatus = compareIntegrations(
installedCCRIntegrations,
installedIntegrationsByName
@ -189,7 +204,8 @@ const compareIntegrations = (
const fetchAndCompareCustomAssets = async (
esClient: ElasticsearchClient,
logger: Logger,
ccrCustomAssets: { [key: string]: CustomAssetsData }
ccrCustomAssets: { [key: string]: CustomAssetsData },
installedIntegrationsByName: Record<string, SavedObjectsFindResult<Installation>>
): Promise<RemoteSyncedCustomAssetsRecord | undefined> => {
if (!ccrCustomAssets) return;
@ -232,6 +248,7 @@ const fetchAndCompareCustomAssets = async (
ccrCustomAsset,
ingestPipelines: installedPipelines,
componentTemplates,
installedIntegration: installedIntegrationsByName[ccrCustomAsset.package_name],
});
result[ccrCustomName] = res;
});
@ -246,10 +263,12 @@ const compareCustomAssets = ({
ccrCustomAsset,
ingestPipelines,
componentTemplates,
installedIntegration,
}: {
ccrCustomAsset: CustomAssetsData;
ingestPipelines?: IngestGetPipelineResponse;
componentTemplates?: Record<string, ClusterComponentTemplateSummary>;
installedIntegration: SavedObjectsFindResult<Installation> | undefined;
}): RemoteSyncedCustomAssetsStatus => {
const result = {
name: ccrCustomAsset.name,
@ -258,30 +277,68 @@ const compareCustomAssets = ({
package_version: ccrCustomAsset.package_version,
};
const latestCustomAssetError =
installedIntegration?.attributes?.latest_custom_asset_install_failed_attempts?.[
`${ccrCustomAsset.type}:${ccrCustomAsset.name}`
];
const latestFailedAttemptTime = latestCustomAssetError?.created_at
? `at ${new Date(latestCustomAssetError?.created_at).toUTCString()}`
: '';
const latestFailedAttempt = latestCustomAssetError?.error?.message
? `- reason: ${latestCustomAssetError.error.message}`
: '';
const latestFailedErrorMessage = `Failed to update ${ccrCustomAsset.type.replaceAll('_', ' ')} ${
ccrCustomAsset.name
} ${latestFailedAttempt} ${latestFailedAttemptTime}`;
if (ccrCustomAsset.type === 'ingest_pipeline') {
if (!ingestPipelines) {
const installedPipeline = ingestPipelines?.[ccrCustomAsset.name];
if (!installedPipeline) {
if (ccrCustomAsset.is_deleted === true) {
return {
...result,
is_deleted: true,
sync_status: SyncStatus.COMPLETED,
};
}
if (latestCustomAssetError) {
return {
...result,
sync_status: SyncStatus.FAILED,
error: latestFailedErrorMessage,
};
}
return {
...result,
sync_status: SyncStatus.SYNCHRONIZING,
};
}
const installedPipeline = ingestPipelines[ccrCustomAsset?.name];
if (ccrCustomAsset.is_deleted === true && installedPipeline) {
if (latestCustomAssetError) {
return {
...result,
is_deleted: true,
sync_status: SyncStatus.FAILED,
error: latestFailedErrorMessage,
};
}
return {
...result,
is_deleted: true,
sync_status: SyncStatus.SYNCHRONIZING,
};
} else if (
installedPipeline?.version &&
installedPipeline.version < ccrCustomAsset.pipeline.version
) {
if (latestCustomAssetError) {
return {
...result,
sync_status: SyncStatus.FAILED,
error: latestFailedErrorMessage,
};
}
return {
...result,
sync_status: SyncStatus.SYNCHRONIZING,
@ -292,17 +349,11 @@ const compareCustomAssets = ({
sync_status: SyncStatus.COMPLETED,
};
} else {
return {
...result,
sync_status: SyncStatus.SYNCHRONIZING,
};
}
} else if (ccrCustomAsset.type === 'component_template') {
if (!componentTemplates) {
if (ccrCustomAsset.is_deleted === true) {
if (latestCustomAssetError) {
return {
...result,
sync_status: SyncStatus.COMPLETED,
sync_status: SyncStatus.FAILED,
error: latestFailedErrorMessage,
};
}
return {
@ -310,19 +361,55 @@ const compareCustomAssets = ({
sync_status: SyncStatus.SYNCHRONIZING,
};
}
const installedCompTemplate = componentTemplates[ccrCustomAsset?.name];
if (ccrCustomAsset.is_deleted === true && installedCompTemplate) {
} else if (ccrCustomAsset.type === 'component_template') {
const installedCompTemplate = componentTemplates?.[ccrCustomAsset.name];
if (!installedCompTemplate) {
if (ccrCustomAsset.is_deleted === true) {
return {
...result,
is_deleted: true,
sync_status: SyncStatus.COMPLETED,
};
}
if (latestCustomAssetError) {
return {
...result,
sync_status: SyncStatus.FAILED,
error: latestFailedErrorMessage,
};
}
return {
...result,
sync_status: SyncStatus.SYNCHRONIZING,
};
}
if (ccrCustomAsset.is_deleted === true && installedCompTemplate) {
if (latestCustomAssetError) {
return {
...result,
is_deleted: true,
sync_status: SyncStatus.FAILED,
error: latestFailedErrorMessage,
};
}
return {
...result,
is_deleted: true,
sync_status: SyncStatus.SYNCHRONIZING,
};
} else if (isEqual(installedCompTemplate, ccrCustomAsset?.template)) {
return {
...result,
sync_status: SyncStatus.COMPLETED,
};
} else {
if (latestCustomAssetError) {
return {
...result,
sync_status: SyncStatus.FAILED,
error: latestFailedErrorMessage,
};
}
return {
...result,
sync_status: SyncStatus.SYNCHRONIZING,

2
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/get_remote_status.ts
View file

@ -83,6 +83,8 @@ export const getRemoteSyncedIntegrationsInfoByOutputId = async (
} catch (error) {
if (error.isBoom && error.output.statusCode === 404) {
throw new FleetNotFoundError(`No output found with id ${outputId}`);
} else if (error.type === 'system' && error.code === 'ECONNREFUSED') {
throw new FleetError(`${error.message}${error.code}`);
}
logger.error(`${error}`);
throw error;

2
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/model.ts
View file

@ -17,9 +17,9 @@ export interface BaseCustomAssetsData {
name: string;
package_name: string;
package_version: string;
is_deleted?: boolean;
}
export interface CustomAssetsData extends BaseCustomAssetsData {
is_deleted: boolean;
deleted_at?: string;
[key: string]: any;
}

26
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_on_remote.test.ts
View file

@ -24,6 +24,7 @@ describe('syncIntegrationsOnRemote', () => {
let searchMock: jest.Mock;
let packageClientMock: any;
let loggerMock: any;
let soClientMock: any;
beforeEach(() => {
getIndicesMock = jest.fn();
@ -53,6 +54,9 @@ describe('syncIntegrationsOnRemote', () => {
info: jest.fn(),
};
(installCustomAsset as jest.Mock).mockClear();
soClientMock = {
update: jest.fn(),
};
});
it('should throw error if multiple synced integrations ccr indices exist', async () => {
@ -62,7 +66,7 @@ describe('syncIntegrationsOnRemote', () => {
});
await expect(
syncIntegrationsOnRemote(esClientMock, {} as any, {} as any, abortController, loggerMock)
syncIntegrationsOnRemote(esClientMock, soClientMock, {} as any, abortController, loggerMock)
).rejects.toThrowError(
'Not supported to sync multiple indices with prefix fleet-synced-integrations-ccr-*'
);
@ -125,7 +129,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -153,7 +157,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -184,7 +188,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -226,7 +230,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -261,7 +265,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -309,7 +313,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -354,7 +358,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -390,7 +394,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -418,7 +422,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock
@ -442,7 +446,7 @@ describe('syncIntegrationsOnRemote', () => {
await syncIntegrationsOnRemote(
esClientMock,
{} as any,
soClientMock,
packageClientMock,
abortController,
loggerMock

58
x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_on_remote.ts
View file

@ -4,11 +4,18 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient, SavedObjectsClient, Logger } from '@kbn/core/server';
import type {
ElasticsearchClient,
SavedObjectsClient,
Logger,
SavedObjectsClientContract,
} from '@kbn/core/server';
import semverEq from 'semver/functions/eq';
import semverGte from 'semver/functions/gte';
import { uniq } from 'lodash';
import type { PackageClient } from '../../services';
import { outputService } from '../../services';
@ -17,8 +24,10 @@ import { FLEET_SYNCED_INTEGRATIONS_CCR_INDEX_PREFIX } from '../../services/setup
import { getInstallation, removeInstallation } from '../../services/epm/packages';
import type { SyncIntegrationsData } from './model';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../constants';
import { installCustomAsset } from './custom_assets';
import type { CustomAssetsData, SyncIntegrationsData } from './model';
const MAX_RETRY_ATTEMPTS = 5;
const RETRY_BACKOFF_MINUTES = [5, 10, 20, 40, 60];
@ -235,6 +244,8 @@ export const syncIntegrationsOnRemote = async (
await uninstallPackageIfInstalled(esClient, soClient, pkg, logger);
}
await clearCustomAssetFailedAttempts(soClient, syncIntegrationsDoc);
for (const customAsset of Object.values(syncIntegrationsDoc?.custom_assets ?? {})) {
if (abortController.signal.aborted) {
throw new Error('Task was aborted');
@ -243,6 +254,49 @@ export const syncIntegrationsOnRemote = async (
await installCustomAsset(customAsset, esClient, abortController, logger);
} catch (error) {
logger.error(`Failed to install ${customAsset.type} ${customAsset.name}, error: ${error}`);
await updateCustomAssetFailedAttempts(soClient, customAsset, error, logger);
}
}
};
async function clearCustomAssetFailedAttempts(
soClient: SavedObjectsClientContract,
syncIntegrationsDoc?: SyncIntegrationsData
) {
const customAssetPackages = uniq(
Object.values(syncIntegrationsDoc?.custom_assets ?? {}).map((customAsset) => {
return customAsset.package_name;
})
);
for (const pkgName of customAssetPackages) {
await soClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
latest_custom_asset_install_failed_attempts: {},
});
}
}
async function updateCustomAssetFailedAttempts(
savedObjectsClient: SavedObjectsClientContract,
customAsset: CustomAssetsData,
error: Error,
logger: Logger
) {
try {
await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, customAsset.package_name, {
latest_custom_asset_install_failed_attempts: {
[`${customAsset.type}:${customAsset.name}`]: {
type: customAsset.type,
name: customAsset.name,
error: {
name: error.name,
message: error.message,
stack: error.stack,
},
created_at: new Date().toISOString(),
},
},
});
} catch (err) {
logger.warn(`Error occurred while updating custom asset failed attempts: ${err}`);
}
}

2
x-pack/platform/plugins/shared/fleet/server/types/index.tsx
View file

@ -42,7 +42,7 @@ export type {
EpmPackageInstallStatus,
InstallationStatus,
InstallFailedAttempt,
UninstallFailedAttempt,
FailedAttempt,
PackageInfo,
ArchivePackage,
RegistryVarsEntry,

22
x-pack/platform/plugins/shared/fleet/server/types/models/agent_policy.ts
View file

@ -266,15 +266,19 @@ const BaseSSLSchema = schema.object({
renegotiation: schema.maybe(schema.string()),
});
const BaseSecretsSchema = schema.object({
ssl: schema.maybe(
schema.object({
key: schema.object({
id: schema.maybe(schema.string()),
}),
})
),
});
const BaseSecretsSchema = schema
.object({
ssl: schema.maybe(
schema.object({
key: schema.object({
id: schema.maybe(schema.string()),
}),
})
),
})
.extendsDeep({
unknowns: 'allow',
});
export const NewAgentPolicySchema = schema.object({
...AgentPolicyBaseSchema,

2
x-pack/platform/plugins/shared/fleet/server/types/models/synced_integrations.ts
View file

@ -35,6 +35,8 @@ export const CustomAssetsDataSchema = schema.object({
schema.literal(SyncStatus.SYNCHRONIZING),
schema.literal(SyncStatus.FAILED),
]),
error: schema.maybe(schema.string()),
is_deleted: schema.maybe(schema.boolean()),
});
export const GetRemoteSyncedIntegrationsStatusResponseSchema = schema.object({

4
x-pack/platform/plugins/shared/fleet/server/types/rest_spec/agent.ts
View file

@ -141,8 +141,8 @@ export const AgentResponseSchema = schema.object({
schema.recordOf(
schema.string(),
schema.object({
api_key_id: schema.string(),
type: schema.string(),
api_key_id: schema.maybe(schema.string()),
type: schema.maybe(schema.string()),
to_retire_api_key_ids: schema.maybe(
schema.arrayOf(
schema.object({