[Fleet] Sync Remote integrations: Show warning when package is not found in remote (#223248)

Closes https://github.com/elastic/kibana/issues/220850

## Summary
Add logic that handles the case of custom integrations that cannot be
installed in the remote because of error
` PackageNotFoundError: [pkgName] package not found in registry`

Currently, when the installation is attempted, we just log the error and
exit. I added some logic that creates an "empty" installation SO in
`epm-packages` with status `install_failed` and populates
the`latest_install_failed_attempts` field so we know what happened.
**NOTE**: we could add this logic in the regular install process as
well, it would be useful for debugging purpose

- I also added some logic to `compareSyncIntegrations` to make this
error a warning and because I also made the logic for the warning more
generic

### Testing 
- Install a custom integration in the main cluster, let the
syncIntegrationsTask run and after a while check `GET
.kibana_ingest/_doc/epm-packages:PKGNAME`. You should see the error
saved under `latest_install_failed_attempts`:

<img width="1804" alt="Screenshot 2025-06-10 at 16 20 09"
src="https://github.com/user-attachments/assets/2cffbf01-15f2-4091-bcbd-660fbb390c56"
/>

- In the remote, query `GET
kbn:/api/fleet/remote_synced_integrations/status` and verify that it
returns a warning for the custom integration. Example:
```
 "integrations": [
    {
      "package_name": "agentless_package_links",
      "package_version": "0.0.2",
      "install_status": {
        "main": "installed",
        "remote": "not_installed"
      },
      "updated_at": "2025-05-21T08:55:47.981Z",
      "sync_status": "warning",
      "warning": {
        "title": "agentless_package_links can't be automatically synced",
        "message": "This integration must be manually installed on the remote cluster. Automatic updates and remote installs are not supported."
      }
    },
...
```

- Test the UI - go to Fleet settings and verify that the custom
integration shows a warning:

![Uploading Screenshot 2025-06-11 at 16.26.47.png…]()


### Checklist

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

oas_docs/bundle.json

@@ -45416,7 +45416,19 @@
                             "type": "string"
                           },
                           "warning": {
+                            "additionalProperties": false,
+                            "properties": {
+                              "message": {
                                 "type": "string"
+                              },
+                              "title": {
+                                "type": "string"
+                              }
+                            },
+                            "required": [
+                              "title"
+                            ],
+                            "type": "object"
                           }
                         },
                         "required": [
@@ -45428,7 +45440,19 @@
                       "type": "array"
                     },
                     "warning": {
+                      "additionalProperties": false,
+                      "properties": {
+                        "message": {
                           "type": "string"
+                        },
+                        "title": {
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "title"
+                      ],
+                      "type": "object"
                     }
                   },
                   "required": [
@@ -45587,7 +45611,19 @@
                             "type": "string"
                           },
                           "warning": {
+                            "additionalProperties": false,
+                            "properties": {
+                              "message": {
                                 "type": "string"
+                              },
+                              "title": {
+                                "type": "string"
+                              }
+                            },
+                            "required": [
+                              "title"
+                            ],
+                            "type": "object"
                           }
                         },
                         "required": [
@@ -45599,7 +45635,19 @@
                       "type": "array"
                     },
                     "warning": {
+                      "additionalProperties": false,
+                      "properties": {
+                        "message": {
                           "type": "string"
+                        },
+                        "title": {
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "title"
+                      ],
+                      "type": "object"
                     }
                   },
                   "required": [

oas_docs/bundle.serverless.json

@@ -45416,7 +45416,19 @@
                             "type": "string"
                           },
                           "warning": {
+                            "additionalProperties": false,
+                            "properties": {
+                              "message": {
                                 "type": "string"
+                              },
+                              "title": {
+                                "type": "string"
+                              }
+                            },
+                            "required": [
+                              "title"
+                            ],
+                            "type": "object"
                           }
                         },
                         "required": [
@@ -45428,7 +45440,19 @@
                       "type": "array"
                     },
                     "warning": {
+                      "additionalProperties": false,
+                      "properties": {
+                        "message": {
                           "type": "string"
+                        },
+                        "title": {
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "title"
+                      ],
+                      "type": "object"
                     }
                   },
                   "required": [
@@ -45587,7 +45611,19 @@
                             "type": "string"
                           },
                           "warning": {
+                            "additionalProperties": false,
+                            "properties": {
+                              "message": {
                                 "type": "string"
+                              },
+                              "title": {
+                                "type": "string"
+                              }
+                            },
+                            "required": [
+                              "title"
+                            ],
+                            "type": "object"
                           }
                         },
                         "required": [
@@ -45599,7 +45635,19 @@
                       "type": "array"
                     },
                     "warning": {
+                      "additionalProperties": false,
+                      "properties": {
+                        "message": {
                           "type": "string"
+                        },
+                        "title": {
+                          "type": "string"
+                        }
+                      },
+                      "required": [
+                        "title"
+                      ],
+                      "type": "object"
                     }
                   },
                   "required": [

oas_docs/output/kibana.serverless.yaml

@@ -39855,13 +39855,29 @@ paths:
                         updated_at:
                           type: string
                         warning:
+                          additionalProperties: false
+                          type: object
+                          properties:
+                            message:
                               type: string
+                            title:
+                              type: string
+                          required:
+                            - title
                       required:
                         - sync_status
                         - install_status
                     type: array
                   warning:
+                    additionalProperties: false
+                    type: object
+                    properties:
+                      message:
                         type: string
+                      title:
+                        type: string
+                    required:
+                      - title
                 required:
                   - integrations
         '400':
@@ -39966,13 +39982,29 @@ paths:
                         updated_at:
                           type: string
                         warning:
+                          additionalProperties: false
+                          type: object
+                          properties:
+                            message:
                               type: string
+                            title:
+                              type: string
+                          required:
+                            - title
                       required:
                         - sync_status
                         - install_status
                     type: array
                   warning:
+                    additionalProperties: false
+                    type: object
+                    properties:
+                      message:
                         type: string
+                      title:
+                        type: string
+                    required:
+                      - title
                 required:
                   - integrations
         '400':

oas_docs/output/kibana.yaml

@@ -42097,13 +42097,29 @@ paths:
                         updated_at:
                           type: string
                         warning:
+                          additionalProperties: false
+                          type: object
+                          properties:
+                            message:
                               type: string
+                            title:
+                              type: string
+                          required:
+                            - title
                       required:
                         - sync_status
                         - install_status
                     type: array
                   warning:
+                    additionalProperties: false
+                    type: object
+                    properties:
+                      message:
                         type: string
+                      title:
+                        type: string
+                    required:
+                      - title
                 required:
                   - integrations
         '400':
@@ -42208,13 +42224,29 @@ paths:
                         updated_at:
                           type: string
                         warning:
+                          additionalProperties: false
+                          type: object
+                          properties:
+                            message:
                               type: string
+                            title:
+                              type: string
+                          required:
+                            - title
                       required:
                         - sync_status
                         - install_status
                     type: array
                   warning:
+                    additionalProperties: false
+                    type: object
+                    properties:
+                      message:
                         type: string
+                      title:
+                        type: string
+                    required:
+                      - title
                 required:
                   - integrations
         '400':

x-pack/platform/plugins/shared/fleet/common/types/models/remote_synced_integrations.ts

@@ -21,7 +21,10 @@ export interface RemoteSyncedIntegrationsBase {
 export interface RemoteSyncedIntegrationsStatus extends RemoteSyncedIntegrationsBase {
   sync_status: SyncStatus;
   error?: string;
-  warning?: string;
+  warning?: {
+    title: string;
+    message?: string;
+  };
   updated_at?: string;
   install_status: {
     main: string;

x-pack/platform/plugins/shared/fleet/common/types/rest_spec/remote_synced_integrations.ts

@@ -10,5 +10,8 @@ export interface GetRemoteSyncedIntegrationsStatusResponse {
   integrations: RemoteSyncedIntegrationsStatus[];
   custom_assets?: RemoteSyncedCustomAssetsRecord;
   error?: string;
-  warning?: string;
+  warning?: {
+    title: string;
+    message?: string;
+  };
 }

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/outputs_table/integration_status.tsx

@@ -202,34 +202,30 @@ export const IntegrationStatus: React.FunctionComponent<{
             </>
           )}
 
-          {integration.sync_status === 'warning' && (
+          {integration.sync_status === 'warning' && integration?.warning && (
             <>
               <EuiCallOut
                 title={
-                  syncUninstalledIntegrations ? (
                   <FormattedMessage
-                      id="xpack.fleet.integrationSyncStatus.integrationWarningContent"
+                    id="xpack.fleet.integrationSyncStatus.integrationWarningTitle"
-                      defaultMessage="Integration was uninstalled, but removal from remote cluster failed."
+                    defaultMessage="{Warning}"
+                    values={{
+                      Warning: integration.warning?.title,
+                    }}
                   />
-                  ) : (
-                    <FormattedMessage
-                      id="xpack.fleet.integrationSyncStatus.integrationErrorTitle"
-                      defaultMessage="Warning"
-                    />
-                  )
                 }
                 color="warning"
                 iconType="warning"
                 size="s"
                 data-test-subj="integrationSyncIntegrationWarningCallout"
               >
-                {integration?.warning && (
+                {integration?.warning?.message && (
                   <EuiText size="s">
                     <FormattedMessage
                       id="xpack.fleet.integrationSyncStatus.integrationWarningContent"
                       defaultMessage="{uninstallWarning}"
                       values={{
-                        uninstallWarning: integration?.warning,
+                        uninstallWarning: integration.warning.message,
                       }}
                     />
                   </EuiText>

x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/outputs_table/integration_sync_flyout.test.tsx

@@ -64,7 +64,7 @@ describe('IntegrationSyncFlyout', () => {
         },
         updated_at: '2025-05-19T15:40:26.554Z',
         sync_status: SyncStatus.WARNING,
-        warning: 'Unable to remove package 1password:1.32.0',
+        warning: { message: 'Unable to remove package 1password:1.32.0', title: 'warning' },
       },
       {
         package_name: 'apache',

x-pack/platform/plugins/shared/fleet/server/services/epm/packages/get.ts

@@ -141,6 +141,11 @@ export async function getPackages(
               );
               return null;
             }
+            // ignoring errors of type PackageNotFoundError to avoid blocking the UI over a package not found in the registry
+            if (err instanceof PackageNotFoundError) {
+              logger.warn(`Package ${pkg.id} ${pkg.attributes.version} not found in registry`);
+              return null;
+            }
             throw err;
           }
         } else {

x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_errors_helpers.test.ts

@@ -4,12 +4,19 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import { savedObjectsClientMock } from '@kbn/core/server/mocks';
+import { loggerMock } from '@kbn/logging-mocks';
+
+import type { Logger } from '@kbn/core/server';
 
 import type { InstallFailedAttempt } from '../../../types';
 
+import { getInstallationObject } from './get';
+
 import {
   clearLatestFailedAttempts,
   addErrorToLatestFailedAttempts,
+  createOrUpdateFailedInstallStatus,
 } from './install_errors_helpers';
 
 const generateFailedAttempt = (version: string) => ({
@@ -20,8 +27,18 @@ const generateFailedAttempt = (version: string) => ({
     message: 'test',
   },
 });
+let mockedLogger: jest.Mocked<Logger>;
 
-const mapFailledAttempsToTargetVersion = (attemps: InstallFailedAttempt[]) =>
+jest.mock('../../audit_logging');
+jest.mock('./get', () => {
+  return { getInstallationObject: jest.fn() };
+});
+
+const getInstallationObjectMock = getInstallationObject as jest.MockedFunction<
+  typeof getInstallationObject
+>;
+
+const mapFailedAttempsToTargetVersion = (attemps: InstallFailedAttempt[]) =>
   attemps.map((attempt) => attempt.target_version);
 
 describe('Install error helpers', () => {
@@ -30,16 +47,16 @@ describe('Install error helpers', () => {
       generateFailedAttempt('0.1.0'),
       generateFailedAttempt('0.2.0'),
     ];
-    it('should clear previous error on succesfull upgrade', () => {
+    it('should clear previous error on successful upgrade', () => {
-      const currentFailledAttemps = clearLatestFailedAttempts('0.2.0', previousFailedAttemps);
+      const currentFailedAttemps = clearLatestFailedAttempts('0.2.0', previousFailedAttemps);
 
-      expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual([]);
+      expect(mapFailedAttempsToTargetVersion(currentFailedAttemps)).toEqual([]);
     });
 
-    it('should not clear previous upgrade error on succesfull rollback', () => {
+    it('should not clear previous upgrade error on successful rollback', () => {
-      const currentFailledAttemps = clearLatestFailedAttempts('0.1.0', previousFailedAttemps);
+      const currentFailedAttempts = clearLatestFailedAttempts('0.1.0', previousFailedAttemps);
 
-      expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual(['0.2.0']);
+      expect(mapFailedAttempsToTargetVersion(currentFailedAttempts)).toEqual(['0.2.0']);
     });
   });
 
@@ -52,14 +69,14 @@ describe('Install error helpers', () => {
         generateFailedAttempt('0.2.2'),
         generateFailedAttempt('0.2.1'),
       ];
-      const currentFailledAttemps = addErrorToLatestFailedAttempts({
+      const currentFailedAttempts = addErrorToLatestFailedAttempts({
         targetVersion: '0.2.6',
         createdAt: new Date().toISOString(),
         error: new Error('new test'),
         latestAttempts: previousFailedAttemps,
       });
 
-      expect(mapFailledAttempsToTargetVersion(currentFailledAttemps)).toEqual([
+      expect(mapFailedAttempsToTargetVersion(currentFailedAttempts)).toEqual([
         '0.2.6',
         '0.2.5',
         '0.2.4',
@@ -68,4 +85,99 @@ describe('Install error helpers', () => {
       ]);
     });
   });
+
+  describe('createOrUpdateFailedInstallStatus', () => {
+    const soClientMock = savedObjectsClientMock.create();
+    mockedLogger = loggerMock.create();
+
+    it('should create installation object with latest_install_failed_attempts if none was found', async () => {
+      getInstallationObjectMock.mockResolvedValue(undefined);
+
+      await createOrUpdateFailedInstallStatus({
+        logger: mockedLogger,
+        savedObjectsClient: soClientMock,
+        pkgName: 'test-package',
+        pkgVersion: '0.1.0',
+        error: new Error('test error'),
+        installSource: 'registry',
+      });
+      expect(soClientMock.create).toHaveBeenCalledWith(
+        'epm-packages',
+        {
+          es_index_patterns: {},
+          install_source: 'registry',
+          install_started_at: expect.any(String),
+          install_status: 'install_failed',
+          install_version: '0.1.0',
+          installed_es: [],
+          installed_kibana: [],
+          name: 'test-package',
+          package_assets: [],
+          verification_status: 'unknown',
+          version: '0.1.0',
+          latest_install_failed_attempts: [
+            {
+              created_at: expect.any(String),
+              target_version: '0.1.0',
+              error: {
+                message: 'test error',
+                name: 'Error',
+                stack: expect.any(String),
+              },
+            },
+          ],
+        },
+        { id: 'test-package', overwrite: true }
+      );
+    });
+
+    it('should update latest_install_failed_attempts in the installation object if it exists', async () => {
+      getInstallationObjectMock.mockResolvedValue({
+        es_index_patterns: {},
+        install_source: 'registry',
+        install_started_at: '2025-06-11T07:15:06.838Z',
+        install_status: 'install_failed',
+        install_version: '0.1.0',
+        installed_es: [],
+        installed_kibana: [],
+        name: 'test-package',
+        package_assets: [],
+        verification_status: 'unknown',
+        version: '0.1.0',
+        latest_install_failed_attempts: [
+          {
+            created_at: '2025-06-11T07:15:06.838Z',
+            target_version: '0.1.0',
+            error: {
+              message: 'test error',
+              name: 'Error',
+              stack: 'test error - stacktrace',
+            },
+          },
+        ],
+      } as any);
+
+      await createOrUpdateFailedInstallStatus({
+        logger: mockedLogger,
+        savedObjectsClient: soClientMock,
+        pkgName: 'test-package',
+        pkgVersion: '0.1.0',
+        error: new Error('test error'),
+        installSource: 'registry',
+      });
+      expect(soClientMock.update).toHaveBeenCalledWith('epm-packages', 'test-package', {
+        latest_install_failed_attempts: [
+          {
+            created_at: expect.any(String),
+            target_version: '0.1.0',
+            error: {
+              message: 'test error',
+              name: 'Error',
+              stack: expect.any(String),
+            },
+          },
+        ],
+      });
+    });
+  });
 });

x-pack/platform/plugins/shared/fleet/server/services/epm/packages/install_errors_helpers.ts

@@ -7,7 +7,17 @@
 
 import { lt } from 'semver';
 
-import type { InstallFailedAttempt } from '../../../types';
+import { SavedObjectsErrorHelpers } from '@kbn/core/server';
+
+import type { SavedObjectsClientContract } from '@kbn/core/server';
+
+import type { Logger } from '@kbn/core/server';
+
+import type { InstallFailedAttempt, InstallSource, Installation } from '../../../types';
+import { auditLoggingService } from '../../audit_logging';
+import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
+
+import { getInstallationObject } from './get';
 
 const MAX_ATTEMPTS_TO_KEEP = 5;
 
@@ -42,3 +52,88 @@ export function addErrorToLatestFailedAttempts({
     ...latestAttempts,
   ].slice(0, MAX_ATTEMPTS_TO_KEEP);
 }
+
+export const createOrUpdateFailedInstallStatus = async ({
+  logger,
+  savedObjectsClient,
+  pkgName,
+  pkgVersion,
+  error,
+  installSource = 'registry',
+}: {
+  logger: Logger;
+  savedObjectsClient: SavedObjectsClientContract;
+  pkgName: string;
+  pkgVersion: string;
+  error: Error;
+  installSource?: InstallSource;
+}) => {
+  const installation = await getInstallationObject({
+    pkgName,
+    savedObjectsClient,
+  });
+
+  if (installation) {
+    auditLoggingService.writeCustomSoAuditLog({
+      action: 'update',
+      id: pkgName,
+      name: pkgName,
+      savedObjectType: PACKAGES_SAVED_OBJECT_TYPE,
+    });
+    const latestInstallFailedAttempts = addErrorToLatestFailedAttempts({
+      error,
+      targetVersion: pkgVersion,
+      createdAt: new Date().toISOString(),
+      latestAttempts: installation?.attributes?.latest_install_failed_attempts,
+    });
+
+    try {
+      return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
+        latest_install_failed_attempts: latestInstallFailedAttempts,
+      });
+    } catch (err) {
+      logger.warn(`Error occurred while updating installation failed attempts: ${err}`);
+    }
+  } else {
+    auditLoggingService.writeCustomSoAuditLog({
+      action: 'create',
+      id: pkgName,
+      name: pkgName,
+      savedObjectType: PACKAGES_SAVED_OBJECT_TYPE,
+    });
+    const installFailedAttempts = addErrorToLatestFailedAttempts({
+      error,
+      targetVersion: pkgVersion,
+      createdAt: new Date().toISOString(),
+      latestAttempts: [],
+    });
+    const savedObject: Installation = {
+      installed_kibana: [],
+      installed_es: [],
+      package_assets: [],
+      name: pkgName,
+      version: pkgVersion,
+      install_version: pkgVersion,
+      install_status: 'install_failed',
+      install_started_at: new Date().toISOString(),
+      verification_status: 'unknown',
+      latest_install_failed_attempts: installFailedAttempts,
+      es_index_patterns: {},
+      install_source: installSource,
+    };
+    try {
+      return await savedObjectsClient.create<Installation>(
+        PACKAGES_SAVED_OBJECT_TYPE,
+        savedObject,
+        {
+          id: pkgName,
+          overwrite: true,
+        }
+      );
+    } catch (err) {
+      if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {
+        logger.error(`Failed to create package installation: ${err}`);
+      }
+    }
+  }
+};

x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/compare_synced_integrations.test.ts

@@ -216,7 +216,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
     });
   });
 
-  it('should return status = synchronizing if there are integrations on sync index but none are installed yet', async () => {
+  it('should return synchronizing status if there are integrations on sync index but none are installed yet', async () => {
     esClientMock.search.mockResolvedValueOnce({
       hits: {
         hits: [
@@ -279,7 +279,76 @@ describe('fetchAndCompareSyncedIntegrations', () => {
       ],
     });
   });
-  it('should compare integrations installed on remote with the ones on sync index', async () => {
+  it('should return a warning for integrations not found in registry', async () => {
+    esClientMock.search.mockResolvedValueOnce({
+      hits: {
+        hits: [
+          {
+            _source: {
+              integrations: [
+                {
+                  package_name: 'custom-pkg',
+                  package_version: '1.0.0',
+                  updated_at: '2025-03-20T14:18:40.111Z',
+                  install_status: 'installed',
+                },
+              ],
+            },
+          },
+        ],
+      },
+    } as any);
+    (getPackageSavedObjects as jest.MockedFunction<any>).mockReturnValue({
+      page: 1,
+      per_page: 10000,
+      total: 1,
+      saved_objects: [
+        {
+          type: 'epm-packages',
+          id: 'custom-pkg',
+          attributes: {
+            version: '1.0.0',
+            install_status: 'install_failed',
+            latest_install_failed_attempts: [
+              {
+                created_at: '2025-06-10T15:30:31.614Z',
+                target_version: '0.0.2',
+                error: {
+                  name: 'PackageNotFoundError',
+                  message: '[agentless_package_links] package not found in registry',
+                },
+              },
+            ],
+          },
+          updated_at: '2025-03-26T14:06:27.611Z',
+        },
+      ],
+    });
+    const res = await fetchAndCompareSyncedIntegrations(
+      esClientMock,
+      soClientMock,
+      'fleet-synced-integrations-ccr-*',
+      mockedLogger
+    );
+    expect(res).toEqual({
+      integrations: [
+        {
+          sync_status: 'warning',
+          package_name: 'custom-pkg',
+          package_version: '1.0.0',
+          install_status: { main: 'installed', remote: 'not_installed' },
+          updated_at: expect.any(String),
+          warning: {
+            message:
+              'This integration must be manually installed on the remote cluster. Automatic updates and remote installs are not supported.',
+            title: "Integration can't be automatically synced",
+          },
+        },
+      ],
+    });
+  });
+
+  it('should return completed state when the integrations are correctly synced', async () => {
     esClientMock.search.mockResolvedValueOnce({
       hits: {
         hits: [
@@ -461,7 +530,10 @@ describe('fetchAndCompareSyncedIntegrations', () => {
           updated_at: expect.any(String),
         },
         {
-          warning: 'failed to uninstall at Wed, 26 Mar 2025 14:06:27 GMT',
+          warning: {
+            message: 'failed to uninstall at Wed, 26 Mar 2025 14:06:27 GMT',
+            title: 'Integration was uninstalled, but removal from remote cluster failed.',
+          },
           install_status: {
             main: 'not_installed',
             remote: 'installed',
@@ -645,7 +717,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
           error: 'Found incorrect installed version 1.67.2',
         },
         {
-          error: `Installation status: install_failed error: installation failure at Tue, 20 Jun 2023 08:47:31 GMT`,
+          error: `Installation status: install_failed installation failure at Tue, 20 Jun 2023 08:47:31 GMT`,
           package_name: 'synthetics',
           package_version: '1.4.1',
           install_status: { main: 'installed', remote: 'install_failed' },
@@ -942,7 +1014,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
       });
     });
 
-    it('should return status = completed if custom assets are equal', async () => {
+    it('should return completed status if custom assets are equal', async () => {
       (getPipelineMock as jest.MockedFunction<any>).mockResolvedValueOnce({
         'logs-system.auth@custom': {
           processors: [
@@ -1042,7 +1114,7 @@ describe('fetchAndCompareSyncedIntegrations', () => {
         },
       });
     });
-    it('should return status = synchronizing if versions do not match', async () => {
+    it('should return synchronizing status if versions do not match', async () => {
       const searchMockWithVersionedPipeline = jest.fn().mockResolvedValue({
         hits: {
           hits: [

x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/compare_synced_integrations.ts

@@ -180,6 +180,7 @@ const compareIntegrations = (
           updated_at: ccrIntegration?.updated_at,
         };
       }
+
       if (!localIntegrationSO) {
         return {
           ...baseIntegrationData,
@@ -216,11 +217,26 @@ const compareIntegrations = (
           latestFailedAttemptTime = `at ${new Date(
             latestInstallFailedAttempts.created_at
           ).toUTCString()}`;
-          latestFailedAttempt = latestInstallFailedAttempts.error?.message
-            ? `error: ${latestInstallFailedAttempts.error?.message}`
-            : '';
-        }
 
+          // handling special case for those integrations that cannot be found in registry
+          if (latestInstallFailedAttempts.error?.name === 'PackageNotFoundError') {
+            return {
+              ...baseIntegrationData,
+              install_status: {
+                main: ccrIntegration.install_status,
+                remote: 'not_installed',
+              },
+              updated_at: ccrIntegration.updated_at,
+              sync_status: SyncStatus.WARNING,
+              warning: {
+                title: `Integration can't be automatically synced`,
+                message: `This integration must be manually installed on the remote cluster. Automatic updates and remote installs are not supported.`,
+              },
+            };
+          } else {
+            latestFailedAttempt = latestInstallFailedAttempts.error?.message ?? '';
+          }
+        }
         return {
           ...baseIntegrationData,
           install_status: {
@@ -246,9 +262,7 @@ const compareIntegrations = (
           latestUninstallFailedAttemptTime = `at ${new Date(
             latestInstallFailedAttempts.created_at
           ).toUTCString()}`;
-          latestUninstallFailedAttempt = latestInstallFailedAttempts.error?.message
+          latestUninstallFailedAttempt = latestInstallFailedAttempts.error?.message ?? '';
-            ? `${latestInstallFailedAttempts.error?.message}`
-            : '';
         }
         return {
           ...baseIntegrationData,
@@ -259,7 +273,12 @@ const compareIntegrations = (
           updated_at: ccrIntegration.updated_at,
           sync_status: SyncStatus.WARNING,
           ...(localIntegrationSO?.attributes.latest_uninstall_failed_attempts !== undefined
-            ? { warning: `${latestUninstallFailedAttempt} ${latestUninstallFailedAttemptTime}` }
+            ? {
+                warning: {
+                  message: `${latestUninstallFailedAttempt} ${latestUninstallFailedAttemptTime}`,
+                  title: 'Integration was uninstalled, but removal from remote cluster failed.',
+                },
+              }
             : {}),
         };
       }

x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/model.ts

@@ -5,11 +5,14 @@
  * 2.0.
  */
 
+import type { InstallSource } from '../../types';
+
 export interface IntegrationsData {
   package_name: string;
   package_version: string;
   updated_at: string;
   install_status: string;
+  install_source?: InstallSource;
 }
 
 export interface BaseCustomAssetsData {

x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_on_remote.test.ts

@@ -8,14 +8,20 @@
 import { PackageNotFoundError } from '../../errors';
 import { outputService } from '../../services';
 
+import { createOrUpdateFailedInstallStatus } from '../../services/epm/packages/install_errors_helpers';
+
 import { installCustomAsset } from './custom_assets';
 
 import { syncIntegrationsOnRemote } from './sync_integrations_on_remote';
 
 jest.mock('../../services');
 jest.mock('./custom_assets');
+jest.mock('../../services/epm/packages/install_errors_helpers');
 
 const outputServiceMock = outputService as jest.Mocked<typeof outputService>;
+const createOrUpdateFailedInstallStatusMock = createOrUpdateFailedInstallStatus as jest.Mocked<
+  typeof createOrUpdateFailedInstallStatus
+>;
 
 describe('syncIntegrationsOnRemote', () => {
   const abortController = new AbortController();
@@ -89,11 +95,19 @@ describe('syncIntegrationsOnRemote', () => {
                   package_name: 'nginx',
                   package_version: '2.2.0',
                   updated_at: '2021-01-01T00:00:00.000Z',
+                  install_source: 'registry',
                 },
                 {
                   package_name: 'system',
                   package_version: '2.2.0',
                   updated_at: '2021-01-01T00:00:00.000Z',
+                  install_source: 'registry',
+                },
+                {
+                  package_name: 'custom-pkg',
+                  package_version: '1.0.0',
+                  updated_at: '2021-01-01T00:00:00.000Z',
+                  install_source: 'custom',
                 },
               ],
               custom_assets: {
@@ -273,7 +287,100 @@ describe('syncIntegrationsOnRemote', () => {
 
     expect(packageClientMock.installPackage).toHaveBeenCalledTimes(2);
   });
+  it('should call createOrUpdateFailedInstallStatus if installation failed', async () => {
+    getIndicesMock.mockResolvedValue({
+      'fleet-synced-integrations-ccr-remote1': {},
+    });
+    searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
+    packageClientMock.getInstallation.mockImplementation((packageName: string) =>
+      packageName === 'custom-pkg'
+        ? undefined
+        : {
+            install_status: 'installed',
+            version: '2.2.0',
+          }
+    );
+    packageClientMock.installPackage.mockImplementation(({ pkgName, pkgVersion }: any) => {
+      if (pkgName === 'custom-pkg') {
+        throw new PackageNotFoundError('package not found in registry');
+      }
+      return {
+        status: 'installed',
+      };
+    });
 
+    await syncIntegrationsOnRemote(
+      esClientMock,
+      soClientMock,
+      packageClientMock,
+      abortController,
+      loggerMock
+    );
+
+    expect(packageClientMock.installPackage).toHaveBeenCalledTimes(1);
+    expect(createOrUpdateFailedInstallStatusMock).toHaveBeenCalledWith({
+      error: new PackageNotFoundError('package not found in registry'),
+      installSource: 'custom',
+      pkgName: 'custom-pkg',
+      pkgVersion: '1.0.0',
+      logger: expect.anything(),
+      savedObjectsClient: expect.anything(),
+    });
+  });
+
+  it('should do nothing if sync enabled and the package is installing', async () => {
+    getIndicesMock.mockResolvedValue({
+      'fleet-synced-integrations-ccr-remote1': {},
+    });
+    searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
+    packageClientMock.getInstallation.mockImplementation((packageName: string) =>
+      packageName === 'nginx'
+        ? {
+            install_status: 'installing',
+            version: '2.1.0',
+          }
+        : {
+            install_status: 'installed',
+            version: '2.3.0',
+          }
+    );
+
+    await syncIntegrationsOnRemote(
+      esClientMock,
+      soClientMock,
+      packageClientMock,
+      abortController,
+      loggerMock
+    );
+
+    expect(packageClientMock.installPackage).not.toHaveBeenCalled();
+  });
+
+  it('should install custom assets', async () => {
+    getIndicesMock.mockResolvedValue({
+      'fleet-synced-integrations-ccr-remote1': {},
+    });
+    searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
+    packageClientMock.getInstallation.mockImplementation(() => ({
+      install_status: 'installed',
+      version: '2.2.0',
+    }));
+    packageClientMock.installPackage.mockResolvedValue({
+      status: 'installed',
+    });
+
+    await syncIntegrationsOnRemote(
+      esClientMock,
+      soClientMock,
+      packageClientMock,
+      abortController,
+      loggerMock
+    );
+
+    expect(installCustomAsset).toHaveBeenCalledTimes(2);
+  });
+
+  describe('Retry logic', () => {
     it('should not retry if max retry attempts reached', async () => {
       getIndicesMock.mockResolvedValue({
         'fleet-synced-integrations-ccr-remote1': {},
@@ -403,43 +510,27 @@ describe('syncIntegrationsOnRemote', () => {
       expect(packageClientMock.installPackage).toHaveBeenCalled();
     });
 
-  it('should do nothing if sync enabled and the package is installing', async () => {
+    it('should not retry if package has install_source custom even if retry time has passed', async () => {
       getIndicesMock.mockResolvedValue({
         'fleet-synced-integrations-ccr-remote1': {},
       });
       searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
       packageClientMock.getInstallation.mockImplementation((packageName: string) =>
-      packageName === 'nginx'
+        packageName === 'custom'
           ? {
-            install_status: 'installing',
+              install_status: 'install_failed',
               version: '2.1.0',
+              latest_install_failed_attempts: [
+                {
+                  created_at: '2025-02-28T04:11:44.395Z',
+                },
+              ],
             }
           : {
               install_status: 'installed',
-            version: '2.3.0',
+              version: '2.2.0',
             }
       );
-
-    await syncIntegrationsOnRemote(
-      esClientMock,
-      soClientMock,
-      packageClientMock,
-      abortController,
-      loggerMock
-    );
-
-    expect(packageClientMock.installPackage).not.toHaveBeenCalled();
-  });
-
-  it('should install custom assets', async () => {
-    getIndicesMock.mockResolvedValue({
-      'fleet-synced-integrations-ccr-remote1': {},
-    });
-    searchMock.mockResolvedValue(getSyncedIntegrationsCCRDoc(true));
-    packageClientMock.getInstallation.mockImplementation(() => ({
-      install_status: 'installed',
-      version: '2.2.0',
-    }));
       packageClientMock.installPackage.mockResolvedValue({
         status: 'installed',
       });
@@ -452,6 +543,7 @@ describe('syncIntegrationsOnRemote', () => {
         loggerMock
       );
 
-    expect(installCustomAsset).toHaveBeenCalledTimes(2);
+      expect(packageClientMock.installPackage).not.toHaveBeenCalled();
+    });
   });
 });

x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_on_remote.ts

@@ -26,6 +26,10 @@ import { getInstallation, removeInstallation } from '../../services/epm/packages
 
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../constants';
 
+import { createOrUpdateFailedInstallStatus } from '../../services/epm/packages/install_errors_helpers';
+
+import type { InstallSource } from '../../types';
+
 import { installCustomAsset } from './custom_assets';
 import type { CustomAssetsData, SyncIntegrationsData } from './model';
 
@@ -93,7 +97,8 @@ async function getSyncIntegrationsEnabled(
 }
 
 async function installPackageIfNotInstalled(
-  pkg: { package_name: string; package_version: string },
+  savedObjectsClient: SavedObjectsClientContract,
+  pkg: { package_name: string; package_version: string; install_source?: InstallSource },
   packageClient: PackageClient,
   logger: Logger,
   abortController: AbortController
@@ -123,12 +128,13 @@ async function installPackageIfNotInstalled(
     }
     const lastRetryAttemptTime = installation.latest_install_failed_attempts?.[0].created_at;
     // retry install if backoff time has passed since the last attempt
+    // excluding custom and upload packages from retries
     const shouldRetryInstall =
       attempt > 0 &&
       lastRetryAttemptTime &&
       Date.now() - Date.parse(lastRetryAttemptTime) >
-        RETRY_BACKOFF_MINUTES[attempt - 1] * 60 * 1000;
+        RETRY_BACKOFF_MINUTES[attempt - 1] * 60 * 1000 &&
-
+      (pkg.install_source === 'registry' || pkg.install_source === 'bundled');
     if (!shouldRetryInstall) {
       return;
     }
@@ -165,6 +171,16 @@ async function installPackageIfNotInstalled(
     logger.error(
       `Failed to install package ${pkg.package_name} with version ${pkg.package_version}, error: ${error}`
     );
+    if (error instanceof PackageNotFoundError && error.message.includes('not found in registry')) {
+      await createOrUpdateFailedInstallStatus({
+        logger,
+        savedObjectsClient,
+        pkgName: pkg.package_name,
+        pkgVersion: pkg.package_version,
+        error,
+        installSource: pkg?.install_source,
+      });
+    }
   }
 }
 
@@ -231,7 +247,7 @@ export const syncIntegrationsOnRemote = async (
     if (abortController.signal.aborted) {
       throw new Error('Task was aborted');
     }
-    await installPackageIfNotInstalled(pkg, packageClient, logger, abortController);
+    await installPackageIfNotInstalled(soClient, pkg, packageClient, logger, abortController);
   }
 
   const uninstalledIntegrations =

x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_task.ts

@@ -230,6 +230,7 @@ export class SyncIntegrationsTask {
         package_version: item.attributes.version,
         updated_at: item.updated_at ?? new Date().toISOString(),
         install_status: item.attributes.install_status,
+        install_source: item.attributes.install_source,
       };
     });
 

x-pack/platform/plugins/shared/fleet/server/types/models/synced_integrations.ts

@@ -23,7 +23,12 @@ export const RemoteSyncedIntegrationsStatusSchema = RemoteSyncedIntegrationsBase
     schema.literal(SyncStatus.WARNING),
   ]),
   error: schema.maybe(schema.string()),
-  warning: schema.maybe(schema.string()),
+  warning: schema.maybe(
+    schema.object({
+      title: schema.string(),
+      message: schema.maybe(schema.string()),
+    })
+  ),
   updated_at: schema.maybe(schema.string()),
   install_status: schema.object({
     main: schema.string(),
@@ -50,5 +55,10 @@ export const GetRemoteSyncedIntegrationsStatusResponseSchema = schema.object({
   integrations: schema.arrayOf(RemoteSyncedIntegrationsStatusSchema),
   custom_assets: schema.maybe(schema.recordOf(schema.string(), CustomAssetsDataSchema)),
   error: schema.maybe(schema.string()),
-  warning: schema.maybe(schema.string()),
+  warning: schema.maybe(
+    schema.object({
+      title: schema.string(),
+      message: schema.maybe(schema.string()),
+    })
+  ),
 });