[Fleet] Sync uninstalled integrations on remote clusters (#217144)

## Summary Closes https://github.com/elastic/kibana/issues/206556 This PR adds a setting to remote ES outputs for also uninstalling integrations on remote clusters when integrations sync is enabled. This new setting can be toggled in the UI with a new switch: <img width="1728" alt="Screenshot 2025-04-09 at 11 53 43" src="https://github.com/user-attachments/assets/34544aa9-28fd-4360-a32f-5031e3d4293f" /> ### Testing * Follow the steps in https://github.com/elastic/kibana/blob/main/x-pack/platform/plugins/shared/fleet/dev_docs/local_setup/remote_clusters_ccr.md to set up two clusters with integrations syncing. * Add some integrations in your main cluster and check that they are also installed in the remote cluster. * Disable uninstalling integrations on remote. * Remove an integration in your main cluster and check that it is NOT removed from the remote cluster. * Enable uninstalling integrations on remote. * Remove an integration in your main cluster and check that it is also removed from the remote cluster. * In your remote cluster, enroll an agent onto a policy that points to at least 1 package policy of the installed integrations (cf. Docker commands below if using dockerized fleet-server/agent). * In your main cluster, uninstall the integration that is used by the agent policy in the remote. This should cause the uninstall to fail into the remote cluster. * In your remote cluster, inspect the package SO of that integration with `GET .kibana_ingest/_search?q=type:epm-packages`: the `latest_uninstall_failed_attempts` field should be populated. Docker command for running a fleet-server in your remote cluster: ``` docker run \ -e ELASTICSEARCH_HOST=http://host.docker.internal:9500 \ -e KIBANA_HOST=http://host.docker.internal:5701/<path> \ -e KIBANA_USERNAME=elastic \ -e KIBANA_PASSWORD=changeme \ -e KIBANA_FLEET_SETUP=1 \ -e FLEET_INSECURE=1 \ -e FLEET_SERVER_ENABLE=1 \ -e FLEET_SERVER_POLICY_ID=fleet-server-policy \ -p 8220:8220 \ --rm docker.elastic.co/beats/elastic-agent:9.0.0-SNAPSHOT ``` Docker command for enrolling an agent in your remote cluster: ``` docker run \ -e ELASTICSEARCH_HOST=http://host.docker.internal:9500 \ -e KIBANA_HOST=http://host.docker.internal:5701/<path> \ -e FLEET_URL=https://host.docker.internal:8220 \ -e FLEET_ENROLL=1 \ -e FLEET_ENROLLMENT_TOKEN=<token> \ -e FLEET_INSECURE=1 \ --rm docker.elastic.co/beats/elastic-agent:9.0.0-SNAPSHOT ``` ### Checklist - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) ### Identify risks This feature is currently in development and behind the `enableSyncIntegrationsOnRemote` feature flag. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
2025-04-24 09:48:58 -04:00 · 2025-04-10 17:36:15 +02:00 · 2025-04-10 17:36:15 +02:00 · 79058c6529
commit 79058c6529
parent 0cf0e75c9c
26 changed files with 401 additions and 7 deletions
--- a/oas_docs/bundle.json
+++ b/oas_docs/bundle.json
@ -29967,6 +29967,9 @@
                              "sync_integrations": {
                                "type": "boolean"
                              },
+                              "sync_uninstalled_integrations": {
+                                "type": "boolean"
+                              },
                              "type": {
                                "enum": [
                                  "remote_elasticsearch"
@ -31129,6 +31132,9 @@
                      "sync_integrations": {
                        "type": "boolean"
                      },
+                      "sync_uninstalled_integrations": {
+                        "type": "boolean"
+                      },
                      "type": {
                        "enum": [
                          "remote_elasticsearch"
@ -32224,6 +32230,9 @@
                            "sync_integrations": {
                              "type": "boolean"
                            },
+                            "sync_uninstalled_integrations": {
+                              "type": "boolean"
+                            },
                            "type": {
                              "enum": [
                                "remote_elasticsearch"
@ -33484,6 +33493,9 @@
                            "sync_integrations": {
                              "type": "boolean"
                            },
+                            "sync_uninstalled_integrations": {
+                              "type": "boolean"
+                            },
                            "type": {
                              "enum": [
                                "remote_elasticsearch"
@ -34631,6 +34643,9 @@
                      "sync_integrations": {
                        "type": "boolean"
                      },
+                      "sync_uninstalled_integrations": {
+                        "type": "boolean"
+                      },
                      "type": {
                        "enum": [
                          "remote_elasticsearch"
@ -35711,6 +35726,9 @@
                            "sync_integrations": {
                              "type": "boolean"
                            },
+                            "sync_uninstalled_integrations": {
+                              "type": "boolean"
+                            },
                            "type": {
                              "enum": [
                                "remote_elasticsearch"
--- a/oas_docs/bundle.serverless.json
+++ b/oas_docs/bundle.serverless.json
@ -29967,6 +29967,9 @@
                              "sync_integrations": {
                                "type": "boolean"
                              },
+                              "sync_uninstalled_integrations": {
+                                "type": "boolean"
+                              },
                              "type": {
                                "enum": [
                                  "remote_elasticsearch"
@ -31129,6 +31132,9 @@
                      "sync_integrations": {
                        "type": "boolean"
                      },
+                      "sync_uninstalled_integrations": {
+                        "type": "boolean"
+                      },
                      "type": {
                        "enum": [
                          "remote_elasticsearch"
@ -32224,6 +32230,9 @@
                            "sync_integrations": {
                              "type": "boolean"
                            },
+                            "sync_uninstalled_integrations": {
+                              "type": "boolean"
+                            },
                            "type": {
                              "enum": [
                                "remote_elasticsearch"
@ -33484,6 +33493,9 @@
                            "sync_integrations": {
                              "type": "boolean"
                            },
+                            "sync_uninstalled_integrations": {
+                              "type": "boolean"
+                            },
                            "type": {
                              "enum": [
                                "remote_elasticsearch"
@ -34631,6 +34643,9 @@
                      "sync_integrations": {
                        "type": "boolean"
                      },
+                      "sync_uninstalled_integrations": {
+                        "type": "boolean"
+                      },
                      "type": {
                        "enum": [
                          "remote_elasticsearch"
@ -35711,6 +35726,9 @@
                            "sync_integrations": {
                              "type": "boolean"
                            },
+                            "sync_uninstalled_integrations": {
+                              "type": "boolean"
+                            },
                            "type": {
                              "enum": [
                                "remote_elasticsearch"
--- a/oas_docs/output/kibana.serverless.yaml
+++ b/oas_docs/output/kibana.serverless.yaml
@ -29409,6 +29409,8 @@ paths:
                                  type: string
                            sync_integrations:
                              type: boolean
+                            sync_uninstalled_integrations:
+                              type: boolean
                            type:
                              enum:
                                - remote_elasticsearch
@ -30179,6 +30181,8 @@ paths:
                          type: string
                    sync_integrations:
                      type: boolean
+                    sync_uninstalled_integrations:
+                      type: boolean
                    type:
                      enum:
                        - remote_elasticsearch
@ -30908,6 +30912,8 @@ paths:
                                type: string
                          sync_integrations:
                            type: boolean
+                          sync_uninstalled_integrations:
+                            type: boolean
                          type:
                            enum:
                              - remote_elasticsearch
@ -31743,6 +31749,8 @@ paths:
                                type: string
                          sync_integrations:
                            type: boolean
+                          sync_uninstalled_integrations:
+                            type: boolean
                          type:
                            enum:
                              - remote_elasticsearch
@ -32500,6 +32508,8 @@ paths:
                          type: string
                    sync_integrations:
                      type: boolean
+                    sync_uninstalled_integrations:
+                      type: boolean
                    type:
                      enum:
                        - remote_elasticsearch
@ -33216,6 +33226,8 @@ paths:
                                type: string
                          sync_integrations:
                            type: boolean
+                          sync_uninstalled_integrations:
+                            type: boolean
                          type:
                            enum:
                              - remote_elasticsearch
--- a/oas_docs/output/kibana.yaml
+++ b/oas_docs/output/kibana.yaml
@ -31646,6 +31646,8 @@ paths:
                                  type: string
                            sync_integrations:
                              type: boolean
+                            sync_uninstalled_integrations:
+                              type: boolean
                            type:
                              enum:
                                - remote_elasticsearch
@ -32416,6 +32418,8 @@ paths:
                          type: string
                    sync_integrations:
                      type: boolean
+                    sync_uninstalled_integrations:
+                      type: boolean
                    type:
                      enum:
                        - remote_elasticsearch
@ -33145,6 +33149,8 @@ paths:
                                type: string
                          sync_integrations:
                            type: boolean
+                          sync_uninstalled_integrations:
+                            type: boolean
                          type:
                            enum:
                              - remote_elasticsearch
@ -33980,6 +33986,8 @@ paths:
                                type: string
                          sync_integrations:
                            type: boolean
+                          sync_uninstalled_integrations:
+                            type: boolean
                          type:
                            enum:
                              - remote_elasticsearch
@ -34737,6 +34745,8 @@ paths:
                          type: string
                    sync_integrations:
                      type: boolean
+                    sync_uninstalled_integrations:
+                      type: boolean
                    type:
                      enum:
                        - remote_elasticsearch
@ -35453,6 +35463,8 @@ paths:
                                type: string
                          sync_integrations:
                            type: boolean
+                          sync_uninstalled_integrations:
+                            type: boolean
                          type:
                            enum:
                              - remote_elasticsearch
--- a/packages/kbn-check-mappings-update-cli/current_mappings.json
+++ b/packages/kbn-check-mappings-update-cli/current_mappings.json
@ -1101,6 +1101,7 @@
    }
  },
  "epm-packages": {
+    "dynamic": false,
    "properties": {
      "additional_spaces_installed_kibana": {
        "index": false,
--- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
+++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@ -98,7 +98,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
        "entity-definition": "1c6bff35c423d5dc5650bc806cf2899e4706a0bc",
        "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
        "entity-engine-status": "09f6a617020708e4f638137e5ef35bd9534133be",
-        "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
+        "epm-packages": "5a9f55e38d424f5b5ebbfeac802788b5b05d867f",
        "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
        "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",
        "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88",
--- a/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts
+++ b/x-pack/platform/plugins/shared/fleet/common/types/models/epm.ts
@ -621,6 +621,15 @@ export interface InstallFailedAttempt {
  };
 }

+export interface UninstallFailedAttempt {
+  created_at: string;
+  error: {
+    name: string;
+    message: string;
+    stack?: string;
+  };
+}
+
 export enum INSTALL_STATES {
  CREATE_RESTART_INSTALLATION = 'create_restart_installation',
  INSTALL_KIBANA_ASSETS = 'install_kibana_assets',
@ -672,6 +681,7 @@ export interface Installation {
  internal?: boolean;
  removable?: boolean;
  latest_install_failed_attempts?: InstallFailedAttempt[];
+  latest_uninstall_failed_attempts?: UninstallFailedAttempt[];
  latest_executed_state?: InstallLatestExecutedState;
 }

--- a/x-pack/platform/plugins/shared/fleet/common/types/models/output.ts
+++ b/x-pack/platform/plugins/shared/fleet/common/types/models/output.ts
@ -62,6 +62,7 @@ export interface NewRemoteElasticsearchOutput extends NewBaseOutput {
  sync_integrations?: boolean;
  kibana_url?: string | null;
  kibana_api_key?: string | null;
+  sync_uninstalled_integrations?: boolean;
 }

 export interface NewLogstashOutput extends NewBaseOutput {
--- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx
+++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/output_form_remote_es.tsx
@ -231,6 +231,28 @@ export const OutputFormRemoteEsSection: React.FunctionComponent<Props> = (props)
          </EuiFormRow>
          {inputs.syncIntegrationsInput.value === true && (
            <>
+              <EuiSpacer size="m" />
+              <EuiFormRow
+                fullWidth
+                helpText={
+                  <FormattedMessage
+                    id="xpack.fleet.settings.editOutputFlyout.syncUninstalledIntegrationsFormRowLabel"
+                    defaultMessage="If enabled, uninstalled integrations will also be installed on the remote Elasticsearch cluster"
+                  />
+                }
+                {...inputs.syncUninstalledIntegrationsInput.formRowProps}
+              >
+                <EuiSwitch
+                  {...inputs.syncUninstalledIntegrationsInput.props}
+                  data-test-subj="syncUninstalledIntegrationsSwitch"
+                  label={
+                    <FormattedMessage
+                      id="xpack.fleet.settings.editOutputFlyout.syncUninstalledIntegrationsSwitchLabel"
+                      defaultMessage="Uninstall integrations on remote"
+                    />
+                  }
+                />
+              </EuiFormRow>
              <EuiSpacer size="m" />
              <EuiCallOut
                iconType="iInCircle"
--- a/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx
+++ b/x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/settings/components/edit_output_flyout/use_output_form.tsx
@ -103,6 +103,7 @@ export interface OutputFormInputsType {
  syncIntegrationsInput: ReturnType<typeof useSwitchInput>;
  kibanaURLInput: ReturnType<typeof useInput>;
  kibanaAPIKeyInput: ReturnType<typeof useInput>;
+  syncUninstalledIntegrationsInput: ReturnType<typeof useSwitchInput>;
  sslCertificateInput: ReturnType<typeof useInput>;
  sslKeyInput: ReturnType<typeof useInput>;
  sslKeySecretInput: ReturnType<typeof useSecretInput>;
@ -297,6 +298,11 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOupu
    (val) => validateKibanaURL(val, syncIntegrationsInput.value),
    isDisabled('kibana_url')
  );
+
+  const syncUninstalledIntegrationsInput = useSwitchInput(
+    (output as NewRemoteElasticsearchOutput)?.sync_uninstalled_integrations ?? false,
+    isDisabled('sync_uninstalled_integrations')
+  );
  /*
  Shipper feature flag - currently depends on the content of the yaml
  # Enables the shipper:
@ -594,6 +600,7 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOupu
    kibanaAPIKeyInput,
    syncIntegrationsInput,
    kibanaURLInput,
+    syncUninstalledIntegrationsInput,
    sslCertificateInput,
    sslKeyInput,
    sslKeySecretInput,
@ -992,6 +999,7 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOupu
              ...(secrets ? { secrets } : {}),
              sync_integrations: syncIntegrationsInput.value,
              kibana_url: kibanaURLInput.value || null,
+              sync_uninstalled_integrations: syncUninstalledIntegrationsInput.value,
              proxy_id: proxyIdValue,
              ...shipperParams,
              ssl: {
@ -1125,6 +1133,7 @@ export function useOutputForm(onSucess: () => void, output?: Output, defaultOupu
    serviceTokenSecretInput.value,
    kibanaAPIKeyInput.value,
    syncIntegrationsInput.value,
+    syncUninstalledIntegrationsInput.value,
    kibanaURLInput.value,
    caTrustedFingerprintInput.value,
    confirm,
--- a/x-pack/platform/plugins/shared/fleet/server/saved_objects/index.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/saved_objects/index.ts
@ -988,6 +988,7 @@ export const getSavedObjectTypes = (
        importableAndExportable: false,
      },
      mappings: {
+        dynamic: false,
        properties: {
          name: { type: 'keyword' },
          version: { type: 'keyword' },
@ -1075,6 +1076,14 @@ export const getSavedObjectTypes = (
            },
          ],
        },
+        '4': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {}, // Empty to add dynamic:false
+            },
+          ],
+        },
      },
      migrations: {
        '7.14.0': migrateInstallationToV7140,
--- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/snapshots/full_agent_policy.test.ts.snap
+++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/snapshots/full_agent_policy.test.ts.snap
@ -312,6 +312,7 @@ Object {
      "preset": "balanced",
      "service_token": undefined,
      "sync_integrations": undefined,
+      "sync_uninstalled_integrations": undefined,
      "type": "remote_elasticsearch",
    },
  },
--- a/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/full_agent_policy.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/services/agent_policies/full_agent_policy.ts
@ -617,6 +617,7 @@ export function transformOutputToFullPolicyOutput(
  if (output.type === outputType.RemoteElasticsearch) {
    newOutput.service_token = output.service_token;
    newOutput.sync_integrations = output.sync_integrations;
+    newOutput.sync_uninstalled_integrations = output.sync_uninstalled_integrations;
  }

  if (outputTypeSupportPresets(output.type)) {
--- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/remove.test.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/remove.test.ts
@ -50,6 +50,7 @@ describe('removeInstallation', () => {
  beforeEach(() => {
    soClientMock = {
      get: jest.fn().mockResolvedValue({ attributes: { installed_kibana: [], installed_es: [] } }),
+      update: jest.fn(),
      delete: jest.fn(),
      find: jest.fn().mockResolvedValue({ saved_objects: [] }),
      bulkResolve: jest.fn().mockResolvedValue({ resolved_objects: [] }),
@ -81,7 +82,7 @@ describe('removeInstallation', () => {
        force: false,
      })
    ).rejects.toThrowError(
-      `Unable to remove package with existing package policy(s) in use by agent(s)`
+      `Unable to remove package system:1.0.0 with existing package policy(s) in use by agent(s)`
    );
  });

--- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/remove.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/remove.ts
@ -54,6 +54,7 @@ import { populatePackagePolicyAssignedAgentsCount } from '../../package_policies
 import * as Registry from '../registry';

 import { getInstallation, kibanaSavedObjectTypes } from '.';
+import { updateUninstallFailedAttempts } from './uninstall_errors_helpers';

 const MAX_ASSETS_TO_DELETE = 1000;

@ -64,7 +65,7 @@ export async function removeInstallation(options: {
  esClient: ElasticsearchClient;
  force?: boolean;
 }): Promise<AssetReference[]> {
-  const { savedObjectsClient, pkgName, esClient } = options;
+  const { savedObjectsClient, pkgName, pkgVersion, esClient } = options;
  const installation = await getInstallation({ savedObjectsClient, pkgName });
  if (!installation) {
    throw new PackageRemovalError(`${pkgName} is not installed`);
@ -91,9 +92,11 @@ export async function removeInstallation(options: {
        force: options.force,
      });
    } else {
-      throw new PackageRemovalError(
-        `Unable to remove package with existing package policy(s) in use by agent(s)`
+      const error = new PackageRemovalError(
+        `Unable to remove package ${pkgName}:${pkgVersion} with existing package policy(s) in use by agent(s)`
      );
+      await updateUninstallStatusToFailed(savedObjectsClient, pkgName, error);
+      throw error;
    }
  }

@ -484,3 +487,25 @@ export function cleanupTransforms(
    .map((asset) => asset.id);
  return deleteTransforms(esClient, idsToDelete);
 }
+
+async function updateUninstallStatusToFailed(
+  savedObjectsClient: SavedObjectsClientContract,
+  pkgName: string,
+  error: Error
+) {
+  const pkgSo = await savedObjectsClient.get<Installation>(PACKAGES_SAVED_OBJECT_TYPE, pkgName);
+  const updatedLatestUninstallFailedAttempts = updateUninstallFailedAttempts({
+    error,
+    createdAt: new Date().toISOString(),
+    latestAttempts: pkgSo.attributes.latest_uninstall_failed_attempts ?? [],
+  });
+  await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
+    latest_uninstall_failed_attempts: updatedLatestUninstallFailedAttempts,
+  });
+  auditLoggingService.writeCustomSoAuditLog({
+    action: 'update',
+    id: pkgName,
+    name: pkgName,
+    savedObjectType: PACKAGES_SAVED_OBJECT_TYPE,
+  });
+}
--- a/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/uninstall_errors_helpers.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/services/epm/packages/uninstall_errors_helpers.ts
@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { UninstallFailedAttempt } from '../../../types';
+
+const MAX_ATTEMPTS_TO_KEEP = 5;
+
+export function updateUninstallFailedAttempts({
+  error,
+  createdAt,
+  latestAttempts = [],
+}: {
+  error: Error;
+  createdAt: string;
+  latestAttempts?: UninstallFailedAttempt[];
+}): UninstallFailedAttempt[] {
+  return [
+    {
+      created_at: createdAt,
+      error: {
+        name: error.name,
+        message: error.message,
+        stack: error.stack,
+      },
+    },
+    ...latestAttempts,
+  ].slice(0, MAX_ATTEMPTS_TO_KEEP);
+}
--- a/x-pack/platform/plugins/shared/fleet/server/services/preconfiguration/outputs.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/services/preconfiguration/outputs.ts
@ -342,7 +342,11 @@ async function isPreconfiguredOutputDifferentFromCurrent(
        existingOutput.secrets?.service_token
      )) ||
      isDifferent(existingOutput.kibana_url, preconfiguredOutput.kibana_url) ||
-      isDifferent(existingOutput.sync_integrations, preconfiguredOutput.sync_integrations);
+      isDifferent(existingOutput.sync_integrations, preconfiguredOutput.sync_integrations) ||
+      isDifferent(
+        existingOutput.sync_uninstalled_integrations,
+        preconfiguredOutput.sync_uninstalled_integrations
+      );

    return serviceTokenIsDifferent;
  };
--- a/x-pack/platform/plugins/shared/fleet/server/services/uninstall_errors_helpers.test.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/services/uninstall_errors_helpers.test.ts
@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { UninstallFailedAttempt } from '../types';
+
+import { updateUninstallFailedAttempts } from './epm/packages/uninstall_errors_helpers';
+
+const generateFailedAttempt = () => ({
+  created_at: new Date().toISOString(),
+  error: {
+    name: 'test',
+    message: 'test',
+  },
+});
+
+describe('Uninstall error helpers', () => {
+  describe('updateUninstallFailedAttempts', () => {
+    it('should only keep 5 errors', () => {
+      const previousFailedAttempts: UninstallFailedAttempt[] = Array(5)
+        .fill(null)
+        .map((_) => generateFailedAttempt());
+      const updatedLatestUninstallFailedAttempts = updateUninstallFailedAttempts({
+        error: new Error('new test'),
+        createdAt: new Date().toISOString(),
+        latestAttempts: previousFailedAttempts,
+      });
+
+      expect(updatedLatestUninstallFailedAttempts.length).toEqual(5);
+      expect(updatedLatestUninstallFailedAttempts[0].error.message).toEqual('new test');
+    });
+  });
+});
--- a/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/custom_assets.test.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/custom_assets.test.ts
@ -13,16 +13,19 @@ describe('custom assets', () => {
      package_name: 'system',
      package_version: '0.1.0',
      updated_at: new Date().toISOString(),
+      install_status: 'installed',
    },
    {
      package_name: 'endpoint',
      package_version: '0.1.0',
      updated_at: new Date().toISOString(),
+      install_status: 'installed',
    },
    {
      package_name: 'synthetics',
      package_version: '0.1.0',
      updated_at: new Date().toISOString(),
+      install_status: 'installed',
    },
  ];

--- a/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/model.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/model.ts
@ -9,6 +9,7 @@ export interface IntegrationsData {
  package_name: string;
  package_version: string;
  updated_at: string;
+  install_status: string;
 }

 export interface BaseCustomAssetsData {
--- a/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_on_remote.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_on_remote.ts
@ -6,6 +6,7 @@
 */
 import type { ElasticsearchClient, SavedObjectsClient, Logger } from '@kbn/core/server';

+import semverEq from 'semver/functions/eq';
 import semverGte from 'semver/functions/gte';

 import type { PackageClient } from '../../services';
@ -14,6 +15,8 @@ import { outputService } from '../../services';
 import { PackageNotFoundError } from '../../errors';
 import { FLEET_SYNCED_INTEGRATIONS_CCR_INDEX_PREFIX } from '../../services/setup/fleet_synced_integrations';

+import { getInstallation, removeInstallation } from '../../services/epm/packages';
+
 import type { SyncIntegrationsData } from './model';
 import { installCustomAsset } from './custom_assets';

@ -155,6 +158,43 @@ async function installPackageIfNotInstalled(
  }
 }

+async function uninstallPackageIfInstalled(
+  esClient: ElasticsearchClient,
+  savedObjectsClient: SavedObjectsClient,
+  pkg: { package_name: string; package_version: string },
+  logger: Logger
+) {
+  const installation = await getInstallation({ savedObjectsClient, pkgName: pkg.package_name });
+  if (!installation) {
+    return;
+  }
+  if (
+    !(
+      installation.install_status === 'installed' &&
+      semverEq(installation.version, pkg.package_version)
+    )
+  ) {
+    return;
+  }
+
+  try {
+    await removeInstallation({
+      savedObjectsClient,
+      pkgName: pkg.package_name,
+      pkgVersion: pkg.package_version,
+      esClient,
+      force: false,
+    });
+    logger.info(
+      `Package ${pkg.package_name} with version ${pkg.package_version} uninstalled via integration syncing`
+    );
+  } catch (error) {
+    logger.error(
+      `Failed to uninstall package ${pkg.package_name} with version ${pkg.package_version} via integration syncing: ${error.message}`
+    );
+  }
+}
+
 export const syncIntegrationsOnRemote = async (
  esClient: ElasticsearchClient,
  soClient: SavedObjectsClient,
@ -173,13 +213,28 @@ export const syncIntegrationsOnRemote = async (
    return;
  }

-  for (const pkg of syncIntegrationsDoc?.integrations ?? []) {
+  const installedIntegrations =
+    syncIntegrationsDoc?.integrations.filter(
+      (integration) => integration.install_status !== 'not_installed'
+    ) ?? [];
+  for (const pkg of installedIntegrations) {
    if (abortController.signal.aborted) {
      throw new Error('Task was aborted');
    }
    await installPackageIfNotInstalled(pkg, packageClient, logger, abortController);
  }

+  const uninstalledIntegrations =
+    syncIntegrationsDoc?.integrations.filter(
+      (integration) => integration.install_status === 'not_installed'
+    ) ?? [];
+  for (const pkg of uninstalledIntegrations) {
+    if (abortController.signal.aborted) {
+      throw new Error('Task was aborted');
+    }
+    await uninstallPackageIfInstalled(esClient, soClient, pkg, logger);
+  }
+
  for (const customAsset of Object.values(syncIntegrationsDoc?.custom_assets ?? {})) {
    if (abortController.signal.aborted) {
      throw new Error('Task was aborted');
--- a/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_task.test.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_task.test.ts
@ -40,6 +40,7 @@ jest.mock('../../services/epm/packages/get', () => ({
          name: 'system',
          version: '0.1.0',
          updated_at: new Date().toISOString(),
+          install_status: 'installed',
        },
      },
      {
@ -47,6 +48,7 @@ jest.mock('../../services/epm/packages/get', () => ({
          name: 'package-2',
          version: '0.2.0',
          updated_at: new Date().toISOString(),
+          install_status: 'installed',
        },
      },
    ],
@ -183,11 +185,13 @@ describe('SyncIntegrationsTask', () => {
                package_name: 'system',
                package_version: '0.1.0',
                updated_at: expect.any(String),
+                install_status: 'installed',
              },
              {
                package_name: 'package-2',
                package_version: '0.2.0',
                updated_at: expect.any(String),
+                install_status: 'installed',
              },
            ],
            remote_es_hosts: [
@ -244,11 +248,13 @@ describe('SyncIntegrationsTask', () => {
                package_name: 'system',
                package_version: '0.1.0',
                updated_at: expect.any(String),
+                install_status: 'installed',
              },
              {
                package_name: 'package-2',
                package_version: '0.2.0',
                updated_at: expect.any(String),
+                install_status: 'installed',
              },
            ],
            remote_es_hosts: [
@ -346,5 +352,102 @@ describe('SyncIntegrationsTask', () => {

      expect(esClient.index).not.toHaveBeenCalled();
    });
+
+    it('Should mark removed integrations as uninstalled if uninstall syncing is enabled', async () => {
+      mockOutputService.list.mockResolvedValue({
+        items: [
+          {
+            type: 'remote_elasticsearch',
+            name: 'remote1',
+            hosts: ['https://remote1:9200'],
+            sync_integrations: true,
+            sync_uninstalled_integrations: true,
+          },
+        ],
+      } as any);
+      esClient.get.mockResolvedValue({
+        _source: {
+          remote_es_hosts: [
+            { hosts: ['https://remote1:9200'], name: 'remote1', sync_integrations: false },
+          ],
+          integrations: [
+            {
+              package_name: 'system',
+              package_version: '0.1.0',
+              updated_at: new Date().toISOString(),
+              install_status: 'installed',
+            },
+            {
+              package_name: 'package-2',
+              package_version: '0.2.0',
+              updated_at: new Date().toISOString(),
+              install_status: 'installed',
+            },
+            {
+              package_name: 'package-3',
+              package_version: '0.3.0',
+              updated_at: new Date().toISOString(),
+              install_status: 'installed',
+            },
+          ],
+          custom_assets: {},
+          custom_assets_error: {},
+        },
+      } as any);
+      await runTask();
+
+      expect(esClient.index).toHaveBeenCalledWith(
+        {
+          id: 'fleet-synced-integrations',
+          index: 'fleet-synced-integrations',
+          body: {
+            integrations: [
+              {
+                package_name: 'system',
+                package_version: '0.1.0',
+                updated_at: expect.any(String),
+                install_status: 'installed',
+              },
+              {
+                package_name: 'package-2',
+                package_version: '0.2.0',
+                updated_at: expect.any(String),
+                install_status: 'installed',
+              },
+              {
+                package_name: 'package-3',
+                package_version: '0.3.0',
+                updated_at: expect.any(String),
+                install_status: 'not_installed',
+              },
+            ],
+            remote_es_hosts: [
+              { hosts: ['https://remote1:9200'], name: 'remote1', sync_integrations: true },
+            ],
+            custom_assets: {
+              'component_template:logs-system.auth@custom': {
+                is_deleted: false,
+                name: 'logs-system.auth@custom',
+                package_name: 'system',
+                package_version: '0.1.0',
+                template: {},
+                type: 'component_template',
+              },
+              'ingest_pipeline:logs-system.auth@custom': {
+                is_deleted: false,
+                name: 'logs-system.auth@custom',
+                package_name: 'system',
+                package_version: '0.1.0',
+                pipeline: {
+                  processors: [],
+                },
+                type: 'ingest_pipeline',
+              },
+            },
+          },
+        },
+        expect.anything()
+      );
+    });
  });
 });
--- a/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_task.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/tasks/sync_integrations/sync_integrations_task.ts
@ -240,9 +240,25 @@ export class SyncIntegrationsTask {
        package_name: item.attributes.name,
        package_version: item.attributes.version,
        updated_at: item.updated_at ?? new Date().toISOString(),
+        install_status: item.attributes.install_status,
      };
    });

+    const isSyncUninstalledEnabled = remoteESOutputs.some(
+      (output) => (output as NewRemoteElasticsearchOutput).sync_uninstalled_integrations
+    );
+    if (isSyncUninstalledEnabled && previousSyncIntegrationsData) {
+      const removedIntegrations = previousSyncIntegrationsData.integrations.filter(
+        (item) =>
+          !packageSavedObjects.saved_objects
+            .map((data) => data.attributes.name)
+            .includes(item.package_name)
+      );
+      newDoc.integrations.push(
+        ...removedIntegrations.map((item) => ({ ...item, install_status: 'not_installed' }))
+      );
+    }
+
    try {
      const customAssets = await getCustomAssets(
        esClient,
--- a/x-pack/platform/plugins/shared/fleet/server/types/index.tsx
+++ b/x-pack/platform/plugins/shared/fleet/server/types/index.tsx
@ -42,6 +42,7 @@ export type {
  EpmPackageInstallStatus,
  InstallationStatus,
  InstallFailedAttempt,
+  UninstallFailedAttempt,
  PackageInfo,
  ArchivePackage,
  RegistryVarsEntry,
--- a/x-pack/platform/plugins/shared/fleet/server/types/models/output.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/types/models/output.ts
@ -163,6 +163,7 @@ export const RemoteElasticSearchSchema = {
  sync_integrations: schema.maybe(schema.boolean()),
  kibana_url: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])),
  kibana_api_key: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])),
+  sync_uninstalled_integrations: schema.maybe(schema.boolean()),
 };

 const RemoteElasticSearchUpdateSchema = {
@ -178,6 +179,7 @@ const RemoteElasticSearchUpdateSchema = {
  sync_integrations: schema.maybe(schema.boolean()),
  kibana_url: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])),
  kibana_api_key: schema.maybe(schema.oneOf([schema.literal(null), schema.string()])),
+  sync_uninstalled_integrations: schema.maybe(schema.boolean()),
 };

 /**
--- a/x-pack/platform/plugins/shared/fleet/server/types/so_attributes.ts
+++ b/x-pack/platform/plugins/shared/fleet/server/types/so_attributes.ts
@ -190,6 +190,7 @@ export interface OutputSoRemoteElasticsearchAttributes extends OutputSoBaseAttri
  sync_integrations?: boolean;
  kibana_url?: string;
  kibana_api_key?: string;
+  sync_uninstalled_integrations?: boolean;
 }

 interface OutputSoLogstashAttributes extends OutputSoBaseAttributes {