[Fleet] Handle output deletion for agent policy (#127042)

x-pack/plugins/fleet/server/services/agent_policy.test.ts

@@ -204,6 +204,59 @@ describe('agent policy', () => {
     });
   });
 
+  describe('removeOutputFromAll', () => {
+    let mockedAgentPolicyServiceUpdate: jest.SpyInstance<
+      ReturnType<typeof agentPolicyService['update']>
+    >;
+    beforeEach(() => {
+      mockedAgentPolicyServiceUpdate = jest
+        .spyOn(agentPolicyService, 'update')
+        .mockResolvedValue({} as any);
+    });
+
+    afterEach(() => {
+      mockedAgentPolicyServiceUpdate.mockRestore();
+    });
+    it('should update policies using deleted output', async () => {
+      const soClient = savedObjectsClientMock.create();
+      const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+      soClient.find.mockResolvedValue({
+        saved_objects: [
+          {
+            id: 'test1',
+            attributes: {
+              data_output_id: 'output-id-123',
+              monitoring_output_id: 'output-id-another-output',
+            },
+          },
+          {
+            id: 'test2',
+            attributes: {
+              data_output_id: 'output-id-another-output',
+              monitoring_output_id: 'output-id-123',
+            },
+          },
+        ],
+      } as any);
+
+      await agentPolicyService.removeOutputFromAll(soClient, esClient, 'output-id-123');
+
+      expect(mockedAgentPolicyServiceUpdate).toHaveBeenCalledTimes(2);
+      expect(mockedAgentPolicyServiceUpdate).toHaveBeenCalledWith(
+        expect.anything(),
+        expect.anything(),
+        'test1',
+        { data_output_id: null, monitoring_output_id: 'output-id-another-output' }
+      );
+      expect(mockedAgentPolicyServiceUpdate).toHaveBeenCalledWith(
+        expect.anything(),
+        expect.anything(),
+        'test2',
+        { data_output_id: 'output-id-another-output', monitoring_output_id: null }
+      );
+    });
+  });
+
   describe('update', () => {
     it('should update is_managed property, if given', async () => {
       // ignore unrelated unique name constraint

x-pack/plugins/fleet/server/services/agent_policy.ts

@@ -414,6 +414,49 @@ class AgentPolicyService {
     return res;
   }
 
+  /**
+   * Remove an output from all agent policies that are using it, and replace the output by the default ones.
+   * @param soClient
+   * @param esClient
+   * @param outputId
+   */
+  public async removeOutputFromAll(
+    soClient: SavedObjectsClientContract,
+    esClient: ElasticsearchClient,
+    outputId: string
+  ) {
+    const agentPolicies = (
+      await soClient.find<AgentPolicySOAttributes>({
+        type: SAVED_OBJECT_TYPE,
+        fields: ['revision', 'data_output_id', 'monitoring_output_id'],
+        searchFields: ['data_output_id', 'monitoring_output_id'],
+        search: escapeSearchQueryPhrase(outputId),
+        perPage: SO_SEARCH_LIMIT,
+      })
+    ).saved_objects.map((so) => ({
+      id: so.id,
+      ...so.attributes,
+    }));
+
+    if (agentPolicies.length > 0) {
+      await pMap(
+        agentPolicies,
+        (agentPolicy) =>
+          this.update(soClient, esClient, agentPolicy.id, {
+            data_output_id:
+              agentPolicy.data_output_id === outputId ? null : agentPolicy.data_output_id,
+            monitoring_output_id:
+              agentPolicy.monitoring_output_id === outputId
+                ? null
+                : agentPolicy.monitoring_output_id,
+          }),
+        {
+          concurrency: 50,
+        }
+      );
+    }
+  }
+
   public async bumpAllAgentPoliciesForOutput(
     soClient: SavedObjectsClientContract,
     esClient: ElasticsearchClient,

x-pack/plugins/fleet/server/services/output.test.ts

@@ -10,10 +10,13 @@ import type { OutputSOAttributes } from '../types';
 
 import { outputService, outputIdToUuid } from './output';
 import { appContextService } from './app_context';
+import { agentPolicyService } from './agent_policy';
 
 jest.mock('./app_context');
+jest.mock('./agent_policy');
 
 const mockedAppContextService = appContextService as jest.Mocked<typeof appContextService>;
+const mockedAgentPolicyService = agentPolicyService as jest.Mocked<typeof agentPolicyService>;
 
 const CLOUD_ID =
   'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==';
@@ -145,6 +148,9 @@ function getMockedSoClient(
 }
 
 describe('Output Service', () => {
+  beforeEach(() => {
+    mockedAgentPolicyService.removeOutputFromAll.mockReset();
+  });
   describe('create', () => {
     it('work with a predefined id', async () => {
       const soClient = getMockedSoClient();
@@ -447,6 +453,15 @@ describe('Output Service', () => {
 
       expect(soClient.delete).toBeCalled();
     });
+
+    it('Call removeOutputFromAll before deleting the output', async () => {
+      const soClient = getMockedSoClient();
+      await outputService.delete(soClient, 'existing-preconfigured-default-output', {
+        fromPreconfiguration: true,
+      });
+      expect(mockedAgentPolicyService.removeOutputFromAll).toBeCalled();
+      expect(soClient.delete).toBeCalled();
+    });
   });
 
   describe('get', () => {

x-pack/plugins/fleet/server/services/output.ts

@@ -13,6 +13,7 @@ import { DEFAULT_OUTPUT, DEFAULT_OUTPUT_ID, OUTPUT_SAVED_OBJECT_TYPE } from '../
 import { decodeCloudId, normalizeHostsForAgents, SO_SEARCH_LIMIT, outputType } from '../../common';
 import { OutputUnauthorizedError } from '../errors';
 
+import { agentPolicyService } from './agent_policy';
 import { appContextService } from './app_context';
 
 const SAVED_OBJECT_TYPE = OUTPUT_SAVED_OBJECT_TYPE;
@@ -241,6 +242,12 @@ class OutputService {
       throw new OutputUnauthorizedError(`Default monitoring output ${id} cannot be deleted.`);
     }
 
+    await agentPolicyService.removeOutputFromAll(
+      soClient,
+      appContextService.getInternalUserESClient(),
+      id
+    );
+
     return soClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id));
   }
 

x-pack/plugins/fleet/server/services/preconfiguration.test.ts

@@ -16,7 +16,6 @@ import type {
   InstallResult,
   PackagePolicy,
   PreconfiguredAgentPolicy,
-  PreconfiguredOutput,
   RegistrySearchResult,
 } from '../../common/types';
 import type { AgentPolicy, NewPackagePolicy, Output } from '../types';
@@ -28,10 +27,7 @@ import * as agentPolicy from './agent_policy';
 import {
   ensurePreconfiguredPackagesAndPolicies,
   comparePreconfiguredPolicyToCurrent,
-  ensurePreconfiguredOutputs,
-  cleanPreconfiguredOutputs,
 } from './preconfiguration';
-import { outputService } from './output';
 import { packagePolicyService } from './package_policy';
 import { getBundledPackages } from './epm/packages/bundled_packages';
 import type { InstallPackageParams } from './epm/packages/install';
@@ -41,7 +37,6 @@ jest.mock('./output');
 jest.mock('./epm/packages/bundled_packages');
 jest.mock('./epm/archive');
 
-const mockedOutputService = outputService as jest.Mocked<typeof outputService>;
 const mockedPackagePolicyService = packagePolicyService as jest.Mocked<typeof packagePolicyService>;
 const mockedGetBundledPackages = getBundledPackages as jest.MockedFunction<
   typeof getBundledPackages
@@ -936,201 +931,3 @@ describe('comparePreconfiguredPolicyToCurrent', () => {
     expect(hasChanged).toBe(false);
   });
 });
-
-describe('output preconfiguration', () => {
-  beforeEach(() => {
-    mockedOutputService.create.mockReset();
-    mockedOutputService.update.mockReset();
-    mockedOutputService.delete.mockReset();
-    mockedOutputService.getDefaultDataOutputId.mockReset();
-    mockedOutputService.getDefaultESHosts.mockReturnValue(['http://default-es:9200']);
-    mockedOutputService.bulkGet.mockImplementation(async (soClient, id): Promise<Output[]> => {
-      return [
-        {
-          id: 'existing-output-1',
-          is_default: false,
-          is_default_monitoring: false,
-          name: 'Output 1',
-          // @ts-ignore
-          type: 'elasticsearch',
-          hosts: ['http://es.co:80'],
-          is_preconfigured: true,
-        },
-      ];
-    });
-  });
-
-  it('should create preconfigured output that does not exists', async () => {
-    const soClient = savedObjectsClientMock.create();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
-    await ensurePreconfiguredOutputs(soClient, esClient, [
-      {
-        id: 'non-existing-output-1',
-        name: 'Output 1',
-        type: 'elasticsearch',
-        is_default: false,
-        is_default_monitoring: false,
-        hosts: ['http://test.fr'],
-      },
-    ]);
-
-    expect(mockedOutputService.create).toBeCalled();
-    expect(mockedOutputService.update).not.toBeCalled();
-    expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled();
-  });
-
-  it('should set default hosts if hosts is not set output that does not exists', async () => {
-    const soClient = savedObjectsClientMock.create();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
-    await ensurePreconfiguredOutputs(soClient, esClient, [
-      {
-        id: 'non-existing-output-1',
-        name: 'Output 1',
-        type: 'elasticsearch',
-        is_default: false,
-        is_default_monitoring: false,
-      },
-    ]);
-
-    expect(mockedOutputService.create).toBeCalled();
-    expect(mockedOutputService.create.mock.calls[0][1].hosts).toEqual(['http://default-es:9200']);
-  });
-
-  it('should update output if preconfigured output exists and changed', async () => {
-    const soClient = savedObjectsClientMock.create();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
-    soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 });
-    await ensurePreconfiguredOutputs(soClient, esClient, [
-      {
-        id: 'existing-output-1',
-        is_default: false,
-        is_default_monitoring: false,
-        name: 'Output 1',
-        type: 'elasticsearch',
-        hosts: ['http://newhostichanged.co:9201'], // field that changed
-      },
-    ]);
-
-    expect(mockedOutputService.create).not.toBeCalled();
-    expect(mockedOutputService.update).toBeCalled();
-    expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled();
-  });
-
-  it('should not delete default output if preconfigured default output exists and changed', async () => {
-    const soClient = savedObjectsClientMock.create();
-    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
-    soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 });
-    mockedOutputService.getDefaultDataOutputId.mockResolvedValue('existing-output-1');
-    await ensurePreconfiguredOutputs(soClient, esClient, [
-      {
-        id: 'existing-output-1',
-        is_default: true,
-        is_default_monitoring: false,
-        name: 'Output 1',
-        type: 'elasticsearch',
-        hosts: ['http://newhostichanged.co:9201'], // field that changed
-      },
-    ]);
-
-    expect(mockedOutputService.delete).not.toBeCalled();
-    expect(mockedOutputService.create).not.toBeCalled();
-    expect(mockedOutputService.update).toBeCalled();
-    expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled();
-  });
-
-  const SCENARIOS: Array<{ name: string; data: PreconfiguredOutput }> = [
-    {
-      name: 'no changes',
-      data: {
-        id: 'existing-output-1',
-        is_default: false,
-        is_default_monitoring: false,
-        name: 'Output 1',
-        type: 'elasticsearch',
-        hosts: ['http://es.co:80'],
-      },
-    },
-    {
-      name: 'hosts without port',
-      data: {
-        id: 'existing-output-1',
-        is_default: false,
-        is_default_monitoring: false,
-        name: 'Output 1',
-        type: 'elasticsearch',
-        hosts: ['http://es.co'],
-      },
-    },
-  ];
-  SCENARIOS.forEach((scenario) => {
-    const { data, name } = scenario;
-    it(`should do nothing if preconfigured output exists and did not changed (${name})`, async () => {
-      const soClient = savedObjectsClientMock.create();
-      const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
-      await ensurePreconfiguredOutputs(soClient, esClient, [data]);
-
-      expect(mockedOutputService.create).not.toBeCalled();
-      expect(mockedOutputService.update).not.toBeCalled();
-    });
-  });
-
-  it('should not delete non deleted preconfigured output', async () => {
-    const soClient = savedObjectsClientMock.create();
-    mockedOutputService.list.mockResolvedValue({
-      items: [
-        { id: 'output1', is_preconfigured: true } as Output,
-        { id: 'output2', is_preconfigured: true } as Output,
-      ],
-      page: 1,
-      perPage: 10000,
-      total: 1,
-    });
-    await cleanPreconfiguredOutputs(soClient, [
-      {
-        id: 'output1',
-        is_default: false,
-        is_default_monitoring: false,
-        name: 'Output 1',
-        type: 'elasticsearch',
-        hosts: ['http://es.co:9201'],
-      },
-      {
-        id: 'output2',
-        is_default: false,
-        is_default_monitoring: false,
-        name: 'Output 2',
-        type: 'elasticsearch',
-        hosts: ['http://es.co:9201'],
-      },
-    ]);
-
-    expect(mockedOutputService.delete).not.toBeCalled();
-  });
-
-  it('should delete deleted preconfigured output', async () => {
-    const soClient = savedObjectsClientMock.create();
-    mockedOutputService.list.mockResolvedValue({
-      items: [
-        { id: 'output1', is_preconfigured: true } as Output,
-        { id: 'output2', is_preconfigured: true } as Output,
-      ],
-      page: 1,
-      perPage: 10000,
-      total: 1,
-    });
-    await cleanPreconfiguredOutputs(soClient, [
-      {
-        id: 'output1',
-        is_default: false,
-        is_default_monitoring: false,
-        name: 'Output 1',
-        type: 'elasticsearch',
-        hosts: ['http://es.co:9201'],
-      },
-    ]);
-
-    expect(mockedOutputService.delete).toBeCalled();
-    expect(mockedOutputService.delete).toBeCalledTimes(1);
-    expect(mockedOutputService.delete.mock.calls[0][1]).toEqual('output2');
-  });
-});

x-pack/plugins/fleet/server/services/preconfiguration.ts

@@ -8,7 +8,6 @@
 import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
 import { i18n } from '@kbn/i18n';
 import { groupBy, omit, pick, isEqual } from 'lodash';
-import { safeDump } from 'js-yaml';
 
 import type {
   NewPackagePolicy,
@@ -18,11 +17,9 @@ import type {
   PreconfiguredAgentPolicy,
   PreconfiguredPackage,
   PreconfigurationError,
-  PreconfiguredOutput,
   PackagePolicy,
 } from '../../common';
 import { PRECONFIGURATION_LATEST_KEYWORD } from '../../common';
-import { normalizeHostsForAgents } from '../../common';
 import { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE } from '../constants';
 
 import { escapeSearchQueryPhrase } from './saved_object';
@@ -35,7 +32,6 @@ import type { InputsOverride } from './package_policy';
 import { preconfigurePackageInputs } from './package_policy';
 import { appContextService } from './app_context';
 import type { UpgradeManagedPackagePoliciesResult } from './managed_package_policies';
-import { outputService } from './output';
 
 interface PreconfigurationResult {
   policies: Array<{ id: string; updated_at: string }>;
@@ -43,100 +39,6 @@ interface PreconfigurationResult {
   nonFatalErrors: Array<PreconfigurationError | UpgradeManagedPackagePoliciesResult>;
 }
 
-function isPreconfiguredOutputDifferentFromCurrent(
-  existingOutput: Output,
-  preconfiguredOutput: Partial<Output>
-): boolean {
-  return (
-    existingOutput.is_default !== preconfiguredOutput.is_default ||
-    existingOutput.is_default_monitoring !== preconfiguredOutput.is_default_monitoring ||
-    existingOutput.name !== preconfiguredOutput.name ||
-    existingOutput.type !== preconfiguredOutput.type ||
-    (preconfiguredOutput.hosts &&
-      !isEqual(
-        existingOutput.hosts?.map(normalizeHostsForAgents),
-        preconfiguredOutput.hosts.map(normalizeHostsForAgents)
-      )) ||
-    (preconfiguredOutput.ssl && !isEqual(preconfiguredOutput.ssl, existingOutput.ssl)) ||
-    existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 ||
-    existingOutput.ca_trusted_fingerprint !== preconfiguredOutput.ca_trusted_fingerprint ||
-    existingOutput.config_yaml !== preconfiguredOutput.config_yaml
-  );
-}
-
-export async function ensurePreconfiguredOutputs(
-  soClient: SavedObjectsClientContract,
-  esClient: ElasticsearchClient,
-  outputs: PreconfiguredOutput[]
-) {
-  const logger = appContextService.getLogger();
-
-  if (outputs.length === 0) {
-    return;
-  }
-
-  const existingOutputs = await outputService.bulkGet(
-    soClient,
-    outputs.map(({ id }) => id),
-    { ignoreNotFound: true }
-  );
-
-  await Promise.all(
-    outputs.map(async (output) => {
-      const existingOutput = existingOutputs.find((o) => o.id === output.id);
-
-      const { id, config, ...outputData } = output;
-
-      const configYaml = config ? safeDump(config) : undefined;
-
-      const data = {
-        ...outputData,
-        config_yaml: configYaml,
-        is_preconfigured: true,
-      };
-
-      if (!data.hosts || data.hosts.length === 0) {
-        data.hosts = outputService.getDefaultESHosts();
-      }
-
-      const isCreate = !existingOutput;
-      const isUpdateWithNewData =
-        existingOutput && isPreconfiguredOutputDifferentFromCurrent(existingOutput, data);
-
-      if (isCreate) {
-        logger.debug(`Creating output ${output.id}`);
-        await outputService.create(soClient, data, { id, fromPreconfiguration: true });
-      } else if (isUpdateWithNewData) {
-        logger.debug(`Updating output ${output.id}`);
-        await outputService.update(soClient, id, data, { fromPreconfiguration: true });
-        // Bump revision of all policies using that output
-        if (outputData.is_default || outputData.is_default_monitoring) {
-          await agentPolicyService.bumpAllAgentPolicies(soClient, esClient);
-        } else {
-          await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, id);
-        }
-      }
-    })
-  );
-}
-
-export async function cleanPreconfiguredOutputs(
-  soClient: SavedObjectsClientContract,
-  outputs: PreconfiguredOutput[]
-) {
-  const existingPreconfiguredOutput = (await outputService.list(soClient)).items.filter(
-    (o) => o.is_preconfigured === true
-  );
-  const logger = appContextService.getLogger();
-
-  for (const output of existingPreconfiguredOutput) {
-    if (!outputs.find(({ id }) => output.id === id)) {
-      logger.info(`Deleting preconfigured output ${output.id}`);
-      await outputService.delete(soClient, output.id, { fromPreconfiguration: true });
-    }
-  }
-}
-
 export async function ensurePreconfiguredPackagesAndPolicies(
   soClient: SavedObjectsClientContract,
   esClient: ElasticsearchClient,

x-pack/plugins/fleet/server/services/preconfiguration/index.ts

@@ -6,3 +6,5 @@
  */
 
 export { resetPreconfiguredAgentPolicies } from './reset_agent_policies';
+
+export { ensurePreconfiguredOutputs } from './outputs';

x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts

@@ -0,0 +1,315 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks';
+
+import type { PreconfiguredOutput } from '../../../common/types';
+import type { Output } from '../../types';
+
+import * as agentPolicy from '../agent_policy';
+import { outputService } from '../output';
+
+import { createOrUpdatePreconfiguredOutputs, cleanPreconfiguredOutputs } from './outputs';
+
+jest.mock('../agent_policy_update');
+jest.mock('../output');
+jest.mock('../epm/packages/bundled_packages');
+jest.mock('../epm/archive');
+
+const mockedOutputService = outputService as jest.Mocked<typeof outputService>;
+
+jest.mock('../app_context', () => ({
+  appContextService: {
+    getLogger: () =>
+      new Proxy(
+        {},
+        {
+          get() {
+            return jest.fn();
+          },
+        }
+      ),
+  },
+}));
+
+const spyAgentPolicyServicBumpAllAgentPoliciesForOutput = jest.spyOn(
+  agentPolicy.agentPolicyService,
+  'bumpAllAgentPoliciesForOutput'
+);
+
+describe('output preconfiguration', () => {
+  beforeEach(() => {
+    mockedOutputService.create.mockReset();
+    mockedOutputService.update.mockReset();
+    mockedOutputService.delete.mockReset();
+    mockedOutputService.getDefaultDataOutputId.mockReset();
+    mockedOutputService.getDefaultESHosts.mockReturnValue(['http://default-es:9200']);
+    mockedOutputService.bulkGet.mockImplementation(async (soClient, id): Promise<Output[]> => {
+      return [
+        {
+          id: 'existing-output-1',
+          is_default: false,
+          is_default_monitoring: false,
+          name: 'Output 1',
+          // @ts-ignore
+          type: 'elasticsearch',
+          hosts: ['http://es.co:80'],
+          is_preconfigured: true,
+        },
+      ];
+    });
+  });
+
+  it('should create preconfigured output that does not exists', async () => {
+    const soClient = savedObjectsClientMock.create();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    await createOrUpdatePreconfiguredOutputs(soClient, esClient, [
+      {
+        id: 'non-existing-output-1',
+        name: 'Output 1',
+        type: 'elasticsearch',
+        is_default: false,
+        is_default_monitoring: false,
+        hosts: ['http://test.fr'],
+      },
+    ]);
+
+    expect(mockedOutputService.create).toBeCalled();
+    expect(mockedOutputService.update).not.toBeCalled();
+    expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled();
+  });
+
+  it('should set default hosts if hosts is not set output that does not exists', async () => {
+    const soClient = savedObjectsClientMock.create();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    await createOrUpdatePreconfiguredOutputs(soClient, esClient, [
+      {
+        id: 'non-existing-output-1',
+        name: 'Output 1',
+        type: 'elasticsearch',
+        is_default: false,
+        is_default_monitoring: false,
+      },
+    ]);
+
+    expect(mockedOutputService.create).toBeCalled();
+    expect(mockedOutputService.create.mock.calls[0][1].hosts).toEqual(['http://default-es:9200']);
+  });
+
+  it('should update output if non preconfigured output with the same id exists', async () => {
+    const soClient = savedObjectsClientMock.create();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 });
+    mockedOutputService.bulkGet.mockResolvedValue([
+      {
+        id: 'existing-output-1',
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'Output 1',
+        // @ts-ignore
+        type: 'elasticsearch',
+        hosts: ['http://es.co:80'],
+        is_preconfigured: false,
+      },
+    ]);
+    await createOrUpdatePreconfiguredOutputs(soClient, esClient, [
+      {
+        id: 'existing-output-1',
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'Output 1',
+        type: 'elasticsearch',
+        hosts: ['http://es.co:80'],
+      },
+    ]);
+
+    expect(mockedOutputService.create).not.toBeCalled();
+    expect(mockedOutputService.update).toBeCalled();
+    expect(mockedOutputService.update).toBeCalledWith(
+      expect.anything(),
+      'existing-output-1',
+      expect.objectContaining({
+        is_preconfigured: true,
+      }),
+      { fromPreconfiguration: true }
+    );
+    expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled();
+  });
+
+  it('should update output if preconfigured output exists and changed', async () => {
+    const soClient = savedObjectsClientMock.create();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 });
+    await createOrUpdatePreconfiguredOutputs(soClient, esClient, [
+      {
+        id: 'existing-output-1',
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'Output 1',
+        type: 'elasticsearch',
+        hosts: ['http://newhostichanged.co:9201'], // field that changed
+      },
+    ]);
+
+    expect(mockedOutputService.create).not.toBeCalled();
+    expect(mockedOutputService.update).toBeCalled();
+    expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled();
+  });
+
+  it('should not delete default output if preconfigured default output exists and changed', async () => {
+    const soClient = savedObjectsClientMock.create();
+    const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 });
+    mockedOutputService.getDefaultDataOutputId.mockResolvedValue('existing-output-1');
+    await createOrUpdatePreconfiguredOutputs(soClient, esClient, [
+      {
+        id: 'existing-output-1',
+        is_default: true,
+        is_default_monitoring: false,
+        name: 'Output 1',
+        type: 'elasticsearch',
+        hosts: ['http://newhostichanged.co:9201'], // field that changed
+      },
+    ]);
+
+    expect(mockedOutputService.delete).not.toBeCalled();
+    expect(mockedOutputService.create).not.toBeCalled();
+    expect(mockedOutputService.update).toBeCalled();
+    expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled();
+  });
+
+  const SCENARIOS: Array<{ name: string; data: PreconfiguredOutput }> = [
+    {
+      name: 'no changes',
+      data: {
+        id: 'existing-output-1',
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'Output 1',
+        type: 'elasticsearch',
+        hosts: ['http://es.co:80'],
+      },
+    },
+    {
+      name: 'hosts without port',
+      data: {
+        id: 'existing-output-1',
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'Output 1',
+        type: 'elasticsearch',
+        hosts: ['http://es.co'],
+      },
+    },
+  ];
+  SCENARIOS.forEach((scenario) => {
+    const { data, name } = scenario;
+    it(`should do nothing if preconfigured output exists and did not changed (${name})`, async () => {
+      const soClient = savedObjectsClientMock.create();
+      const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+      await createOrUpdatePreconfiguredOutputs(soClient, esClient, [data]);
+
+      expect(mockedOutputService.create).not.toBeCalled();
+      expect(mockedOutputService.update).not.toBeCalled();
+    });
+  });
+
+  describe('cleanPreconfiguredOutputs', () => {
+    it('should not delete non deleted preconfigured output', async () => {
+      const soClient = savedObjectsClientMock.create();
+      mockedOutputService.list.mockResolvedValue({
+        items: [
+          { id: 'output1', is_preconfigured: true } as Output,
+          { id: 'output2', is_preconfigured: true } as Output,
+        ],
+        page: 1,
+        perPage: 10000,
+        total: 1,
+      });
+      await cleanPreconfiguredOutputs(soClient, [
+        {
+          id: 'output1',
+          is_default: false,
+          is_default_monitoring: false,
+          name: 'Output 1',
+          type: 'elasticsearch',
+          hosts: ['http://es.co:9201'],
+        },
+        {
+          id: 'output2',
+          is_default: false,
+          is_default_monitoring: false,
+          name: 'Output 2',
+          type: 'elasticsearch',
+          hosts: ['http://es.co:9201'],
+        },
+      ]);
+
+      expect(mockedOutputService.delete).not.toBeCalled();
+    });
+
+    it('should delete deleted preconfigured output', async () => {
+      const soClient = savedObjectsClientMock.create();
+      mockedOutputService.list.mockResolvedValue({
+        items: [
+          { id: 'output1', is_preconfigured: true } as Output,
+          { id: 'output2', is_preconfigured: true } as Output,
+        ],
+        page: 1,
+        perPage: 10000,
+        total: 1,
+      });
+      await cleanPreconfiguredOutputs(soClient, [
+        {
+          id: 'output1',
+          is_default: false,
+          is_default_monitoring: false,
+          name: 'Output 1',
+          type: 'elasticsearch',
+          hosts: ['http://es.co:9201'],
+        },
+      ]);
+
+      expect(mockedOutputService.delete).toBeCalled();
+      expect(mockedOutputService.delete).toBeCalledTimes(1);
+      expect(mockedOutputService.delete.mock.calls[0][1]).toEqual('output2');
+    });
+
+    it('should update default deleted preconfigured output', async () => {
+      const soClient = savedObjectsClientMock.create();
+      mockedOutputService.list.mockResolvedValue({
+        items: [
+          { id: 'output1', is_preconfigured: true, is_default: true } as Output,
+          { id: 'output2', is_preconfigured: true, is_default_monitoring: true } as Output,
+        ],
+        page: 1,
+        perPage: 10000,
+        total: 1,
+      });
+      await cleanPreconfiguredOutputs(soClient, []);
+
+      expect(mockedOutputService.delete).not.toBeCalled();
+      expect(mockedOutputService.update).toBeCalledTimes(2);
+      expect(mockedOutputService.update).toBeCalledWith(
+        expect.anything(),
+        'output1',
+        expect.objectContaining({
+          is_preconfigured: false,
+        }),
+        { fromPreconfiguration: true }
+      );
+      expect(mockedOutputService.update).toBeCalledWith(
+        expect.anything(),
+        'output2',
+        expect.objectContaining({
+          is_preconfigured: false,
+        }),
+        { fromPreconfiguration: true }
+      );
+    });
+  });
+});

x-pack/plugins/fleet/server/services/preconfiguration/outputs.ts

@@ -0,0 +1,148 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
+import { isEqual } from 'lodash';
+import { safeDump } from 'js-yaml';
+
+import type { PreconfiguredOutput, Output } from '../../../common';
+import { normalizeHostsForAgents } from '../../../common';
+import { outputService } from '../output';
+import { agentPolicyService } from '../agent_policy';
+
+import { appContextService } from '../app_context';
+
+export async function ensurePreconfiguredOutputs(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  outputs: PreconfiguredOutput[]
+) {
+  await createOrUpdatePreconfiguredOutputs(soClient, esClient, outputs);
+  await cleanPreconfiguredOutputs(soClient, outputs);
+}
+
+export async function createOrUpdatePreconfiguredOutputs(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  outputs: PreconfiguredOutput[]
+) {
+  const logger = appContextService.getLogger();
+
+  if (outputs.length === 0) {
+    return;
+  }
+
+  const existingOutputs = await outputService.bulkGet(
+    soClient,
+    outputs.map(({ id }) => id),
+    { ignoreNotFound: true }
+  );
+
+  await Promise.all(
+    outputs.map(async (output) => {
+      const existingOutput = existingOutputs.find((o) => o.id === output.id);
+
+      const { id, config, ...outputData } = output;
+
+      const configYaml = config ? safeDump(config) : undefined;
+
+      const data = {
+        ...outputData,
+        config_yaml: configYaml,
+        is_preconfigured: true,
+      };
+
+      if (!data.hosts || data.hosts.length === 0) {
+        data.hosts = outputService.getDefaultESHosts();
+      }
+
+      const isCreate = !existingOutput;
+      const isUpdateWithNewData =
+        existingOutput && isPreconfiguredOutputDifferentFromCurrent(existingOutput, data);
+
+      if (isCreate) {
+        logger.debug(`Creating output ${output.id}`);
+        await outputService.create(soClient, data, { id, fromPreconfiguration: true });
+      } else if (isUpdateWithNewData) {
+        logger.debug(`Updating output ${output.id}`);
+        await outputService.update(soClient, id, data, { fromPreconfiguration: true });
+        // Bump revision of all policies using that output
+        if (outputData.is_default || outputData.is_default_monitoring) {
+          await agentPolicyService.bumpAllAgentPolicies(soClient, esClient);
+        } else {
+          await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, id);
+        }
+      }
+    })
+  );
+}
+
+export async function cleanPreconfiguredOutputs(
+  soClient: SavedObjectsClientContract,
+  outputs: PreconfiguredOutput[]
+) {
+  const existingOutputs = await outputService.list(soClient);
+  const existingPreconfiguredOutput = existingOutputs.items.filter(
+    (o) => o.is_preconfigured === true
+  );
+
+  const logger = appContextService.getLogger();
+
+  for (const output of existingPreconfiguredOutput) {
+    const hasBeenDelete = !outputs.find(({ id }) => output.id === id);
+    if (!hasBeenDelete) {
+      continue;
+    }
+
+    if (output.is_default) {
+      logger.info(`Updating default preconfigured output ${output.id} is no longer preconfigured`);
+      await outputService.update(
+        soClient,
+        output.id,
+        { is_preconfigured: false },
+        {
+          fromPreconfiguration: true,
+        }
+      );
+    } else if (output.is_default_monitoring) {
+      logger.info(`Updating default preconfigured output ${output.id} is no longer preconfigured`);
+      await outputService.update(
+        soClient,
+        output.id,
+        { is_preconfigured: false },
+        {
+          fromPreconfiguration: true,
+        }
+      );
+    } else {
+      logger.info(`Deleting preconfigured output ${output.id}`);
+      await outputService.delete(soClient, output.id, { fromPreconfiguration: true });
+    }
+  }
+}
+
+function isPreconfiguredOutputDifferentFromCurrent(
+  existingOutput: Output,
+  preconfiguredOutput: Partial<Output>
+): boolean {
+  return (
+    !existingOutput.is_preconfigured ||
+    existingOutput.is_default !== preconfiguredOutput.is_default ||
+    existingOutput.is_default_monitoring !== preconfiguredOutput.is_default_monitoring ||
+    existingOutput.name !== preconfiguredOutput.name ||
+    existingOutput.type !== preconfiguredOutput.type ||
+    (preconfiguredOutput.hosts &&
+      !isEqual(
+        existingOutput.hosts?.map(normalizeHostsForAgents),
+        preconfiguredOutput.hosts.map(normalizeHostsForAgents)
+      )) ||
+    (preconfiguredOutput.ssl && !isEqual(preconfiguredOutput.ssl, existingOutput.ssl)) ||
+    existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 ||
+    existingOutput.ca_trusted_fingerprint !== preconfiguredOutput.ca_trusted_fingerprint ||
+    existingOutput.config_yaml !== preconfiguredOutput.config_yaml
+  );
+}

x-pack/plugins/fleet/server/services/setup.test.ts

@@ -18,6 +18,7 @@ import { upgradeManagedPackagePolicies } from './managed_package_policies';
 import { setupFleet } from './setup';
 
 jest.mock('./preconfiguration');
+jest.mock('./preconfiguration/index');
 jest.mock('./settings');
 jest.mock('./output');
 jest.mock('./epm/packages');

x-pack/plugins/fleet/server/services/setup.ts

@@ -17,11 +17,8 @@ import { DEFAULT_SPACE_ID } from '../../../spaces/common/constants';
 
 import { appContextService } from './app_context';
 import { agentPolicyService } from './agent_policy';
-import {
-  cleanPreconfiguredOutputs,
-  ensurePreconfiguredOutputs,
-  ensurePreconfiguredPackagesAndPolicies,
-} from './preconfiguration';
+import { ensurePreconfiguredPackagesAndPolicies } from './preconfiguration';
+import { ensurePreconfiguredOutputs } from './preconfiguration/index';
 import { outputService } from './output';
 
 import { generateEnrollmentAPIKey, hasEnrollementAPIKeysForPolicy } from './api_keys';
@@ -115,9 +112,6 @@ async function createSetupSideEffects(
 
   const nonFatalErrors = [...preconfiguredPackagesNonFatalErrors, ...packagePolicyUpgradeErrors];
 
-  logger.debug('Cleaning up Fleet outputs');
-  await cleanPreconfiguredOutputs(soClient, outputsOrUndefined ?? []);
-
   logger.debug('Setting up Fleet enrollment keys');
   await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient);