[UII] Remove references to .fleet-servers index (#183868)

## Summary

Resolves https://github.com/elastic/kibana/issues/173537.

This PR removes all usages and references to `.fleet-servers` index.

A small refactoring was done that moves helper functions in enrollment
settings API to fleet server services. It's a better place for them
because they are related to fleet server anyway. With this change, we
now use these functions to calculate the readiness of Fleet
(`hasFleetServers` was previously reading on `.fleet-servers` index).

The readiness of a fleet server is defined as at least one online agent
enrolled into an agent policy that contains a fleet server policy.

### Changes in Security Solution (@paul-tavares)

- Updated `enableFleetServerIfNecessary()` utility _(used for testing,
dev)_ to not use `.fleet-servers` index and instead write a Agent record
to the `.fleet-agents` index for Fleet Server. This record writing is
skipped when running tests in serverless mode, instead,
`xpack.fleet.internal.fleetServerStandalone=true` is added to mimic
skipping checks for Fleet Server in real serverless.


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Paul Tavares <paul.tavares@elastic.co>

x-pack/plugins/fleet/common/constants/index.ts

@@ -37,8 +37,6 @@ export const FLEET_SERVER_INDICES_VERSION = 1;
 
 export const FLEET_SERVER_ARTIFACTS_INDEX = '.fleet-artifacts';
 
-export const FLEET_SERVER_SERVERS_INDEX = '.fleet-servers';
-
 export const FLEET_SERVER_INDICES = [
   '.fleet-actions',
   '.fleet-actions-results',
@@ -47,7 +45,6 @@ export const FLEET_SERVER_INDICES = [
   '.fleet-enrollment-api-keys',
   '.fleet-policies',
   '.fleet-policies-leader',
-  FLEET_SERVER_SERVERS_INDEX,
 ];
 
 // Nodes that can be queried by datastreams API

x-pack/plugins/fleet/common/index.ts

@@ -32,7 +32,6 @@ export {
   MESSAGE_SIGNING_KEYS_SAVED_OBJECT_TYPE,
   UNINSTALL_TOKENS_SAVED_OBJECT_TYPE,
   // Fleet server index
-  FLEET_SERVER_SERVERS_INDEX,
   FLEET_SERVER_ARTIFACTS_INDEX,
   AGENTS_INDEX,
   AGENT_POLICY_INDEX,

x-pack/plugins/fleet/cypress/e2e/agents/agent_list.cy.ts

@@ -9,7 +9,7 @@ import { FLEET_AGENT_LIST_PAGE } from '../../screens/fleet';
 
 import { createAgentDoc } from '../../tasks/agents';
 import { setupFleetServer } from '../../tasks/fleet_server';
-import { deleteFleetServerDocs, deleteAgentDocs, cleanupAgentPolicies } from '../../tasks/cleanup';
+import { deleteAgentDocs, cleanupAgentPolicies } from '../../tasks/cleanup';
 import type { CreateAgentPolicyRequest } from '../../../common/types';
 import { setUISettings } from '../../tasks/ui_settings';
 
@@ -87,7 +87,6 @@ function assertTableIsEmpty() {
 
 describe('View agents list', () => {
   before(() => {
-    deleteFleetServerDocs(true);
     deleteAgentDocs(true);
     cleanupAgentPolicies();
     setupFleetServer();
@@ -103,7 +102,6 @@ describe('View agents list', () => {
     }
   });
   after(() => {
-    deleteFleetServerDocs(true);
     deleteAgentDocs(true);
     cleanupAgentPolicies();
   });

x-pack/plugins/fleet/cypress/e2e/fleet_agent_flyout.cy.ts

@@ -6,7 +6,7 @@
  */
 
 import { ADD_AGENT_BUTTON, AGENT_FLYOUT } from '../screens/fleet';
-import { cleanupAgentPolicies, deleteFleetServerDocs, deleteAgentDocs } from '../tasks/cleanup';
+import { cleanupAgentPolicies, deleteAgentDocs } from '../tasks/cleanup';
 import { createAgentDoc } from '../tasks/agents';
 import { setFleetServerHost } from '../tasks/fleet_server';
 import { FLEET, navigateTo } from '../tasks/navigation';
@@ -18,7 +18,6 @@ import { login } from '../tasks/login';
 const FLEET_SERVER_POLICY_ID = 'fleet-server-policy';
 
 function cleanUp() {
-  deleteFleetServerDocs(true);
   deleteAgentDocs(true);
   cleanupAgentPolicies();
 }
@@ -53,14 +52,6 @@ describe('Fleet add agent flyout', () => {
           index: '.fleet-agents',
           docs: [createAgentDoc('agent1', policyId, 'online', kibanaVersion)],
         });
-        cy.task('insertDocs', {
-          index: '.fleet-servers',
-          docs: [
-            {
-              '@timestamp': new Date().toISOString(),
-            },
-          ],
-        });
         setFleetServerHost();
       });
 

x-pack/plugins/fleet/cypress/e2e/fleet_startup.cy.ts

@@ -19,7 +19,7 @@ import {
 import { cleanupAgentPolicies, unenrollAgent } from '../tasks/cleanup';
 import { request } from '../tasks/common';
 import { verifyPolicy, verifyAgentPackage, navigateToTab } from '../tasks/fleet';
-import { deleteFleetServer, setFleetServerHost } from '../tasks/fleet_server';
+import { setFleetServerHost } from '../tasks/fleet_server';
 import { login } from '../tasks/login';
 import { FLEET, navigateTo } from '../tasks/navigation';
 
@@ -28,8 +28,6 @@ describe('Fleet startup', () => {
     before(() => {
       unenrollAgent();
       cleanupAgentPolicies();
-      deleteFleetServer();
-
       setFleetServerHost();
     });
 

x-pack/plugins/fleet/cypress/tasks/cleanup.ts

@@ -48,13 +48,6 @@ export function cleanupDownloadSources() {
   });
 }
 
-export function deleteFleetServerDocs(ignoreUnavailable: boolean = false) {
-  cy.task('deleteDocsByQuery', {
-    index: '.fleet-servers',
-    query: { match_all: {} },
-    ignoreUnavailable,
-  });
-}
 export function deleteAgentDocs(ignoreUnavailable: boolean = false) {
   cy.task('deleteDocsByQuery', {
     index: '.fleet-agents',

x-pack/plugins/fleet/cypress/tasks/fleet_server.ts

@@ -44,26 +44,10 @@ export async function setupFleetServer() {
       index: '.fleet-agents',
       docs: [createAgentDoc('fleet-server', policyId, 'online', kibanaVersion)],
     });
-    cy.task('insertDocs', {
-      index: '.fleet-servers',
-      docs: [
-        {
-          '@timestamp': new Date().toISOString(),
-        },
-      ],
-    });
     setFleetServerHost();
   });
 }
 
-export function deleteFleetServer() {
-  cy.task('deleteDocsByQuery', {
-    index: '.fleet-servers',
-    query: { match_all: {} },
-    ignoreUnavailable: true,
-  });
-}
-
 export function setFleetServerHost(host = 'https://fleetserver:8220') {
   request({
     method: 'POST',

x-pack/plugins/fleet/dev_docs/data_model.md

@@ -58,10 +58,6 @@ The total schema for actions is represented by the `FleetServerAgentAction` type
 
 - Cleanup model: N/A
 
-### `.fleet-servers`
-
-- Cleanup model: N/A
-
 ### `.fleet-artifacts`
 
 - Cleanup model: N/A

x-pack/plugins/fleet/public/applications/fleet/sections/debug/components/fleet_index_debugger.tsx

@@ -43,7 +43,6 @@ export const FleetIndexDebugger = () => {
   const indices = [
     { label: '.fleet-agents', value: '.fleet-agents' },
     { label: '.fleet-actions', value: '.fleet-actions' },
-    { label: '.fleet-servers', value: '.fleet-servers' },
     { label: '.fleet-enrollment-api-keys', value: '.fleet-enrollment-api-keys' },
   ];
   const [index, setIndex] = useState<string | undefined>();

x-pack/plugins/fleet/server/constants/index.ts

@@ -58,7 +58,6 @@ export {
   PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES,
   AGENT_POLICY_DEFAULT_MONITORING_DATASETS,
   // Fleet Server index
-  FLEET_SERVER_SERVERS_INDEX,
   ENROLLMENT_API_KEYS_INDEX,
   AGENTS_INDEX,
   // Preconfiguration

x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.test.ts

@@ -4,21 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks';
+import { savedObjectsClientMock } from '@kbn/core/server/mocks';
 
-import { packagePolicyService, agentPolicyService } from '../../services';
+import { agentPolicyService } from '../../services';
-import { getAgentStatusForAgentPolicy } from '../../services/agents';
+import { getFleetServerPolicies } from '../../services/fleet_server';
 
-import {
+import { getFleetServerOrAgentPolicies, getDownloadSource } from './enrollment_settings_handler';
-  getFleetServerPolicies,
-  hasActiveFleetServersForPolicies,
-  getDownloadSource,
-} from './enrollment_settings_handler';
 
 jest.mock('../../services', () => ({
-  packagePolicyService: {
-    list: jest.fn(),
-  },
   agentPolicyService: {
     get: jest.fn(),
     getByIDs: jest.fn(),
@@ -44,13 +37,12 @@ jest.mock('../../services', () => ({
   },
 }));
 
-jest.mock('../../services/agents', () => ({
+jest.mock('../../services/fleet_server', () => ({
-  getAgentStatusForAgentPolicy: jest.fn(),
+  getFleetServerPolicies: jest.fn(),
 }));
 
 describe('EnrollmentSettingsHandler utils', () => {
   const mockSoClient = savedObjectsClientMock.create();
-  const mockEsClient = elasticsearchServiceMock.createInternalClient();
   const mockAgentPolicies = [
     {
       id: 'agent-policy-1',
@@ -124,20 +116,21 @@ describe('EnrollmentSettingsHandler utils', () => {
     },
   ];
 
-  describe('getFleetServerPolicies', () => {
+  describe('getFleetServerOrAgentPolicies', () => {
     it('returns only fleet server policies if there are any when no agent policy ID is provided', async () => {
-      (packagePolicyService.list as jest.Mock).mockResolvedValueOnce({
+      (getFleetServerPolicies as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies);
-        items: [{ policy_id: 'fs-policy-1' }, { policy_id: 'fs-policy-2' }],
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
-      });
+        mockSoClient
-      (agentPolicyService.getByIDs as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies);
+      );
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(mockSoClient);
       expect(fleetServerPolicies).toEqual(mockFleetServerPolicies);
       expect(scopedAgentPolicy).toBeUndefined();
     });
 
     it('returns no fleet server policies when there are none and no agent policy ID is provided', async () => {
-      (packagePolicyService.list as jest.Mock).mockResolvedValueOnce({ items: [] });
+      (getFleetServerPolicies as jest.Mock).mockResolvedValueOnce([]);
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(mockSoClient);
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
+        mockSoClient
+      );
       expect(fleetServerPolicies).toEqual([]);
       expect(scopedAgentPolicy).toBeUndefined();
     });
@@ -147,7 +140,7 @@ describe('EnrollmentSettingsHandler utils', () => {
         ...mockFleetServerPolicies[1],
         package_policies: [mockPackagePolicies[1]],
       });
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
         mockSoClient,
         'fs-policy-2'
       );
@@ -160,7 +153,7 @@ describe('EnrollmentSettingsHandler utils', () => {
         ...mockAgentPolicies[1],
         package_policies: [mockPackagePolicies[2]],
       });
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
         mockSoClient,
         'agent-policy-2'
       );
@@ -170,7 +163,7 @@ describe('EnrollmentSettingsHandler utils', () => {
 
     it('returns no policies when specified agent policy ID is not found', async () => {
       (agentPolicyService.get as jest.Mock).mockResolvedValueOnce(undefined);
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
         mockSoClient,
         'agent-policy-3'
       );
@@ -179,73 +172,6 @@ describe('EnrollmentSettingsHandler utils', () => {
     });
   });
 
-  describe('hasActiveFleetServersForPolicies', () => {
-    it('returns false when no agent IDs are provided', async () => {
-      const hasActive = await hasActiveFleetServersForPolicies(mockEsClient, mockSoClient, []);
-      expect(hasActive).toBe(false);
-    });
-
-    it('returns true when at least one agent is online', async () => {
-      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
-        other: 0,
-        events: 0,
-        total: 1,
-        all: 1,
-        active: 0,
-        updating: 0,
-        offline: 0,
-        inactive: 0,
-        unenrolled: 0,
-        online: 1,
-        error: 0,
-      });
-      const hasActive = await hasActiveFleetServersForPolicies(mockEsClient, mockSoClient, [
-        'policy-1',
-      ]);
-      expect(hasActive).toBe(true);
-    });
-
-    it('returns true when at least one agent is updating', async () => {
-      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
-        other: 0,
-        events: 0,
-        total: 1,
-        all: 1,
-        active: 0,
-        updating: 1,
-        offline: 0,
-        inactive: 0,
-        unenrolled: 0,
-        online: 0,
-        error: 0,
-      });
-      const hasActive = await hasActiveFleetServersForPolicies(mockEsClient, mockSoClient, [
-        'policy-1',
-      ]);
-      expect(hasActive).toBe(true);
-    });
-
-    it('returns false when no agents are updating or online', async () => {
-      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
-        other: 0,
-        events: 0,
-        total: 3,
-        all: 3,
-        active: 1,
-        updating: 0,
-        offline: 1,
-        inactive: 1,
-        unenrolled: 1,
-        online: 0,
-        error: 1,
-      });
-      const hasActive = await hasActiveFleetServersForPolicies(mockEsClient, mockSoClient, [
-        'policy-1',
-      ]);
-      expect(hasActive).toBe(false);
-    });
-  });
-
   describe('getDownloadSource', () => {
     it('returns the default download source when no id is specified', async () => {
       const source = await getDownloadSource(mockSoClient);

x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.ts

@@ -7,9 +7,9 @@
 
 import type { TypeOf } from '@kbn/config-schema';
 
-import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
+import type { SavedObjectsClientContract } from '@kbn/core/server';
 
-import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, FLEET_SERVER_PACKAGE } from '../../../common/constants';
+import { FLEET_SERVER_PACKAGE } from '../../../common/constants';
 
 import type {
   GetEnrollmentSettingsResponse,
@@ -18,10 +18,10 @@ import type {
 } from '../../../common/types';
 import type { FleetRequestHandler, GetEnrollmentSettingsRequestSchema } from '../../types';
 import { defaultFleetErrorHandler } from '../../errors';
-import { agentPolicyService, packagePolicyService, downloadSourceService } from '../../services';
+import { agentPolicyService, downloadSourceService } from '../../services';
-import { getAgentStatusForAgentPolicy } from '../../services/agents';
 import { getFleetServerHostsForAgentPolicy } from '../../services/fleet_server_host';
 import { getFleetProxy } from '../../services/fleet_proxies';
+import { getFleetServerPolicies, hasFleetServersForPolicies } from '../../services/fleet_server';
 
 export const getEnrollmentSettingsHandler: FleetRequestHandler<
   undefined,
@@ -40,7 +40,7 @@ export const getEnrollmentSettingsHandler: FleetRequestHandler<
   try {
     // Get all possible fleet server or scoped normal agent policies
     const { fleetServerPolicies, scopedAgentPolicy: scopedAgentPolicyResponse } =
-      await getFleetServerPolicies(soClient, agentPolicyId);
+      await getFleetServerOrAgentPolicies(soClient, agentPolicyId);
     const scopedAgentPolicy = scopedAgentPolicyResponse || {
       id: undefined,
       name: undefined,
@@ -51,10 +51,11 @@ export const getEnrollmentSettingsHandler: FleetRequestHandler<
     // Check if there is any active fleet server enrolled into the fleet server policies policies
     if (fleetServerPolicies) {
       settingsResponse.fleet_server.policies = fleetServerPolicies;
-      settingsResponse.fleet_server.has_active = await hasActiveFleetServersForPolicies(
+      settingsResponse.fleet_server.has_active = await hasFleetServersForPolicies(
         esClient,
         soClient,
-        fleetServerPolicies.map((p) => p.id)
+        fleetServerPolicies.map((p) => p.id),
+        true
       );
     }
 
@@ -99,7 +100,7 @@ export const getEnrollmentSettingsHandler: FleetRequestHandler<
   }
 };
 
-export const getFleetServerPolicies = async (
+export const getFleetServerOrAgentPolicies = async (
   soClient: SavedObjectsClientContract,
   agentPolicyId?: string
 ): Promise<{
@@ -134,42 +135,9 @@ export const getFleetServerPolicies = async (
     return {};
   }
 
-  // If an agent policy is not specified, perform default behavior to retrieve all fleet server policies
+  // If an agent policy is not specified, return all fleet server policies
-  const fleetServerPackagePolicies = await packagePolicyService.list(soClient, {
+  const fleetServerPolicies = (await getFleetServerPolicies(soClient)).map(mapPolicy);
-    kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${FLEET_SERVER_PACKAGE}`,
+  return { fleetServerPolicies };
-  });
-
-  // Extract associated fleet server agent policy IDs
-  const fleetServerAgentPolicyIds = [
-    ...new Set(fleetServerPackagePolicies.items.map((p) => p.policy_id)),
-  ];
-
-  // Retrieve associated agent policies
-  const fleetServerAgentPolicies = fleetServerAgentPolicyIds.length
-    ? await agentPolicyService.getByIDs(soClient, fleetServerAgentPolicyIds)
-    : [];
-
-  return {
-    fleetServerPolicies: fleetServerAgentPolicies.map(mapPolicy),
-  };
-};
-
-export const hasActiveFleetServersForPolicies = async (
-  esClient: ElasticsearchClient,
-  soClient: SavedObjectsClientContract,
-  agentPolicyIds: string[]
-): Promise<boolean> => {
-  if (agentPolicyIds.length > 0) {
-    const agentStatusesRes = await getAgentStatusForAgentPolicy(
-      esClient,
-      soClient,
-      undefined,
-      agentPolicyIds.map((id) => `policy_id:${id}`).join(' or ')
-    );
-
-    return agentStatusesRes.online > 0 || agentStatusesRes.updating > 0;
-  }
-  return false;
 };
 
 export const getDownloadSource = async (

x-pack/plugins/fleet/server/routes/setup/handlers.ts

@@ -25,9 +25,7 @@ export const getFleetStatusHandler: FleetRequestHandler = async (context, reques
     const isApiKeysEnabled = await appContextService
       .getSecurity()
       .authc.apiKeys.areAPIKeysEnabled();
-    const isFleetServerMissing = !(await hasFleetServers(
+    const isFleetServerMissing = !(await hasFleetServers(esClient, soClient));
-      coreContext.elasticsearch.client.asInternalUser
-    ));
 
     const isFleetServerStandalone =
       appContextService.getConfig()?.internal?.fleetServerStandalone ?? false;

x-pack/plugins/fleet/server/services/agents/status.ts

@@ -130,14 +130,14 @@ export async function getAgentStatusForAgentPolicy(
   const allActive = allStatuses - combinedStatuses.unenrolled - combinedStatuses.inactive;
   return {
     ...combinedStatuses,
+    all: allStatuses,
+    active: allActive,
     /* @deprecated no agents will have other status */
     other: 0,
     /* @deprecated Agent events do not exists anymore */
     events: 0,
     /* @deprecated use active instead */
     total: allActive,
-    all: allStatuses,
-    active: allActive,
   };
 }
 export async function getIncomingDataByAgentsId(

x-pack/plugins/fleet/server/services/fleet_server/index.test.ts

@@ -15,9 +15,13 @@ import { createAppContextStartContractMock } from '../../mocks';
 
 import { agentPolicyService } from '../agent_policy';
 import { packagePolicyService } from '../package_policy';
-import { getAgentsByKuery, getAgentStatusById } from '../agents';
+import { getAgentsByKuery, getAgentStatusById, getAgentStatusForAgentPolicy } from '../agents';
 
-import { checkFleetServerVersionsForSecretsStorage } from '.';
+import {
+  checkFleetServerVersionsForSecretsStorage,
+  hasFleetServersForPolicies,
+  getFleetServerPolicies,
+} from '.';
 
 jest.mock('../agent_policy');
 jest.mock('../package_policy');
@@ -111,3 +115,177 @@ describe('checkFleetServerVersionsForSecretsStorage', () => {
     expect(result).toBe(true);
   });
 });
+
+describe('getFleetServerPolicies', () => {
+  const soClient = savedObjectsClientMock.create();
+  const mockPackagePolicies = [
+    {
+      id: 'package-policy-1',
+      name: 'Package Policy 1',
+      package: {
+        name: 'fleet_server',
+        title: 'Fleet Server',
+        version: '1.0.0',
+      },
+      policy_id: 'fs-policy-1',
+    },
+    {
+      id: 'package-policy-2',
+      name: 'Package Policy 2',
+      package: {
+        name: 'fleet_server',
+        title: 'Fleet Server',
+        version: '1.0.0',
+      },
+      policy_id: 'fs-policy-2',
+    },
+    {
+      id: 'package-policy-3',
+      name: 'Package Policy 3',
+      package: {
+        name: 'system',
+        title: 'System',
+        version: '1.0.0',
+      },
+      policy_id: 'agent-policy-2',
+    },
+  ];
+  const mockFleetServerPolicies = [
+    {
+      id: 'fs-policy-1',
+      name: 'FS Policy 1',
+      is_managed: true,
+      is_default_fleet_server: true,
+      has_fleet_server: true,
+      download_source_id: undefined,
+      fleet_server_host_id: undefined,
+    },
+    {
+      id: 'fs-policy-2',
+      name: 'FS Policy 2',
+      is_managed: true,
+      is_default_fleet_server: false,
+      has_fleet_server: false,
+      download_source_id: undefined,
+      fleet_server_host_id: undefined,
+    },
+  ];
+
+  it('should return no policies if there are no fleet server package policies', async () => {
+    (mockedPackagePolicyService.list as jest.Mock).mockResolvedValueOnce({
+      items: [],
+    });
+    const result = await getFleetServerPolicies(soClient);
+    expect(result).toEqual([]);
+  });
+
+  it('should return agent policies with fleet server package policies', async () => {
+    (mockedPackagePolicyService.list as jest.Mock).mockResolvedValueOnce({
+      items: mockPackagePolicies,
+    });
+    (mockedAgentPolicyService.getByIDs as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies);
+    const result = await getFleetServerPolicies(soClient);
+    expect(result).toEqual(mockFleetServerPolicies);
+  });
+});
+
+describe('hasActiveFleetServersForPolicies', () => {
+  const mockSoClient = savedObjectsClientMock.create();
+  const mockEsClient = elasticsearchServiceMock.createInternalClient();
+
+  it('returns false when no agent IDs are provided', async () => {
+    const hasFs = await hasFleetServersForPolicies(mockEsClient, mockSoClient, []);
+    expect(hasFs).toBe(false);
+  });
+
+  describe('activeOnly is true', () => {
+    it('returns true when at least one agent is online', async () => {
+      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
+        other: 0,
+        events: 0,
+        total: 1,
+        all: 1,
+        active: 0,
+        updating: 0,
+        offline: 0,
+        inactive: 0,
+        unenrolled: 0,
+        online: 1,
+        error: 0,
+      });
+      const hasFs = await hasFleetServersForPolicies(
+        mockEsClient,
+        mockSoClient,
+        ['policy-1'],
+        true
+      );
+      expect(hasFs).toBe(true);
+    });
+
+    it('returns true when at least one agent is updating', async () => {
+      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
+        other: 0,
+        events: 0,
+        total: 1,
+        all: 1,
+        active: 0,
+        updating: 1,
+        offline: 0,
+        inactive: 0,
+        unenrolled: 0,
+        online: 0,
+        error: 0,
+      });
+      const hasFs = await hasFleetServersForPolicies(
+        mockEsClient,
+        mockSoClient,
+        ['policy-1'],
+        true
+      );
+      expect(hasFs).toBe(true);
+    });
+
+    it('returns false when no agents are updating or online', async () => {
+      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
+        other: 0,
+        events: 0,
+        total: 3,
+        all: 3,
+        active: 1,
+        updating: 0,
+        offline: 1,
+        inactive: 1,
+        unenrolled: 1,
+        online: 0,
+        error: 1,
+      });
+      const hasFs = await hasFleetServersForPolicies(
+        mockEsClient,
+        mockSoClient,
+        ['policy-1'],
+        true
+      );
+      expect(hasFs).toBe(false);
+    });
+  });
+
+  describe('activeOnly is false', () => {
+    it('returns true when at least one agent is found regardless of its status', async () => {
+      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
+        other: 0,
+        events: 0,
+        total: 0,
+        all: 1,
+        active: 0,
+        updating: 0,
+        offline: 1,
+        inactive: 0,
+        unenrolled: 0,
+        online: 0,
+        error: 0,
+      });
+      const hasFs = await hasFleetServersForPolicies(mockEsClient, mockSoClient, ['policy-1']);
+      expect(hasFs).toBe(true);
+    });
+  });
+});

x-pack/plugins/fleet/server/services/fleet_server/index.ts

@@ -9,26 +9,78 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/
 import semverGte from 'semver/functions/gte';
 import semverCoerce from 'semver/functions/coerce';
 
-import { FLEET_SERVER_SERVERS_INDEX, SO_SEARCH_LIMIT } from '../../constants';
+import type { AgentPolicy } from '../../../common/types';
+import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, FLEET_SERVER_PACKAGE } from '../../../common/constants';
+
+import { SO_SEARCH_LIMIT } from '../../constants';
 import { getAgentsByKuery, getAgentStatusById } from '../agents';
 
 import { packagePolicyService } from '../package_policy';
 import { agentPolicyService } from '../agent_policy';
+import { getAgentStatusForAgentPolicy } from '../agents';
 import { appContextService } from '..';
 
 /**
- * Check if at least one fleet server is connected
+ * Retrieve all agent policies which has a Fleet Server package policy
  */
-export async function hasFleetServers(esClient: ElasticsearchClient) {
+export const getFleetServerPolicies = async (
-  const res = await esClient.search<{}, {}>({
+  soClient: SavedObjectsClientContract
-    index: FLEET_SERVER_SERVERS_INDEX,
+): Promise<AgentPolicy[]> => {
-    ignore_unavailable: true,
+  const fleetServerPackagePolicies = await packagePolicyService.list(soClient, {
-    filter_path: 'hits.total',
+    kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${FLEET_SERVER_PACKAGE}`,
-    track_total_hits: true,
-    rest_total_hits_as_int: true,
   });
 
-  return (res.hits.total as number) > 0;
+  // Extract associated fleet server agent policy IDs
+  const fleetServerAgentPolicyIds = [
+    ...new Set(fleetServerPackagePolicies.items.map((p) => p.policy_id)),
+  ];
+
+  // Retrieve associated agent policies
+  const fleetServerAgentPolicies = fleetServerAgentPolicyIds.length
+    ? await agentPolicyService.getByIDs(soClient, fleetServerAgentPolicyIds)
+    : [];
+
+  return fleetServerAgentPolicies;
+};
+
+/**
+ * Check if there is at least one agent enrolled into the given agent policies.
+ * Assumes that `agentPolicyIds` contains list of Fleet Server agent policies.
+ * `activeOnly` flag can be used to filter only active agents.
+ */
+export const hasFleetServersForPolicies = async (
+  esClient: ElasticsearchClient,
+  soClient: SavedObjectsClientContract,
+  agentPolicyIds: string[],
+  activeOnly: boolean = false
+): Promise<boolean> => {
+  if (agentPolicyIds.length > 0) {
+    const agentStatusesRes = await getAgentStatusForAgentPolicy(
+      esClient,
+      soClient,
+      undefined,
+      agentPolicyIds.map((id) => `policy_id:${id}`).join(' or ')
+    );
+
+    return activeOnly
+      ? agentStatusesRes.online > 0 || agentStatusesRes.updating > 0
+      : agentStatusesRes.all > 0;
+  }
+  return false;
+};
+
+/**
+ * Check if at least one fleet server agent exists, regardless of its online status
+ */
+export async function hasFleetServers(
+  esClient: ElasticsearchClient,
+  soClient: SavedObjectsClientContract
+) {
+  return await hasFleetServersForPolicies(
+    esClient,
+    soClient,
+    (await getFleetServerPolicies(soClient)).map((policy) => policy.id)
+  );
 }
 
 /**

x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts

@@ -84,6 +84,7 @@ export class FleetAgentGenerator extends BaseDataGenerator<Agent> {
     const hostname = this.randomHostname();
     const now = new Date().toISOString();
     const osFamily = this.randomOSFamily();
+    const version = overrides?._source?.agent?.version ?? this.randomVersion();
     const componentStatus = this.randomChoice<FleetServerAgentComponentStatus>(
       FleetServerAgentComponentStatuses
     );
@@ -113,19 +114,19 @@ export class FleetAgentGenerator extends BaseDataGenerator<Agent> {
           enrolled_at: now,
           agent: {
             id: agentId,
-            version: this.randomVersion(),
+            version,
           },
           local_metadata: {
             elastic: {
               agent: {
-                'build.original': `8.0.0-SNAPSHOT (build: ${this.randomString(
+                'build.original': `${version} (build: ${this.randomString(
                   5
                 )} at 2021-05-07 18:42:49 +0000 UTC)`,
                 id: agentId,
                 log_level: 'info',
                 snapshot: true,
                 upgradeable: true,
-                version: '8.0.0',
+                version,
               },
             },
             host: {

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_agent.ts

@@ -6,15 +6,20 @@
  */
 
 import type { Client } from '@elastic/elasticsearch';
-import type { DeleteByQueryResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type {
+  DeleteByQueryResponse,
+  IndexRequest,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { KbnClient } from '@kbn/test';
 import type { FleetServerAgent } from '@kbn/fleet-plugin/common';
 import { AGENTS_INDEX } from '@kbn/fleet-plugin/common';
 import type { BulkRequest } from '@elastic/elasticsearch/lib/api/types';
+import type { DeepPartial } from 'utility-types';
+import type { ToolingLog } from '@kbn/tooling-log';
 import { usageTracker } from './usage_tracker';
 import type { HostMetadata } from '../types';
 import { FleetAgentGenerator } from '../data_generators/fleet_agent_generator';
-import { wrapErrorAndRejectPromise } from './utils';
+import { createToolingLogger, wrapErrorAndRejectPromise } from './utils';
 
 const defaultFleetAgentGenerator = new FleetAgentGenerator();
 
@@ -189,3 +194,31 @@ export const deleteIndexedFleetAgents = async (
 
   return response;
 };
+
+export const indexFleetServerAgent = async (
+  esClient: Client,
+  log: ToolingLog = createToolingLogger(),
+  overrides: DeepPartial<FleetServerAgent> = {}
+): Promise<IndexedFleetAgentResponse> => {
+  const doc = defaultFleetAgentGenerator.generateEsHit({
+    _source: overrides,
+  });
+
+  const indexRequest: IndexRequest<FleetServerAgent> = {
+    index: doc._index,
+    id: doc._id,
+    body: doc._source,
+    op_type: 'create',
+    refresh: 'wait_for',
+  };
+
+  log.verbose(`Indexing new fleet agent with:\n${JSON.stringify(indexRequest, null, 2)}`);
+
+  await esClient.index<FleetServerAgent>(indexRequest).catch(wrapErrorAndRejectPromise);
+
+  return {
+    fleetAgentsIndex: doc._index,
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    agents: [doc._source!],
+  };
+};

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_fleet_server.ts

@@ -6,53 +6,155 @@
  */
 
 import type { Client } from '@elastic/elasticsearch';
-import { FLEET_SERVER_SERVERS_INDEX } from '@kbn/fleet-plugin/common';
+import { kibanaPackageJson } from '@kbn/repo-info';
+import type { KbnClient } from '@kbn/test';
+import type {
+  GetPackagePoliciesResponse,
+  AgentPolicy,
+  GetOneAgentPolicyResponse,
+  CreateAgentPolicyResponse,
+} from '@kbn/fleet-plugin/common';
+import {
+  AGENT_POLICY_API_ROUTES,
+  agentPolicyRouteService,
+  AGENTS_INDEX,
+  FLEET_SERVER_PACKAGE,
+  PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+  packagePolicyRouteService,
+} from '@kbn/fleet-plugin/common';
+import type { ToolingLog } from '@kbn/tooling-log';
+import { indexFleetServerAgent } from './index_fleet_agent';
+import { catchAxiosErrorFormatAndThrow } from '../format_axios_error';
 import { usageTracker } from './usage_tracker';
-import { wrapErrorAndRejectPromise } from './utils';
+import { createToolingLogger, wrapErrorAndRejectPromise } from './utils';
 
 /**
- * Will ensure that at least one fleet server is present in the `.fleet-servers` index. This will
+ * Will ensure that at least one fleet server is present in the `.fleet-agents` index. This will
- * enable the `Agent` section of kibana Fleet to be displayed
+ * enable the `Agent` section of kibana Fleet to be displayed. We skip on serverless because
+ * Fleet Server agents are not checked against there.
  *
  * @param esClient
+ * @param kbnClient
+ * @param log
  * @param version
  */
 export const enableFleetServerIfNecessary = usageTracker.track(
   'enableFleetServerIfNecessary',
-  async (esClient: Client, version: string = '8.0.0') => {
+  async (
-    const res = await esClient.search({
+    esClient: Client,
-      index: FLEET_SERVER_SERVERS_INDEX,
+    isServerless: boolean = false,
-      ignore_unavailable: true,
+    kbnClient: KbnClient,
-      rest_total_hits_as_int: true,
+    log: ToolingLog = createToolingLogger(),
+    version: string = kibanaPackageJson.version
+  ) => {
+    const agentPolicy = await getOrCreateFleetServerAgentPolicy(kbnClient, log);
+
+    if (!isServerless && !(await hasFleetServerAgent(esClient, agentPolicy.id))) {
+      log.debug(`Indexing a new fleet server agent`);
+      const lastCheckin = new Date();
+      lastCheckin.setFullYear(lastCheckin.getFullYear() + 1);
+
+      const indexedAgent = await indexFleetServerAgent(esClient, log, {
+        policy_id: agentPolicy.id,
+        agent: { version },
+        last_checkin_status: 'online',
+        last_checkin: lastCheckin.toISOString(),
       });
 
-    if (res.hits.total) {
+      log.verbose(`New fleet server agent indexed:\n${JSON.stringify(indexedAgent)}`);
-      return;
+    } else {
+      log.debug(`Nothing to do. A Fleet Server agent is already registered with Fleet`);
     }
-
-    // Create a Fake fleet-server in this kibana instance
-    await esClient
-      .index({
-        index: FLEET_SERVER_SERVERS_INDEX,
-        refresh: 'wait_for',
-        body: {
-          agent: {
-            id: '12988155-475c-430d-ac89-84dc84b67cd1',
-            version,
-          },
-          host: {
-            architecture: 'linux',
-            id: 'c3e5f4f690b4a3ff23e54900701a9513',
-            ip: ['127.0.0.1', '::1', '10.201.0.213', 'fe80::4001:aff:fec9:d5'],
-            name: 'endpoint-data-generator',
-          },
-          server: {
-            id: '12988155-475c-430d-ac89-84dc84b67cd1',
-            version,
-          },
-          '@timestamp': '2021-05-12T18:42:52.009482058Z',
-        },
-      })
-      .catch(wrapErrorAndRejectPromise);
   }
 );
+
+const getOrCreateFleetServerAgentPolicy = async (
+  kbnClient: KbnClient,
+  log: ToolingLog = createToolingLogger()
+): Promise<AgentPolicy> => {
+  const packagePolicies = await kbnClient
+    .request<GetPackagePoliciesResponse>({
+      method: 'GET',
+      headers: { 'elastic-api-version': '2023-10-31' },
+      path: packagePolicyRouteService.getListPath(),
+      query: {
+        perPage: 1,
+        kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name: "${FLEET_SERVER_PACKAGE}"`,
+      },
+    })
+    .catch(catchAxiosErrorFormatAndThrow);
+
+  if (packagePolicies.data.items[0]) {
+    log.debug(`Found an existing package policy - fetching associated agent policy`);
+    log.verbose(JSON.stringify(packagePolicies.data.items[0]));
+
+    return kbnClient
+      .request<GetOneAgentPolicyResponse>({
+        headers: { 'elastic-api-version': '2023-10-31' },
+        method: 'GET',
+        path: agentPolicyRouteService.getInfoPath(packagePolicies.data.items[0].policy_id),
+      })
+      .catch(catchAxiosErrorFormatAndThrow)
+      .then((response) => {
+        log.verbose(
+          `Existing agent policy for Fleet Server:\n${JSON.stringify(response.data.item)}`
+        );
+
+        return response.data.item;
+      });
+  }
+
+  log.debug(`Creating a new fleet server agent policy`);
+
+  // create new Fleet Server agent policy
+  return kbnClient
+    .request<CreateAgentPolicyResponse>({
+      method: 'POST',
+      path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN,
+      headers: { 'elastic-api-version': '2023-10-31' },
+      body: {
+        name: `Fleet Server policy (${Math.random().toString(32).substring(2)})`,
+        description: `Created by CLI Tool via: ${__filename}`,
+        namespace: 'default',
+        monitoring_enabled: [],
+        // This will ensure the Fleet Server integration policy
+        // is also created and added to the agent policy
+        has_fleet_server: true,
+      },
+    })
+    .then((response) => {
+      log.verbose(
+        `No fleet server agent policy found. Created a new one:\n${JSON.stringify(
+          response.data.item
+        )}`
+      );
+
+      return response.data.item;
+    })
+    .catch(catchAxiosErrorFormatAndThrow);
+};
+
+const hasFleetServerAgent = async (
+  esClient: Client,
+  fleetServerAgentPolicyId: string
+): Promise<boolean> => {
+  const searchResponse = await esClient
+    .search(
+      {
+        index: AGENTS_INDEX,
+        ignore_unavailable: true,
+        rest_total_hits_as_int: true,
+        size: 1,
+        _source: false,
+        query: {
+          match: {
+            policy_id: fleetServerAgentPolicyId,
+          },
+        },
+      },
+      { ignore: [404] }
+    )
+    .catch(wrapErrorAndRejectPromise);
+
+  return Boolean(searchResponse?.hits.total);
+};

x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts

@@ -10,13 +10,14 @@ import type { ToolingLogTextWriterConfig } from '@kbn/tooling-log';
 import { ToolingLog } from '@kbn/tooling-log';
 import type { Flags } from '@kbn/dev-cli-runner';
 import moment from 'moment/moment';
+import { EndpointError } from '../errors';
 
 export const RETRYABLE_TRANSIENT_ERRORS: Readonly<Array<string | RegExp>> = [
   'no_shard_available_action_exception',
   'illegal_index_shard_state_exception',
 ];
 
-export class EndpointDataLoadingError extends Error {
+export class EndpointDataLoadingError extends EndpointError {
   constructor(message: string, public meta?: unknown) {
     super(message);
   }
@@ -88,7 +89,8 @@ export const retryOnError = async <T>(
 
       return result;
     } catch (err) {
-      log.warning(msg(`attempt ${thisAttempt} failed with: ${err.message}`), err);
+      log.warning(msg(`attempt ${thisAttempt} failed with: ${err.message.split('\n').at(0)}`));
+      log.verbose(err);
 
       // If not an error that is retryable, then end loop here and return that error;
       if (!isRetryableError(err)) {

x-pack/plugins/security_solution/common/endpoint/index_data.ts

@@ -74,6 +74,7 @@ export const indexHostsAndAlerts = usageTracker.track(
     withResponseActions = true,
     numResponseActions?: number,
     alertIds?: string[],
+    isServerless: boolean = false,
     logger_?: ToolingLog
   ): Promise<IndexedHostsAndAlertsResponse> => {
     const random = seedrandom(seed);
@@ -103,7 +104,7 @@ export const indexHostsAndAlerts = usageTracker.track(
 
     // If `fleet` integration is true, then ensure a (fake) fleet-server is connected
     if (fleet) {
-      await enableFleetServerIfNecessary(client);
+      await enableFleetServerIfNecessary(client, isServerless, kbnClient, logger);
     }
 
     // Keep a map of host applied policy ids (fake) to real ingest package configs (policy record)

x-pack/plugins/security_solution/public/management/cypress/support/common.ts

@@ -17,6 +17,9 @@ export const setupStackServicesUsingCypressConfig = async (config: Cypress.Plugi
     return RUNTIME_SERVICES_CACHE.get(config)!;
   }
 
+  const isServerless = config.env.IS_SERVERLESS;
+  const isCloudServerless = config.env.CLOUD_SERVERLESS;
+
   const stackServices = await createRuntimeServices({
     kibanaUrl: config.env.KIBANA_URL,
     elasticsearchUrl: config.env.ELASTICSEARCH_URL,
@@ -25,7 +28,8 @@ export const setupStackServicesUsingCypressConfig = async (config: Cypress.Plugi
     password: config.env.KIBANA_PASSWORD,
     esUsername: config.env.ELASTICSEARCH_USERNAME,
     esPassword: config.env.ELASTICSEARCH_PASSWORD,
-    asSuperuser: !config.env.CLOUD_SERVERLESS,
+    asSuperuser: !isCloudServerless,
+    useCertForSsl: !isCloudServerless && isServerless,
   }).then(({ log, ...others }) => {
     return {
       ...others,

x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts

@@ -217,6 +217,7 @@ export const dataLoaders = (
         withResponseActions,
         numResponseActions,
         alertIds,
+        isServerless,
       });
     },
 

x-pack/plugins/security_solution/public/management/cypress/support/plugin_handlers/endpoint_data_loader.ts

@@ -45,6 +45,7 @@ export interface CyLoadEndpointDataOptions
   isolation: boolean;
   bothIsolatedAndNormalEndpoints?: boolean;
   alertIds?: string[];
+  isServerless?: boolean;
 }
 
 /**
@@ -73,6 +74,7 @@ export const cyLoadEndpointDataHandler = async (
     isolation,
     numResponseActions,
     alertIds,
+    isServerless = false,
   } = options;
 
   const DocGenerator = EndpointDocGenerator.custom({
@@ -80,6 +82,8 @@ export const cyLoadEndpointDataHandler = async (
   });
 
   if (waitUntilTransformed) {
+    log.info(`Stopping transforms...`);
+
     // need this before indexing docs so that the united transform doesn't
     // create a checkpoint with a timestamp after the doc timestamps
     await stopTransform(esClient, log, metadataTransformPrefix);
@@ -88,6 +92,8 @@ export const cyLoadEndpointDataHandler = async (
     await stopTransform(esClient, log, METADATA_UNITED_TRANSFORM_V2);
   }
 
+  log.info(`Calling indexHostAndAlerts() to index [${numHosts}] endpoint hosts...`);
+
   // load data into the system
   const indexedData = await indexHostsAndAlerts(
     esClient as Client,
@@ -106,25 +112,31 @@ export const cyLoadEndpointDataHandler = async (
     withResponseActions,
     numResponseActions,
     alertIds,
+    isServerless,
     log
   );
 
+  log.info(`Hosts have been indexed`);
+
   if (waitUntilTransformed) {
+    log.info(`starting transforms...`);
+
     // missing transforms are ignored, start either name
-    await startTransform(esClient, metadataTransformPrefix);
+    await startTransform(esClient, log, metadataTransformPrefix);
-    await startTransform(esClient, METADATA_CURRENT_TRANSFORM_V2);
+    await startTransform(esClient, log, METADATA_CURRENT_TRANSFORM_V2);
 
     const metadataIds = Array.from(new Set(indexedData.hosts.map((host) => host.agent.id)));
-    await waitForEndpoints(esClient, 'endpoint_index', metadataIds);
+    await waitForEndpoints(esClient, log, 'endpoint_index', metadataIds);
 
-    await startTransform(esClient, METADATA_UNITED_TRANSFORM);
+    await startTransform(esClient, log, METADATA_UNITED_TRANSFORM);
-    await startTransform(esClient, METADATA_UNITED_TRANSFORM_V2);
+    await startTransform(esClient, log, METADATA_UNITED_TRANSFORM_V2);
 
     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
     const agentIds = Array.from(new Set(indexedData.agents.map((agent) => agent.agent!.id)));
-    await waitForEndpoints(esClient, 'united_index', agentIds);
+    await waitForEndpoints(esClient, log, 'united_index', agentIds);
   }
 
+  log.info(`Done - [${numHosts}] endpoint hosts have been indexed and are now available in kibana`);
   return indexedData;
 };
 
@@ -133,6 +145,8 @@ const stopTransform = async (
   log: ToolingLog,
   transformId: string
 ): Promise<void> => {
+  log.debug(`Stopping transform id: ${transformId}`);
+
   await esClient.transform
     .stopTransform({
       transform_id: `${transformId}*`,
@@ -147,17 +161,27 @@ const stopTransform = async (
     });
 };
 
-const startTransform = async (esClient: Client, transformId: string): Promise<void> => {
+const startTransform = async (
+  esClient: Client,
+  log: ToolingLog,
+  transformId: string
+): Promise<void> => {
   const transformsResponse = await esClient.transform.getTransformStats({
     transform_id: `${transformId}*`,
   });
 
+  log.verbose(
+    `Transform status found for [${transformId}*] returned:\n${dump(transformsResponse)}`
+  );
+
   await Promise.all(
     transformsResponse.transforms.map((transform) => {
       if (STARTED_TRANSFORM_STATES.has(transform.state)) {
         return Promise.resolve();
       }
 
+      log.debug(`Staring transform id: [${transform.id}]`);
+
       return esClient.transform.startTransform({ transform_id: transform.id });
     })
   );
@@ -173,6 +197,7 @@ const startTransform = async (esClient: Client, transformId: string): Promise<vo
  */
 const waitForEndpoints = async (
   esClient: Client,
+  log: ToolingLog,
   location: 'endpoint_index' | 'united_index',
   ids: string[] = []
 ): Promise<void> => {
@@ -218,8 +243,13 @@ const waitForEndpoints = async (
 
   const expectedSize = ids.length || 1;
 
+  log.info(`Waiting for [${expectedSize}] endpoint hosts to be available`);
+  log.verbose(`Query for searching index [${index}]:\n${dump(body, 10)}`);
+
   await pRetry(
-    async () => {
+    async (attemptCount) => {
+      log.debug(`Attempt [${attemptCount}]: Searching [${index}] to check if hosts are availble`);
+
       const response = await esClient.search({
         index,
         size: expectedSize,
@@ -227,12 +257,16 @@ const waitForEndpoints = async (
         rest_total_hits_as_int: true,
       });
 
+      log.verbose(`Attempt [${attemptCount}]: Search response:\n${dump(response, 10)}`);
+
       // If not the expected number of Endpoints, then throw an error so we keep trying
       if (response.hits.total !== expectedSize) {
         throw new Error(
           `Expected number of endpoints not found. Looking for ${expectedSize} but received ${response.hits.total}`
         );
       }
+
+      log.info(`Attempt [${attemptCount}]: Done - [${expectedSize}] host are now available`);
     },
     { forever: false }
   );

x-pack/plugins/security_solution/scripts/endpoint/agent_emulator/services/endpoint_loader.ts

@@ -21,6 +21,7 @@ import { METADATA_DATASTREAM } from '../../../../common/endpoint/constants';
 import { EndpointMetadataGenerator } from '../../../../common/endpoint/data_generators/endpoint_metadata_generator';
 import { getEndpointPackageInfo } from '../../../../common/endpoint/utils/package';
 import { ENDPOINT_ALERTS_INDEX, ENDPOINT_EVENTS_INDEX } from '../../common/constants';
+import { isServerlessKibanaFlavor } from '../../common/stack_services';
 
 let WAS_FLEET_SETUP_DONE = false;
 
@@ -90,8 +91,9 @@ export const loadEndpoints = async ({
   }
 
   if (!WAS_FLEET_SETUP_DONE) {
+    const isServerless = await isServerlessKibanaFlavor(kbnClient);
     await setupFleetForEndpoint(kbnClient);
-    await enableFleetServerIfNecessary(esClient);
+    await enableFleetServerIfNecessary(esClient, isServerless, kbnClient, log);
     // eslint-disable-next-line require-atomic-updates
     WAS_FLEET_SETUP_DONE = true;
   }

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_server/fleet_server_services.ts

@@ -16,7 +16,6 @@ import {
   AGENT_POLICY_API_ROUTES,
   API_VERSIONS,
   FLEET_SERVER_PACKAGE,
-  FLEET_SERVER_SERVERS_INDEX,
   PACKAGE_POLICY_SAVED_OBJECT_TYPE,
 } from '@kbn/fleet-plugin/common';
 import type {
@@ -42,13 +41,8 @@ import {
 } from '@kbn/dev-utils';
 import { maybeCreateDockerNetwork, SERVERLESS_NODES, verifyDockerInstalled } from '@kbn/es';
 import { resolve } from 'path';
-import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { captureCallingStack, dump, prefixedOutputLogger } from '../utils';
-import {
+import { createToolingLogger } from '../../../../common/endpoint/data_loaders/utils';
-  createToolingLogger,
-  RETRYABLE_TRANSIENT_ERRORS,
-  retryOnError,
-} from '../../../../common/endpoint/data_loaders/utils';
 import { isServerlessKibanaFlavor } from '../stack_services';
 import type { FormattedAxiosError } from '../../../../common/endpoint/format_axios_error';
 import { catchAxiosErrorFormatAndThrow } from '../../../../common/endpoint/format_axios_error';
@@ -329,9 +323,12 @@ const startFleetServerWithDocker = async ({
         await updateFleetElasticsearchOutputHostNames(kbnClient, log);
 
         if (isServerless) {
-          log.info(`Waiting for server [${hostname}] to register with Elasticsearch`);
+          log.info(`Waiting for Fleet Server [${hostname}] to start running`);
-          await waitForFleetServerToRegisterWithElasticsearch(kbnClient, hostname, 180000);
+          if (!(await isFleetServerRunning(kbnClient, log))) {
+            throw Error(`Unable to start Fleet Server [${hostname}]`);
+          }
         } else {
+          log.info(`Waiting for Fleet Server [${hostname}] to enroll with Fleet`);
           await waitForHostToEnroll(kbnClient, log, hostname, 120000);
         }
 
@@ -683,7 +680,7 @@ export const isFleetServerRunning = async (
   const url = new URL(fleetServerUrl);
   url.pathname = '/api/status';
 
-  return pRetry<boolean>(
+  return pRetry(
     async () => {
       return axios
         .request({
@@ -698,75 +695,20 @@ export const isFleetServerRunning = async (
             `Fleet server is up and running at [${fleetServerUrl}]. Status: `,
             response.data
           );
-          return true;
         })
-        .catch(catchAxiosErrorFormatAndThrow)
-        .catch((e) => {
-          log.debug(`Fleet server not up at [${fleetServerUrl}]`);
-          log.verbose(`Call to [${url.toString()}] failed with:`, e);
-          return false;
-        });
-    },
-    { maxTimeout: 10000 }
-  );
-};
-
-/**
- * Checks and waits until the given fleet server hostname has been registered into elasticsearch.
- * This check can be used when enrolling a standalone fleet-server, since those would not show up
- * in Kibana's Fleet UI.
- */
-const waitForFleetServerToRegisterWithElasticsearch = async (
-  kbnClient: KbnClient,
-  fleetServerHostname: string,
-  timeoutMs: number = 30000
-): Promise<void> => {
-  const started = new Date();
-  const hasTimedOut = (): boolean => {
-    const elapsedTime = Date.now() - started.getTime();
-    return elapsedTime > timeoutMs;
-  };
-  let found = false;
-
-  while (!found && !hasTimedOut()) {
-    found = await retryOnError(async () => {
-      const fleetServerRecord = await kbnClient
-        .request<estypes.SearchResponse>({
-          method: 'POST',
-          path: '/api/console/proxy',
-          query: {
-            path: `${FLEET_SERVER_SERVERS_INDEX}/_search`,
-            method: 'GET',
-          },
-          body: {
-            query: {
-              bool: {
-                filter: [
-                  {
-                    term: {
-                      'host.name': fleetServerHostname,
-                    },
-                  },
-                ],
-              },
-            },
-          },
-        })
-        .then((response) => response.data)
         .catch(catchAxiosErrorFormatAndThrow);
-
+    },
-      return ((fleetServerRecord?.hits?.total as estypes.SearchTotalHits)?.value ?? 0) === 1;
+    {
-    }, RETRYABLE_TRANSIENT_ERRORS);
+      maxTimeout: 10000,
-
+      retries: 5,
-    if (!found) {
+      onFailedAttempt: (e) => {
-      // sleep and check again
+        log.warning(
-      await new Promise((r) => setTimeout(r, 2000));
+          `Fleet server not (yet) up at [${fleetServerUrl}]. Retrying... (attempt #${e.attemptNumber}, ${e.retriesLeft} retries left)`
-    }
-  }
-
-  if (!found) {
-    throw new Error(
-      `Timed out waiting for fleet server [${fleetServerHostname}] to register with Elasticsearch`
         );
+        log.verbose(`Call to [${url.toString()}] failed with:`, e);
+      },
     }
+  )
+    .then(() => true)
+    .catch(() => false);
 };

x-pack/test/defend_workflows_cypress/serverless_config.ts

@@ -51,6 +51,11 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
         `--xpack.fleet.agents.elasticsearch.host=http://${hostIp}:${defendWorkflowsCypressConfig.get(
           'servers.elasticsearch.port'
         )}`,
+
+        // Enable Fleet server standalone so that no checks are done to see if fleet-server has
+        // registered with Kibana and we are able to access the Agents page of Fleet
+        '--xpack.fleet.internal.fleetServerStandalone=true',
+
         // set the packagerTaskInterval to 5s in order to speed up test executions when checking fleet artifacts
         '--xpack.securitySolution.packagerTaskInterval=5s',
         `--xpack.securitySolution.enableExperimental=${JSON.stringify(enabledFeatureFlags)}`,

x-pack/test/functional/es_archives/fleet/agents/mappings.json

@@ -375,67 +375,3 @@
     }
   }
 }
-
-{
-  "type": "index",
-  "value": {
-    "aliases": {
-      ".fleet-servers": {
-      }
-    },
-    "index": ".fleet-servers-7",
-    "mappings": {
-      "_meta": {
-        "migrationHash": "e2782448c7235ec9af66ca7997e867d715ac379c"
-      },
-      "dynamic": "false",
-      "properties": {
-        "@timestamp": {
-          "type": "date"
-        },
-        "agent": {
-          "properties": {
-            "id": {
-              "type": "keyword"
-            },
-            "version": {
-              "type": "keyword"
-            }
-          }
-        },
-        "host": {
-          "properties": {
-            "architecture": {
-              "type": "keyword"
-            },
-            "id": {
-              "type": "keyword"
-            },
-            "ip": {
-              "type": "keyword"
-            },
-            "name": {
-              "type": "keyword"
-            }
-          }
-        },
-        "server": {
-          "properties": {
-            "id": {
-              "type": "keyword"
-            },
-            "version": {
-              "type": "keyword"
-            }
-          }
-        }
-      }
-    },
-    "settings": {
-      "index": {
-        "number_of_replicas": "1",
-        "number_of_shards": "1"
-      }
-    }
-  }
-}

x-pack/test/functional/es_archives/fleet/empty_fleet_server/mappings.json

@@ -375,66 +375,3 @@
   }
 }
 
-{
-  "type": "index",
-  "value": {
-    "aliases": {
-      ".fleet-servers": {
-      }
-    },
-    "index": ".fleet-servers-7",
-    "mappings": {
-      "_meta": {
-        "migrationHash": "e2782448c7235ec9af66ca7997e867d715ac379c"
-      },
-      "dynamic": "false",
-      "properties": {
-        "@timestamp": {
-          "type": "date"
-        },
-        "agent": {
-          "properties": {
-            "id": {
-              "type": "keyword"
-            },
-            "version": {
-              "type": "keyword"
-            }
-          }
-        },
-        "host": {
-          "properties": {
-            "architecture": {
-              "type": "keyword"
-            },
-            "id": {
-              "type": "keyword"
-            },
-            "ip": {
-              "type": "keyword"
-            },
-            "name": {
-              "type": "keyword"
-            }
-          }
-        },
-        "server": {
-          "properties": {
-            "id": {
-              "type": "keyword"
-            },
-            "version": {
-              "type": "keyword"
-            }
-          }
-        }
-      }
-    },
-    "settings": {
-      "index": {
-        "number_of_replicas": "1",
-        "number_of_shards": "1"
-      }
-    }
-  }
-}

x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/services/endpoint.ts

@@ -160,6 +160,7 @@ export class EndpointTestResources extends FtrService {
           undefined,
           undefined,
           undefined,
+          undefined,
           this.log
         );