[Security Solution] fix endpoint data generator (#154383)

x-pack/plugins/security_solution/common/endpoint/data_loaders/setup_fleet_for_endpoint.ts

@@ -64,6 +64,14 @@ export const setupFleetForEndpoint = async (kbnClient: KbnClient): Promise<void>
     log.error(error);
     throw error;
   }
+
+  // Install/upgrade the endpoint package
+  try {
+    await installOrUpgradeEndpointFleetPackage(kbnClient);
+  } catch (error) {
+    log.error(error);
+    throw error;
+  }
 };
 
 /**

x-pack/plugins/security_solution/common/endpoint/index_data.ts

@@ -62,8 +62,7 @@ export async function indexHostsAndAlerts(
   alertsPerHost: number,
   fleet: boolean,
   options: TreeOptions = {},
-  DocGenerator: typeof EndpointDocGenerator = EndpointDocGenerator,
-  startTransform = true
+  DocGenerator: typeof EndpointDocGenerator = EndpointDocGenerator
 ): Promise<IndexedHostsAndAlertsResponse> {
   const random = seedrandom(seed);
   const epmEndpointPackage = await getEndpointPackageInfo(kbnClient);
@@ -97,8 +96,11 @@ export async function indexHostsAndAlerts(
   // Keep a map of host applied policy ids (fake) to real ingest package configs (policy record)
   const realPolicies: Record<string, CreatePackagePolicyResponse['item']> = {};
 
-  await waitForMetadataTransformsReady(client);
-  await stopMetadataTransforms(client);
+  const shouldWaitForEndpointMetadataDocs = fleet;
+  if (shouldWaitForEndpointMetadataDocs) {
+    await waitForMetadataTransformsReady(client);
+    await stopMetadataTransforms(client);
+  }
 
   for (let i = 0; i < numHosts; i++) {
     const generator = new DocGenerator(random);
@@ -126,7 +128,7 @@ export async function indexHostsAndAlerts(
     });
   }
 
-  if (startTransform) {
+  if (shouldWaitForEndpointMetadataDocs) {
     await startMetadataTransforms(
       client,
       response.agents.map((agent) => agent.id)

x-pack/plugins/security_solution/common/endpoint/utils/transforms.ts

@@ -16,10 +16,7 @@ import {
 } from '../constants';
 
 export async function waitForMetadataTransformsReady(esClient: Client): Promise<void> {
-  await waitFor(
-    () => areMetadataTransformsReady(esClient),
-    'failed while waiting for transform to start'
-  );
+  await waitFor(() => areMetadataTransformsReady(esClient));
 }
 
 export async function stopMetadataTransforms(esClient: Client): Promise<void> {
@@ -40,7 +37,7 @@ export async function stopMetadataTransforms(esClient: Client): Promise<void> {
 export async function startMetadataTransforms(
   esClient: Client,
   // agentIds to wait for
-  agentIds?: string[]
+  agentIds: string[]
 ): Promise<void> {
   const transformIds = await getMetadataTransformIds(esClient);
   const currentTransformId = transformIds.find((transformId) =>
@@ -50,7 +47,9 @@ export async function startMetadataTransforms(
     transformId.startsWith(METADATA_UNITED_TRANSFORM)
   );
   if (!currentTransformId || !unitedTransformId) {
-    throw new Error('failed to start metadata transforms, transforms not found');
+    // eslint-disable-next-line no-console
+    console.warn('metadata transforms not found, skipping transform start');
+    return;
   }
 
   try {
@@ -102,8 +101,8 @@ async function areMetadataTransformsReady(esClient: Client): Promise<boolean> {
   );
 }
 
-async function waitForCurrentMetdataDocs(esClient: Client, agentIds?: string[]) {
-  const query = agentIds?.length
+async function waitForCurrentMetdataDocs(esClient: Client, agentIds: string[]) {
+  const query = agentIds.length
     ? {
         bool: {
           filter: [
@@ -116,12 +115,9 @@ async function waitForCurrentMetdataDocs(esClient: Client, agentIds?: string[])
         },
       }
     : {
-        size: 1,
-        query: {
-          match_all: {},
-        },
+        match_all: {},
       };
-  const size = agentIds?.length ? agentIds.length : 1;
+  const size = agentIds.length ?? 1;
   await waitFor(
     async () =>
       (
@@ -131,16 +127,14 @@ async function waitForCurrentMetdataDocs(esClient: Client, agentIds?: string[])
           size,
           rest_total_hits_as_int: true,
         })
-      ).hits.total === size,
-    'failed while waiting for current metadata docs to populate'
+      ).hits.total === size
   );
 }
 
 async function waitFor(
   cb: () => Promise<boolean>,
-  errorMessage: string,
   interval: number = 20000,
-  maxAttempts = 5
+  maxAttempts = 6
 ): Promise<void> {
   let attempts = 0;
   let isReady = false;
@@ -151,7 +145,7 @@ async function waitFor(
     attempts++;
 
     if (attempts > maxAttempts) {
-      throw new Error(errorMessage);
+      return;
     }
   }
 }

x-pack/plugins/security_solution/public/management/cypress/tasks/run_endpoint_loader.ts

@@ -26,5 +26,5 @@ export const runEndpointLoaderScript = () => {
   // FIXME: remove use of cli script and use instead data loaders
   const script = `node scripts/endpoint/resolver_generator.js --node="${ES_URL.toString()}" --kibana="${KBN_URL.toString()}" --delete --numHosts=1 --numDocs=1 --fleet --withNewUser=santaEndpoint:changeme --anc=1 --gen=1 --ch=1 --related=1 --relAlerts=1`;
 
-  cy.exec(script, { env: { NODE_TLS_REJECT_UNAUTHORIZED: 1 } });
+  cy.exec(script, { env: { NODE_TLS_REJECT_UNAUTHORIZED: 1 }, timeout: 180000 });
 };

x-pack/test/security_solution_endpoint/services/endpoint.ts

@@ -116,7 +116,7 @@ export class EndpointTestResources extends FtrService {
       customIndexFn,
     } = options;
 
-    if (waitUntilTransformed) {
+    if (waitUntilTransformed && customIndexFn) {
       // need this before indexing docs so that the united transform doesn't
       // create a checkpoint with a timestamp after the doc timestamps
       await this.stopTransform(metadataTransformPrefix);
@@ -139,15 +139,17 @@ export class EndpointTestResources extends FtrService {
           alertsPerHost,
           enableFleetIntegration,
           undefined,
-          CurrentKibanaVersionDocGenerator,
-          false
+          CurrentKibanaVersionDocGenerator
         );
 
-    if (waitUntilTransformed) {
+    if (waitUntilTransformed && customIndexFn) {
       await this.startTransform(metadataTransformPrefix);
       const metadataIds = Array.from(new Set(indexedData.hosts.map((host) => host.agent.id)));
       await this.waitForEndpoints(metadataIds, waitTimeout);
       await this.startTransform(METADATA_UNITED_TRANSFORM);
+    }
+
+    if (waitUntilTransformed) {
       const agentIds = Array.from(new Set(indexedData.agents.map((agent) => agent.agent!.id)));
       await this.waitForUnitedEndpoints(agentIds, waitTimeout);
     }