Add keyword builder pipeline

.buildkite/ftr_security_stateful_configs.yml

@@ -105,3 +105,4 @@ enabled:
   - x-pack/test/cloud_security_posture_functional/data_views/config.ts
   - x-pack/test/automatic_import_api_integration/apis/config_basic.ts
   - x-pack/test/automatic_import_api_integration/apis/config_graphs.ts
+  - x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/configs/ess.config.ts

.github/CODEOWNERS

@@ -2514,6 +2514,9 @@ x-pack/solutions/security/plugins/security_solution/public/common/components/ses
 x-pack/solutions/security/plugins/security_solution/public/cloud_defend @elastic/kibana-cloud-security-posture
 x-pack/solutions/security/plugins/security_solution/public/cloud_security_posture @elastic/kibana-cloud-security-posture
 x-pack/solutions/security/plugins/security_solution/public/kubernetes @elastic/kibana-cloud-security-posture
+x-pack/test/security_solution_api_integration/test_suites/asset_inventory @elastic/kibana-cloud-security-posture
+x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory @elastic/kibana-cloud-security-posture
+
 ## Fleet plugin (co-owned with Fleet team)
 x-pack/platform/plugins/shared/fleet/public/components/cloud_security_posture @elastic/fleet @elastic/kibana-cloud-security-posture
 x-pack/platform/plugins/shared/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/components/cloud_security_posture @elastic/fleet @elastic/kibana-cloud-security-posture

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/asset_inventory_data_client.mock.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { AssetInventoryDataClient } from './asset_inventory_data_client';
+
+const createAssetInventoryDataClientMock = () =>
+  ({
+    init: jest.fn(),
+    enable: jest.fn(),
+    delete: jest.fn(),
+  } as unknown as jest.Mocked<AssetInventoryDataClient>);
+
+export const AssetInventoryDataClientMock = { create: createAssetInventoryDataClientMock };

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/asset_inventory_data_client.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger, ElasticsearchClient, IScopedClusterClient } from '@kbn/core/server';
+
+import type { ExperimentalFeatures } from '../../../common';
+
+import { createKeywordBuilderPipeline, deleteKeywordBuilderPipeline } from './ingest_pipelines';
+
+interface AssetInventoryClientOpts {
+  logger: Logger;
+  clusterClient: IScopedClusterClient;
+  experimentalFeatures: ExperimentalFeatures;
+}
+
+// AssetInventoryDataClient is responsible for managing the asset inventory,
+// including initializing and cleaning up resources such as Elasticsearch ingest pipelines.
+export class AssetInventoryDataClient {
+  private esClient: ElasticsearchClient;
+
+  constructor(private readonly options: AssetInventoryClientOpts) {
+    const { clusterClient } = options;
+    this.esClient = clusterClient.asCurrentUser;
+  }
+
+  // Enables the asset inventory by deferring the initialization to avoid blocking the main thread.
+  public async enable() {
+    // Utility function to defer execution to the next tick using setTimeout.
+    const run = <T>(fn: () => Promise<T>) =>
+      new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
+
+    // Defer and execute the initialization process.
+    await run(() => this.init());
+
+    return { succeeded: true };
+  }
+
+  // Initializes the asset inventory by validating experimental feature flags and triggering asynchronous setup.
+  public async init() {
+    const { experimentalFeatures, logger } = this.options;
+
+    if (!experimentalFeatures.assetInventoryStoreEnabled) {
+      throw new Error('Universal entity store is not enabled');
+    }
+
+    logger.debug(`Initializing asset inventory`);
+
+    this.asyncSetup().catch((e) =>
+      logger.error(`Error during async setup of asset inventory: ${e.message}`)
+    );
+  }
+
+  // Sets up the necessary resources for asset inventory, including creating Elasticsearch ingest pipelines.
+  private async asyncSetup() {
+    const { logger } = this.options;
+    try {
+      logger.debug('creating keyword builder pipeline');
+      await createKeywordBuilderPipeline({
+        logger,
+        esClient: this.esClient,
+      });
+      logger.debug('keyword builder pipeline created');
+    } catch (err) {
+      logger.error(`Error initializing asset inventory: ${err.message}`);
+      await this.delete();
+    }
+  }
+
+  // Cleans up the resources associated with the asset inventory, such as removing the ingest pipeline.
+  public async delete() {
+    const { logger } = this.options;
+
+    logger.debug(`Deleting asset inventory`);
+
+    try {
+      logger.debug(`Deleting asset inventory keyword builder pipeline`);
+
+      await deleteKeywordBuilderPipeline({
+        logger,
+        esClient: this.esClient,
+      }).catch((err) => {
+        logger.error('Error on deleting keyword builder pipeline', err);
+      });
+
+      logger.debug(`Deleted asset inventory`);
+      return { deleted: true };
+    } catch (err) {
+      logger.error(`Error deleting asset inventory: ${err.message}`);
+      throw err;
+    }
+  }
+}

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './keyword_builder_ingest_pipeline';

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/ingest_pipelines/keyword_builder_ingest_pipeline.ts

@@ -0,0 +1,167 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient, Logger } from '@kbn/core/server';
+import type { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/types';
+
+const PIPELINE_ID = 'entity-keyword-builder@platform';
+
+export const buildIngestPipeline = (): IngestProcessorContainer[] => {
+  return [
+    {
+      script: {
+        lang: 'painless',
+        on_failure: [
+          {
+            set: {
+              field: 'error.message',
+              value:
+                'Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message }}',
+            },
+          },
+        ],
+
+        // There are two layers of language to string scape on this script.
+        // - One is in javascript
+        // - Another one is in painless.
+        //
+        // .e.g, in painless we want the following line:
+        //    entry.getKey().replace("\"", "\\\"");
+        //
+        // To do so we must scape each backslash in javascript, otherwise the backslashes will only scape the next character
+        // and the backslashes won't end up in the painless layer
+        //
+        // The code then becomes:
+        //    entry.getKey().replace("\\"", "\\\\\\"");
+        // That is one extra backslash per backslash (there is no need to scape quotes in the javascript layer)
+        source: `
+            String jsonFromMap(Map map) {
+              StringBuilder json = new StringBuilder("{");
+              boolean first = true;
+              for (entry in map.entrySet()) {
+                  if (!first) {
+                      json.append(",");
+                  }
+                  first = false;
+                  String key = entry.getKey().replace("\\"", "\\\\\\"");
+                  Object value = entry.getValue();
+                  json.append("\\"").append(key).append("\\":");
+                  
+                  if (value instanceof String) {
+                      String escapedValue = ((String) value).replace("\\"", "\\\\\\"").replace("=", ":");
+                      json.append("\\"").append(escapedValue).append("\\"");
+                  } else if (value instanceof Map) {
+                      json.append(jsonFromMap((Map) value));
+                  } else if (value instanceof List) {
+                      json.append(jsonFromList((List) value));
+                  } else if (value instanceof Boolean || value instanceof Number) {
+                      json.append(value.toString());
+                  } else {
+                      // For other types, treat as string
+                      String escapedValue = value.toString().replace("\\"", "\\\\\\"").replace("=", ":");
+                      json.append("\\"").append(escapedValue).append("\\"");
+                  }
+              }
+              json.append("}");
+              return json.toString();
+          }
+
+          String jsonFromList(List list) {
+              StringBuilder json = new StringBuilder("[");
+              boolean first = true;
+              for (item in list) {
+                  if (!first) {
+                      json.append(",");
+                  }
+                  first = false;
+                  if (item instanceof String) {
+                      String escapedItem = ((String) item).replace("\\"", "\\\\\\"").replace("=", ":");
+                      json.append("\\"").append(escapedItem).append("\\"");
+                  } else if (item instanceof Map) {
+                      json.append(jsonFromMap((Map) item));
+                  } else if (item instanceof List) {
+                      json.append(jsonFromList((List) item));
+                  } else if (item instanceof Boolean || item instanceof Number) {
+                      json.append(item.toString());
+                  } else {
+                      // For other types, treat as string
+                      String escapedItem = item.toString().replace("\\"", "\\\\\\"").replace("=", ":");
+                      json.append("\\"").append(escapedItem).append("\\"");
+                  }
+              }
+              json.append("]");
+              return json.toString();
+          }
+
+          if (ctx.entities?.metadata == null) {
+            return;
+          }
+
+          def keywords = [];
+          for (key in ctx.entities.metadata.keySet()) {
+            def value = ctx.entities.metadata[key];
+            def metadata = jsonFromMap([key: value]);  
+            keywords.add(metadata);
+          }
+
+          ctx['entities']['keyword'] = keywords;
+          `,
+      },
+    },
+    {
+      set: {
+        field: 'event.ingested',
+        value: '{{{_ingest.timestamp}}}',
+      },
+    },
+  ];
+};
+
+// developing the pipeline is a bit tricky, so we have a debug mode
+// set  xpack.securitySolution.entityAnalytics.entityStore.developer.pipelineDebugMode
+// to true to keep the enrich field and the context field in the document to help with debugging.
+export const createKeywordBuilderPipeline = async ({
+  logger,
+  esClient,
+}: {
+  logger: Logger;
+  esClient: ElasticsearchClient;
+}) => {
+  const pipeline = {
+    id: PIPELINE_ID,
+    body: {
+      _meta: {
+        managed_by: 'entity_store',
+        managed: true,
+      },
+      description: `Serialize entities.metadata into a keyword field`,
+      processors: buildIngestPipeline(),
+    },
+  };
+
+  logger.debug(`Attempting to create pipeline: ${JSON.stringify(pipeline)}`);
+
+  await esClient.ingest.putPipeline(pipeline);
+};
+
+export const deleteKeywordBuilderPipeline = ({
+  logger,
+  esClient,
+}: {
+  logger: Logger;
+  esClient: ElasticsearchClient;
+}) => {
+  logger.debug(`Attempting to delete pipeline: ${PIPELINE_ID}`);
+  return esClient.ingest.deletePipeline(
+    {
+      id: PIPELINE_ID,
+    },
+    {
+      ignore: [404],
+    }
+  );
+};

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/routes/delete.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils';
+import { transformError } from '@kbn/securitysolution-es-utils';
+import { API_VERSIONS } from '../../../../common/constants';
+import type { AssetInventoryRoutesDeps } from '../types';
+
+export const deleteAssetInventoryRoute = (
+  router: AssetInventoryRoutesDeps['router'],
+  logger: Logger
+) => {
+  router.versioned
+    .delete({
+      access: 'public',
+      path: '/api/asset_inventory/delete',
+      security: {
+        authz: {
+          requiredPrivileges: ['securitySolution'],
+        },
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        // TODO: create validation
+        validate: false,
+      },
+
+      async (context, request, response) => {
+        const siemResponse = buildSiemResponse(response);
+
+        try {
+          const secSol = await context.securitySolution;
+          const body = await secSol.getAssetInventoryClient().delete();
+
+          return response.ok({ body });
+        } catch (e) {
+          logger.error('Error in DeleteEntityEngine:', e);
+          const error = transformError(e);
+          return siemResponse.error({
+            statusCode: error.statusCode,
+            body: error.message,
+          });
+        }
+      }
+    );
+};

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/routes/enablement.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils';
+import { transformError } from '@kbn/securitysolution-es-utils';
+import type { Logger } from '@kbn/core/server';
+import { API_VERSIONS } from '../../../../common/constants';
+
+import type { AssetInventoryRoutesDeps } from '../types';
+
+export const enableAssetInventoryRoute = (
+  router: AssetInventoryRoutesDeps['router'],
+  logger: Logger,
+  config: AssetInventoryRoutesDeps['config']
+) => {
+  router.versioned
+    .post({
+      access: 'public',
+      path: '/api/asset_inventory/enable',
+      security: {
+        authz: {
+          requiredPrivileges: ['securitySolution'],
+        },
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.public.v1,
+        // TODO: create validation
+        validate: false,
+      },
+
+      async (context, request, response) => {
+        const siemResponse = buildSiemResponse(response);
+        const secSol = await context.securitySolution;
+
+        try {
+          const body = await secSol.getAssetInventoryClient().enable();
+
+          return response.ok({ body });
+        } catch (e) {
+          const error = transformError(e);
+          logger.error(`Error initializing asset inventory: ${error.message}`);
+          return siemResponse.error({
+            statusCode: error.statusCode,
+            body: error.message,
+          });
+        }
+      }
+    );
+};

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/routes/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { registerAssetInventoryRoutes } from './register_asset_inventory_routes';

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/routes/register_asset_inventory_routes.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { AssetInventoryRoutesDeps } from '../types';
+import { deleteAssetInventoryRoute } from './delete';
+import { enableAssetInventoryRoute } from './enablement';
+
+export const registerAssetInventoryRoutes = ({
+  router,
+  logger,
+  config,
+}: AssetInventoryRoutesDeps) => {
+  enableAssetInventoryRoute(router, logger, config);
+  deleteAssetInventoryRoute(router, logger);
+};

x-pack/solutions/security/plugins/security_solution/server/lib/asset_inventory/types.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import type { SecuritySolutionPluginRouter } from '../../types';
+import type { ConfigType } from '../../config';
+
+export interface AssetInventoryRoutesDeps {
+  router: SecuritySolutionPluginRouter;
+  logger: Logger;
+  config: ConfigType;
+}

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts

@@ -43,6 +43,7 @@ import { detectionRulesClientMock } from '../../rule_management/logic/detection_
 import { packageServiceMock } from '@kbn/fleet-plugin/server/services/epm/package_service.mock';
 import type { EndpointInternalFleetServicesInterface } from '../../../../endpoint/services/fleet';
 import { siemMigrationsServiceMock } from '../../../siem_migrations/__mocks__/mocks';
+import { AssetInventoryDataClientMock } from '../../../asset_inventory/asset_inventory_data_client.mock';
 
 export const createMockClients = () => {
   const core = coreMock.createRequestHandlerContext();
@@ -81,6 +82,7 @@ export const createMockClients = () => {
     },
     siemRuleMigrationsClient: siemMigrationsServiceMock.createRulesClient(),
     getInferenceClient: jest.fn(),
+    assetInventoryDataClient: AssetInventoryDataClientMock.create(),
   };
 };
 
@@ -169,6 +171,7 @@ const createSecuritySolutionRequestContextMock = (
     getEntityStoreDataClient: jest.fn(() => clients.entityStoreDataClient),
     getSiemRuleMigrationsClient: jest.fn(() => clients.siemRuleMigrationsClient),
     getInferenceClient: jest.fn(() => clients.getInferenceClient()),
+    getAssetInventoryClient: jest.fn(() => clients.assetInventoryDataClient),
   };
 };
 

x-pack/solutions/security/plugins/security_solution/server/request_context_factory.ts

@@ -33,6 +33,7 @@ import { createDetectionRulesClient } from './lib/detection_engine/rule_manageme
 import { buildMlAuthz } from './lib/machine_learning/authz';
 import { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client';
 import type { SiemMigrationsService } from './lib/siem_migrations/siem_migrations_service';
+import { AssetInventoryDataClient } from './lib/asset_inventory/asset_inventory_data_client';
 
 export interface IRequestContextFactory {
   create(
@@ -248,6 +249,15 @@ export class RequestContextFactory implements IRequestContextFactory {
           telemetry: core.analytics,
         });
       }),
+      getAssetInventoryClient: memoize(() => {
+        const clusterClient = coreContext.elasticsearch.client;
+        const logger = options.logger;
+        return new AssetInventoryDataClient({
+          clusterClient,
+          logger,
+          experimentalFeatures: config.experimentalFeatures,
+        });
+      }),
     };
   }
 }

x-pack/solutions/security/plugins/security_solution/server/routes/index.ts

@@ -53,6 +53,7 @@ import { registerTimelineRoutes } from '../lib/timeline/routes';
 import { getFleetManagedIndexTemplatesRoute } from '../lib/security_integrations/cribl/routes';
 import { registerEntityAnalyticsRoutes } from '../lib/entity_analytics/register_entity_analytics_routes';
 import { registerSiemMigrationsRoutes } from '../lib/siem_migrations/routes';
+import { registerAssetInventoryRoutes } from '../lib/asset_inventory/routes';
 
 export const initRoutes = (
   router: SecuritySolutionPluginRouter,
@@ -138,4 +139,6 @@ export const initRoutes = (
   getFleetManagedIndexTemplatesRoute(router);
 
   registerWorkflowInsightsRoutes(router, config, endpointContext);
+
+  registerAssetInventoryRoutes({ router, config, logger });
 };

x-pack/solutions/security/plugins/security_solution/server/types.ts

@@ -38,6 +38,7 @@ import type { AssetCriticalityDataClient } from './lib/entity_analytics/asset_cr
 import type { IDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface';
 import type { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client';
 import type { SiemRuleMigrationsClient } from './lib/siem_migrations/rules/siem_rule_migrations_service';
+import type { AssetInventoryDataClient } from './lib/asset_inventory/asset_inventory_data_client';
 export { AppClient };
 
 export interface SecuritySolutionApiRequestHandlerContext {
@@ -63,6 +64,7 @@ export interface SecuritySolutionApiRequestHandlerContext {
   getEntityStoreDataClient: () => EntityStoreDataClient;
   getSiemRuleMigrationsClient: () => SiemRuleMigrationsClient;
   getInferenceClient: () => InferenceClient;
+  getAssetInventoryClient: () => AssetInventoryDataClient;
 }
 
 export type SecuritySolutionRequestHandlerContext = CustomRequestHandlerContext<{

x-pack/test/security_solution_api_integration/package.json

@@ -17,6 +17,12 @@
     "initialize-server:ea:basic_essentials": "node ./scripts/index.js server entity_analytics basic_license_essentials_tier",
     "run-tests:ea:basic_essentials": "node ./scripts/index.js runner entity_analytics basic_license_essentials_tier",
 
+    "initialize-server:asset_inventory:trial_complete": "node ./scripts/index.js server asset_inventory trial_license_complete_tier",
+    "run-tests:asset_inventory:trial_complete": "node ./scripts/index.js runner asset_inventory trial_license_complete_tier",
+
+    "asset_inventory:entity_store:server:ess": "npm run initialize-server:asset_inventory:trial_complete entity_store ess",
+    "asset_inventory:entity_store:runner:ess": "npm run run-tests:asset_inventory:trial_complete entity_store ess essEnv --",
+
     "initialize-server:dr": "node ./scripts/index.js server detections_response trial_license_complete_tier",
     "run-tests:dr": "node ./scripts/index.js runner detections_response trial_license_complete_tier",
 

x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/configs/ess.config.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(
+    require.resolve('../../../../../config/ess/config.base.trial')
+  );
+
+  return {
+    ...functionalConfig.getAll(),
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...functionalConfig.get('kbnTestServer.serverArgs'),
+        `--xpack.securitySolution.enableExperimental=${JSON.stringify([])}`,
+      ],
+    },
+    testFiles: [require.resolve('..')],
+    junit: {
+      reportName: 'Asset Inventory Integration Tests - ESS Env - Basic License',
+    },
+  };
+}

x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Asset Inventory - Entity Store', function () {
+    loadTestFile(require.resolve('./keyword_builder_ingest_pipeline'));
+  });
+}

x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/trial_license_complete_tier/keyword_builder_ingest_pipeline.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+
+import { buildIngestPipeline } from '@kbn/security-solution-plugin/server/lib/asset_inventory/ingest_pipelines';
+import { applyIngestProcessorToDoc } from '../utils/ingest';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
+export default ({ getService }: FtrProviderContext) => {
+  const es = getService('es');
+  const log = getService('log');
+
+  const applyOperatorToDoc = async (docSource: any): Promise<any> => {
+    const steps = buildIngestPipeline();
+
+    return applyIngestProcessorToDoc(steps, docSource, es, log);
+  };
+
+  describe('@ess @skipInServerlessMKI Asset Inventory - Entity store - Keyword builder pipeline', () => {
+    it('should build entities.keyword when entities.metadata is provided ', async () => {
+      const doc = {
+        related: {
+          entity: ['entity-id-1', 'entity-id-2', 'entity-id-3'],
+        },
+        entities: {
+          metadata: {
+            'entity-id-1': {
+              entity: {
+                type: 'SomeType',
+              },
+              cloud: {
+                region: 'us-east-1',
+              },
+              someECSfield: 'someECSfieldValue',
+              SomeNonECSField: 'SomeNonECSValue',
+            },
+            'entity-id-2': {
+              entity: {
+                type: 'SomeOtherType',
+              },
+              SomeNonECSField: 'SomeNonECSValue',
+            },
+            'entity-id-3': {
+              someECSfield: 'someECSfieldValue',
+            },
+          },
+        },
+      };
+
+      const resultDoc = await applyOperatorToDoc(doc);
+
+      expect(resultDoc.entities.keyword).to.eql([
+        '{"entity-id-3":{"someECSfield":"someECSfieldValue"}}',
+        '{"entity-id-2":{"SomeNonECSField":"SomeNonECSValue","entity":{"type":"SomeOtherType"}}}',
+        '{"entity-id-1":{"cloud":{"region":"us-east-1"},"SomeNonECSField":"SomeNonECSValue","someECSfield":"someECSfieldValue","entity":{"type":"SomeType"}}}',
+      ]);
+    });
+  });
+};

x-pack/test/security_solution_api_integration/test_suites/asset_inventory/entity_store/utils/ingest.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { IngestProcessorContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { ToolingLog } from '@kbn/tooling-log';
+
+export const applyIngestProcessorToDoc = async (
+  steps: IngestProcessorContainer[],
+  docSource: any,
+  es: Client,
+  log: ToolingLog
+): Promise<any> => {
+  const doc = {
+    _index: 'index',
+    _id: 'id',
+    _source: docSource,
+  };
+
+  const res = await es.ingest.simulate({
+    pipeline: {
+      description: 'test',
+      processors: steps,
+    },
+    docs: [doc],
+  });
+
+  const firstDoc = res.docs?.[0];
+
+  const error = firstDoc?.error;
+  if (error) {
+    log.error('Full painless error below: ');
+    log.error(JSON.stringify(error, null, 2));
+    throw new Error('Painless error running pipeline see logs for full detail : ' + error?.type);
+  }
+
+  return firstDoc?.doc?._source;
+};