[Security Solution] Use docker for endpoint tests (#73092)

* Copying api integration tests into their own directory

* Removing api integration tests and using ingest docker image

* Fixing typo

* Fixing type errors and empty string and reenabling tests

* Rebuilding docs

* Renaming url override variable

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/developer/architecture/code-exploration.asciidoc

@@ -524,9 +524,9 @@ WARNING: Missing README.
 See Configuring security in Kibana.
 
 
-- {kib-repo}blob/{branch}/x-pack/plugins/security_solution[securitySolution]
+- {kib-repo}blob/{branch}/x-pack/plugins/security_solution/README.md[securitySolution]
 
-WARNING: Missing README.
+Welcome to the Kibana Security Solution plugin! This README will go over getting started with development and testing.
 
 
 - {kib-repo}blob/{branch}/x-pack/plugins/snapshot_restore/README.md[snapshotRestore]

x-pack/plugins/security_solution/README.md

@@ -0,0 +1,130 @@
+# Security Solution
+
+Welcome to the Kibana Security Solution plugin! This README will go over getting started with development and testing.
+
+## Development
+
+## Tests
+
+The endpoint specific tests leverage the ingest manager to install the endpoint package. Before the api integration
+and functional tests are run the ingest manager is initialized. This initialization process includes reaching out to
+a package registry service to install the endpoint package. The endpoint tests support three different ways to run
+the tests given the constraint on an available package registry.
+
+1. Using Docker
+2. Running your own local package registry
+3. Using the default external package registry
+
+These scenarios will be outlined the sections below.
+
+### Endpoint API Integration Tests Location
+
+The endpoint api integration tests are located [here](../../test/security_solution_endpoint_api_int)
+
+### Endpoint Functional Tests Location
+
+The endpoint functional tests are located [here](../../test/security_solution_endpoint)
+
+### Using Docker
+
+To run the tests using the recommended docker image version you must have `docker` installed. The testing infrastructure
+will stand up a docker container using the image defined [here](../../test/ingest_manager_api_integration/config.ts#L15)
+
+Make sure you're in the Kibana root directory.
+
+#### Endpoint API Integration Tests
+
+In one terminal, run:
+
+```bash
+INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+In another terminal, run:
+
+```bash
+INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+#### Endpoint Functional Tests
+
+In one terminal, run:
+
+```bash
+INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:server --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+In another terminal, run:
+
+```bash
+INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+### Running your own package registry
+
+If you are doing endpoint package development it will be useful to run your own package registry to serve the latest package you're building.
+To do this use the following commands:
+
+Make sure you're in the Kibana root directory.
+
+#### Endpoint API Integration Tests
+
+In one terminal, run:
+
+```bash
+PACKAGE_REGISTRY_URL_OVERRIDE=<url to your package registry like http://localhost:8080> yarn test:ftr:server --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+In another terminal, run:
+
+```bash
+PACKAGE_REGISTRY_URL_OVERRIDE=<url to your package registry like http://localhost:8080>  yarn test:ftr:runner --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+#### Endpoint Functional Tests
+
+In one terminal, run:
+
+```bash
+PACKAGE_REGISTRY_URL_OVERRIDE=<url to your package registry like http://localhost:8080> yarn test:ftr:server --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+In another terminal, run:
+
+```bash
+PACKAGE_REGISTRY_URL_OVERRIDE=<url to your package registry like http://localhost:8080>  yarn test:ftr:runner --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+### Using the default public registry
+
+If you don't have docker installed and don't want to run your own registry, you can run the tests using the ingest manager's default public package registry. The actual package registry used is [here](../../plugins/ingest_manager/common/constants/epm.ts#L9)
+
+Make sure you're in the Kibana root directory.
+
+#### Endpoint API Integration Tests
+
+In one terminal, run:
+
+```bash
+yarn test:ftr:server --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+In another terminal, run:
+
+```bash
+yarn test:ftr:runner --config x-pack/test/security_solution_endpoint_api_int/config.ts
+```
+
+#### Endpoint Functional Tests
+
+In one terminal, run:
+
+```bash
+yarn test:ftr:server --config x-pack/test/security_solution_endpoint/config.ts
+```
+
+In another terminal, run:
+
+```bash
+yarn test:ftr:runner --config x-pack/test/security_solution_endpoint/config.ts
+```

x-pack/scripts/functional_tests.js

@@ -53,6 +53,7 @@ const onlyNotInCoverageTests = [
   require.resolve('../test/licensing_plugin/config.legacy.ts'),
   require.resolve('../test/endpoint_api_integration_no_ingest/config.ts'),
   require.resolve('../test/reporting_api_integration/config.js'),
+  require.resolve('../test/security_solution_endpoint_api_int/config.ts'),
   require.resolve('../test/ingest_manager_api_integration/config.ts'),
 ];
 

x-pack/test/api_integration/apis/endpoint/index.ts

@@ -1,22 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-import { FtrProviderContext } from '../../ftr_provider_context';
-
-export default function endpointAPIIntegrationTests({
-  loadTestFile,
-  getService,
-}: FtrProviderContext) {
-  describe('Endpoint plugin', function () {
-    const ingestManager = getService('ingestManager');
-    this.tags(['endpoint']);
-    before(async () => {
-      await ingestManager.setup();
-    });
-    loadTestFile(require.resolve('./resolver'));
-    loadTestFile(require.resolve('./metadata'));
-    loadTestFile(require.resolve('./policy'));
-  });
-}

x-pack/test/api_integration/apis/index.js

@@ -28,7 +28,6 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./lens'));
     loadTestFile(require.resolve('./ml'));
     loadTestFile(require.resolve('./transform'));
-    loadTestFile(require.resolve('./endpoint'));
     loadTestFile(require.resolve('./lists'));
     loadTestFile(require.resolve('./upgrade_assistant'));
   });

x-pack/test/api_integration/services/index.ts

@@ -27,7 +27,6 @@ import { InfraOpsSourceConfigurationProvider } from './infraops_source_configura
 import { InfraLogSourceConfigurationProvider } from './infra_log_source_configuration';
 import { MachineLearningProvider } from './ml';
 import { IngestManagerProvider } from '../../common/services/ingest_manager';
-import { ResolverGeneratorProvider } from './resolver';
 import { TransformProvider } from './transform';
 
 export const services = {
@@ -48,6 +47,5 @@ export const services = {
   usageAPI: UsageAPIProvider,
   ml: MachineLearningProvider,
   ingestManager: IngestManagerProvider,
-  resolverGenerator: ResolverGeneratorProvider,
   transform: TransformProvider,
 };

x-pack/test/ingest_manager_api_integration/config.ts

@@ -9,6 +9,11 @@ import path from 'path';
 import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
 import { defineDockerServersConfig } from '@kbn/test';
 
+// Docker image to use for Ingest Manager API integration tests.
+// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit/48f3935a72b0c5aacc6fec8ef36d559b089a238b
+export const dockerImage =
+  'docker.elastic.co/package-registry/distribution:48f3935a72b0c5aacc6fec8ef36d559b089a238b';
+
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
 
@@ -29,10 +34,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
     )}:/packages/test-packages`,
   ];
 
-  // Docker image to use for Ingest Manager API integration tests.
-  const dockerImage =
-    'docker.elastic.co/package-registry/distribution:184b85f19e8fd14363e36150173d338ff9659f01';
-
   return {
     testFiles: [require.resolve('./apis')],
     servers: xPackAPITestsConfig.get('servers'),

x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts

@@ -6,7 +6,7 @@
 
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../ftr_provider_context';
-import { deleteMetadataStream } from '../../../api_integration/apis/endpoint/data_stream_helper';
+import { deleteMetadataStream } from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
 
 export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const pageObjects = getPageObjects(['common', 'endpoint', 'header', 'endpointPageUtils']);

x-pack/test/security_solution_endpoint/apps/endpoint/index.ts

@@ -3,12 +3,28 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { DEFAULT_REGISTRY_URL } from '../../../../plugins/ingest_manager/common';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import {
+  isRegistryEnabled,
+  getRegistryUrl,
+} from '../../../security_solution_endpoint_api_int/registry';
+
+export default function (providerContext: FtrProviderContext) {
+  const { loadTestFile, getService } = providerContext;
 
-export default function ({ loadTestFile, getService }: FtrProviderContext) {
   describe('endpoint', function () {
     this.tags('ciGroup7');
     const ingestManager = getService('ingestManager');
+    const log = getService('log');
+
+    if (!isRegistryEnabled()) {
+      log.warning('These tests are being run with an external package registry');
+    }
+
+    const registryUrl = getRegistryUrl() ?? DEFAULT_REGISTRY_URL;
+    log.info(`Package registry URL for tests: ${registryUrl}`);
+
     before(async () => {
       await ingestManager.setup();
     });

x-pack/test/security_solution_endpoint/config.ts

@@ -8,6 +8,10 @@ import { resolve } from 'path';
 import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
 import { pageObjects } from './page_objects';
 import { services } from './services';
+import {
+  getRegistryUrlAsArray,
+  createEndpointDockerConfig,
+} from '../security_solution_endpoint_api_int/registry';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xpackFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js'));
@@ -16,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
     ...xpackFunctionalConfig.getAll(),
     pageObjects,
     testFiles: [resolve(__dirname, './apps/endpoint')],
+    dockerServers: createEndpointDockerConfig(),
     junit: {
       reportName: 'X-Pack Endpoint Functional Tests',
     },
@@ -31,6 +36,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       serverArgs: [
         ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
         '--xpack.ingestManager.enabled=true',
+        // if you return an empty string here the kibana server will not start properly but an empty array works
+        ...getRegistryUrlAsArray(),
       ],
     },
   };

x-pack/test/security_solution_endpoint_api_int/apis/artifacts/index.ts

@@ -8,11 +8,8 @@ import expect from '@kbn/expect';
 import { createHash } from 'crypto';
 import { inflateSync } from 'zlib';
 
-import { FtrProviderContext } from '../../../ftr_provider_context';
-import {
-  getSupertestWithoutAuth,
-  setupIngest,
-} from '../../../../ingest_manager_api_integration/apis/fleet/agents/services';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { getSupertestWithoutAuth } from '../../../ingest_manager_api_integration/apis/fleet/agents/services';
 
 export default function (providerContext: FtrProviderContext) {
   const { getService } = providerContext;
@@ -22,7 +19,6 @@ export default function (providerContext: FtrProviderContext) {
   let agentAccessAPIKey: string;
 
   describe('artifact download', () => {
-    setupIngest(providerContext);
     before(async () => {
       await esArchiver.load('endpoint/artifacts/api_feature', { useCreate: true });
 

x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts

@@ -10,7 +10,7 @@ import {
   eventsIndexPattern,
   alertsIndexPattern,
   policyIndexPattern,
-} from '../../../../plugins/security_solution/common/endpoint/constants';
+} from '../../../plugins/security_solution/common/endpoint/constants';
 
 export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
   const client = getService('es');

x-pack/test/security_solution_endpoint_api_int/apis/fixtures/package_registry_config.yml

@@ -0,0 +1,2 @@
+package_paths:
+  - /packages/production

x-pack/test/security_solution_endpoint_api_int/apis/index.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { FtrProviderContext } from '../ftr_provider_context';
+import { isRegistryEnabled, getRegistryUrl } from '../registry';
+import { DEFAULT_REGISTRY_URL } from '../../../plugins/ingest_manager/common';
+
+export default function endpointAPIIntegrationTests(providerContext: FtrProviderContext) {
+  const { loadTestFile, getService } = providerContext;
+
+  describe('Endpoint plugin', function () {
+    const ingestManager = getService('ingestManager');
+
+    this.tags('ciGroup7');
+    const log = getService('log');
+
+    if (!isRegistryEnabled()) {
+      log.warning('These tests are being run with an external package registry');
+    }
+
+    const registryUrl = getRegistryUrl() ?? DEFAULT_REGISTRY_URL;
+    log.info(`Package registry URL for tests: ${registryUrl}`);
+
+    before(async () => {
+      await ingestManager.setup();
+    });
+    loadTestFile(require.resolve('./resolver'));
+    loadTestFile(require.resolve('./metadata'));
+    loadTestFile(require.resolve('./policy'));
+    loadTestFile(require.resolve('./artifacts'));
+  });
+}

x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import expect from '@kbn/expect/expect.js';
-import { FtrProviderContext } from '../../ftr_provider_context';
+import { FtrProviderContext } from '../ftr_provider_context';
 import { deleteMetadataStream } from './data_stream_helper';
 
 /**

x-pack/test/security_solution_endpoint_api_int/apis/policy.ts

@@ -5,7 +5,7 @@
  */
 
 import expect from '@kbn/expect/expect.js';
-import { FtrProviderContext } from '../../ftr_provider_context';
+import { FtrProviderContext } from '../ftr_provider_context';
 import { deletePolicyStream } from './data_stream_helper';
 
 export default function ({ getService }: FtrProviderContext) {

x-pack/test/security_solution_endpoint_api_int/apis/resolver.ts

@@ -16,12 +16,12 @@ import {
   LegacyEndpointEvent,
   ResolverNodeStats,
   ResolverRelatedAlerts,
-} from '../../../../plugins/security_solution/common/endpoint/types';
+} from '../../../plugins/security_solution/common/endpoint/types';
 import {
   parentEntityId,
   eventId,
-} from '../../../../plugins/security_solution/common/endpoint/models/event';
-import { FtrProviderContext } from '../../ftr_provider_context';
+} from '../../../plugins/security_solution/common/endpoint/models/event';
+import { FtrProviderContext } from '../ftr_provider_context';
 import {
   Event,
   Tree,
@@ -29,8 +29,8 @@ import {
   RelatedEventCategory,
   RelatedEventInfo,
   categoryMapping,
-} from '../../../../plugins/security_solution/common/endpoint/generate_data';
-import { Options, GeneratedTrees } from '../../services/resolver';
+} from '../../../plugins/security_solution/common/endpoint/generate_data';
+import { Options, GeneratedTrees } from '../services/resolver';
 
 /**
  * Check that the given lifecycle is in the resolver tree's corresponding map

x-pack/test/security_solution_endpoint_api_int/config.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
+import { createEndpointDockerConfig, getRegistryUrlAsArray } from './registry';
+import { services } from './services';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
+
+  return {
+    ...xPackAPITestsConfig.getAll(),
+    testFiles: [require.resolve('./apis')],
+    dockerServers: createEndpointDockerConfig(),
+    services,
+    junit: {
+      reportName: 'X-Pack Endpoint API Integration Tests',
+    },
+    kbnTestServer: {
+      ...xPackAPITestsConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'),
+        // if you return an empty string here the kibana server will not start properly but an empty array works
+        ...getRegistryUrlAsArray(),
+      ],
+    },
+  };
+}

x-pack/test/security_solution_endpoint_api_int/ftr_provider_context.d.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test/types/ftr';
+
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, {}>;

x-pack/test/security_solution_endpoint_api_int/registry.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import path from 'path';
+
+import { defineDockerServersConfig } from '@kbn/test';
+import { dockerImage as ingestDockerImage } from '../ingest_manager_api_integration/config';
+
+/**
+ * This is used by CI to set the docker registry port
+ * you can also define this environment variable locally when running tests which
+ * will spin up a local docker package registry locally for you
+ * if this is defined it takes precedence over the `packageRegistryOverride` variable
+ */
+const dockerRegistryPort: string | undefined = process.env.INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT;
+
+/**
+ * If you don't want to use the docker image version pinned below and instead want to run your own
+ * registry or use an external registry you can define this environment variable when running
+ * the tests to use that registry url instead.
+ *
+ * This is particularly useful when a developer needs to test a new package against the kibana
+ * integration or functional tests. Instead of having to publish a whole new docker image we
+ * can set this environment variable which will point to the location of where your package registry
+ * is serving the updated package.
+ *
+ * This variable will not and should not be used by CI. CI should always use the pinned docker image below.
+ */
+const packageRegistryOverride: string | undefined = process.env.PACKAGE_REGISTRY_URL_OVERRIDE;
+
+const defaultRegistryConfigPath = path.join(
+  __dirname,
+  './apis/fixtures/package_registry_config.yml'
+);
+
+export function createEndpointDockerConfig(
+  packageRegistryConfig: string = defaultRegistryConfigPath,
+  dockerImage: string = ingestDockerImage,
+  dockerArgs: string[] = []
+) {
+  const args: string[] = [
+    '-v',
+    `${packageRegistryConfig}:/package-registry/config.yml`,
+    ...dockerArgs,
+  ];
+  return defineDockerServersConfig({
+    registry: {
+      enabled: !!dockerRegistryPort,
+      image: dockerImage,
+      portInContainer: 8080,
+      port: dockerRegistryPort,
+      args,
+      waitForLogLine: 'package manifests loaded',
+    },
+  });
+}
+
+export function getRegistryUrl(): string | undefined {
+  let registryUrl: string | undefined;
+  if (dockerRegistryPort !== undefined) {
+    registryUrl = `--xpack.ingestManager.registryUrl=http://localhost:${dockerRegistryPort}`;
+  } else if (packageRegistryOverride !== undefined) {
+    registryUrl = `--xpack.ingestManager.registryUrl=${packageRegistryOverride}`;
+  }
+  return registryUrl;
+}
+
+export function getRegistryUrlAsArray(): string[] {
+  const registryUrl: string | undefined = getRegistryUrl();
+  return registryUrl !== undefined ? [registryUrl] : [];
+}
+
+export function isRegistryEnabled() {
+  return getRegistryUrl() !== undefined;
+}

x-pack/test/security_solution_endpoint_api_int/services/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { services as xPackAPIServices } from '../../api_integration/services';
+import { ResolverGeneratorProvider } from './resolver';
+
+export const services = {
+  ...xPackAPIServices,
+  resolverGenerator: ResolverGeneratorProvider,
+};