Add mock identity provider for serverless (2nd attempt) (#171513)

Attempting to merge #170852 again now that the release artefact step has
been fixed as part of https://github.com/elastic/kibana/pull/171457

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Aleh Zasypkin <aleh.zasypkin@gmail.com>
Co-authored-by: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>

.github/CODEOWNERS

@@ -538,6 +538,7 @@ x-pack/packages/ml/string_hash @elastic/ml-ui
 x-pack/packages/ml/trained_models_utils @elastic/ml-ui
 x-pack/packages/ml/ui_actions @elastic/ml-ui
 x-pack/packages/ml/url_state @elastic/ml-ui
+packages/kbn-mock-idp-plugin @elastic/kibana-security
 packages/kbn-monaco @elastic/appex-sharedux
 x-pack/plugins/monitoring_collection @elastic/obs-ux-infra_services-team
 x-pack/plugins/monitoring @elastic/obs-ux-infra_services-team

package.json

@@ -1238,6 +1238,7 @@
     "@kbn/managed-vscode-config": "link:packages/kbn-managed-vscode-config",
     "@kbn/managed-vscode-config-cli": "link:packages/kbn-managed-vscode-config-cli",
     "@kbn/management-storybook-config": "link:packages/kbn-management/storybook/config",
+    "@kbn/mock-idp-plugin": "link:packages/kbn-mock-idp-plugin",
     "@kbn/openapi-generator": "link:packages/kbn-openapi-generator",
     "@kbn/optimizer": "link:packages/kbn-optimizer",
     "@kbn/optimizer-webpack-helpers": "link:packages/kbn-optimizer-webpack-helpers",

packages/kbn-es/src/cli_commands/serverless.ts

@@ -38,6 +38,7 @@ export const serverless: Command = {
       --host              Publish ES docker container on additional host IP
       --port              The port to bind to on 127.0.0.1 [default: ${DEFAULT_PORT}]
       --ssl               Enable HTTP SSL on the ES cluster
+      --kibanaUrl         Fully qualified URL where Kibana is hosted (including base path). [default: https://localhost:5601/]
       --skipTeardown      If this process exits, leave the ES cluster running in the background
       --waitForReady      Wait for the ES cluster to be ready to serve requests
       --resources         Overrides resources under ES 'config/' directory, which are by default
@@ -73,7 +74,7 @@ export const serverless: Command = {
         files: 'F',
       },
 
-      string: ['tag', 'image', 'basePath', 'resources', 'host'],
+      string: ['tag', 'image', 'basePath', 'resources', 'host', 'kibanaUrl'],
       boolean: ['clean', 'ssl', 'kill', 'background', 'skipTeardown', 'waitForReady'],
 
       default: defaults,

packages/kbn-es/src/paths.ts

@@ -8,6 +8,7 @@
 
 import Os from 'os';
 import { resolve } from 'path';
+import { REPO_ROOT } from '@kbn/repo-info';
 
 function maybeUseBat(bin: string) {
   return Os.platform().startsWith('win') ? `${bin}.bat` : bin;
@@ -51,6 +52,8 @@ export const SERVERLESS_SECRETS_SSL_PATH = resolve(
 
 export const SERVERLESS_JWKS_PATH = resolve(__dirname, './serverless_resources/jwks.json');
 
+export const SERVERLESS_IDP_METADATA_PATH = resolve(REPO_ROOT, '.es', 'idp_metadata.xml');
+
 export const SERVERLESS_RESOURCES_PATHS = [
   SERVERLESS_OPERATOR_USERS_PATH,
   SERVERLESS_ROLE_MAPPING_PATH,

packages/kbn-es/src/utils/docker.test.ts

@@ -32,15 +32,17 @@ import {
   ServerlessOptions,
 } from './docker';
 import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log';
-import { ES_P12_PATH } from '@kbn/dev-utils';
+import { CA_CERT_PATH, ES_P12_PATH } from '@kbn/dev-utils';
 import {
   SERVERLESS_CONFIG_PATH,
   SERVERLESS_RESOURCES_PATHS,
   SERVERLESS_SECRETS_PATH,
   SERVERLESS_JWKS_PATH,
+  SERVERLESS_IDP_METADATA_PATH,
 } from '../paths';
 import * as waitClusterUtil from './wait_until_cluster_ready';
 import * as waitForSecurityIndexUtil from './wait_for_security_index';
+import * as mockIdpPluginUtil from '@kbn/mock-idp-plugin/common';
 
 jest.mock('execa');
 const execa = jest.requireMock('execa');
@@ -58,6 +60,8 @@ jest.mock('./wait_for_security_index', () => ({
   waitForSecurityIndex: jest.fn(),
 }));
 
+jest.mock('@kbn/mock-idp-plugin/common');
+
 const log = new ToolingLog();
 const logWriter = new ToolingLogCollectingWriter();
 log.setWriters([logWriter]);
@@ -69,6 +73,8 @@ const serverlessObjectStorePath = `${baseEsPath}/${serverlessDir}`;
 
 const waitUntilClusterReadyMock = jest.spyOn(waitClusterUtil, 'waitUntilClusterReady');
 const waitForSecurityIndexMock = jest.spyOn(waitForSecurityIndexUtil, 'waitForSecurityIndex');
+const ensureSAMLRoleMappingMock = jest.spyOn(mockIdpPluginUtil, 'ensureSAMLRoleMapping');
+const createMockIdpMetadataMock = jest.spyOn(mockIdpPluginUtil, 'createMockIdpMetadata');
 
 beforeEach(() => {
   jest.resetAllMocks();
@@ -423,6 +429,66 @@ describe('resolveEsArgs()', () => {
       ]
     `);
   });
+
+  test('should add SAML realm args when kibanaUrl and SSL are passed', () => {
+    const esArgs = resolveEsArgs([], {
+      ssl: true,
+      kibanaUrl: 'https://localhost:5601/',
+    });
+
+    expect(esArgs).toHaveLength(26);
+    expect(esArgs).toMatchInlineSnapshot(`
+      Array [
+        "--env",
+        "xpack.security.http.ssl.enabled=true",
+        "--env",
+        "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elasticsearch.p12",
+        "--env",
+        "xpack.security.http.ssl.verification_mode=certificate",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.order=0",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.idp.metadata.path=/usr/share/elasticsearch/config/secrets/idp_metadata.xml",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.idp.entity_id=urn:mock-idp",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.sp.entity_id=https://localhost:5601",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.sp.acs=https://localhost:5601/api/security/saml/callback",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.sp.logout=https://localhost:5601/logout",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.attributes.principal=http://saml.elastic-cloud.com/attributes/principal",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.attributes.groups=http://saml.elastic-cloud.com/attributes/roles",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.attributes.name=http://saml.elastic-cloud.com/attributes/email",
+        "--env",
+        "xpack.security.authc.realms.saml.mock-idp.attributes.mail=http://saml.elastic-cloud.com/attributes/name",
+      ]
+    `);
+  });
+
+  test('should not add SAML realm args when security is disabled', () => {
+    const esArgs = resolveEsArgs([['xpack.security.enabled', 'false']], {
+      ssl: true,
+      kibanaUrl: 'https://localhost:5601/',
+    });
+
+    expect(esArgs).toHaveLength(8);
+    expect(esArgs).toMatchInlineSnapshot(`
+      Array [
+        "--env",
+        "xpack.security.enabled=false",
+        "--env",
+        "xpack.security.http.ssl.enabled=true",
+        "--env",
+        "xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elasticsearch.p12",
+        "--env",
+        "xpack.security.http.ssl.verification_mode=certificate",
+      ]
+    `);
+  });
 });
 
 describe('setupServerlessVolumes()', () => {
@@ -463,21 +529,29 @@ describe('setupServerlessVolumes()', () => {
     expect(existsSync(`${serverlessObjectStorePath}/cluster_state/lease`)).toBe(false);
   });
 
-  test('should add SSL volumes when ssl is passed', async () => {
+  test('should add SSL and IDP metadata volumes when ssl and kibanaUrl are passed', async () => {
     mockFs(existingObjectStore);
+    createMockIdpMetadataMock.mockResolvedValue('<xml/>');
 
-    const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath, ssl: true });
+    const volumeCmd = await setupServerlessVolumes(log, {
+      basePath: baseEsPath,
+      ssl: true,
+      kibanaUrl: 'https://localhost:5603/',
+    });
+
+    expect(createMockIdpMetadataMock).toHaveBeenCalledTimes(1);
+    expect(createMockIdpMetadataMock).toHaveBeenCalledWith('https://localhost:5603/');
 
     const requiredPaths = [
       `${baseEsPath}:/objectstore:z`,
+      SERVERLESS_IDP_METADATA_PATH,
       ES_P12_PATH,
       ...SERVERLESS_RESOURCES_PATHS,
     ];
     const pathsNotIncludedInCmd = requiredPaths.filter(
       (path) => !volumeCmd.some((cmd) => cmd.includes(path))
     );
-
-    expect(volumeCmd).toHaveLength(20);
+    expect(volumeCmd).toHaveLength(22);
     expect(pathsNotIncludedInCmd).toEqual([]);
   });
 
@@ -543,6 +617,7 @@ describe('runServerlessEsNode()', () => {
 
 describe('runServerlessCluster()', () => {
   test('should start 3 serverless nodes', async () => {
+    waitUntilClusterReadyMock.mockResolvedValue();
     mockFs({
       [baseEsPath]: {},
     });
@@ -567,7 +642,27 @@ describe('runServerlessCluster()', () => {
     expect(waitUntilClusterReadyMock.mock.calls[0][0].readyTimeout).toEqual(undefined);
   });
 
+  test(`should create SAML role mapping when ssl and kibanaUrl are passed`, async () => {
+    waitUntilClusterReadyMock.mockResolvedValue();
+    mockFs({
+      [CA_CERT_PATH]: '',
+      [baseEsPath]: {},
+    });
+    execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
+    createMockIdpMetadataMock.mockResolvedValue('<xml/>');
+
+    await runServerlessCluster(log, {
+      basePath: baseEsPath,
+      waitForReady: true,
+      ssl: true,
+      kibanaUrl: 'https://localhost:5601/',
+    });
+
+    expect(ensureSAMLRoleMappingMock).toHaveBeenCalledTimes(1);
+  });
+
   test(`should wait for the security index`, async () => {
+    waitUntilClusterReadyMock.mockResolvedValue();
     waitForSecurityIndexMock.mockResolvedValue();
     mockFs({
       [baseEsPath]: {},
@@ -580,6 +675,7 @@ describe('runServerlessCluster()', () => {
   });
 
   test(`should not wait for the security index when security is disabled`, async () => {
+    waitUntilClusterReadyMock.mockResolvedValue();
     mockFs({
       [baseEsPath]: {},
     });

packages/kbn-es/src/utils/docker.ts

@@ -14,12 +14,17 @@ import { Client, ClientOptions, HttpConnection } from '@elastic/elasticsearch';
 
 import { ToolingLog } from '@kbn/tooling-log';
 import { kibanaPackageJson as pkg, REPO_ROOT } from '@kbn/repo-info';
+import { CA_CERT_PATH, ES_P12_PASSWORD, ES_P12_PATH } from '@kbn/dev-utils';
 import {
-  CA_CERT_PATH,
-  ES_P12_PASSWORD,
-  ES_P12_PATH,
-  kibanaDevServiceAccount,
-} from '@kbn/dev-utils';
+  MOCK_IDP_REALM_NAME,
+  MOCK_IDP_ENTITY_ID,
+  MOCK_IDP_ATTRIBUTE_PRINCIPAL,
+  MOCK_IDP_ATTRIBUTE_ROLES,
+  MOCK_IDP_ATTRIBUTE_EMAIL,
+  MOCK_IDP_ATTRIBUTE_NAME,
+  ensureSAMLRoleMapping,
+  createMockIdpMetadata,
+} from '@kbn/mock-idp-plugin/common';
 
 import { waitForSecurityIndex } from './wait_for_security_index';
 import { createCliError } from '../errors';
@@ -28,6 +33,7 @@ import {
   SERVERLESS_RESOURCES_PATHS,
   SERVERLESS_SECRETS_PATH,
   SERVERLESS_JWKS_PATH,
+  SERVERLESS_IDP_METADATA_PATH,
   SERVERLESS_CONFIG_PATH,
   SERVERLESS_FILES_PATH,
   SERVERLESS_SECRETS_SSL_PATH,
@@ -69,6 +75,8 @@ export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions {
   background?: boolean;
   /** Wait for the ES cluster to be ready to serve requests */
   waitForReady?: boolean;
+  /** Fully qualified URL where Kibana is hosted (including base path) */
+  kibanaUrl?: string;
   /**
    * Resource file(s) to overwrite
    * (see list of files that can be overwritten under `packages/kbn-es/src/serverless_resources/users`)
@@ -460,6 +468,54 @@ export function resolveEsArgs(
     esArgs.set('ELASTIC_PASSWORD', password);
   }
 
+  // Configure mock identify provider (ES only supports SAML when running in SSL mode)
+  if (
+    ssl &&
+    'kibanaUrl' in options &&
+    options.kibanaUrl &&
+    esArgs.get('xpack.security.enabled') !== 'false'
+  ) {
+    const trimTrailingSlash = (url: string) => (url.endsWith('/') ? url.slice(0, -1) : url);
+
+    esArgs.set(`xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.order`, '0');
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.metadata.path`,
+      `${SERVERLESS_CONFIG_PATH}secrets/idp_metadata.xml`
+    );
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.entity_id`,
+      MOCK_IDP_ENTITY_ID
+    );
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.entity_id`,
+      trimTrailingSlash(options.kibanaUrl)
+    );
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.acs`,
+      `${trimTrailingSlash(options.kibanaUrl)}/api/security/saml/callback`
+    );
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.logout`,
+      `${trimTrailingSlash(options.kibanaUrl)}/logout`
+    );
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.principal`,
+      MOCK_IDP_ATTRIBUTE_PRINCIPAL
+    );
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.groups`,
+      MOCK_IDP_ATTRIBUTE_ROLES
+    );
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.name`,
+      MOCK_IDP_ATTRIBUTE_EMAIL
+    );
+    esArgs.set(
+      `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.mail`,
+      MOCK_IDP_ATTRIBUTE_NAME
+    );
+  }
+
   return Array.from(esArgs).flatMap((e) => ['--env', e.join('=')]);
 }
 
@@ -480,7 +536,7 @@ export function getDockerFileMountPath(hostPath: string) {
  * Setup local volumes for Serverless ES
  */
 export async function setupServerlessVolumes(log: ToolingLog, options: ServerlessOptions) {
-  const { basePath, clean, ssl, files, resources } = options;
+  const { basePath, clean, ssl, kibanaUrl, files, resources } = options;
   const objectStorePath = resolve(basePath, 'stateless');
 
   log.info(chalk.bold(`Checking for local serverless ES object store at ${objectStorePath}`));
@@ -551,6 +607,16 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
     );
   }
 
+  // Create and add meta data for mock identity provider
+  if (ssl && kibanaUrl) {
+    const metadata = await createMockIdpMetadata(kibanaUrl);
+    await Fsp.writeFile(SERVERLESS_IDP_METADATA_PATH, metadata);
+    volumeCmds.push(
+      '--volume',
+      `${SERVERLESS_IDP_METADATA_PATH}:${SERVERLESS_CONFIG_PATH}secrets/idp_metadata.xml:z`
+    );
+  }
+
   volumeCmds.push(
     ...getESp12Volume(),
     ...serverlessResources,
@@ -559,7 +625,6 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
     `${
       ssl ? SERVERLESS_SECRETS_SSL_PATH : SERVERLESS_SECRETS_PATH
     }:${SERVERLESS_CONFIG_PATH}secrets/secrets.json:z`,
-
     '--volume',
     `${SERVERLESS_JWKS_PATH}:${SERVERLESS_CONFIG_PATH}secrets/jwks.json:z`
   );
@@ -661,33 +726,52 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
     process.on('SIGINT', () => teardownServerlessClusterSync(log, options));
   }
 
+  const esNodeUrl = `${options.ssl ? 'https' : 'http'}://${portCmd[1].substring(
+    0,
+    portCmd[1].lastIndexOf(':')
+  )}`;
+
+  const client = getESClient({
+    node: esNodeUrl,
+    auth: {
+      username: ELASTIC_SERVERLESS_SUPERUSER,
+      password: ELASTIC_SERVERLESS_SUPERUSER_PASSWORD,
+    },
+    ...(options.ssl
+      ? {
+          tls: {
+            ca: [fs.readFileSync(CA_CERT_PATH)],
+            // NOTE: Even though we've added ca into the tls options, we are using 127.0.0.1 instead of localhost
+            // for the ip which is not validated. As such we are getting the error
+            // Hostname/IP does not match certificate's altnames: IP: 127.0.0.1 is not in the cert's list:
+            // To work around that we are overriding the function checkServerIdentity too
+            checkServerIdentity: () => {
+              return undefined;
+            },
+          },
+        }
+      : {}),
+  });
+
+  const readyPromise = waitUntilClusterReady({ client, expectedStatus: 'green', log }).then(
+    async () => {
+      if (!options.ssl || !options.kibanaUrl) {
+        return;
+      }
+
+      await ensureSAMLRoleMapping(client);
+
+      log.success(
+        `Created role mapping for mock identity provider. You can now login using ${chalk.bold.cyan(
+          MOCK_IDP_REALM_NAME
+        )} realm`
+      );
+    }
+  );
+
   if (options.waitForReady) {
     log.info('Waiting until ES is ready to serve requests...');
-
-    const esNodeUrl = `${options.ssl ? 'https' : 'http'}://${portCmd[1].substring(
-      0,
-      portCmd[1].lastIndexOf(':')
-    )}`;
-
-    const client = getESClient({
-      node: esNodeUrl,
-      auth: { bearer: kibanaDevServiceAccount.token },
-      ...(options.ssl
-        ? {
-            tls: {
-              ca: [fs.readFileSync(CA_CERT_PATH)],
-              // NOTE: Even though we've added ca into the tls options, we are using 127.0.0.1 instead of localhost
-              // for the ip which is not validated. As such we are getting the error
-              // Hostname/IP does not match certificate's altnames: IP: 127.0.0.1 is not in the cert's list:
-              // To work around that we are overriding the function checkServerIdentity too
-              checkServerIdentity: () => {
-                return undefined;
-              },
-            },
-          }
-        : {}),
-    });
-    await waitUntilClusterReady({ client, expectedStatus: 'green', log });
+    await readyPromise;
     if (!options.esArgs || !options.esArgs.includes('xpack.security.enabled=false')) {
       // If security is not disabled, make sure the security index exists before running the test to avoid flakiness
       await waitForSecurityIndex({ client, log });

packages/kbn-es/tsconfig.json

@@ -3,13 +3,20 @@
   "compilerOptions": {
     "outDir": "target/types"
   },
-  "include": ["**/*.ts", "**/*.js", "**/*.json"],
-  "exclude": ["target/**/*"],
+  "include": [
+    "**/*.ts",
+    "**/*.js",
+    "**/*.json"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
   "kbn_references": [
     "@kbn/tooling-log",
     "@kbn/dev-utils",
     "@kbn/dev-proc-runner",
     "@kbn/ci-stats-reporter",
+    "@kbn/mock-idp-plugin",
     "@kbn/jest-serializers",
     "@kbn/repo-info"
   ]

packages/kbn-mock-idp-plugin/common/constants.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { resolve } from 'path';
+
+export const MOCK_IDP_PLUGIN_PATH = resolve(__dirname, '..');
+export const MOCK_IDP_METADATA_PATH = resolve(MOCK_IDP_PLUGIN_PATH, 'metadata.xml');
+
+export const MOCK_IDP_LOGIN_PATH = '/mock_idp/login';
+export const MOCK_IDP_LOGOUT_PATH = '/mock_idp/logout';
+
+export const MOCK_IDP_REALM_NAME = 'mock-idp';
+export const MOCK_IDP_ENTITY_ID = 'urn:mock-idp'; // Must match `entityID` in `metadata.xml`
+export const MOCK_IDP_ROLE_MAPPING_NAME = 'mock-idp-mapping';
+
+export const MOCK_IDP_ATTRIBUTE_PRINCIPAL = 'http://saml.elastic-cloud.com/attributes/principal';
+export const MOCK_IDP_ATTRIBUTE_ROLES = 'http://saml.elastic-cloud.com/attributes/roles';
+export const MOCK_IDP_ATTRIBUTE_EMAIL = 'http://saml.elastic-cloud.com/attributes/email';
+export const MOCK_IDP_ATTRIBUTE_NAME = 'http://saml.elastic-cloud.com/attributes/name';

packages/kbn-mock-idp-plugin/common/index.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export {
+  MOCK_IDP_PLUGIN_PATH,
+  MOCK_IDP_METADATA_PATH,
+  MOCK_IDP_LOGIN_PATH,
+  MOCK_IDP_LOGOUT_PATH,
+  MOCK_IDP_REALM_NAME,
+  MOCK_IDP_ENTITY_ID,
+  MOCK_IDP_ROLE_MAPPING_NAME,
+  MOCK_IDP_ATTRIBUTE_PRINCIPAL,
+  MOCK_IDP_ATTRIBUTE_ROLES,
+  MOCK_IDP_ATTRIBUTE_EMAIL,
+  MOCK_IDP_ATTRIBUTE_NAME,
+} from './constants';
+export {
+  createMockIdpMetadata,
+  createSAMLResponse,
+  ensureSAMLRoleMapping,
+  parseSAMLAuthnRequest,
+} from './utils';

packages/kbn-mock-idp-plugin/common/utils.ts

@@ -0,0 +1,231 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { SignedXml } from 'xml-crypto';
+import { KBN_KEY_PATH, KBN_CERT_PATH } from '@kbn/dev-utils';
+import { readFile } from 'fs/promises';
+import zlib from 'zlib';
+import { promisify } from 'util';
+import { parseString } from 'xml2js';
+import { X509Certificate } from 'crypto';
+
+import {
+  MOCK_IDP_REALM_NAME,
+  MOCK_IDP_ENTITY_ID,
+  MOCK_IDP_ROLE_MAPPING_NAME,
+  MOCK_IDP_ATTRIBUTE_PRINCIPAL,
+  MOCK_IDP_ATTRIBUTE_ROLES,
+  MOCK_IDP_ATTRIBUTE_EMAIL,
+  MOCK_IDP_ATTRIBUTE_NAME,
+  MOCK_IDP_LOGIN_PATH,
+  MOCK_IDP_LOGOUT_PATH,
+} from './constants';
+
+const inflateRawAsync = promisify(zlib.inflateRaw);
+const parseStringAsync = promisify(parseString);
+
+/**
+ * Creates XML metadata for our mock identity provider.
+ *
+ * This can be saved to file and used to configure Elasticsearch SAML realm.
+ *
+ * @param kibanaUrl Fully qualified URL where Kibana is hosted (including base path)
+ */
+export async function createMockIdpMetadata(kibanaUrl: string) {
+  const signingKey = await readFile(KBN_CERT_PATH);
+  const cert = new X509Certificate(signingKey);
+  const trimTrailingSlash = (url: string) => (url.endsWith('/') ? url.slice(0, -1) : url);
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+  <md:EntityDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata"
+    entityID="${MOCK_IDP_ENTITY_ID}">
+    <md:IDPSSODescriptor WantAuthnRequestsSigned="false"
+      protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol">
+      <md:KeyDescriptor use="signing">
+        <ds:KeyInfo xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
+          <ds:X509Data>
+            <ds:X509Certificate>${cert.raw.toString('base64')}</ds:X509Certificate>
+          </ds:X509Data>
+        </ds:KeyInfo>
+      </md:KeyDescriptor>
+      <md:SingleLogoutService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
+        Location="${trimTrailingSlash(kibanaUrl)}${MOCK_IDP_LOGOUT_PATH}" />
+      <md:SingleLogoutService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
+        Location="${trimTrailingSlash(kibanaUrl)}${MOCK_IDP_LOGOUT_PATH}" />
+      <md:SingleSignOnService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
+        Location="${trimTrailingSlash(kibanaUrl)}${MOCK_IDP_LOGIN_PATH}" />
+      <md:SingleSignOnService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
+        Location="${trimTrailingSlash(kibanaUrl)}${MOCK_IDP_LOGIN_PATH}" />
+    </md:IDPSSODescriptor>
+  </md:EntityDescriptor>
+  `;
+}
+
+/**
+ * Creates a SAML response that can be passed directly to the Kibana ACS endpoint to authenticate a user.
+ *
+ * @example Create a SAML response.
+ *
+ * ```ts
+ * const samlResponse = await createSAMLResponse({
+ *    username: '1234567890',
+ *    email: 'mail@elastic.co',
+ *    fullname: 'Test User',
+ *    roles: ['t1_analyst', 'editor'],
+ *  })
+ * ```
+ *
+ * @example Authenticate user with SAML response.
+ *
+ * ```ts
+ * fetch('/api/security/saml/callback', {
+ *   method: 'POST',
+ *   body: JSON.stringify({ SAMLResponse: samlResponse }),
+ *   redirect: 'manual'
+ * })
+ * ```
+ */
+export async function createSAMLResponse(options: {
+  /** Fully qualified URL where Kibana is hosted (including base path) */
+  kibanaUrl: string;
+  /** ID from SAML authentication request */
+  authnRequestId?: string;
+  username: string;
+  fullname?: string;
+  email?: string;
+  roles: string[];
+}) {
+  const issueInstant = new Date().toISOString();
+  const notOnOrAfter = new Date(Date.now() + 3600 * 1000).toISOString();
+
+  const samlAssertionTemplateXML = `
+    <saml:Assertion xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" Version="2.0" ID="_RPs1WfOkul8lZ72DtJtes0BKyPgaCamg" IssueInstant="${issueInstant}">
+      <saml:Issuer>${MOCK_IDP_ENTITY_ID}</saml:Issuer>
+      <saml:Subject>
+        <saml:NameID Format="urn:oasis:names:tc:SAML:1.1:nameid-format:transient">_643ec1b3f5673583b9f9a1e9e73a36daa2a3748f</saml:NameID>
+        <saml:SubjectConfirmation Method="urn:oasis:names:tc:SAML:2.0:cm:bearer">
+          <saml:SubjectConfirmationData NotOnOrAfter="${notOnOrAfter}" ${
+    options.authnRequestId ? `InResponseTo="${options.authnRequestId}"` : ''
+  } Recipient="${options.kibanaUrl}" />
+        </saml:SubjectConfirmation>
+      </saml:Subject>
+      <saml:AuthnStatement AuthnInstant="${issueInstant}" SessionIndex="4464894646681600">
+        <saml:AuthnContext>
+          <saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:unspecified</saml:AuthnContextClassRef>
+        </saml:AuthnContext>
+      </saml:AuthnStatement>
+      <saml:AttributeStatement xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+        <saml:Attribute Name="${MOCK_IDP_ATTRIBUTE_PRINCIPAL}" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">
+          <saml:AttributeValue xsi:type="xs:string">${options.username}</saml:AttributeValue>
+        </saml:Attribute>
+        <saml:Attribute Name="${MOCK_IDP_ATTRIBUTE_ROLES}" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">
+          ${options.roles
+            .map(
+              (role) => `<saml:AttributeValue xsi:type="xs:string">${role}</saml:AttributeValue>`
+            )
+            .join('')}
+        </saml:Attribute>
+        ${
+          options.email
+            ? `<saml:Attribute Name="${MOCK_IDP_ATTRIBUTE_EMAIL}" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">
+        <saml:AttributeValue xsi:type="xs:string">${options.email}</saml:AttributeValue>
+      </saml:Attribute>`
+            : ''
+        }
+        ${
+          options.fullname
+            ? `<saml:Attribute Name="${MOCK_IDP_ATTRIBUTE_NAME}" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:uri">
+        <saml:AttributeValue xsi:type="xs:string">${options.fullname}</saml:AttributeValue>
+      </saml:Attribute>`
+            : ''
+        }
+      </saml:AttributeStatement>
+    </saml:Assertion>
+  `;
+
+  const signature = new SignedXml();
+  signature.signatureAlgorithm = 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha256';
+  signature.signingKey = await readFile(KBN_KEY_PATH);
+
+  // Adds a reference to a `Assertion` xml element and an array of transform algorithms to be used during signing.
+  signature.addReference(
+    `//*[local-name(.)='Assertion']`,
+    [
+      'http://www.w3.org/2000/09/xmldsig#enveloped-signature',
+      'http://www.w3.org/2001/10/xml-exc-c14n#',
+    ],
+    'http://www.w3.org/2001/04/xmlenc#sha256'
+  );
+
+  signature.computeSignature(samlAssertionTemplateXML, {
+    location: { reference: `//*[local-name(.)='Issuer']`, action: 'after' },
+  });
+
+  const value = await Buffer.from(
+    `
+    <samlp:Response xmlns:samlp="urn:oasis:names:tc:SAML:2.0:protocol" ID="_bdf1d51245ed0f71aa23" Version="2.0" IssueInstant="${issueInstant}" ${
+      options.authnRequestId ? `InResponseTo="${options.authnRequestId}"` : ''
+    } Destination="${options.kibanaUrl}">
+      <saml:Issuer xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion">${MOCK_IDP_ENTITY_ID}</saml:Issuer>
+      <samlp:Status>
+        <samlp:StatusCode Value="urn:oasis:names:tc:SAML:2.0:status:Success"/>
+      </samlp:Status>${signature.getSignedXml()}
+    </samlp:Response>
+  `
+  ).toString('base64');
+
+  return value;
+}
+
+/**
+ * Creates the role mapping required for developers to authenticate using SAML.
+ */
+export async function ensureSAMLRoleMapping(client: Client) {
+  return client.transport.request({
+    method: 'PUT',
+    path: `/_security/role_mapping/${MOCK_IDP_ROLE_MAPPING_NAME}`,
+    body: {
+      enabled: true,
+      role_templates: [
+        {
+          template: '{"source":"{{#tojson}}groups{{/tojson}}"}',
+          format: 'json',
+        },
+      ],
+      rules: {
+        all: [
+          {
+            field: {
+              'realm.name': MOCK_IDP_REALM_NAME,
+            },
+          },
+        ],
+      },
+    },
+  });
+}
+
+interface SAMLAuthnRequest {
+  'saml2p:AuthnRequest': {
+    $: {
+      AssertionConsumerServiceURL: string;
+      Destination: string;
+      ID: string;
+      IssueInstant: string;
+    };
+  };
+}
+
+export async function parseSAMLAuthnRequest(samlRequest: string) {
+  const inflatedSAMLRequest = (await inflateRawAsync(Buffer.from(samlRequest, 'base64'))) as Buffer;
+  const parsedSAMLRequest = (await parseStringAsync(
+    inflatedSAMLRequest.toString()
+  )) as SAMLAuthnRequest;
+  return parsedSAMLRequest['saml2p:AuthnRequest'].$;
+}

packages/kbn-mock-idp-plugin/kibana.jsonc

@@ -0,0 +1,11 @@
+{
+  "type": "plugin",
+  "id": "@kbn/mock-idp-plugin",
+  "owner": "@elastic/kibana-security",
+  "devOnly": true,
+  "plugin": {
+    "id": "mockIdpPlugin",
+    "server": true,
+    "browser": false
+  }
+}

packages/kbn-mock-idp-plugin/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/mock-idp-plugin",
+  "private": true,
+  "version": "1.0.0",
+  "license": "SSPL-1.0 OR Elastic License 2.0"
+}

packages/kbn-mock-idp-plugin/server/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { plugin } from './plugin';

packages/kbn-mock-idp-plugin/server/plugin.ts

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { PluginInitializer, Plugin } from '@kbn/core-plugins-server';
+import { schema } from '@kbn/config-schema';
+
+import {
+  MOCK_IDP_LOGIN_PATH,
+  MOCK_IDP_LOGOUT_PATH,
+  createSAMLResponse,
+  parseSAMLAuthnRequest,
+} from '../common';
+
+export const plugin: PluginInitializer<void, void> = async (): Promise<Plugin> => ({
+  setup(core) {
+    core.http.resources.register(
+      {
+        path: MOCK_IDP_LOGIN_PATH,
+        validate: {
+          query: schema.object({
+            SAMLRequest: schema.string(),
+          }),
+        },
+        options: { authRequired: false },
+      },
+      async (context, request, response) => {
+        let samlRequest: Awaited<ReturnType<typeof parseSAMLAuthnRequest>>;
+        try {
+          samlRequest = await parseSAMLAuthnRequest(request.query.SAMLRequest);
+        } catch (error) {
+          return response.badRequest({
+            body: '[request query.SAMLRequest]: value is not valid SAMLRequest.',
+          });
+        }
+
+        const userRoles: Array<[string, string]> = [
+          ['system_indices_superuser', 'system_indices_superuser'],
+          ['t1_analyst', 't1_analyst'],
+          ['t2_analyst', 't2_analyst'],
+          ['t3_analyst', 't3_analyst'],
+          ['threat_intelligence_analyst', 'threat_intelligence_analyst'],
+          ['rule_author', 'rule_author'],
+          ['soc_manager', 'soc_manager'],
+          ['detections_admin', 'detections_admin'],
+          ['platform_engineer', 'platform_engineer'],
+          ['endpoint_operations_analyst', 'endpoint_operations_analyst'],
+          ['endpoint_policy_manager', 'endpoint_policy_manager'],
+        ];
+
+        const samlResponses = await Promise.all(
+          userRoles.map(([username, role]) =>
+            createSAMLResponse({
+              authnRequestId: samlRequest.ID,
+              kibanaUrl: samlRequest.AssertionConsumerServiceURL,
+              username,
+              roles: [role],
+            })
+          )
+        );
+
+        return response.renderHtml({
+          body: `
+            <!DOCTYPE html>
+            <title>Mock Identity Provider</title>
+            <link rel="icon" href="data:,">
+            <body>
+              <h2>Mock Identity Provider</h2>
+              <form id="loginForm" method="post" action="${
+                samlRequest.AssertionConsumerServiceURL
+              }">
+                <h3>Pick a role:<h3>
+                <ul>
+                  ${userRoles
+                    .map(
+                      ([username], i) =>
+                        `
+                    <li>
+                      <button name="SAMLResponse" value="${samlResponses[i]}">${username}</button>
+                    </li>
+                    `
+                    )
+                    .join('')}
+                </ul>
+              </form>
+            </body>
+          `,
+        });
+      }
+    );
+
+    core.http.resources.register(
+      {
+        path: `${MOCK_IDP_LOGIN_PATH}/submit.js`,
+        validate: false,
+        options: { authRequired: false },
+      },
+      (context, request, response) => {
+        return response.renderJs({ body: 'document.getElementById("loginForm").submit();' });
+      }
+    );
+
+    core.http.resources.register(
+      {
+        path: MOCK_IDP_LOGOUT_PATH,
+        validate: false,
+        options: { authRequired: false },
+      },
+      async (context, request, response) => {
+        return response.redirected({ headers: { location: '/' } });
+      }
+    );
+  },
+  start() {},
+  stop() {},
+});

packages/kbn-mock-idp-plugin/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types"
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/core-plugins-server",
+    "@kbn/config-schema",
+    "@kbn/dev-utils"
+  ]
+}

scripts/es.js

@@ -20,6 +20,7 @@ kbnEs
     'source-path': resolve(__dirname, '../../elasticsearch'),
     'base-path': resolve(__dirname, '../.es'),
     ssl: false,
+    kibanaUrl: 'https://localhost:5601/',
   })
   .catch(function (e) {
     console.error(e);

src/cli/serve/serve.js

@@ -109,6 +109,38 @@ export function applyConfigOverrides(rawConfig, opts, extraCliOptions) {
   if (opts.dev) {
     if (opts.serverless) {
       setServerlessKibanaDevServiceAccountIfPossible(get, set, opts);
+
+      // Load mock identity provider plugin and configure realm if supported (ES only supports SAML when run with SSL)
+      if (opts.ssl && canRequire('@kbn/mock-idp-plugin/common')) {
+        // Ensure the plugin is loaded in dynamically to exclude from production build
+        const {
+          MOCK_IDP_PLUGIN_PATH,
+          MOCK_IDP_REALM_NAME,
+        } = require('@kbn/mock-idp-plugin/common');
+
+        if (has('server.basePath')) {
+          console.log(
+            `Custom base path is not supported when running in Serverless, it will be removed.`
+          );
+          _.unset(rawConfig, 'server.basePath');
+        }
+
+        set('plugins.paths', _.compact([].concat(get('plugins.paths'), MOCK_IDP_PLUGIN_PATH)));
+        set(`xpack.security.authc.providers.saml.${MOCK_IDP_REALM_NAME}`, {
+          order: Number.MAX_SAFE_INTEGER,
+          realm: MOCK_IDP_REALM_NAME,
+          icon: 'user',
+          description: 'Continue as Test User',
+          hint: 'Allows testing serverless user roles',
+        });
+        // Add basic realm since defaults won't be applied when a provider has been configured
+        if (!has('xpack.security.authc.providers.basic')) {
+          set('xpack.security.authc.providers.basic.basic', {
+            order: 0,
+            enabled: true,
+          });
+        }
+      }
     }
 
     if (!has('elasticsearch.serviceAccountToken') && opts.devCredentials !== false) {
@@ -274,7 +306,9 @@ export default function (program) {
       // We can tell users they only have to run with `yarn start --run-examples` to get those
       // local links to work.  Similar to what we do for "View in Console" links in our
       // elastic.co links.
-      basePath: opts.runExamples ? false : !!opts.basePath,
+      // We also want to run without base path when running in serverless mode so that Elasticsearch can
+      // connect to Kibana's mock identity provider.
+      basePath: opts.runExamples || isServerlessMode ? false : !!opts.basePath,
       optimize: !!opts.optimize,
       disableOptimizer: !opts.optimizer,
       oss: !!opts.oss,

src/cli/tsconfig.json

@@ -17,6 +17,7 @@
     "@kbn/config",
     "@kbn/dev-utils",
     "@kbn/apm-config-loader",
+    "@kbn/mock-idp-plugin",
   ],
   "exclude": [
     "target/**/*",

tsconfig.base.json

@@ -1070,6 +1070,8 @@
       "@kbn/ml-ui-actions/*": ["x-pack/packages/ml/ui_actions/*"],
       "@kbn/ml-url-state": ["x-pack/packages/ml/url_state"],
       "@kbn/ml-url-state/*": ["x-pack/packages/ml/url_state/*"],
+      "@kbn/mock-idp-plugin": ["packages/kbn-mock-idp-plugin"],
+      "@kbn/mock-idp-plugin/*": ["packages/kbn-mock-idp-plugin/*"],
       "@kbn/monaco": ["packages/kbn-monaco"],
       "@kbn/monaco/*": ["packages/kbn-monaco/*"],
       "@kbn/monitoring-collection-plugin": ["x-pack/plugins/monitoring_collection"],

yarn.lock

@@ -5077,6 +5077,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/mock-idp-plugin@link:packages/kbn-mock-idp-plugin":
+  version "0.0.0"
+  uid ""
+
 "@kbn/monaco@link:packages/kbn-monaco":
   version "0.0.0"
   uid ""