Implement new platform plugin discovery. (#24256)

src/core/server/config/__snapshots__/env.test.ts.snap

@@ -17,7 +17,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": false,
   "logDir": "/test/cwd/log",
@@ -53,7 +52,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": false,
   "logDir": "/test/cwd/log",
@@ -88,7 +86,6 @@ Env {
   "configs": Array [
     "/test/cwd/config/kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": true,
   "logDir": "/test/cwd/log",
@@ -123,7 +120,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": false,
   "logDir": "/test/cwd/log",
@@ -158,7 +154,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": false,
   "logDir": "/test/cwd/log",
@@ -193,7 +188,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/some/home/dir/core_plugins",
   "homeDir": "/some/home/dir",
   "isDevClusterMaster": false,
   "logDir": "/some/home/dir/log",

src/core/server/config/env.ts

@@ -22,7 +22,7 @@ import process from 'process';
 
 import { pkg } from '../../../utils/package_json';
 
-interface PackageInfo {
+export interface PackageInfo {
   version: string;
   branch: string;
   buildNum: number;
@@ -61,7 +61,6 @@ export class Env {
   }
 
   public readonly configDir: string;
-  public readonly corePluginsDir: string;
   public readonly binDir: string;
   public readonly logDir: string;
   public readonly staticFilesDir: string;
@@ -96,7 +95,6 @@ export class Env {
    */
   constructor(readonly homeDir: string, options: EnvOptions) {
     this.configDir = resolve(this.homeDir, 'config');
-    this.corePluginsDir = resolve(this.homeDir, 'core_plugins');
     this.binDir = resolve(this.homeDir, 'bin');
     this.logDir = resolve(this.homeDir, 'log');
     this.staticFilesDir = resolve(this.homeDir, 'ui');

src/core/server/config/index.ts

@@ -22,5 +22,5 @@ export { RawConfigService } from './raw_config_service';
 export { Config, ConfigPath } from './config';
 /** @internal */
 export { ObjectToConfigAdapter } from './object_to_config_adapter';
-export { Env, CliArgs } from './env';
+export { Env, CliArgs, PackageInfo } from './env';
 export { ConfigWithSchema } from './config_with_schema';

src/core/server/index.test.ts

@@ -22,6 +22,11 @@ jest.mock('./http/http_service', () => ({
   HttpService: jest.fn(() => mockHttpService),
 }));
 
+const mockPluginsService = { start: jest.fn(), stop: jest.fn() };
+jest.mock('./plugins/plugins_service', () => ({
+  PluginsService: jest.fn(() => mockPluginsService),
+}));
+
 const mockLegacyService = { start: jest.fn(), stop: jest.fn() };
 jest.mock('./legacy_compat/legacy_service', () => ({
   LegacyService: jest.fn(() => mockLegacyService),
@@ -45,6 +50,8 @@ afterEach(() => {
   mockConfigService.atPath.mockReset();
   mockHttpService.start.mockReset();
   mockHttpService.stop.mockReset();
+  mockPluginsService.start.mockReset();
+  mockPluginsService.stop.mockReset();
   mockLegacyService.start.mockReset();
   mockLegacyService.stop.mockReset();
 });
@@ -56,11 +63,13 @@ test('starts services on "start"', async () => {
   const server = new Server(mockConfigService as any, logger, env);
 
   expect(mockHttpService.start).not.toHaveBeenCalled();
+  expect(mockPluginsService.start).not.toHaveBeenCalled();
   expect(mockLegacyService.start).not.toHaveBeenCalled();
 
   await server.start();
 
   expect(mockHttpService.start).toHaveBeenCalledTimes(1);
+  expect(mockPluginsService.start).toHaveBeenCalledTimes(1);
   expect(mockLegacyService.start).toHaveBeenCalledTimes(1);
   expect(mockLegacyService.start).toHaveBeenCalledWith(mockHttpServiceStartContract);
 });
@@ -112,10 +121,12 @@ test('stops services on "stop"', async () => {
   await server.start();
 
   expect(mockHttpService.stop).not.toHaveBeenCalled();
+  expect(mockPluginsService.stop).not.toHaveBeenCalled();
   expect(mockLegacyService.stop).not.toHaveBeenCalled();
 
   await server.stop();
 
   expect(mockHttpService.stop).toHaveBeenCalledTimes(1);
+  expect(mockPluginsService.stop).toHaveBeenCalledTimes(1);
   expect(mockLegacyService.stop).toHaveBeenCalledTimes(1);
 });

src/core/server/index.ts

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { PluginsModule } from './plugins';
+
 export { bootstrap } from './bootstrap';
 
 import { first } from 'rxjs/operators';
@@ -27,6 +29,7 @@ import { Logger, LoggerFactory } from './logging';
 
 export class Server {
   private readonly http: HttpModule;
+  private readonly plugins: PluginsModule;
   private readonly legacy: LegacyCompatModule;
   private readonly log: Logger;
 
@@ -38,6 +41,7 @@ export class Server {
     this.log = logger.get('server');
 
     this.http = new HttpModule(configService.atPath('server', HttpConfig), logger);
+    this.plugins = new PluginsModule(configService, logger, env);
     this.legacy = new LegacyCompatModule(configService, logger, env);
   }
 
@@ -54,6 +58,7 @@ export class Server {
       httpServerInfo = await this.http.service.start();
     }
 
+    await this.plugins.service.start();
     await this.legacy.service.start(httpServerInfo);
 
     const unhandledConfigPaths = await this.configService.getUnusedPaths();
@@ -70,6 +75,7 @@ export class Server {
     this.log.debug('stopping server');
 
     await this.legacy.service.stop();
+    await this.plugins.service.stop();
     await this.http.service.stop();
   }
 }

src/core/server/plugins/discovery/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { PluginDiscoveryErrorType } from './plugin_discovery_error';
+export { discover } from './plugins_discovery';

src/core/server/plugins/discovery/plugin_discovery.test.ts

@@ -0,0 +1,159 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const mockReaddir = jest.fn();
+const mockReadFile = jest.fn();
+const mockStat = jest.fn();
+jest.mock('fs', () => ({
+  readdir: mockReaddir,
+  readFile: mockReadFile,
+  stat: mockStat,
+}));
+
+import { resolve } from 'path';
+import { map, toArray } from 'rxjs/operators';
+import { logger } from '../../logging/__mocks__';
+import { discover } from './plugins_discovery';
+
+const TEST_PATHS = {
+  scanDirs: {
+    nonEmpty: resolve('scan', 'non-empty'),
+    nonEmpty2: resolve('scan', 'non-empty-2'),
+    nonExistent: resolve('scan', 'non-existent'),
+    empty: resolve('scan', 'empty'),
+  },
+  paths: {
+    existentDir: resolve('path', 'existent-dir'),
+    existentDir2: resolve('path', 'existent-dir-2'),
+    nonDir: resolve('path', 'non-dir'),
+    nonExistent: resolve('path', 'non-existent'),
+  },
+};
+
+beforeEach(() => {
+  mockReaddir.mockImplementation((path, cb) => {
+    if (path === TEST_PATHS.scanDirs.nonEmpty) {
+      cb(null, ['1', '2-no-manifest', '3', '4-incomplete-manifest']);
+    } else if (path === TEST_PATHS.scanDirs.nonEmpty2) {
+      cb(null, ['5-invalid-manifest', '6', '7-non-dir', '8-incompatible-manifest']);
+    } else if (path === TEST_PATHS.scanDirs.nonExistent) {
+      cb(new Error('ENOENT'));
+    } else {
+      cb(null, []);
+    }
+  });
+
+  mockStat.mockImplementation((path, cb) => {
+    if (path.includes('non-existent')) {
+      cb(new Error('ENOENT'));
+    } else {
+      cb(null, { isDirectory: () => !path.includes('non-dir') });
+    }
+  });
+
+  mockReadFile.mockImplementation((path, cb) => {
+    if (path.includes('no-manifest')) {
+      cb(new Error('ENOENT'));
+    } else if (path.includes('invalid-manifest')) {
+      cb(null, Buffer.from('not-json'));
+    } else if (path.includes('incomplete-manifest')) {
+      cb(null, Buffer.from(JSON.stringify({ version: '1' })));
+    } else if (path.includes('incompatible-manifest')) {
+      cb(null, Buffer.from(JSON.stringify({ id: 'plugin', version: '1' })));
+    } else {
+      cb(null, Buffer.from(JSON.stringify({ id: 'plugin', version: '1', kibanaVersion: '1.2.3' })));
+    }
+  });
+});
+
+afterEach(() => {
+  jest.clearAllMocks();
+});
+
+test('properly scans folders and paths', async () => {
+  const { plugin$, error$ } = discover(
+    {
+      initialize: true,
+      scanDirs: Object.values(TEST_PATHS.scanDirs),
+      paths: Object.values(TEST_PATHS.paths),
+    },
+    {
+      branch: 'master',
+      buildNum: 1,
+      buildSha: '',
+      version: '1.2.3',
+    },
+    logger.get()
+  );
+
+  await expect(plugin$.pipe(toArray()).toPromise()).resolves.toEqual(
+    [
+      resolve(TEST_PATHS.scanDirs.nonEmpty, '1'),
+      resolve(TEST_PATHS.scanDirs.nonEmpty, '3'),
+      resolve(TEST_PATHS.scanDirs.nonEmpty2, '6'),
+      resolve(TEST_PATHS.paths.existentDir),
+      resolve(TEST_PATHS.paths.existentDir2),
+    ].map(path => ({
+      manifest: {
+        id: 'plugin',
+        version: '1',
+        kibanaVersion: '1.2.3',
+        optionalPlugins: [],
+        requiredPlugins: [],
+        ui: false,
+      },
+      path,
+    }))
+  );
+
+  await expect(
+    error$
+      .pipe(
+        map(error => error.toString()),
+        toArray()
+      )
+      .toPromise()
+  ).resolves.toEqual([
+    `Error: ENOENT (invalid-scan-dir, ${resolve(TEST_PATHS.scanDirs.nonExistent)})`,
+    `Error: ${resolve(TEST_PATHS.paths.nonDir)} is not a directory. (invalid-plugin-dir, ${resolve(
+      TEST_PATHS.paths.nonDir
+    )})`,
+    `Error: ENOENT (invalid-plugin-dir, ${resolve(TEST_PATHS.paths.nonExistent)})`,
+    `Error: ENOENT (missing-manifest, ${resolve(
+      TEST_PATHS.scanDirs.nonEmpty,
+      '2-no-manifest',
+      'kibana.json'
+    )})`,
+    `Error: Plugin manifest must contain an "id" property. (invalid-manifest, ${resolve(
+      TEST_PATHS.scanDirs.nonEmpty,
+      '4-incomplete-manifest',
+      'kibana.json'
+    )})`,
+    `Error: Unexpected token o in JSON at position 1 (invalid-manifest, ${resolve(
+      TEST_PATHS.scanDirs.nonEmpty2,
+      '5-invalid-manifest',
+      'kibana.json'
+    )})`,
+    `Error: Plugin "plugin" is only compatible with Kibana version "1", but used Kibana version is "1.2.3". (incompatible-version, ${resolve(
+      TEST_PATHS.scanDirs.nonEmpty2,
+      '8-incompatible-manifest',
+      'kibana.json'
+    )})`,
+  ]);
+});

src/core/server/plugins/discovery/plugin_discovery_error.ts

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export enum PluginDiscoveryErrorType {
+  IncompatibleVersion = 'incompatible-version',
+  InvalidScanDirectory = 'invalid-scan-dir',
+  InvalidPluginDirectory = 'invalid-plugin-dir',
+  InvalidManifest = 'invalid-manifest',
+  MissingManifest = 'missing-manifest',
+}
+
+export class PluginDiscoveryError extends Error {
+  public static incompatibleVersion(path: string, cause: Error) {
+    return new PluginDiscoveryError(PluginDiscoveryErrorType.IncompatibleVersion, path, cause);
+  }
+
+  public static invalidScanDirectory(path: string, cause: Error) {
+    return new PluginDiscoveryError(PluginDiscoveryErrorType.InvalidScanDirectory, path, cause);
+  }
+
+  public static invalidPluginDirectory(path: string, cause: Error) {
+    return new PluginDiscoveryError(PluginDiscoveryErrorType.InvalidPluginDirectory, path, cause);
+  }
+
+  public static invalidManifest(path: string, cause: Error) {
+    return new PluginDiscoveryError(PluginDiscoveryErrorType.InvalidManifest, path, cause);
+  }
+
+  public static missingManifest(path: string, cause: Error) {
+    return new PluginDiscoveryError(PluginDiscoveryErrorType.MissingManifest, path, cause);
+  }
+
+  /**
+   * @param type Type of the discovery error (invalid directory, invalid manifest etc.)
+   * @param path Path at which discovery error occurred.
+   * @param cause "Raw" error object that caused discovery error.
+   */
+  constructor(
+    public readonly type: PluginDiscoveryErrorType,
+    public readonly path: string,
+    public readonly cause: Error
+  ) {
+    super(`${cause.message} (${type}, ${path})`);
+
+    // Set the prototype explicitly, see:
+    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work
+    Object.setPrototypeOf(this, PluginDiscoveryError.prototype);
+  }
+}

src/core/server/plugins/discovery/plugin_manifest_parser.test.ts

@@ -0,0 +1,215 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { PluginDiscoveryErrorType } from './plugin_discovery_error';
+
+const mockReadFile = jest.fn();
+jest.mock('fs', () => ({ readFile: mockReadFile }));
+
+import { resolve } from 'path';
+import { parseManifest } from './plugin_manifest_parser';
+
+const pluginPath = resolve('path', 'existent-dir');
+const pluginManifestPath = resolve(pluginPath, 'kibana.json');
+const packageInfo = {
+  branch: 'master',
+  buildNum: 1,
+  buildSha: '',
+  version: '7.0.0-alpha1',
+};
+
+afterEach(() => {
+  jest.clearAllMocks();
+});
+
+test('return error when manifest is empty', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(null, Buffer.from(''));
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({
+    message: `Unexpected end of JSON input (invalid-manifest, ${pluginManifestPath})`,
+    type: PluginDiscoveryErrorType.InvalidManifest,
+    path: pluginManifestPath,
+  });
+});
+
+test('return error when manifest content is null', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(null, Buffer.from('null'));
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({
+    message: `Plugin manifest must contain a JSON encoded object. (invalid-manifest, ${pluginManifestPath})`,
+    type: PluginDiscoveryErrorType.InvalidManifest,
+    path: pluginManifestPath,
+  });
+});
+
+test('return error when manifest content is not a valid JSON', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(null, Buffer.from('not-json'));
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({
+    message: `Unexpected token o in JSON at position 1 (invalid-manifest, ${pluginManifestPath})`,
+    type: PluginDiscoveryErrorType.InvalidManifest,
+    path: pluginManifestPath,
+  });
+});
+
+test('return error when plugin id is missing', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(null, Buffer.from(JSON.stringify({ version: 'some-version' })));
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({
+    message: `Plugin manifest must contain an "id" property. (invalid-manifest, ${pluginManifestPath})`,
+    type: PluginDiscoveryErrorType.InvalidManifest,
+    path: pluginManifestPath,
+  });
+});
+
+test('return error when plugin version is missing', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(null, Buffer.from(JSON.stringify({ id: 'some-id' })));
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({
+    message: `Plugin manifest for "some-id" must contain a "version" property. (invalid-manifest, ${pluginManifestPath})`,
+    type: PluginDiscoveryErrorType.InvalidManifest,
+    path: pluginManifestPath,
+  });
+});
+
+test('return error when plugin expected Kibana version is lower than actual version', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(null, Buffer.from(JSON.stringify({ id: 'some-id', version: '6.4.2' })));
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({
+    message: `Plugin "some-id" is only compatible with Kibana version "6.4.2", but used Kibana version is "7.0.0-alpha1". (incompatible-version, ${pluginManifestPath})`,
+    type: PluginDiscoveryErrorType.IncompatibleVersion,
+    path: pluginManifestPath,
+  });
+});
+
+test('return error when plugin expected Kibana version is higher than actual version', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(null, Buffer.from(JSON.stringify({ id: 'some-id', version: '7.0.1' })));
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).rejects.toMatchObject({
+    message: `Plugin "some-id" is only compatible with Kibana version "7.0.1", but used Kibana version is "7.0.0-alpha1". (incompatible-version, ${pluginManifestPath})`,
+    type: PluginDiscoveryErrorType.IncompatibleVersion,
+    path: pluginManifestPath,
+  });
+});
+
+test('set defaults for all missing optional fields', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(null, Buffer.from(JSON.stringify({ id: 'some-id', version: '7.0.0' })));
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).resolves.toEqual({
+    id: 'some-id',
+    version: '7.0.0',
+    kibanaVersion: '7.0.0',
+    optionalPlugins: [],
+    requiredPlugins: [],
+    ui: false,
+  });
+});
+
+test('return all set optional fields as they are in manifest', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(
+      null,
+      Buffer.from(
+        JSON.stringify({
+          id: 'some-id',
+          version: 'some-version',
+          kibanaVersion: '7.0.0',
+          requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'],
+          optionalPlugins: ['some-optional-plugin'],
+          ui: true,
+        })
+      )
+    );
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).resolves.toEqual({
+    id: 'some-id',
+    version: 'some-version',
+    kibanaVersion: '7.0.0',
+    optionalPlugins: ['some-optional-plugin'],
+    requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'],
+    ui: true,
+  });
+});
+
+test('return manifest when plugin expected Kibana version matches actual version', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(
+      null,
+      Buffer.from(
+        JSON.stringify({
+          id: 'some-id',
+          version: 'some-version',
+          kibanaVersion: '7.0.0-alpha2',
+          requiredPlugins: ['some-required-plugin'],
+        })
+      )
+    );
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).resolves.toEqual({
+    id: 'some-id',
+    version: 'some-version',
+    kibanaVersion: '7.0.0-alpha2',
+    optionalPlugins: [],
+    requiredPlugins: ['some-required-plugin'],
+    ui: false,
+  });
+});
+
+test('return manifest when plugin expected Kibana version is `kibana`', async () => {
+  mockReadFile.mockImplementation((path, cb) => {
+    cb(
+      null,
+      Buffer.from(
+        JSON.stringify({
+          id: 'some-id',
+          version: 'some-version',
+          kibanaVersion: 'kibana',
+          requiredPlugins: ['some-required-plugin'],
+        })
+      )
+    );
+  });
+
+  await expect(parseManifest(pluginPath, packageInfo)).resolves.toEqual({
+    id: 'some-id',
+    version: 'some-version',
+    kibanaVersion: 'kibana',
+    optionalPlugins: [],
+    requiredPlugins: ['some-required-plugin'],
+    ui: false,
+  });
+});

src/core/server/plugins/discovery/plugin_manifest_parser.ts

@@ -0,0 +1,176 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { readFile } from 'fs';
+import { resolve } from 'path';
+import { promisify } from 'util';
+import { PackageInfo } from '../../config';
+import { PluginDiscoveryError } from './plugin_discovery_error';
+
+const fsReadFileAsync = promisify(readFile);
+
+/**
+ * Describes the set of required and optional properties plugin can define in its
+ * mandatory JSON manifest file.
+ */
+export interface PluginManifest {
+  /**
+   * Identifier of the plugin.
+   */
+  readonly id: string;
+
+  /**
+   * Version of the plugin.
+   */
+  readonly version: string;
+
+  /**
+   * The version of Kibana the plugin is compatible with, defaults to "version".
+   */
+  readonly kibanaVersion: string;
+
+  /**
+   * An optional list of the other plugins that **must be** installed and enabled
+   * for this plugin to function properly.
+   */
+  readonly requiredPlugins: ReadonlyArray<string>;
+
+  /**
+   * An optional list of the other plugins that if installed and enabled **may be**
+   * leveraged by this plugin for some additional functionality but otherwise are
+   * not required for this plugin to work properly.
+   */
+  readonly optionalPlugins: ReadonlyArray<string>;
+
+  /**
+   * Specifies whether plugin includes some client/browser specific functionality
+   * that should be included into client bundle via `public/ui_plugin.js` file.
+   */
+  readonly ui: boolean;
+}
+
+/**
+ * Name of the JSON manifest file that should be located in the plugin directory.
+ */
+const MANIFEST_FILE_NAME = 'kibana.json';
+
+/**
+ * The special "kibana" version can be used by the plugins to be always compatible.
+ */
+const ALWAYS_COMPATIBLE_VERSION = 'kibana';
+
+/**
+ * Regular expression used to extract semantic version part from the plugin or
+ * kibana version, e.g. `1.2.3` ---> `1.2.3` and `7.0.0-alpha1` ---> `7.0.0`.
+ */
+const SEM_VER_REGEX = /\d+\.\d+\.\d+/;
+
+/**
+ * Tries to load and parse the plugin manifest file located at the provided plugin
+ * directory path and produces an error result if it fails to do so or plugin manifest
+ * isn't valid.
+ * @param pluginPath Path to the plugin directory where manifest should be loaded from.
+ * @param packageInfo Kibana package info.
+ */
+export async function parseManifest(pluginPath: string, packageInfo: PackageInfo) {
+  const manifestPath = resolve(pluginPath, MANIFEST_FILE_NAME);
+
+  let manifestContent;
+  try {
+    manifestContent = await fsReadFileAsync(manifestPath);
+  } catch (err) {
+    throw PluginDiscoveryError.missingManifest(manifestPath, err);
+  }
+
+  let manifest: Partial<PluginManifest>;
+  try {
+    manifest = JSON.parse(manifestContent.toString());
+  } catch (err) {
+    throw PluginDiscoveryError.invalidManifest(manifestPath, err);
+  }
+
+  if (!manifest || typeof manifest !== 'object') {
+    throw PluginDiscoveryError.invalidManifest(
+      manifestPath,
+      new Error('Plugin manifest must contain a JSON encoded object.')
+    );
+  }
+
+  if (!manifest.id || typeof manifest.id !== 'string') {
+    throw PluginDiscoveryError.invalidManifest(
+      manifestPath,
+      new Error('Plugin manifest must contain an "id" property.')
+    );
+  }
+
+  if (!manifest.version || typeof manifest.version !== 'string') {
+    throw PluginDiscoveryError.invalidManifest(
+      manifestPath,
+      new Error(`Plugin manifest for "${manifest.id}" must contain a "version" property.`)
+    );
+  }
+
+  const expectedKibanaVersion =
+    typeof manifest.kibanaVersion === 'string' && manifest.kibanaVersion
+      ? manifest.kibanaVersion
+      : manifest.version;
+  if (!isVersionCompatible(expectedKibanaVersion, packageInfo.version)) {
+    throw PluginDiscoveryError.incompatibleVersion(
+      manifestPath,
+      new Error(
+        `Plugin "${
+          manifest.id
+        }" is only compatible with Kibana version "${expectedKibanaVersion}", but used Kibana version is "${
+          packageInfo.version
+        }".`
+      )
+    );
+  }
+
+  return {
+    id: manifest.id,
+    version: manifest.version,
+    kibanaVersion: expectedKibanaVersion,
+    requiredPlugins: Array.isArray(manifest.requiredPlugins) ? manifest.requiredPlugins : [],
+    optionalPlugins: Array.isArray(manifest.optionalPlugins) ? manifest.optionalPlugins : [],
+    ui: typeof manifest.ui === 'boolean' ? manifest.ui : false,
+  };
+}
+
+/**
+ * Checks whether plugin expected Kibana version is compatible with the used Kibana version.
+ * @param expectedKibanaVersion Kibana version expected by the plugin.
+ * @param actualKibanaVersion Used Kibana version.
+ */
+function isVersionCompatible(expectedKibanaVersion: string, actualKibanaVersion: string) {
+  if (expectedKibanaVersion === ALWAYS_COMPATIBLE_VERSION) {
+    return true;
+  }
+
+  return extractSemVer(actualKibanaVersion) === extractSemVer(expectedKibanaVersion);
+}
+
+/**
+ * Tries to extract semantic version part from the full version string.
+ * @param version
+ */
+function extractSemVer(version: string) {
+  const semVerMatch = version.match(SEM_VER_REGEX);
+  return semVerMatch === null ? version : semVerMatch[0];
+}

src/core/server/plugins/discovery/plugins_discovery.ts

@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { readdir, stat } from 'fs';
+import { resolve } from 'path';
+import { bindNodeCallback, from, merge, Observable, throwError } from 'rxjs';
+import { catchError, map, mergeMap, shareReplay } from 'rxjs/operators';
+import { PackageInfo } from '../../config';
+import { Logger } from '../../logging';
+import { PluginsConfig } from '../plugins_config';
+import { PluginDiscoveryError } from './plugin_discovery_error';
+import { parseManifest, PluginManifest } from './plugin_manifest_parser';
+
+const fsReadDir$ = bindNodeCallback(readdir);
+const fsStat$ = bindNodeCallback(stat);
+
+interface DiscoveryResult {
+  plugin?: { path: string; manifest: PluginManifest };
+  error?: PluginDiscoveryError;
+}
+
+/**
+ * Tries to discover all possible plugins based on the provided plugin config.
+ * Discovery result consists of two separate streams, the one (`plugin$`) is
+ * for the successfully discovered plugins and the other one (`error$`) is for
+ * all the errors that occurred during discovery process.
+ *
+ * @param config Plugin config instance.
+ * @param packageInfo Kibana package info.
+ * @param log Plugin discovery logger instance.
+ */
+export function discover(config: PluginsConfig, packageInfo: PackageInfo, log: Logger) {
+  log.debug('Discovering plugins...');
+
+  const discoveryResults$ = merge(
+    processScanDirs$(config.scanDirs, log),
+    processPaths$(config.paths, log)
+  ).pipe(
+    mergeMap(pluginPathOrError => {
+      return typeof pluginPathOrError === 'string'
+        ? createPlugin$(pluginPathOrError, packageInfo, log)
+        : [pluginPathOrError];
+    }),
+    shareReplay()
+  );
+
+  return {
+    plugin$: discoveryResults$.pipe(
+      mergeMap(entry => (entry.plugin !== undefined ? [entry.plugin] : []))
+    ),
+    error$: discoveryResults$.pipe(
+      mergeMap(entry => (entry.error !== undefined ? [entry.error] : []))
+    ),
+  };
+}
+
+/**
+ * Iterates over every entry in `scanDirs` and returns a merged stream of all
+ * sub-directories. If directory cannot be read or it's impossible to get stat
+ * for any of the nested entries then error is added into the stream instead.
+ * @param scanDirs List of the top-level directories to process.
+ * @param log Plugin discovery logger instance.
+ */
+function processScanDirs$(scanDirs: string[], log: Logger) {
+  return from(scanDirs).pipe(
+    mergeMap(dir => {
+      log.debug(`Scanning "${dir}" for plugin sub-directories...`);
+
+      return fsReadDir$(dir).pipe(
+        mergeMap(subDirs => subDirs.map(subDir => resolve(dir, subDir))),
+        mergeMap(path =>
+          fsStat$(path).pipe(
+            // Filter out non-directory entries from target directories, it's expected that
+            // these directories may contain files (e.g. `README.md` or `package.json`).
+            // We shouldn't silently ignore the entries we couldn't get stat for though.
+            mergeMap(pathStat => (pathStat.isDirectory() ? [path] : [])),
+            catchError(err => [wrapError(PluginDiscoveryError.invalidPluginDirectory(path, err))])
+          )
+        ),
+        catchError(err => [wrapError(PluginDiscoveryError.invalidScanDirectory(dir, err))])
+      );
+    })
+  );
+}
+
+/**
+ * Iterates over every entry in `paths` and returns a stream of all paths that
+ * are directories. If path is not a directory or it's impossible to get stat
+ * for this path then error is added into the stream instead.
+ * @param paths List of paths to process.
+ * @param log Plugin discovery logger instance.
+ */
+function processPaths$(paths: string[], log: Logger) {
+  return from(paths).pipe(
+    mergeMap(path => {
+      log.debug(`Including "${path}" into the plugin path list.`);
+
+      return fsStat$(path).pipe(
+        // Since every path is specifically provided we should treat non-directory
+        // entries as mistakes we should report of.
+        mergeMap(pathStat => {
+          return pathStat.isDirectory()
+            ? [path]
+            : throwError(new Error(`${path} is not a directory.`));
+        }),
+        catchError(err => [wrapError(PluginDiscoveryError.invalidPluginDirectory(path, err))])
+      );
+    })
+  );
+}
+
+/**
+ * Tries to load and parse the plugin manifest file located at the provided plugin
+ * directory path and produces an error result if it fails to do so or plugin manifest
+ * isn't valid.
+ * @param path Path to the plugin directory where manifest should be loaded from.
+ * @param packageInfo Kibana package info.
+ * @param log Plugin discovery logger instance.
+ */
+function createPlugin$(
+  path: string,
+  packageInfo: PackageInfo,
+  log: Logger
+): Observable<DiscoveryResult> {
+  return from(parseManifest(path, packageInfo)).pipe(
+    map(manifest => {
+      log.debug(`Successfully discovered plugin "${manifest.id}" at "${path}"`);
+      return { plugin: { path, manifest } };
+    }),
+    catchError(err => [wrapError(err)])
+  );
+}
+
+/**
+ * Wraps `PluginDiscoveryError` into `DiscoveryResult` entry.
+ * @param error Instance of the `PluginDiscoveryError` error.
+ */
+function wrapError(error: PluginDiscoveryError): DiscoveryResult {
+  return { error };
+}

src/core/server/plugins/index.ts

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ConfigService, Env } from '../config';
+import { LoggerFactory } from '../logging';
+import { PluginsService } from './plugins_service';
+
+export class PluginsModule {
+  public readonly service: PluginsService;
+
+  constructor(private readonly configService: ConfigService, logger: LoggerFactory, env: Env) {
+    this.service = new PluginsService(env, logger, this.configService);
+  }
+}

src/core/server/plugins/plugins_config.ts

@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+const pluginsSchema = schema.object({
+  initialize: schema.boolean({ defaultValue: true }),
+  scanDirs: schema.arrayOf(schema.string(), {
+    defaultValue: [],
+  }),
+  paths: schema.arrayOf(schema.string(), {
+    defaultValue: [],
+  }),
+});
+
+type PluginsConfigType = TypeOf<typeof pluginsSchema>;
+
+/** @internal */
+export class PluginsConfig {
+  public static schema = pluginsSchema;
+
+  /**
+   * Indicates whether or not plugins should be initialized.
+   */
+  public readonly initialize: boolean;
+
+  /**
+   * Defines directories that we should scan for the plugin subdirectories.
+   */
+  public readonly scanDirs: string[];
+
+  /**
+   * Defines direct paths to specific plugin directories that we should initialize.
+   */
+  public readonly paths: string[];
+
+  constructor(config: PluginsConfigType) {
+    this.initialize = config.initialize;
+    this.scanDirs = config.scanDirs;
+    this.paths = config.paths;
+  }
+}

src/core/server/plugins/plugins_service.test.ts

@@ -0,0 +1,146 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const mockPackage = new Proxy({ raw: {} as any }, { get: (obj, prop) => obj.raw[prop] });
+jest.mock('../../../utils/package_json', () => ({ pkg: mockPackage }));
+
+const mockDiscover = jest.fn();
+jest.mock('./discovery/plugins_discovery', () => ({ discover: mockDiscover }));
+
+import { BehaviorSubject, from } from 'rxjs';
+
+import { Config, ConfigService, Env, ObjectToConfigAdapter } from '../config';
+import { getEnvOptions } from '../config/__mocks__/env';
+import { logger } from '../logging/__mocks__';
+import { PluginDiscoveryError } from './discovery/plugin_discovery_error';
+import { PluginsService } from './plugins_service';
+
+let pluginsService: PluginsService;
+let configService: ConfigService;
+let env: Env;
+beforeEach(() => {
+  mockPackage.raw = {
+    branch: 'feature-v1',
+    version: 'v1',
+    build: {
+      distributable: true,
+      number: 100,
+      sha: 'feature-v1-build-sha',
+    },
+  };
+
+  env = Env.createDefault(getEnvOptions());
+
+  configService = new ConfigService(
+    new BehaviorSubject<Config>(
+      new ObjectToConfigAdapter({
+        plugins: {
+          initialize: true,
+          scanDirs: ['one', 'two'],
+          paths: ['three', 'four'],
+        },
+      })
+    ),
+    env,
+    logger
+  );
+  pluginsService = new PluginsService(env, logger, configService);
+});
+
+afterEach(() => {
+  jest.clearAllMocks();
+});
+
+test('properly invokes `discover` on `start`.', async () => {
+  mockDiscover.mockReturnValue({
+    error$: from([
+      PluginDiscoveryError.invalidManifest('path-1', new Error('Invalid JSON')),
+      PluginDiscoveryError.missingManifest('path-2', new Error('No manifest')),
+      PluginDiscoveryError.invalidScanDirectory('dir-1', new Error('No dir')),
+      PluginDiscoveryError.incompatibleVersion('path-3', new Error('Incompatible version')),
+    ]),
+
+    plugin$: from([
+      {
+        path: 'path-4',
+        manifest: {
+          id: 'some-id',
+          version: 'some-version',
+          kibanaVersion: '7.0.0',
+          requiredPlugins: ['some-required-plugin', 'some-required-plugin-2'],
+          optionalPlugins: ['some-optional-plugin'],
+          ui: true,
+        },
+      },
+      {
+        path: 'path-5',
+        manifest: {
+          id: 'some-other-id',
+          version: 'some-other-version',
+          kibanaVersion: '7.0.0',
+          requiredPlugins: ['some-required-plugin'],
+          optionalPlugins: [],
+          ui: false,
+        },
+      },
+    ]),
+  });
+
+  await pluginsService.start();
+
+  expect(mockDiscover).toHaveBeenCalledTimes(1);
+  expect(mockDiscover).toHaveBeenCalledWith(
+    { initialize: true, paths: ['three', 'four'], scanDirs: ['one', 'two'] },
+    { branch: 'feature-v1', buildNum: 100, buildSha: 'feature-v1-build-sha', version: 'v1' },
+    expect.objectContaining({
+      debug: expect.any(Function),
+      error: expect.any(Function),
+      info: expect.any(Function),
+    })
+  );
+
+  expect(logger.mockCollect()).toMatchInlineSnapshot(`
+Object {
+  "debug": Array [
+    Array [
+      "starting plugins service",
+    ],
+    Array [
+      "Marking config path as handled: plugins",
+    ],
+    Array [
+      "Discovered 2 plugins.",
+    ],
+  ],
+  "error": Array [
+    Array [
+      [Error: Invalid JSON (invalid-manifest, path-1)],
+    ],
+    Array [
+      [Error: Incompatible version (incompatible-version, path-3)],
+    ],
+  ],
+  "fatal": Array [],
+  "info": Array [],
+  "log": Array [],
+  "trace": Array [],
+  "warn": Array [],
+}
+`);
+});

src/core/server/plugins/plugins_service.ts

@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { filter, first, map, tap, toArray } from 'rxjs/operators';
+import { CoreService } from '../../types/core_service';
+import { ConfigService, Env } from '../config';
+import { Logger, LoggerFactory } from '../logging';
+import { discover, PluginDiscoveryErrorType } from './discovery';
+import { PluginsConfig } from './plugins_config';
+
+export class PluginsService implements CoreService {
+  private readonly log: Logger;
+
+  constructor(
+    private readonly env: Env,
+    private readonly logger: LoggerFactory,
+    private readonly configService: ConfigService
+  ) {
+    this.log = logger.get('plugins', 'service');
+  }
+
+  public async start() {
+    this.log.debug('starting plugins service');
+
+    // At this stage we report only errors that can occur when new platform plugin
+    // manifest is present, otherwise we can't be sure that the plugin is for the new
+    // platform and let legacy platform to handle it.
+    const errorTypesToReport = [
+      PluginDiscoveryErrorType.IncompatibleVersion,
+      PluginDiscoveryErrorType.InvalidManifest,
+    ];
+
+    const { error$, plugin$ } = await this.configService
+      .atPath('plugins', PluginsConfig)
+      .pipe(
+        first(),
+        map(config =>
+          discover(config, this.env.packageInfo, this.logger.get('plugins', 'discovery'))
+        )
+      )
+      .toPromise();
+
+    await error$
+      .pipe(
+        filter(error => errorTypesToReport.includes(error.type)),
+        tap(invalidManifestError => this.log.error(invalidManifestError))
+      )
+      .toPromise();
+
+    await plugin$
+      .pipe(
+        toArray(),
+        tap(plugins => this.log.debug(`Discovered ${plugins.length} plugins.`))
+      )
+      .toPromise();
+  }
+
+  public async stop() {
+    this.log.debug('stopping plugins service');
+  }
+}