Prefer third-party plugin development in ./plugins instead of ../kibana-extra (#31748)

* Prefer third-party plugin development in plugins instead of kibana-extra

* Fix failing recursive directory creation and removal

* Add new built version of kbn-pm

docs/development/plugin/development-plugin-resources.asciidoc

@@ -20,7 +20,7 @@ Many Kibana developers hang out on `irc.freenode.net` in the `#kibana` channel.
 [float]
 ==== Plugin Generator
 
-It is recommended that you kick-start your plugin by generating it with the {repo}tree/{branch}/packages/kbn-plugin-generator[Kibana Plugin Generator]. Run the following within the Kibana repo and you will be asked a couple questions, see some progress bars, and have a freshly generated plugin ready for you to play within Kibana's sibling `kibana-extra` folder.
+It is recommended that you kick-start your plugin by generating it with the {repo}tree/{branch}/packages/kbn-plugin-generator[Kibana Plugin Generator]. Run the following within the Kibana repo and you will be asked a couple questions, see some progress bars, and have a freshly generated plugin ready for you to play within Kibana's `plugins` folder.
 
 ["source","shell"]
 -----------
@@ -31,14 +31,15 @@ node scripts/generate_plugin my_plugin_name # replace "my_plugin_name" with your
 [float]
 ==== Directory structure for plugins
 
-The Kibana directory must be named `kibana`, and your plugin directory must be located within the sibling `kibana-extra` folder, for example:
+The Kibana directory must be named `kibana`, and your plugin directory should be located in the root of `kibana` in a `plugins` directory, for example:
 
 ["source","shell"]
 -----------
 .
-├── kibana
-├── kibana-extra/foo-plugin
-└── kibana-extra/bar-plugin
+└── kibana
+    └── plugins
+        ├── foo-plugin
+        └── bar-plugin
 -----------
 
 [float]

packages/kbn-eslint-import-resolver-kibana/lib/get_kibana_path.js

@@ -33,7 +33,8 @@ exports.getKibanaPath = function(config, projectRoot) {
   if (inConfig && config.kibanaPath !== '.') {
     throw new Error(
       'The `kibanaPath` option has been removed from `eslint-import-resolver-kibana`. ' +
-        'During development your plugin must live in `../kibana-extra/{pluginName}` ' +
+        'During development your plugin must live in `./plugins/{pluginName}` ' +
+        'inside the Kibana folder or `../kibana-extra/{pluginName}` ' +
         'relative to the Kibana folder to work with this package.'
     );
   }

packages/kbn-plugin-generator/README.md

@@ -18,7 +18,7 @@ To target the current development version of Kibana just use the default  `maste
 
 ```sh
 node scripts/generate_plugin my_plugin_name
-# generates a plugin in `../kibana-extra/my_plugin_name`
+# generates a plugin in `plugins/my_plugin_name`
 ```
 
 To target 6.3, use the `6.x` branch (until the `6.3` branch is created).

packages/kbn-plugin-generator/index.js

@@ -42,7 +42,7 @@ exports.run = function run(argv) {
       dedent(chalk`
         {dim usage:} node scripts/generate-plugin {bold [name]}
 
-        generate a fresh Kibana plugin in the ../kibana-extra/ directory
+        generate a fresh Kibana plugin in the plugins/ directory
       `) + '\n'
     );
     process.exit(1);
@@ -50,8 +50,8 @@ exports.run = function run(argv) {
 
   const name = options._[0];
   const template = resolve(__dirname, './sao_template');
-  const kibanaExtra = resolve(__dirname, '../../../kibana-extra');
-  const targetPath = resolve(kibanaExtra, snakeCase(name));
+  const kibanaPlugins = resolve(__dirname, '../../plugins');
+  const targetPath = resolve(kibanaPlugins, snakeCase(name));
 
   sao({
     template: template,

packages/kbn-plugin-generator/sao_template/sao.js

@@ -102,7 +102,7 @@ module.exports = function({ name }) {
         cwd: KBN_DIR,
         stdio: 'inherit',
       }).then(() => {
-        const dir = relative(process.cwd(), resolve(KBN_DIR, `../kibana-extra`, snakeCase(name)));
+        const dir = relative(process.cwd(), resolve(KBN_DIR, 'plugins', snakeCase(name)));
 
         log.success(chalk`🎉
 

packages/kbn-pm/README.md

@@ -88,15 +88,16 @@ are running inside of.
 ```
 
 This works because we moved to a strict location of Kibana plugins,
-`../kibana-extra/{pluginName}` relative to Kibana. This is one of the reasons we
-wanted to move towards a setup that looks like this:
+`./plugins/{pluginName}` inside of Kibana, or `../kibana-extra/{pluginName}`
+relative to Kibana. This is one of the reasons we wanted to move towards a setup
+that looks like this:
 
 ```
 elastic
-├── kibana
-└── kibana-extra
-    ├── kibana-canvas
-    └── x-pack-kibana
+└── kibana
+    └── plugins
+        ├── kibana-canvas
+        └── x-pack-kibana
 ```
 
 Relying on `link:` style dependencies means we no longer need to `npm publish`
@@ -119,11 +120,12 @@ yarn kbn bootstrap
 ```
 
 By default, `@kbn/pm` will bootstrap all packages within Kibana, plus all
-Kibana plugins located in `../kibana-extra`. There are several options for
-skipping parts of this, e.g. to skip bootstrapping of Kibana plugins:
+Kibana plugins located in `./plugins` or `../kibana-extra`. There are several
+options for skipping parts of this, e.g. to skip bootstrapping of Kibana
+plugins:
 
 ```
-yarn kbn bootstrap --skip-kibana-extra
+yarn kbn bootstrap --skip-kibana-plugins
 ```
 
 Or just skip few selected packages:
@@ -152,7 +154,7 @@ yarn kbn run build
 ```
 
 And if needed, you can skip packages in the same way as for bootstrapping, e.g.
-with `--exclude` and `--skip-kibana-extra`:
+with `--exclude` and `--skip-kibana-plugins`:
 
 ```
 yarn kbn run build --exclude kibana

packages/kbn-pm/src/cli.ts

@@ -35,7 +35,7 @@ function help() {
     usage: kbn <command> [<args>]
 
     By default commands are run for Kibana itself, all packages in the 'packages/'
-    folder and for all plugins in '../kibana-extra'.
+    folder and for all plugins in './plugins' and '../kibana-extra'.
 
     Available commands:
 
@@ -43,10 +43,10 @@ function help() {
 
     Global options:
 
-       -e, --exclude        Exclude specified project. Can be specified multiple times to exclude multiple projects, e.g. '-e kibana -e @kbn/pm'.
-       -i, --include        Include only specified projects. If left unspecified, it defaults to including all projects.
-       --oss                Do not include the x-pack when running command.
-       --skip-kibana-extra  Filter all plugins in ../kibana-extra when running command.
+       -e, --exclude          Exclude specified project. Can be specified multiple times to exclude multiple projects, e.g. '-e kibana -e @kbn/pm'.
+       -i, --include          Include only specified projects. If left unspecified, it defaults to including all projects.
+       --oss                  Do not include the x-pack when running command.
+       --skip-kibana-plugins  Filter all plugins in ./plugins and ../kibana-extra when running command.
   `);
 }
 

packages/kbn-pm/src/config.ts

@@ -20,7 +20,7 @@
 import { resolve } from 'path';
 
 export interface IProjectPathOptions {
-  'skip-kibana-extra'?: boolean;
+  'skip-kibana-plugins'?: boolean;
   oss?: boolean;
 }
 
@@ -28,7 +28,7 @@ export interface IProjectPathOptions {
  * Returns all the paths where plugins are located
  */
 export function getProjectPaths(rootPath: string, options: IProjectPathOptions) {
-  const skipKibanaExtra = Boolean(options['skip-kibana-extra']);
+  const skipKibanaPlugins = Boolean(options['skip-kibana-plugins']);
   const ossOnly = Boolean(options.oss);
 
   const projectPaths = [rootPath, resolve(rootPath, 'packages/*')];
@@ -49,10 +49,13 @@ export function getProjectPaths(rootPath: string, options: IProjectPathOptions)
     projectPaths.push(resolve(rootPath, 'x-pack/plugins/*'));
   }
 
-  if (!skipKibanaExtra) {
+  if (!skipKibanaPlugins) {
     projectPaths.push(resolve(rootPath, '../kibana-extra/*'));
     projectPaths.push(resolve(rootPath, '../kibana-extra/*/packages/*'));
     projectPaths.push(resolve(rootPath, '../kibana-extra/*/plugins/*'));
+    projectPaths.push(resolve(rootPath, 'plugins/*'));
+    projectPaths.push(resolve(rootPath, 'plugins/*/packages/*'));
+    projectPaths.push(resolve(rootPath, 'plugins/*/plugins/*'));
   }
 
   return projectPaths;

packages/kbn-pm/src/production/prepare_project_dependencies.ts

@@ -21,7 +21,8 @@ import { isLinkDependency } from '../utils/package_json';
 import { Project } from '../utils/project';
 
 /**
- * All external projects are located within `../kibana-extra/{plugin}` relative
+ * All external projects are located within `./plugins/{plugin}` relative
+ * to the Kibana root directory or `../kibana-extra/{plugin}` relative
  * to Kibana itself.
  */
 const isKibanaDep = (depVersion: string) => depVersion.includes('../../kibana/');

packages/kbn-pm/src/utils/__fixtures__/symlinked-plugins/corge/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "corge",
+  "version": "1.0.0"
+}

packages/kbn-pm/src/utils/projects.test.ts

@@ -17,7 +17,10 @@
  * under the License.
  */
 
-import { resolve } from 'path';
+import { mkdir, symlink } from 'fs';
+import { join, resolve } from 'path';
+import rmdir from 'rimraf';
+import { promisify } from 'util';
 
 import { getProjectPaths } from '../config';
 import { Project } from './project';
@@ -31,8 +34,20 @@ import {
 } from './projects';
 
 const rootPath = resolve(`${__dirname}/__fixtures__/kibana`);
+const rootPlugins = join(rootPath, 'plugins');
 
 describe('#getProjects', () => {
+  beforeAll(async () => {
+    await promisify(mkdir)(rootPlugins);
+
+    return promisify(symlink)(
+      join(__dirname, '__fixtures__/symlinked-plugins/corge'),
+      join(rootPlugins, 'corge')
+    );
+  });
+
+  afterAll(() => promisify(rmdir)(rootPlugins));
+
   test('find all packages in the packages directory', async () => {
     const projects = await getProjects(rootPath, ['packages/*']);
 
@@ -68,7 +83,15 @@ describe('#getProjects', () => {
     const projectPaths = getProjectPaths(rootPath, {});
     const projects = await getProjects(rootPath, projectPaths);
 
-    const expectedProjects = ['kibana', 'bar', 'foo', 'with-additional-projects', 'quux', 'baz'];
+    const expectedProjects = [
+      'kibana',
+      'bar',
+      'foo',
+      'with-additional-projects',
+      'quux',
+      'baz',
+      'bar',
+    ];
 
     expect([...projects.keys()]).toEqual(expect.arrayContaining(expectedProjects));
     expect(projects.size).toBe(expectedProjects.length);
@@ -85,7 +108,12 @@ describe('#getProjects', () => {
         exclude: ['foo', 'bar', 'baz'],
       });
 
-      expect([...projects.keys()].sort()).toEqual(['kibana', 'quux', 'with-additional-projects']);
+      expect([...projects.keys()].sort()).toEqual([
+        'corge',
+        'kibana',
+        'quux',
+        'with-additional-projects',
+      ]);
     });
 
     test('ignores unknown projects specified in `exclude` filter', async () => {
@@ -95,6 +123,7 @@ describe('#getProjects', () => {
 
       expect([...projects.keys()].sort()).toEqual([
         'baz',
+        'corge',
         'foo',
         'kibana',
         'quux',
@@ -137,7 +166,7 @@ describe('#getProjects', () => {
 
     test('does not return any project if `exclude` filter is specified for all projects', async () => {
       const projects = await getProjects(rootPath, projectPaths, {
-        exclude: ['kibana', 'bar', 'foo', 'with-additional-projects', 'quux', 'baz'],
+        exclude: ['kibana', 'bar', 'corge', 'foo', 'with-additional-projects', 'quux', 'baz'],
       });
 
       expect(projects.size).toBe(0);

tasks/test.js

@@ -91,7 +91,7 @@ module.exports = function (grunt) {
   function runProjectsTests() {
     const serverCmd = {
       cmd: 'yarn',
-      args: ['kbn', 'run', 'test', '--exclude', 'kibana', '--oss', '--skip-kibana-extra'],
+      args: ['kbn', 'run', 'test', '--exclude', 'kibana', '--oss', '--skip-kibana-plugins'],
       opts: { stdio: 'inherit' },
     };
 

x-pack/plugins/canvas/README.md

@@ -35,7 +35,7 @@ This will prompt you for some input. Generally, you can answer as follows:
 Once this has completed, go to your plugin directory:
 
 ```bash
-cd ../kibana-extra/canvas_example
+cd plugins/canvas_example
 ```
 
 Open that folder in your code editor of choice: `code .`
@@ -113,7 +113,7 @@ kbnInterpreter.register({
 In the terminal, in your plugin's directory, run:
 
 ```bash
-# In kibana-extra/canvas_example
+# In plugins/canvas_example
 yarn start
 ```