[ci] Splits Jest integration tests (#125454) (#125532)

(cherry picked from commit 435b772786)

Co-authored-by: Tyler Smalley <tyler.smalley@elastic.co>

.buildkite/pipelines/hourly.yml

@@ -127,9 +127,10 @@ steps:
 
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
+    parallelism: 2
     agents:
       queue: n2-4
-    timeout_in_minutes: 120
+    timeout_in_minutes: 90
     key: jest-integration
 
   - command: .buildkite/scripts/steps/test/api_integration.sh

.buildkite/pipelines/pull_request/base.yml

@@ -127,9 +127,10 @@ steps:
 
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
+    parallelism: 2
     agents:
       queue: n2-4
-    timeout_in_minutes: 120
+    timeout_in_minutes: 90
     key: jest-integration
 
   - command: .buildkite/scripts/steps/test/api_integration.sh

.buildkite/scripts/steps/test/jest_integration.sh

@@ -9,5 +9,5 @@ is_test_execution_step
 .buildkite/scripts/bootstrap.sh
 
 echo '--- Jest Integration Tests'
-checks-reporter-with-killswitch "Jest Integration Tests" \
-  node --max-old-space-size=6144 scripts/jest_integration --ci
+checks-reporter-with-killswitch "Jest Integration Tests $((BUILDKITE_PARALLEL_JOB+1))" \
+  .buildkite/scripts/steps/test/jest_parallel.sh jest.integration.config.js

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -13,7 +13,7 @@ exitCode=0
 while read -r config; do
   if [ "$((i % JOB_COUNT))" -eq "$JOB" ]; then
     echo "--- $ node scripts/jest --config $config"
-    node --max-old-space-size=14336 ./node_modules/.bin/jest --config="$config" --runInBand --coverage=false
+    node --max-old-space-size=14336 ./node_modules/.bin/jest --config="$config" --runInBand --coverage=false --passWithNoTests
     lastCode=$?
 
     if [ $lastCode -ne 0 ]; then
@@ -25,6 +25,6 @@ while read -r config; do
 
   ((i=i+1))
 # uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode
-done <<< "$(find src x-pack packages -name jest.config.js -not -path "*/__fixtures__/*" | sort)"
+done <<< "$(find src x-pack packages -name ${1:-jest.config.js} -not -path "*/__fixtures__/*" | sort)"
 
-exit $exitCode
+exit $exitCode

dev_docs/tutorials/debugging.mdx

@@ -21,7 +21,9 @@ Next we will go over how to exactly enable the inspector for different aspects o
 
 You will need to run Jest directly from the Node script:
 
-`node --inspect-brk scripts/jest [TestPathPattern]`
+`node --inspect-brk node_modules/.bin/jest --runInBand --config [JestConfig] [TestPathPattern]`
+
+Additional information can be found in the [Jest troubleshooting documentation](https://jestjs.io/docs/troubleshooting).
 
 ### Functional Test Runner
 

packages/kbn-es/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-es'],
+};

packages/kbn-optimizer/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-optimizer'],
+};

packages/kbn-plugin-generator/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-plugin-generator'],
+};

packages/kbn-plugin-helpers/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-plugin-helpers'],
+};

packages/kbn-test/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-test'],
+};

packages/kbn-test/jest_integration/jest-preset.js

@@ -20,7 +20,13 @@ module.exports = {
   ],
   reporters: [
     'default',
-    ['@kbn/test/target_node/jest/junit_reporter', { reportName: 'Jest Integration Tests' }],
+    [
+      '@kbn/test/target_node/jest/junit_reporter',
+      {
+        rootDirectory: '.',
+        reportName: 'Jest Integration Tests',
+      },
+    ],
     [
       '@kbn/test/target_node/jest/ci_stats_jest_reporter',
       {

packages/kbn-test/src/jest/configs/__snapshots__/jest_configs.test.ts.snap

@@ -0,0 +1,3 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`jestConfigs #expected throws if test file outside root 1`] = `[Error: Test file (bad.test.js) can not exist outside roots (packages/b/nested, packages). Move it to a root or configure additional root.]`;

packages/kbn-test/src/jest/configs/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './jest_configs';

packages/kbn-test/src/jest/configs/jest_configs.test.ts

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import mockFs from 'mock-fs';
+import fs from 'fs';
+
+import { JestConfigs } from './jest_configs';
+
+describe('jestConfigs', () => {
+  let jestConfigs: JestConfigs;
+
+  beforeEach(async () => {
+    mockFs({
+      '/kbn-test/packages': {
+        a: {
+          'jest.config.js': '',
+          'a_first.test.js': '',
+          'a_second.test.js': '',
+        },
+        b: {
+          'b.test.js': '',
+          integration_tests: {
+            'b_integration.test.js': '',
+          },
+          nested: {
+            d: {
+              'd.test.js': '',
+            },
+          },
+        },
+        c: {
+          'jest.integration.config.js': '',
+          integration_tests: {
+            'c_integration.test.js': '',
+          },
+        },
+      },
+    });
+    jestConfigs = new JestConfigs('/kbn-test', ['packages/b/nested', 'packages']);
+  });
+
+  afterEach(mockFs.restore);
+
+  describe('#files', () => {
+    it('lists unit test files', async () => {
+      const files = await jestConfigs.files('unit');
+      expect(files).toEqual([
+        'packages/a/a_first.test.js',
+        'packages/a/a_second.test.js',
+        'packages/b/b.test.js',
+        'packages/b/nested/d/d.test.js',
+      ]);
+    });
+
+    it('lists integration test files', async () => {
+      const files = await jestConfigs.files('integration');
+      expect(files).toEqual([
+        'packages/b/integration_tests/b_integration.test.js',
+        'packages/c/integration_tests/c_integration.test.js',
+      ]);
+    });
+  });
+
+  describe('#expected', () => {
+    it('expects unit config files', async () => {
+      const files = await jestConfigs.expected('unit');
+      expect(files).toEqual([
+        'packages/a/jest.config.js',
+        'packages/b/jest.config.js',
+        'packages/b/nested/d/jest.config.js',
+      ]);
+    });
+
+    it('expects integration config files', async () => {
+      const files = await jestConfigs.expected('integration');
+      expect(files).toEqual([
+        'packages/b/jest.integration.config.js',
+        'packages/c/jest.integration.config.js',
+      ]);
+    });
+
+    it('throws if test file outside root', async () => {
+      fs.writeFileSync('/kbn-test/bad.test.js', '');
+      await expect(() => jestConfigs.expected('unit')).rejects.toMatchSnapshot();
+    });
+  });
+
+  describe('#existing', () => {
+    it('lists existing unit test config files', async () => {
+      const files = await jestConfigs.existing('unit');
+      expect(files).toEqual(['packages/a/jest.config.js']);
+    });
+
+    it('lists existing integration test config files', async () => {
+      const files = await jestConfigs.existing('integration');
+      expect(files).toEqual(['packages/c/jest.integration.config.js']);
+    });
+  });
+
+  describe('#missing', () => {
+    it('lists existing unit test config files', async () => {
+      const files = await jestConfigs.missing('unit');
+      expect(files).toEqual(['packages/b/jest.config.js', 'packages/b/nested/d/jest.config.js']);
+    });
+
+    it('lists existing integration test config files', async () => {
+      const files = await jestConfigs.missing('integration');
+      expect(files).toEqual(['packages/b/jest.integration.config.js']);
+    });
+  });
+});

packages/kbn-test/src/jest/configs/jest_configs.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import path from 'path';
+import globby from 'globby';
+
+// @ts-ignore
+import { testMatch } from '../../../jest-preset';
+
+export const CONFIG_NAMES = {
+  unit: 'jest.config.js',
+  integration: 'jest.integration.config.js',
+};
+
+export class JestConfigs {
+  cwd: string;
+  roots: string[];
+  allFiles: string[] | undefined;
+
+  constructor(cwd: string, roots: string[]) {
+    this.cwd = cwd;
+    this.roots = roots;
+  }
+
+  async files(type: 'unit' | 'integration') {
+    if (!this.allFiles) {
+      this.allFiles = await globby(testMatch, {
+        gitignore: true,
+        cwd: this.cwd,
+      });
+    }
+
+    return this.allFiles.filter((f) =>
+      type === 'integration' ? f.includes('integration_tests') : !f.includes('integration_tests')
+    );
+  }
+
+  async expected(type: 'unit' | 'integration') {
+    const filesForType = await this.files(type);
+    const directories: Set<string> = new Set();
+
+    filesForType.forEach((file) => {
+      const root = this.roots.find((r) => file.startsWith(r));
+
+      if (root) {
+        const splitPath = file.substring(root.length).split(path.sep);
+
+        if (splitPath.length > 2) {
+          const name = splitPath[1];
+          directories.add([root, name].join(path.sep));
+        }
+      } else {
+        throw new Error(
+          `Test file (${file}) can not exist outside roots (${this.roots.join(
+            ', '
+          )}). Move it to a root or configure additional root.`
+        );
+      }
+    });
+
+    return [...directories].map((d) => [d, CONFIG_NAMES[type]].join(path.sep));
+  }
+
+  async existing(type: 'unit' | 'integration') {
+    return await globby(`**/${CONFIG_NAMES[type]}`, {
+      gitignore: true,
+      cwd: this.cwd,
+    });
+  }
+
+  async missing(type: 'unit' | 'integration') {
+    const expectedConfigs = await this.expected(type);
+    const existingConfigs = await this.existing(type);
+    return await expectedConfigs.filter((x) => !existingConfigs.includes(x));
+  }
+
+  async allMissing() {
+    return (await this.missing('unit')).concat(await this.missing('integration'));
+  }
+}

packages/kbn-test/src/jest/run_check_jest_configs_cli.ts

@@ -6,26 +6,29 @@
  * Side Public License, v 1.
  */
 
-import { relative, resolve, sep } from 'path';
 import { writeFileSync } from 'fs';
-
-import execa from 'execa';
-import globby from 'globby';
+import path from 'path';
 import Mustache from 'mustache';
 
 import { run } from '@kbn/dev-utils';
 import { REPO_ROOT } from '@kbn/utils';
 
-// @ts-ignore
-import { testMatch } from '../../jest-preset';
+import { JestConfigs, CONFIG_NAMES } from './configs';
 
-const template: string = `module.exports = {
+const unitTestingTemplate: string = `module.exports = {
   preset: '@kbn/test',
   rootDir: '{{{relToRoot}}}',
   roots: ['<rootDir>/{{{modulePath}}}'],
 };
 `;
 
+const integrationTestingTemplate: string = `module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '{{{relToRoot}}}',
+  roots: ['<rootDir>/{{{modulePath}}}'],
+};
+`;
+
 const roots: string[] = [
   'x-pack/plugins/security_solution/public',
   'x-pack/plugins/security_solution/server',
@@ -40,68 +43,43 @@ const roots: string[] = [
 export async function runCheckJestConfigsCli() {
   run(
     async ({ flags: { fix = false }, log }) => {
-      const { stdout: coveredFiles } = await execa(
-        'yarn',
-        ['--silent', 'jest', '--listTests', '--json'],
-        {
-          cwd: REPO_ROOT,
-        }
-      );
+      const jestConfigs = new JestConfigs(REPO_ROOT, roots);
 
-      const allFiles = new Set(
-        await globby(testMatch.concat(['!**/integration_tests/**']), {
-          gitignore: true,
-        })
-      );
+      const missing = await jestConfigs.allMissing();
 
-      JSON.parse(coveredFiles).forEach((file: string) => {
-        const pathFromRoot = relative(REPO_ROOT, file);
-        allFiles.delete(pathFromRoot);
-      });
-
-      if (allFiles.size) {
+      if (missing.length) {
         log.error(
-          `The following files do not belong to a jest.config.js file, or that config is not included from the root jest.config.js\n${[
-            ...allFiles,
+          `The following Jest config files do not exist for which there are test files for:\n${[
+            ...missing,
           ]
             .map((file) => ` - ${file}`)
             .join('\n')}`
         );
-      } else {
-        log.success('All test files are included by a Jest configuration');
-        return;
-      }
 
-      if (fix) {
-        allFiles.forEach((file) => {
-          const root = roots.find((r) => file.startsWith(r));
-
-          if (root) {
-            const name = relative(root, file).split(sep)[0];
-            const modulePath = [root, name].join('/');
+        if (fix) {
+          missing.forEach((file) => {
+            const template = file.endsWith(CONFIG_NAMES.unit)
+              ? unitTestingTemplate
+              : integrationTestingTemplate;
 
+            const modulePath = path.dirname(file);
             const content = Mustache.render(template, {
-              relToRoot: relative(modulePath, '.'),
+              relToRoot: path.relative(modulePath, '.'),
               modulePath,
             });
 
-            const configPath = resolve(root, name, 'jest.config.js');
-            log.info('created %s', configPath);
-            writeFileSync(configPath, content);
-          } else {
-            log.warning(`Unable to determind where to place jest.config.js for ${file}`);
-          }
-        });
-      } else {
-        log.info(
-          `Run 'node scripts/check_jest_configs --fix' to attempt to create the missing config files`
-        );
+            writeFileSync(file, content);
+            log.info('created %s', file);
+          });
+        } else {
+          log.info(
+            `Run 'node scripts/check_jest_configs --fix' to create the missing config files`
+          );
+        }
       }
-
-      process.exit(1);
     },
     {
-      description: 'Check that all test files are covered by a jest.config.js',
+      description: 'Check that all test files are covered by a Jest config',
       flags: {
         boolean: ['fix'],
         help: `

src/cli/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../..',
+  roots: ['<rootDir>/src/cli'],
+};

src/core/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../..',
+  roots: ['<rootDir>/src/core'],
+};

src/dev/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../..',
+  roots: ['<rootDir>/src/dev'],
+};

src/plugins/chart_expressions/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/src/plugins/chart_expressions'],
+};

src/plugins/kibana_usage_collection/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../../..',
+  roots: ['<rootDir>/src/plugins/kibana_usage_collection'],
+};

src/plugins/usage_collection/jest.integration.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../../..',
+  roots: ['<rootDir>/src/plugins/usage_collection'],
+};

src/plugins/vis_types/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/src/plugins/vis_types'],
+};

x-pack/plugins/fleet/jest.integration.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/plugins/fleet'],
+};

x-pack/plugins/global_search/jest.integration.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/plugins/global_search'],
+};

x-pack/plugins/reporting/jest.integration.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/plugins/reporting'],
+};

x-pack/plugins/task_manager/jest.integration.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/plugins/task_manager'],
+};