[ftr/test_serverless] Run common tests as part of each project (#160783)

## Summary

This PR removes standalone `common` serverless tests and instead makes
them run as part of every serverless project's tests.

### Details

Before, the `common` tests ran on a "vanilla" Kibana serverless mode
(i.e. the `serverless` plugin was loaded but none of the
`serverless-PROJECT` plugins). With continued serverless development,
this state of Kibana doesn't work as expected anymore and since this is
not supported officially anyway, it's not worth to invest making it work
properly. So we decided to accept the extra test run time and actually
include `common` tests in every project.

.buildkite/pipelines/pull_request/base.yml

@@ -56,19 +56,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: SERVERLESS_ENVIRONMENT=common .buildkite/scripts/steps/functional/serverless_ftr.sh
-    label: 'Serverless Common Tests'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 40
-    soft_fail:
-      - exit_status: 10
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
   - command: SERVERLESS_ENVIRONMENT=observability .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Observability Tests'
     agents:

.buildkite/scripts/steps/functional/serverless_ftr.sh

@@ -6,12 +6,7 @@ source .buildkite/scripts/steps/functional/common.sh
 
 export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT"
 
-if [[ "$SERVERLESS_ENVIRONMENT" == "common" ]]; then
-  SERVERLESS_CONFIGS=(
-    "x-pack/test_serverless/api_integration/test_suites/common/config.ts"
-    "x-pack/test_serverless/functional/test_suites/common/config.ts"
-  )
-elif [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then
+if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then
   SERVERLESS_CONFIGS=(
     "x-pack/test_serverless/api_integration/test_suites/search/config.ts"
     "x-pack/test_serverless/functional/test_suites/search/config.ts"

x-pack/test_serverless/README.md

@@ -42,6 +42,32 @@ x-pack/test_serverless/
 │  ├─ types
 ```
 
+### Common tests
+
+As outlined above, tests in the `common` API integration and functional test suites are
+covering functionality that's shared across serverless projects. As a result, these tests
+are automatically included in all project specific test configurations and don't have a
+dedicated configuration file. We always run in the context of one of the serverless projects
+and invoke the corresponding set of tests, which then also includes the `common` tests.
+
+In case a common test needs to be skipped for one of the projects, there are the following
+suite tags available to do so: `skipSvlOblt`, `skipSvlSearch`, `skipSvlSec`, which can be
+added like this to a test suite:
+
+```
+describe('my test suite', function () {
+  this.tags(['skipSvlOblt', 'skipSvlSearch', 'skipSvlSec']);
+  // or for a single tag: this.tags('skipSvlSec');
+  [...]
+});
+```
+
+Tests that are designed to only run in one of the projects should be added to the project
+specific test directory and not to `common` with two skips.
+
+Note, that `common` tests are invoked three times in a full test run: once per project to make
+sure the covered shared functionality works correctly in every project. So when writing tests there, be mindful about the test run time.
+
 ### Shared services and page objects
 
 Test services and page objects from `x-pack/test/[api_integration|functional]`
@@ -65,8 +91,8 @@ following namespaces:
 
 As outlined above, serverless tests are separated from stateful tests (except
 the reuse of helper methods), which includes a new base configuration. All
-tests that should run in a serverless environment have to be added to, in the 
-`x-pack/test_serverless` directory.
+tests that should run in a serverless environment have to be added to the 
+`x-pack/test_serverless`.
 
 Tests in this area should be clearly designed for the serverless environment,
 particularly when it comes to timing for API requests and UI interaction.
@@ -76,7 +102,7 @@ Similar to how functional tests are run in `x-pack/test`, you can point the
 functional tests server and test runner to config files in this `x-pack/test_serverless`
 directory, e.g. from the `x-pack` directory run:
 ```
-node scripts/functional_tests_server.js --config test_serverless/api_integration/test_suites/common/config.ts
+node scripts/functional_tests_server.js --config test_serverless/api_integration/test_suites/search/config.ts
 
-node scripts/functional_test_runner.js --config test_serverless/api_integration/test_suites/common/config.ts
+node scripts/functional_test_runner.js --config test_serverless/api_integration/test_suites/search/config.ts
 ```

x-pack/test_serverless/api_integration/config.base.ts

@@ -4,9 +4,6 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { resolve } from 'path';
-
-import { REPO_ROOT } from '@kbn/repo-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 
 import { services } from './services';
@@ -15,7 +12,6 @@ import type { CreateTestConfigOptions } from '../shared/types';
 export function createTestConfig(options: CreateTestConfigOptions) {
   return async ({ readConfigFile }: FtrConfigProviderContext) => {
     const svlSharedConfig = await readConfigFile(require.resolve('../shared/config.base.ts'));
-    const svlBaseConfig = resolve(REPO_ROOT, 'config', 'serverless.yml');
 
     return {
       ...svlSharedConfig.getAll(),
@@ -25,13 +21,12 @@ export function createTestConfig(options: CreateTestConfigOptions) {
         ...svlSharedConfig.get('kbnTestServer'),
         serverArgs: [
           ...svlSharedConfig.get('kbnTestServer.serverArgs'),
-          options.serverlessProject
-            ? `--serverless=${options.serverlessProject}`
-            : `--config=${svlBaseConfig}`,
+          `--serverless=${options.serverlessProject}`,
         ],
       },
       testFiles: options.testFiles,
       junit: options.junit,
+      suiteTags: options.suiteTags,
     };
   };
 }

x-pack/test_serverless/api_integration/test_suites/common/config.ts

@@ -1,16 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { createTestConfig } from '../../config.base';
-
-export default createTestConfig({
-  serverlessProject: undefined,
-  testFiles: [require.resolve('.')],
-  junit: {
-    reportName: 'Serverless Common API Integration Tests',
-  },
-});

x-pack/test_serverless/api_integration/test_suites/observability/config.ts

@@ -9,8 +9,9 @@ import { createTestConfig } from '../../config.base';
 
 export default createTestConfig({
   serverlessProject: 'oblt',
-  testFiles: [require.resolve('.')],
+  testFiles: [require.resolve('../common'), require.resolve('.')],
   junit: {
     reportName: 'Serverless Observability API Integration Tests',
   },
+  suiteTags: { exclude: ['skipSvlOblt'] },
 });

x-pack/test_serverless/api_integration/test_suites/search/config.ts

@@ -9,8 +9,9 @@ import { createTestConfig } from '../../config.base';
 
 export default createTestConfig({
   serverlessProject: 'es',
-  testFiles: [require.resolve('.')],
+  testFiles: [require.resolve('../common'), require.resolve('.')],
   junit: {
     reportName: 'Serverless Search API Integration Tests',
   },
+  suiteTags: { exclude: ['skipSvlSearch'] },
 });

x-pack/test_serverless/api_integration/test_suites/security/config.ts

@@ -9,8 +9,9 @@ import { createTestConfig } from '../../config.base';
 
 export default createTestConfig({
   serverlessProject: 'security',
-  testFiles: [require.resolve('.')],
+  testFiles: [require.resolve('../common'), require.resolve('.')],
   junit: {
     reportName: 'Serverless Security API Integration Tests',
   },
+  suiteTags: { exclude: ['skipSvlSec'] },
 });

x-pack/test_serverless/functional/config.base.ts

@@ -7,7 +7,6 @@
 
 import { resolve } from 'path';
 
-import { REPO_ROOT } from '@kbn/repo-info';
 import { FtrConfigProviderContext } from '@kbn/test';
 
 import { pageObjects } from './page_objects';
@@ -17,7 +16,6 @@ import type { CreateTestConfigOptions } from '../shared/types';
 export function createTestConfig(options: CreateTestConfigOptions) {
   return async ({ readConfigFile }: FtrConfigProviderContext) => {
     const svlSharedConfig = await readConfigFile(require.resolve('../shared/config.base.ts'));
-    const svlBaseConfig = resolve(REPO_ROOT, 'config', 'serverless.yml');
 
     return {
       ...svlSharedConfig.getAll(),
@@ -28,9 +26,7 @@ export function createTestConfig(options: CreateTestConfigOptions) {
         ...svlSharedConfig.get('kbnTestServer'),
         serverArgs: [
           ...svlSharedConfig.get('kbnTestServer.serverArgs'),
-          options.serverlessProject
-            ? `--serverless=${options.serverlessProject}`
-            : `--config=${svlBaseConfig}`,
+          `--serverless=${options.serverlessProject}`,
         ],
       },
       testFiles: options.testFiles,
@@ -62,6 +58,7 @@ export function createTestConfig(options: CreateTestConfigOptions) {
         directory: resolve(__dirname, 'screenshots'),
       },
       junit: options.junit,
+      suiteTags: options.suiteTags,
     };
   };
 }

x-pack/test_serverless/functional/test_suites/common/config.ts

@@ -1,16 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { createTestConfig } from '../../config.base';
-
-export default createTestConfig({
-  serverlessProject: undefined,
-  testFiles: [require.resolve('.')],
-  junit: {
-    reportName: 'Serverless Common Functional Tests',
-  },
-});

x-pack/test_serverless/functional/test_suites/observability/config.ts

@@ -9,8 +9,9 @@ import { createTestConfig } from '../../config.base';
 
 export default createTestConfig({
   serverlessProject: 'oblt',
-  testFiles: [require.resolve('.')],
+  testFiles: [require.resolve('../common'), require.resolve('.')],
   junit: {
     reportName: 'Serverless Observability Functional Tests',
   },
+  suiteTags: { exclude: ['skipSvlOblt'] },
 });

x-pack/test_serverless/functional/test_suites/search/config.ts

@@ -9,8 +9,9 @@ import { createTestConfig } from '../../config.base';
 
 export default createTestConfig({
   serverlessProject: 'es',
-  testFiles: [require.resolve('.')],
+  testFiles: [require.resolve('../common'), require.resolve('.')],
   junit: {
     reportName: 'Serverless Search Functional Tests',
   },
+  suiteTags: { exclude: ['skipSvlSearch'] },
 });

x-pack/test_serverless/functional/test_suites/security/config.ts

@@ -9,8 +9,9 @@ import { createTestConfig } from '../../config.base';
 
 export default createTestConfig({
   serverlessProject: 'security',
-  testFiles: [require.resolve('.')],
+  testFiles: [require.resolve('../common'), require.resolve('.')],
   junit: {
     reportName: 'Serverless Security Functional Tests',
   },
+  suiteTags: { exclude: ['skipSvlSec'] },
 });

x-pack/test_serverless/shared/types/index.ts

@@ -6,7 +6,8 @@
  */
 
 export interface CreateTestConfigOptions {
-  serverlessProject: 'es' | 'oblt' | 'security' | undefined;
+  serverlessProject: 'es' | 'oblt' | 'security';
   testFiles: string[];
   junit: { reportName: string };
+  suiteTags?: { include?: string[]; exclude?: string[] };
 }