[Serverless] Only load current solution's plugins by default (#216088)

## Summary

Addresses https://github.com/elastic/kibana/issues/204227

Instead of having to explicitly disable other solutions' plugins in the
`serverless.(es|o11y|security|chat).yml` files, this PR proposes an
approach to filter them out directly in the plugin discovery mechanism.

The PR had to remove:
* a bunch of project-specific configurations that are no longer defined
in each project.
* some global serverless configurations that refer project-specific
settings.

Risks:
* Serverless deployments overriding configs that are no longer known.
* Serverless project type A relying on plugin from project type B in
some way (side effect?).

Mitigation:
* Since this is a configuration, we can simply override it
(`plugins.allowlistPluginGroups`) if something is broken.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Gerard Soldevila 2025-04-25 09:43:46 +02:00 committed by GitHub
parent 44cad37254
commit f26d5e622f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 209 additions and 117 deletions

7
.github/CODEOWNERS vendored
View file

@ -1890,9 +1890,10 @@ src/platform/plugins/shared/discover/public/context_awareness/profile_providers/
/x-pack/test_serverless/functional/test_suites/security/config.saved_objects_management.ts @elastic/kibana-core
/config/ @elastic/kibana-core
/config/serverless.yml @elastic/kibana-core @elastic/kibana-security
/config/serverless.es.yml @elastic/kibana-core @elastic/kibana-security
/config/serverless.oblt.yml @elastic/kibana-core @elastic/kibana-security
/config/serverless.security.yml @elastic/kibana-core @elastic/kibana-security
/config/serverless.chat.yml @elastic/kibana-core @elastic/kibana-security @elastic/search-kibana
/config/serverless.es.yml @elastic/kibana-core @elastic/kibana-security @elastic/search-kibana
/config/serverless.oblt.yml @elastic/kibana-core @elastic/kibana-security @elastic/observability-ui
/config/serverless.security.yml @elastic/kibana-core @elastic/security-solution @elastic/kibana-security
/config/serverless.security.search_ai_lake.yml @elastic/security-solution @elastic/kibana-security
/config/serverless.security.essentials.yml @elastic/security-solution @elastic/kibana-security
/config/serverless.security.complete.yml @elastic/security-solution @elastic/kibana-security

View file

@ -1,5 +1,8 @@
# Chat Project Config
# Make sure the plugins belonging to this project type are loaded
plugins.allowlistPluginGroups: ['platform', 'chat']
## Enable the Serverless Chat plugin
xpack.serverless.chat.enabled: true
@ -19,32 +22,5 @@ xpack.wciExternalServer.enabled: true
# Disable spaces
xpack.spaces.maxSpaces: 1
## Disable plugins that belong to other solutions
xpack.apm.enabled: false
xpack.infra.enabled: false
xpack.inventory.enabled: false
xpack.observability.enabled: false
xpack.observabilityAIAssistantApp.enabled: false
xpack.observabilityLogsExplorer.enabled: false
xpack.profiling.enabled: false
xpack.serverless.observability.enabled: false
xpack.slo.enabled: false
xpack.uptime.enabled: false
xpack.legacy_uptime.enabled: false
xpack.ux.enabled: false
xpack.search.enabled: false
xpack.searchAssistant.enabled: false
xpack.searchHomepage.enabled: false
xpack.searchIndices.enabled: false
xpack.searchInferenceEndpoints.enabled: false
xpack.searchNotebooks.enabled: false
xpack.searchPlayground.enabled: false
xpack.searchSynonyms.enabled: false
xpack.serverless.search.enabled: false
xpack.cloudSecurityPosture.enabled: false
xpack.securitySolution.enabled: false
xpack.securitySolutionEss.enabled: false
xpack.securitySolutionServerless.enabled: false
## Content Connectors in stack management
xpack.contentConnectors.enabled: false
xpack.contentConnectors.enabled: false

View file

@ -1,14 +1,8 @@
# Search Project Config
## Disable APM and Uptime, enable Enterprise Search
xpack.apm.enabled: false
# Make sure the plugins belonging to this project type are loaded
plugins.allowlistPluginGroups: ['platform', 'search']
xpack.cloudSecurityPosture.enabled: false
xpack.infra.enabled: false
xpack.observabilityLogsExplorer.enabled: false
xpack.observability.enabled: false
xpack.securitySolution.enabled: false
xpack.serverless.observability.enabled: false
xpack.search.enabled: false
xpack.osquery.enabled: false
@ -66,9 +60,6 @@ xpack.features.overrides:
observabilityAIAssistant:
name: "AI Assistant"
category: "enterpriseSearch"
### AI Assistant enables the Inventory feature, moving to Search
inventory:
category: "enterpriseSearch"
## Cloud settings
xpack.cloud.serverless.project_type: search
@ -119,7 +110,6 @@ data_visualizer.resultLinks.fileBeat.enabled: false
xpack.searchNotebooks.catalog.url: https://elastic-enterprise-search.s3.us-east-2.amazonaws.com/serverless/catalog.json
# Semantic text UI
xpack.index_management.dev.enableSemanticText: true
# AI Assistant config
@ -128,12 +118,6 @@ xpack.searchAssistant.enabled: true
xpack.searchAssistant.ui.enabled: true
xpack.observabilityAIAssistant.scope: "search"
aiAssistantManagementSelection.preferredAIAssistantType: "observability"
xpack.observabilityAiAssistantManagement.logSourcesEnabled: false
xpack.observabilityAiAssistantManagement.spacesEnabled: false
xpack.observabilityAiAssistantManagement.visibilityEnabled: false
# Synonyms UI
xpack.searchSynonyms.enabled: true
# Query Rules UI
xpack.searchQueryRules.enabled: false

View file

@ -1,17 +1,14 @@
# Observability Project config
## Disable plugins
xpack.search.enabled: false
xpack.cloudSecurityPosture.enabled: false
xpack.infra.enabled: true
xpack.uptime.enabled: true
xpack.securitySolution.enabled: false
xpack.searchNotebooks.enabled: false
xpack.searchPlayground.enabled: false
xpack.searchInferenceEndpoints.enabled: false
xpack.searchIndices.enabled: false
xpack.searchSynonyms.enabled: false
# Make sure the plugins belonging to this project type are loaded
plugins.allowlistPluginGroups: ['platform', 'observability']
## Enabled plugins
xpack.infra.enabled: true
# Disabled Observability plugins
xpack.ux.enabled: false
xpack.legacy_uptime.enabled: false
## Fine-tune the observability solution feature privileges. Also, refer to `serverless.yml` for the project-agnostic overrides.
xpack.features.overrides:

View file

@ -1,18 +1,13 @@
# Security Project config
# Make sure the plugins belonging to this project type are loaded
plugins.allowlistPluginGroups: ['platform', 'security']
# Ess plugins
xpack.securitySolutionEss.enabled: false
## Disable plugins
xpack.search.enabled: false
xpack.apm.enabled: false
xpack.infra.enabled: false
xpack.observabilityLogsExplorer.enabled: false
xpack.observability.enabled: false
xpack.observabilityAIAssistant.enabled: false
xpack.searchNotebooks.enabled: false
xpack.searchPlayground.enabled: false
xpack.searchInferenceEndpoints.enabled: false
xpack.inventory.enabled: false
xpack.searchIndices.enabled: false
xpack.searchSynonyms.enabled: false
## Fine-tune the security solution feature privileges. Also, refer to `serverless.yml` for the project-agnostic overrides.
xpack.features.overrides:

View file

@ -1,3 +1,5 @@
# Common serverless config
interactiveSetup.enabled: false
newsfeed.enabled: false
xpack.serverless.plugin.enabled: true
@ -8,9 +10,6 @@ xpack.fleet.internal.activeAgentsSoftLimit: 25000
xpack.fleet.internal.onlyAllowAgentUpgradeToKnownVersions: true
xpack.fleet.internal.retrySetupOnBoot: true
xpack.fleet.internal.useMeteringApi: true
xpack.searchSynonyms.enabled: false
xpack.searchQueryRules.enabled: false
## Fine-tune the feature privileges.
xpack.features.overrides:
@ -120,9 +119,6 @@ migrations.batchSize: 250
migrations.zdt:
metaPickupSyncDelaySec: 5
# Ess plugins
xpack.securitySolutionEss.enabled: false
# Management team plugins
xpack.upgrade_assistant.enabled: false
xpack.rollup.enabled: false
@ -257,9 +253,7 @@ xpack.reporting.queue.pollInterval: 3m
xpack.reporting.statefulSettings.enabled: false
xpack.reporting.csv.maxConcurrentShardRequests: 0
# Disabled Observability plugins
xpack.ux.enabled: false
xpack.legacy_uptime.enabled: false
# Disabled Platform plugins
monitoring.enabled: false
monitoring.ui.enabled: false

View file

@ -12,6 +12,11 @@
*/
export const ENABLE_ALL_PLUGINS_CONFIG_PATH = 'forceEnableAllPlugins' as const;
/**
* @internal
*/
export const INCLUDED_PLUGIN_GROUPS = 'allowlistPluginGroups' as const;
/**
* Set this to true in the raw configuration passed to {@link Root} to force
* enable all plugins.

View file

@ -21,10 +21,11 @@ import { resolve } from 'path';
import { ConfigService, Env } from '@kbn/config';
import type { CoreContext } from '@kbn/core-base-server-internal';
import type { NodeInfo } from '@kbn/core-node-server';
import type { KibanaGroup } from '@kbn/projects-solutions-groups';
import { PluginType } from '@kbn/core-base-common';
import { PluginsConfig, PluginsConfigType, config } from '../plugins_config';
import type { InstanceInfo } from '../plugin_context';
import { discover } from './plugins_discovery';
import { PluginType } from '@kbn/core-base-common';
jest.mock('@kbn/repo-packages', () => ({
...jest.requireActual('@kbn/repo-packages'),
@ -38,20 +39,30 @@ jest.mock('./plugin_manifest_from_plugin_package', () => ({
})),
}));
const getPluginPackagesFilter = jest.requireActual('@kbn/repo-packages').getPluginPackagesFilter;
const getPluginPackagesFilterMock: jest.Mock =
jest.requireMock('@kbn/repo-packages').getPluginPackagesFilter;
const pluginManifestFromPluginPackageMock: jest.Mock = jest.requireMock(
'./plugin_manifest_from_plugin_package'
).pluginManifestFromPluginPackage;
function getMockPackage(id: string) {
function getMockPackage(id: string, group: string = 'platform') {
const relativePath = `packages/${id}`;
return {
id,
manifest: {
id,
type: 'plugin',
},
directory: resolve(REPO_ROOT, `packages/${id}`),
group,
isPlugin: () => true,
getPluginCategories: () => ({
oss: false,
}),
getGroup: () => group,
directory: resolve(REPO_ROOT, relativePath),
normalizedRepoRelativeDir: relativePath,
} as Package;
}
@ -649,6 +660,76 @@ describe('plugins discovery system', () => {
value: plugin.manifest,
});
});
it('does not filter if allowlistPluginGroups is not specified', async () => {
const foo = getMockPackage('foo');
const bar = getMockPackage('bar');
const obs = getMockPackage('obs', 'observability');
const sec = getMockPackage('sec', 'security');
coreContext.env = {
...env,
pluginSearchPaths: [],
repoPackages: [foo, bar, obs, sec],
};
getPluginPackagesFilterMock.mockReturnValue(Boolean);
const filteredPluginsConfig: PluginsConfigType = {
...pluginConfig,
allowlistPluginGroups: undefined, // we make it explicit to illustrate the purpose of the test
};
const { plugin$ } = discover({
config: new PluginsConfig(filteredPluginsConfig, coreContext.env),
coreContext,
instanceInfo,
nodeInfo,
});
const plugins = await firstValueFrom(plugin$.pipe(toArray()));
expect(plugins.length).toEqual(4);
// plugin discovery sorts them by name
expect(plugins[0].name).toEqual('bar');
expect(plugins[1].name).toEqual('foo');
expect(plugins[2].name).toEqual('obs');
expect(plugins[3].name).toEqual('sec');
});
it('filters out plugins that do not belong to included groups', async () => {
const foo = getMockPackage('foo');
const bar = getMockPackage('bar');
const obs = getMockPackage('obs', 'observability');
const sec = getMockPackage('sec', 'security');
coreContext.env = {
...env,
pluginSearchPaths: [],
repoPackages: [foo, bar, obs, sec],
};
const allowlistPluginGroups: KibanaGroup[] = ['platform', 'observability'];
const filteredPluginsConfig: PluginsConfigType = {
...pluginConfig,
allowlistPluginGroups,
};
getPluginPackagesFilterMock.mockImplementation(getPluginPackagesFilter);
const { plugin$ } = discover({
config: new PluginsConfig(filteredPluginsConfig, coreContext.env),
coreContext,
instanceInfo,
nodeInfo,
});
const plugins = await firstValueFrom(plugin$.pipe(toArray()));
expect(plugins.length).toEqual(3);
// plugin discovery sorts them by name
expect(plugins[0].name).toEqual('bar');
expect(plugins[1].name).toEqual('foo');
expect(plugins[2].name).toEqual('obs');
});
});
describe('discovery order', () => {

View file

@ -9,15 +9,15 @@
import { from, merge, EMPTY } from 'rxjs';
import { catchError, filter, map, mergeMap, concatMap, shareReplay, toArray } from 'rxjs';
import { Logger } from '@kbn/logging';
import type { Logger } from '@kbn/logging';
import { getPluginPackagesFilter } from '@kbn/repo-packages';
import type { CoreContext } from '@kbn/core-base-server-internal';
import type { NodeInfo } from '@kbn/core-node-server';
import { PluginWrapper } from '../plugin';
import { pluginManifestFromPluginPackage } from './plugin_manifest_from_plugin_package';
import { createPluginInitializerContext, InstanceInfo } from '../plugin_context';
import { PluginsConfig } from '../plugins_config';
import { PluginDiscoveryError } from './plugin_discovery_error';
import type { PluginsConfig } from '../plugins_config';
import type { PluginDiscoveryError } from './plugin_discovery_error';
import { parseManifest } from './plugin_manifest_parser';
import { scanPluginSearchPaths } from './scan_plugin_search_paths';
@ -65,6 +65,7 @@ export function discover({
const pluginPkgDiscovery$ = from(coreContext.env.repoPackages ?? EMPTY).pipe(
filter(
getPluginPackagesFilter({
allowlistPluginGroups: config.allowlistPluginGroups,
oss: coreContext.env.cliArgs.oss,
examples: coreContext.env.cliArgs.runExamples,
paths: config.additionalPluginPaths,

View file

@ -35,12 +35,24 @@ describe('PluginsConfig', () => {
it('retrieves shouldEnableAllPlugins', () => {
const env = Env.createDefault(REPO_ROOT, getEnvOptions({ cliArgs: { dev: true } }));
const rawConfig: any = {
const rawConfig: PluginsConfigType = {
initialize: true,
paths: ['some-path', 'another-path'],
forceEnableAllPlugins: true,
};
const config = new PluginsConfig(rawConfig, env);
expect(config.shouldEnableAllPlugins).toBe(true);
expect(config.shouldEnableAllPlugins).toEqual(true);
});
it('retrieves included plugin groups', () => {
const env = Env.createDefault(REPO_ROOT, getEnvOptions({ cliArgs: { dev: true } }));
const rawConfig: PluginsConfigType = {
initialize: true,
paths: ['some-path', 'another-path'],
forceEnableAllPlugins: true,
allowlistPluginGroups: ['search'],
};
const config = new PluginsConfig(rawConfig, env);
expect(config.allowlistPluginGroups).toEqual(['search']);
});
});

View file

@ -7,12 +7,13 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { schema, TypeOf } from '@kbn/config-schema';
import { schema, type Type, TypeOf } from '@kbn/config-schema';
import { get } from 'lodash';
import { Env } from '@kbn/config';
import type { ServiceConfigDescriptor } from '@kbn/core-base-server-internal';
import { ENABLE_ALL_PLUGINS_CONFIG_PATH } from './constants';
import { KIBANA_GROUPS, type KibanaGroup } from '@kbn/projects-solutions-groups';
import { ENABLE_ALL_PLUGINS_CONFIG_PATH, INCLUDED_PLUGIN_GROUPS } from './constants';
const configSchema = schema.object({
initialize: schema.boolean({ defaultValue: true }),
@ -21,6 +22,19 @@ const configSchema = schema.object({
* Defines an array of directories where another plugin should be loaded from.
*/
paths: schema.arrayOf(schema.string(), { defaultValue: [] }),
/**
* Defines an array of groups to include when loading plugins.
* Plugins from all groups will be taken into account if the parameter is not provided.
*/
allowlistPluginGroups: schema.maybe(
schema.arrayOf(
schema.oneOf(
KIBANA_GROUPS.map((groupName) => schema.literal(groupName)) as [
Type<KibanaGroup> // This cast is needed because it's different to Type<T>[] :sight:
]
)
)
),
/**
* Internal config, not intended to be used by end users. Only for specific
* internal purposes.
@ -61,10 +75,18 @@ export class PluginsConfig {
*/
public readonly shouldEnableAllPlugins: boolean;
/**
* Specify an allowlist of plugin groups.
* Allows reducing the amount of plugins that are taken into account.
* The list will default to "all plugin groups" if the config is not present.
*/
public readonly allowlistPluginGroups?: readonly KibanaGroup[];
constructor(rawConfig: PluginsConfigType, env: Env) {
this.initialize = rawConfig.initialize;
this.pluginSearchPaths = env.pluginSearchPaths;
this.additionalPluginPaths = rawConfig.paths;
this.allowlistPluginGroups = get(rawConfig, INCLUDED_PLUGIN_GROUPS);
this.shouldEnableAllPlugins = get(rawConfig, ENABLE_ALL_PLUGINS_CONFIG_PATH, false);
}
}

View file

@ -135,7 +135,11 @@ async function testSetup() {
coreId = Symbol('core');
env = Env.createDefault(REPO_ROOT, getEnvOptions());
pluginsConfig = { initialize: true, paths: [] };
pluginsConfig = {
initialize: true,
paths: [],
allowlistPluginGroups: ['observability'],
};
config$ = new BehaviorSubject<Record<string, any>>({ plugins: pluginsConfig });
const rawConfigService = rawConfigServiceMock.create({ rawConfig$: config$ });
configService = new ConfigService(rawConfigService, env, logger);
@ -742,6 +746,7 @@ describe('PluginsService', () => {
expect(mockDiscover).toHaveBeenCalledWith({
config: {
additionalPluginPaths: [],
allowlistPluginGroups: ['observability'],
initialize: true,
pluginSearchPaths: [
resolve(REPO_ROOT, '..', 'kibana-extra'),

View file

@ -40,6 +40,7 @@
"@kbn/repo-packages",
"@kbn/utility-types",
"@kbn/core-plugins-contracts-server",
"@kbn/projects-solutions-groups",
],
"exclude": [
"target/**/*",

View file

@ -208,6 +208,7 @@ class Package {
/** @type {import('@kbn/projects-solutions-groups').ModuleVisibility} */
let visibility = 'shared';
// the following checks will only work in dev mode, as production builds create NPM packages under 'node_modules/@kbn-...'
if (dir.startsWith('src/platform/') || dir.startsWith('x-pack/platform/')) {
group = 'platform';
visibility =
@ -228,6 +229,7 @@ class Package {
group = 'chat';
visibility = 'private';
} else {
// this conditional branch is the only one that applies in production
group = this.manifest.group ?? 'common';
// if the group is 'private-only', enforce it
// BOOKMARK - List of Kibana solutions - FIXME we could use KIBANA_SOLUTIONS array here once we modernize this / get rid of Bazel

View file

@ -20,6 +20,7 @@ function getPluginSearchPaths({ rootDir }) {
/**
* @param {import('./types').PluginSelector} selector
* @param {import('./types').PluginCategoryInfo} category
* @returns {boolean}
*/
function matchCategory(selector, category) {
if (!category.oss && selector.oss) {
@ -40,6 +41,7 @@ function matchCategory(selector, category) {
/**
* @param {import('./types').PluginSelector} selector
* @param {string} pkgDir
* @returns {boolean}
*/
function matchPluginPaths(selector, pkgDir) {
if (!selector.paths) {
@ -52,6 +54,7 @@ function matchPluginPaths(selector, pkgDir) {
/**
* @param {import('./types').PluginSelector} selector
* @param {string} pkgDir
* @returns {boolean}
*/
function matchPluginParentDirs(selector, pkgDir) {
if (!selector.parentDirs) {
@ -64,6 +67,7 @@ function matchPluginParentDirs(selector, pkgDir) {
/**
* @param {import('./types').PluginSelector} selector
* @param {string} pkgDir
* @returns {boolean}
*/
function matchParentDirsLimit(selector, pkgDir) {
return !selector.limitParentDirs
@ -74,6 +78,7 @@ function matchParentDirsLimit(selector, pkgDir) {
/**
* @param {import('./types').PluginSelector} selector
* @param {import('./types').PluginPackage} pkg
* @returns {boolean}
*/
function matchBrowserServer(selector, pkg) {
if (selector.browser && !pkg.manifest.plugin.browser) {
@ -88,6 +93,25 @@ function matchBrowserServer(selector, pkg) {
/**
* @param {import('./types').PluginSelector} selector
* @param {import('./types').PluginPackage} pkg
* @returns {boolean}
*/
function matchPluginGroups(selector, pkg) {
if (Array.isArray(selector.allowlistPluginGroups)) {
return (
// if the allowlist is defined, ensure the plugin belongs to one of the groups
selector.allowlistPluginGroups.includes(pkg.group) ||
// add an exception for example and test plugins (they might not have a group)
pkg.group === 'common'
);
}
return true;
}
/**
* @param {import('./types').PluginSelector} selector
* @returns {(pkg: import('./package').Package) => pkg is import('./types').PluginPackage}
*/
function getPluginPackagesFilter(selector = {}) {
/**
@ -100,25 +124,8 @@ function getPluginPackagesFilter(selector = {}) {
matchParentDirsLimit(selector, pkg.directory) &&
(matchCategory(selector, pkg.getPluginCategories()) ||
matchPluginPaths(selector, pkg.directory) ||
matchPluginParentDirs(selector, pkg.directory));
matchPluginParentDirs(selector, pkg.directory)) &&
matchPluginGroups(selector, pkg);
}
/**
* @returns {(pkg: import('./package').Package) => boolean}
*/
function getDistributablePacakgesFilter() {
return (pkg) => {
if (pkg.isDevOnly()) {
return false;
}
if (!pkg.isPlugin()) {
return true;
}
const type = pkg.getPluginCategories();
return !(type.example || type.testPlugin);
};
}
module.exports = { getPluginSearchPaths, getPluginPackagesFilter, getDistributablePacakgesFilter };
module.exports = { getPluginSearchPaths, getPluginPackagesFilter };

View file

@ -181,6 +181,10 @@ export interface PluginSelector {
* When set to true, only select plugins which have browser-side components
*/
browser?: boolean;
/**
* When defined, only select plugins that belong to the specified groups
*/
allowlistPluginGroups?: readonly KibanaGroup[];
}
export interface KbnImportReq {

View file

@ -9,7 +9,7 @@ export interface ServerlessConfig {
developer?: {
projectSwitcher?: {
enabled: boolean;
currentType: 'security' | 'observability' | 'search';
currentType: 'security' | 'observability' | 'search' | 'chat';
};
};
}

View file

@ -39,6 +39,7 @@ const configSchema = schema.object({
schema.literal('security'),
schema.literal('observability'),
schema.literal('search'),
schema.literal('chat'),
]),
})
),

View file

@ -121,11 +121,15 @@ export function createServerlessTestConfig<T extends DeploymentAgnosticCommonSer
...svlSharedConfig.get('kbnTestServer.serverArgs'),
...kbnServerArgsFromController[options.serverlessProject],
`--serverless=${options.serverlessProject}`,
// defined in MKI control plane. Necessary for Synthetics app testing
'--xpack.uptime.service.password=test',
'--xpack.uptime.service.username=localKibanaIntegrationTestsUser',
'--xpack.uptime.service.devUrl=mockDevUrl',
'--xpack.uptime.service.manifestUrl=mockDevUrl',
...(options.serverlessProject === 'oblt'
? [
// defined in MKI control plane. Necessary for Synthetics app testing
'--xpack.uptime.service.password=test',
'--xpack.uptime.service.username=localKibanaIntegrationTestsUser',
'--xpack.uptime.service.devUrl=mockDevUrl',
'--xpack.uptime.service.manifestUrl=mockDevUrl',
]
: []),
],
},
testFiles: options.testFiles,