[On-Week] Hot update of APM/EBT labels (#157093)

This commit is contained in:
Alejandro Fernández Haro 2023-08-31 14:36:20 +02:00 committed by GitHub
parent 81aceaa5c6
commit 0ea37c1b42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 683 additions and 37 deletions

1
.github/CODEOWNERS vendored
View file

@ -1045,6 +1045,7 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
/.github/codeql @elastic/kibana-security
/.github/workflows/codeql.yml @elastic/kibana-security
/src/dev/eslint/security_eslint_rule_tests.ts @elastic/kibana-security
/src/core/server/integration_tests/config/check_dynamic_config.test.ts @elastic/kibana-security
/src/plugins/telemetry/server/config/telemetry_labels.ts @elastic/kibana-security
/test/interactive_setup_api_integration/ @elastic/kibana-security
/test/interactive_setup_functional/ @elastic/kibana-security

View file

@ -90,6 +90,9 @@ elasticsearch.requestHeadersWhitelist: ["authorization", "es-client-authenticati
# Limit maxSockets to 800 as we do in ESS, which improves reliability under high loads.
elasticsearch.maxSockets: 800
# Enable dynamic config to be updated via the internal HTTP requests
coreApp.allowDynamicConfigOverrides: true
# Visualizations editors readonly settings
vis_type_gauge.readOnly: true
vis_type_heatmap.readOnly: true

View file

@ -6,8 +6,9 @@
* Side Public License, v 1.
*/
export { CoreAppsService } from './src';
export { CoreAppsService, config } from './src';
export type {
CoreAppConfigType,
InternalCoreAppsServiceRequestHandlerContext,
InternalCoreAppsServiceRouter,
} from './src';

View file

@ -17,6 +17,7 @@ import { PluginType } from '@kbn/core-base-common';
import type { RequestHandlerContext } from '@kbn/core-http-request-handler-context-server';
import { coreInternalLifecycleMock } from '@kbn/core-lifecycle-server-mocks';
import { CoreAppsService } from './core_app';
import { of } from 'rxjs';
const emptyPlugins = (): UiPlugins => ({
internal: new Map(),
@ -56,10 +57,43 @@ describe('CoreApp', () => {
registerBundleRoutesMock.mockReset();
});
describe('`/internal/core/_settings` route', () => {
it('is not registered by default', async () => {
const routerMock = mockRouter.create();
internalCoreSetup.http.createRouter.mockReturnValue(routerMock);
const localCoreApp = new CoreAppsService(coreContext);
await localCoreApp.setup(internalCoreSetup, emptyPlugins());
expect(routerMock.versioned.put).not.toHaveBeenCalledWith(
expect.objectContaining({
path: '/internal/core/_settings',
})
);
});
it('is registered when enabled', async () => {
const routerMock = mockRouter.create();
internalCoreSetup.http.createRouter.mockReturnValue(routerMock);
coreContext.configService.atPath.mockReturnValue(of({ allowDynamicConfigOverrides: true }));
const localCoreApp = new CoreAppsService(coreContext);
await localCoreApp.setup(internalCoreSetup, emptyPlugins());
expect(routerMock.versioned.put).toHaveBeenCalledWith({
path: '/internal/core/_settings',
access: 'internal',
options: {
tags: ['access:updateDynamicConfig'],
},
});
});
});
describe('`/status` route', () => {
it('is registered with `authRequired: false` is the status page is anonymous', () => {
it('is registered with `authRequired: false` is the status page is anonymous', async () => {
internalCoreSetup.status.isStatusPageAnonymous.mockReturnValue(true);
coreApp.setup(internalCoreSetup, emptyPlugins());
await coreApp.setup(internalCoreSetup, emptyPlugins());
expect(httpResourcesRegistrar.register).toHaveBeenCalledWith(
{
@ -73,9 +107,9 @@ describe('CoreApp', () => {
);
});
it('is registered with `authRequired: true` is the status page is not anonymous', () => {
it('is registered with `authRequired: true` is the status page is not anonymous', async () => {
internalCoreSetup.status.isStatusPageAnonymous.mockReturnValue(false);
coreApp.setup(internalCoreSetup, emptyPlugins());
await coreApp.setup(internalCoreSetup, emptyPlugins());
expect(httpResourcesRegistrar.register).toHaveBeenCalledWith(
{
@ -185,8 +219,8 @@ describe('CoreApp', () => {
});
describe('`/app/{id}/{any*}` route', () => {
it('is registered with the correct parameters', () => {
coreApp.setup(internalCoreSetup, emptyPlugins());
it('is registered with the correct parameters', async () => {
await coreApp.setup(internalCoreSetup, emptyPlugins());
expect(httpResourcesRegistrar.register).toHaveBeenCalledWith(
{
@ -201,9 +235,9 @@ describe('CoreApp', () => {
});
});
it('`setup` calls `registerBundleRoutes` with the correct options', () => {
it('`setup` calls `registerBundleRoutes` with the correct options', async () => {
const uiPlugins = emptyPlugins();
coreApp.setup(internalCoreSetup, uiPlugins);
await coreApp.setup(internalCoreSetup, uiPlugins);
expect(registerBundleRoutesMock).toHaveBeenCalledTimes(1);
expect(registerBundleRoutesMock).toHaveBeenCalledWith({

View file

@ -7,8 +7,8 @@
*/
import { stringify } from 'querystring';
import { Env } from '@kbn/config';
import { schema } from '@kbn/config-schema';
import { Env, IConfigService } from '@kbn/config';
import { schema, ValidationError } from '@kbn/config-schema';
import { fromRoot } from '@kbn/repo-info';
import type { Logger } from '@kbn/logging';
import type { CoreContext } from '@kbn/core-base-server-internal';
@ -22,6 +22,8 @@ import type {
import type { UiPlugins } from '@kbn/core-plugins-base-server-internal';
import type { HttpResources, HttpResourcesServiceToolkit } from '@kbn/core-http-resources-server';
import type { InternalCorePreboot, InternalCoreSetup } from '@kbn/core-lifecycle-server-internal';
import { firstValueFrom, map, type Observable } from 'rxjs';
import { CoreAppConfig, type CoreAppConfigType, CoreAppPath } from './core_app_config';
import { registerBundleRoutes } from './bundle_routes';
import type { InternalCoreAppsServiceRequestHandlerContext } from './internal_types';
@ -41,10 +43,16 @@ interface CommonRoutesParams {
export class CoreAppsService {
private readonly logger: Logger;
private readonly env: Env;
private readonly configService: IConfigService;
private readonly config$: Observable<CoreAppConfig>;
constructor(core: CoreContext) {
this.logger = core.logger.get('core-app');
this.env = core.env;
this.configService = core.configService;
this.config$ = this.configService
.atPath<CoreAppConfigType>(CoreAppPath)
.pipe(map((rawCfg) => new CoreAppConfig(rawCfg)));
}
preboot(corePreboot: InternalCorePreboot, uiPlugins: UiPlugins) {
@ -57,9 +65,10 @@ export class CoreAppsService {
}
}
setup(coreSetup: InternalCoreSetup, uiPlugins: UiPlugins) {
async setup(coreSetup: InternalCoreSetup, uiPlugins: UiPlugins) {
this.logger.debug('Setting up core app.');
this.registerDefaultRoutes(coreSetup, uiPlugins);
const config = await firstValueFrom(this.config$);
this.registerDefaultRoutes(coreSetup, uiPlugins, config);
this.registerStaticDirs(coreSetup);
}
@ -88,7 +97,11 @@ export class CoreAppsService {
});
}
private registerDefaultRoutes(coreSetup: InternalCoreSetup, uiPlugins: UiPlugins) {
private registerDefaultRoutes(
coreSetup: InternalCoreSetup,
uiPlugins: UiPlugins,
config: CoreAppConfig
) {
const httpSetup = coreSetup.http;
const router = httpSetup.createRouter<InternalCoreAppsServiceRequestHandlerContext>('');
const resources = coreSetup.httpResources.createRegistrar(router);
@ -147,6 +160,51 @@ export class CoreAppsService {
}
}
);
if (config.allowDynamicConfigOverrides) {
this.registerInternalCoreSettingsRoute(router);
}
}
/**
* Registers the HTTP API that allows updating in-memory the settings that opted-in to be dynamically updatable.
* @param router {@link IRouter}
* @private
*/
private registerInternalCoreSettingsRoute(router: IRouter) {
router.versioned
.put({
path: '/internal/core/_settings',
access: 'internal',
options: {
tags: ['access:updateDynamicConfig'],
},
})
.addVersion(
{
version: '1',
validate: {
request: {
body: schema.recordOf(schema.string(), schema.any()),
},
response: {
'200': { body: schema.object({ ok: schema.boolean() }) },
},
},
},
async (context, req, res) => {
try {
this.configService.setDynamicConfigOverrides(req.body);
} catch (err) {
if (err instanceof ValidationError) {
return res.badRequest({ body: err });
}
throw err;
}
return res.ok({ body: { ok: true } });
}
);
}
private registerCommonDefaultRoutes({

View file

@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { config, CoreAppConfig } from './core_app_config';
describe('CoreApp Config', () => {
test('set correct defaults', () => {
const configValue = new CoreAppConfig(config.schema.validate({}));
expect(configValue).toMatchInlineSnapshot(`
CoreAppConfig {
"allowDynamicConfigOverrides": false,
}
`);
});
});

View file

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { schema, type TypeOf } from '@kbn/config-schema';
import type { ServiceConfigDescriptor } from '@kbn/core-base-server-internal';
/**
* Validation schema for Core App config.
* @public
*/
export const configSchema = schema.object({
allowDynamicConfigOverrides: schema.boolean({ defaultValue: false }),
});
export type CoreAppConfigType = TypeOf<typeof configSchema>;
export const CoreAppPath = 'coreApp';
export const config: ServiceConfigDescriptor<CoreAppConfigType> = {
path: CoreAppPath,
schema: configSchema,
};
/**
* Wrapper of config schema.
* @internal
*/
export class CoreAppConfig implements CoreAppConfigType {
/**
* @internal
* When true, the HTTP API to dynamically extend the configuration is registered.
*
* @remarks
* You should enable this at your own risk: Settings opted-in to being dynamically
* configurable can be changed at any given point, potentially leading to unexpected behaviours.
* This feature is mostly intended for testing purposes.
*/
public readonly allowDynamicConfigOverrides: boolean;
constructor(rawConfig: CoreAppConfig) {
this.allowDynamicConfigOverrides = rawConfig.allowDynamicConfigOverrides;
}
}

View file

@ -7,6 +7,7 @@
*/
export { CoreAppsService } from './core_app';
export { config, type CoreAppConfigType } from './core_app_config';
export type {
InternalCoreAppsServiceRequestHandlerContext,
InternalCoreAppsServiceRouter,

View file

@ -278,6 +278,14 @@ export class PluginsService implements CoreService<PluginsServiceSetup, PluginsS
getFlattenedObject(configDescriptor.exposeToUsage)
);
}
if (configDescriptor.dynamicConfig) {
const configKeys = Object.entries(getFlattenedObject(configDescriptor.dynamicConfig))
.filter(([, value]) => value === true)
.map(([key]) => key);
if (configKeys.length > 0) {
this.coreContext.configService.addDynamicConfigPaths(plugin.configPath, configKeys);
}
}
this.coreContext.configService.setSchema(plugin.configPath, configDescriptor.schema);
}
}

View file

@ -18,6 +18,7 @@ export type {
SharedGlobalConfig,
MakeUsageFromSchema,
ExposedToBrowserDescriptor,
DynamicConfigDescriptor,
} from './src';
export { SharedGlobalConfigKeys } from './src';

View file

@ -18,6 +18,7 @@ export type {
SharedGlobalConfig,
MakeUsageFromSchema,
ExposedToBrowserDescriptor,
DynamicConfigDescriptor,
} from './types';
export { SharedGlobalConfigKeys } from './shared_global_config';

View file

@ -34,7 +34,7 @@ export type PluginConfigSchema<T> = Type<T>;
/**
* Type defining the list of configuration properties that will be exposed on the client-side
* Object properties can either be fully exposed
* Object properties can either be fully exposed or narrowed down to specific keys.
*
* @public
*/
@ -49,6 +49,23 @@ export type ExposedToBrowserDescriptor<T> = {
boolean;
};
/**
* Type defining the list of configuration properties that can be dynamically updated
* Object properties can either be fully exposed or narrowed down to specific keys.
*
* @public
*/
export type DynamicConfigDescriptor<T> = {
[Key in keyof T]?: T[Key] extends Maybe<any[]>
? // handles arrays as primitive values
boolean
: T[Key] extends Maybe<object>
? // can be nested for objects
DynamicConfigDescriptor<T[Key]> | boolean
: // primitives
boolean;
};
/**
* Describes a plugin configuration properties.
*
@ -88,6 +105,10 @@ export interface PluginConfigDescriptor<T = any> {
* List of configuration properties that will be available on the client-side plugin.
*/
exposeToBrowser?: ExposedToBrowserDescriptor<T>;
/**
* List of configuration properties that can be dynamically changed via the PUT /_settings API.
*/
dynamicConfig?: DynamicConfigDescriptor<T>;
/**
* Schema to use to validate the plugin configuration.
*

View file

@ -16,6 +16,7 @@ import { pidConfig } from '@kbn/core-environment-server-internal';
import { executionContextConfig } from '@kbn/core-execution-context-server-internal';
import { config as httpConfig, cspConfig, externalUrlConfig } from '@kbn/core-http-server-internal';
import { config as elasticsearchConfig } from '@kbn/core-elasticsearch-server-internal';
import { config as coreAppConfig } from '@kbn/core-apps-server-internal';
import { opsConfig } from '@kbn/core-metrics-server-internal';
import {
savedObjectsConfig,
@ -37,6 +38,7 @@ export function registerServiceConfig(configService: ConfigService) {
cspConfig,
deprecationConfig,
elasticsearchConfig,
coreAppConfig,
elasticApmConfig,
executionContextConfig,
externalUrlConfig,

View file

@ -350,7 +350,7 @@ export class Server {
this.#pluginsInitialized = pluginsSetup.initialized;
this.registerCoreContext(coreSetup);
this.coreApp.setup(coreSetup, uiPlugins);
await this.coreApp.setup(coreSetup, uiPlugins);
setupTransaction?.end();
this.uptimePerStep.setup = { start: setupStartUptime, end: performance.now() };

View file

@ -27,6 +27,8 @@ const createConfigServiceMock = ({
validate: jest.fn(),
getHandledDeprecatedConfigs: jest.fn(),
getDeprecatedConfigPath$: jest.fn(),
addDynamicConfigPaths: jest.fn(),
setDynamicConfigOverrides: jest.fn(),
};
mocked.atPath.mockReturnValue(new BehaviorSubject(atPath));

View file

@ -6,8 +6,8 @@
* Side Public License, v 1.
*/
import { BehaviorSubject, Observable } from 'rxjs';
import { first, take } from 'rxjs/operators';
import { BehaviorSubject, firstValueFrom, Observable } from 'rxjs';
import { first, map, take } from 'rxjs/operators';
import {
mockApplyDeprecations,
@ -670,3 +670,43 @@ describe('getDeprecatedConfigPath$', () => {
expect(deprecatedConfigPath).toEqual(mockedChangedPaths);
});
});
describe('Dynamic Overrides', () => {
let configService: ConfigService;
beforeEach(async () => {
const rawConfig$ = new BehaviorSubject<Record<string, any>>({ namespace1: { key: 'value' } });
const rawConfigProvider = createRawConfigServiceMock({ rawConfig$ });
configService = new ConfigService(rawConfigProvider, defaultEnv, logger);
await configService.setSchema('namespace1', schema.object({ key: schema.string() }));
expect(
await firstValueFrom(configService.getConfig$().pipe(map((cfg) => cfg.toRaw())))
).toStrictEqual({ namespace1: { key: 'value' } });
});
test('throws validation error when attempted to set an override that has not been registered as dynamic', () => {
expect(() =>
configService.setDynamicConfigOverrides({ 'namespace1.key': 'another-value' })
).toThrowErrorMatchingInlineSnapshot(`"[namespace1.key]: not a valid dynamic option"`);
});
test('throws validation error when a registered as dynamic option is invalid', () => {
configService.addDynamicConfigPaths('namespace1', ['key']);
expect(() =>
configService.setDynamicConfigOverrides({ 'namespace1.key': 1 })
).toThrowErrorMatchingInlineSnapshot(
`"[config validation of [namespace1].key]: expected value of type [string] but got [number]"`
);
});
test('overrides the static settings with the dynamic ones', async () => {
configService.addDynamicConfigPaths('namespace1', ['key']);
configService.setDynamicConfigOverrides({ 'namespace1.key': 'another-value' });
expect(
await firstValueFrom(configService.getConfig$().pipe(map((cfg) => cfg.toRaw())))
).toStrictEqual({ namespace1: { key: 'another-value' } });
});
});

View file

@ -7,16 +7,18 @@
*/
import type { PublicMethodsOf } from '@kbn/utility-types';
import { Type } from '@kbn/config-schema';
import { isEqual } from 'lodash';
import { SchemaTypeError, Type, ValidationError } from '@kbn/config-schema';
import { cloneDeep, isEqual, merge } from 'lodash';
import { set } from '@kbn/safer-lodash-set';
import { BehaviorSubject, combineLatest, firstValueFrom, Observable } from 'rxjs';
import { distinctUntilChanged, first, map, shareReplay, tap } from 'rxjs/operators';
import { Logger, LoggerFactory } from '@kbn/logging';
import { getDocLinks, DocLinks } from '@kbn/doc-links';
import { getFlattenedObject } from '@kbn/std';
import { Config, ConfigPath, Env } from '..';
import { hasConfigPathIntersection } from './config';
import { RawConfigurationProvider } from './raw/raw_config_service';
import { RawConfigurationProvider } from './raw';
import {
applyDeprecations,
ConfigDeprecationWithContext,
@ -60,6 +62,8 @@ export class ConfigService {
private readonly handledPaths: Set<ConfigPath> = new Set();
private readonly schemas = new Map<string, Type<unknown>>();
private readonly deprecations = new BehaviorSubject<ConfigDeprecationWithContext[]>([]);
private readonly dynamicPaths = new Map<string, string[]>();
private readonly overrides$ = new BehaviorSubject<Record<string, unknown>>({});
private readonly handledDeprecatedConfigs = new Map<string, DeprecatedConfigDetails[]>();
constructor(
@ -71,9 +75,14 @@ export class ConfigService {
this.deprecationLog = logger.get('config', 'deprecation');
this.docLinks = getDocLinks({ kibanaBranch: env.packageInfo.branch });
this.config$ = combineLatest([this.rawConfigProvider.getConfig$(), this.deprecations]).pipe(
map(([rawConfig, deprecations]) => {
const migrated = applyDeprecations(rawConfig, deprecations);
this.config$ = combineLatest([
this.rawConfigProvider.getConfig$(),
this.deprecations,
this.overrides$,
]).pipe(
map(([rawConfig, deprecations, overrides]) => {
const overridden = merge(rawConfig, overrides);
const migrated = applyDeprecations(overridden, deprecations);
this.deprecatedConfigPaths.next(migrated.changedPaths);
return new ObjectToConfigAdapter(migrated.config);
}),
@ -213,6 +222,59 @@ export class ConfigService {
return this.deprecatedConfigPaths.asObservable();
}
/**
* Adds a specific setting to be allowed to change dynamically.
* @param configPath The namespace of the config
* @param dynamicConfigPaths The config keys that can be dynamically changed
*/
public addDynamicConfigPaths(configPath: ConfigPath, dynamicConfigPaths: string[]) {
const _configPath = Array.isArray(configPath) ? configPath.join('.') : configPath;
this.dynamicPaths.set(_configPath, dynamicConfigPaths);
}
/**
* Used for dynamically extending the overrides.
* These overrides are not persisted and will be discarded after restarts.
* @param newOverrides
*/
public setDynamicConfigOverrides(newOverrides: Record<string, unknown>) {
const globalOverrides = cloneDeep(this.overrides$.value);
const flattenedOverrides = getFlattenedObject(newOverrides);
const validateWithNamespace = new Set<string>();
keyLoop: for (const key in flattenedOverrides) {
// this if is enforced by an eslint rule :shrug:
if (key in flattenedOverrides) {
for (const [configPath, dynamicConfigKeys] of this.dynamicPaths.entries()) {
if (
key.startsWith(`${configPath}.`) &&
dynamicConfigKeys.some(
// The key is explicitly allowed OR its prefix is
(dynamicConfigKey) =>
key === `${configPath}.${dynamicConfigKey}` ||
key.startsWith(`${configPath}.${dynamicConfigKey}.`)
)
) {
validateWithNamespace.add(configPath);
set(globalOverrides, key, flattenedOverrides[key]);
continue keyLoop;
}
}
throw new ValidationError(new SchemaTypeError(`not a valid dynamic option`, [key]));
}
}
const globalOverridesAsConfig = new ObjectToConfigAdapter(
merge({}, this.lastConfig, globalOverrides)
);
validateWithNamespace.forEach((ns) => this.validateAtPath(ns, globalOverridesAsConfig.get(ns)));
this.overrides$.next(globalOverrides);
}
private async logDeprecation() {
const rawConfig = await firstValueFrom(this.rawConfigProvider.getConfig$());
const deprecations = await firstValueFrom(this.deprecations);

View file

@ -90,6 +90,7 @@ export function makeFtrConfigProvider(
`--telemetry.labels=${JSON.stringify(telemetryLabels)}`,
'--csp.strict=false',
'--csp.warnLegacyBrowsers=false',
'--coreApp.allowDynamicConfigOverrides=true',
],
env: {

View file

@ -17,6 +17,10 @@ import { asyncMap, asyncForEach } from '@kbn/std';
import { ToolingLog } from '@kbn/tooling-log';
import { Config } from '@kbn/test';
import { EsArchiver, KibanaServer, Es, RetryService } from '@kbn/ftr-common-functional-services';
import {
ELASTIC_HTTP_VERSION_HEADER,
X_ELASTIC_INTERNAL_ORIGIN_REQUEST,
} from '@kbn/core-http-common';
import { Auth } from '../services/auth';
import { getInputDelays } from '../services/input_delays';
@ -55,9 +59,31 @@ export class JourneyFtrHarness {
private apm: apmNode.Agent | null = null;
// Update the Telemetry and APM global labels to link traces with journey
private async updateTelemetryAndAPMLabels(labels: { [k: string]: string }) {
this.log.info(`Updating telemetry & APM labels: ${JSON.stringify(labels)}`);
await this.kibanaServer.request({
path: '/internal/core/_settings',
method: 'PUT',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '1',
[X_ELASTIC_INTERNAL_ORIGIN_REQUEST]: 'ftr',
},
body: { telemetry: { labels } },
});
}
private async setupApm() {
const kbnTestServerEnv = this.config.get(`kbnTestServer.env`);
const journeyLabels: { [k: string]: string } = Object.fromEntries(
kbnTestServerEnv.ELASTIC_APM_GLOBAL_LABELS.split(',').map((kv: string) => kv.split('='))
);
// Update labels before start for consistency b/w APM services
await this.updateTelemetryAndAPMLabels(journeyLabels);
this.apm = apmNode.start({
serviceName: 'functional test runner',
environment: process.env.CI ? 'ci' : 'development',

View file

@ -18,6 +18,7 @@
"@kbn/repo-info",
"@kbn/std",
"@kbn/test-subj-selector",
"@kbn/core-http-common",
],
"exclude": [
"target/**/*",

View file

@ -0,0 +1,74 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { set } from '@kbn/safer-lodash-set';
import { Root } from '@kbn/core-root-server-internal';
import { createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
import { PLUGIN_SYSTEM_ENABLE_ALL_PLUGINS_CONFIG_PATH } from '@kbn/core-plugins-server-internal/src/constants';
describe('checking migration metadata changes on all registered SO types', () => {
let root: Root;
beforeAll(async () => {
const settings = {
logging: {
loggers: [{ name: 'root', level: 'info', appenders: ['console'] }],
},
};
set(settings, PLUGIN_SYSTEM_ENABLE_ALL_PLUGINS_CONFIG_PATH, true);
root = createRootWithCorePlugins(settings, {
basePath: false,
cache: false,
dev: true,
disableOptimizer: true,
silent: false,
dist: false,
oss: false,
runExamples: false,
watch: false,
});
await root.preboot();
await root.setup();
});
afterAll(async () => {
if (root) {
await root.shutdown();
}
});
function getListOfDynamicConfigPaths(): string[] {
// eslint-disable-next-line dot-notation
return [...root['server']['configService']['dynamicPaths'].entries()]
.flatMap(([configPath, dynamicConfigKeys]) => {
return dynamicConfigKeys.map(
(dynamicConfigKey: string) => `${configPath}.${dynamicConfigKey}`
);
})
.sort();
}
/**
* This test is meant to fail when any setting is flagged as capable
* of dynamic configuration {@link PluginConfigDescriptor.dynamicConfig}.
*
* Please, add your settings to the list with a comment of why it's required to be dynamic.
*
* The intent is to trigger a code review from the Core and Security teams to discuss potential issues.
*/
test('detecting all the settings that have opted-in for dynamic in-memory updates', () => {
expect(getListOfDynamicConfigPaths()).toStrictEqual([
// We need this for enriching our Perf tests with more valuable data regarding the steps of the test
// Also helpful in Cloud & Serverless testing because we can't control the labels in those offerings
'telemetry.labels',
]);
});
});

View file

@ -151,6 +151,7 @@
"@kbn/core-elasticsearch-client-server-internal",
"@kbn/tooling-log",
"@kbn/stdio-dev-helpers",
"@kbn/safer-lodash-set",
],
"exclude": [
"target/**/*",

View file

@ -6,6 +6,8 @@
* Side Public License, v 1.
*/
import type { TelemetryConfigLabels } from '../../server/config';
export interface Telemetry {
/** Whether telemetry is enabled */
enabled?: boolean | null;
@ -24,6 +26,7 @@ export interface FetchTelemetryConfigResponse {
optIn: boolean | null;
sendUsageFrom: 'server' | 'browser';
telemetryNotifyUserAboutOptInDefault: boolean;
labels: TelemetryConfigLabels;
}
export interface FetchLastReportedResponse {

View file

@ -6,6 +6,7 @@
"id": "telemetry",
"server": true,
"browser": true,
"enabledOnAnonymousPages": true,
"requiredPlugins": [
"telemetryCollectionManager",
"usageCollection",

View file

@ -6,6 +6,7 @@
* Side Public License, v 1.
*/
import type { ApmBase } from '@elastic/apm-rum';
import type {
Plugin,
CoreStart,
@ -23,7 +24,8 @@ import type {
import type { HomePublicPluginSetup } from '@kbn/home-plugin/public';
import { ElasticV3BrowserShipper } from '@kbn/analytics-shippers-elastic-v3-browser';
import { of } from 'rxjs';
import { BehaviorSubject, map, tap } from 'rxjs';
import type { TelemetryConfigLabels } from '../server/config';
import { FetchTelemetryConfigRoute, INTERNAL_VERSION } from '../common/routes';
import type { v2 } from '../common/types';
import { TelemetrySender, TelemetryService, TelemetryNotifications } from './services';
@ -88,6 +90,12 @@ interface TelemetryPluginStartDependencies {
screenshotMode: ScreenshotModePluginStart;
}
declare global {
interface Window {
elasticApm?: ApmBase;
}
}
/**
* Public-exposed configuration
*/
@ -131,6 +139,7 @@ export class TelemetryPlugin
{
private readonly currentKibanaVersion: string;
private readonly config: TelemetryPluginConfig;
private readonly telemetryLabels$: BehaviorSubject<TelemetryConfigLabels>;
private telemetrySender?: TelemetrySender;
private telemetryNotifications?: TelemetryNotifications;
private telemetryService?: TelemetryService;
@ -139,6 +148,7 @@ export class TelemetryPlugin
constructor(initializerContext: PluginInitializerContext<TelemetryPluginConfig>) {
this.currentKibanaVersion = initializerContext.env.packageInfo.version;
this.config = initializerContext.config.get();
this.telemetryLabels$ = new BehaviorSubject<TelemetryConfigLabels>(this.config.labels);
}
public setup(
@ -163,7 +173,14 @@ export class TelemetryPlugin
analytics.registerContextProvider({
name: 'telemetry labels',
context$: of({ labels: this.config.labels }),
context$: this.telemetryLabels$.pipe(
tap((labels) => {
// Hack to update the APM agent's labels.
// In the future we might want to expose APM as a core service to make reporting metrics much easier.
window.elasticApm?.addLabels(labels);
}),
map((labels) => ({ labels }))
),
schema: {
labels: {
type: 'pass_through',
@ -230,11 +247,6 @@ export class TelemetryPlugin
this.telemetryNotifications = telemetryNotifications;
application.currentAppId$.subscribe(async () => {
const isUnauthenticated = this.getIsUnauthenticated(http);
if (isUnauthenticated) {
return;
}
// Refresh and get telemetry config
const updatedConfig = await this.refreshConfig(http);
@ -242,6 +254,11 @@ export class TelemetryPlugin
global: { enabled: this.telemetryService!.isOptedIn && !screenshotMode.isScreenshotMode() },
});
const isUnauthenticated = this.getIsUnauthenticated(http);
if (isUnauthenticated) {
return;
}
const telemetryBanner = updatedConfig?.banner;
this.maybeStartTelemetryPoller();
@ -285,6 +302,9 @@ export class TelemetryPlugin
if (this.telemetryService) {
this.telemetryService.config = updatedConfig;
}
this.telemetryLabels$.next(updatedConfig.labels);
return updatedConfig;
}
@ -328,8 +348,16 @@ export class TelemetryPlugin
* @private
*/
private async fetchUpdatedConfig(http: HttpStart | HttpSetup): Promise<TelemetryPluginConfig> {
const { allowChangingOptInStatus, optIn, sendUsageFrom, telemetryNotifyUserAboutOptInDefault } =
await http.get<v2.FetchTelemetryConfigResponse>(FetchTelemetryConfigRoute, INTERNAL_VERSION);
const {
allowChangingOptInStatus,
optIn,
sendUsageFrom,
telemetryNotifyUserAboutOptInDefault,
labels,
} = await http.get<v2.FetchTelemetryConfigResponse>(
FetchTelemetryConfigRoute,
INTERNAL_VERSION
);
return {
...this.config,
@ -337,6 +365,7 @@ export class TelemetryPlugin
optIn,
sendUsageFrom,
telemetryNotifyUserAboutOptInDefault,
labels,
userCanChangeSettings: this.canUserChangeSettings,
};
}

View file

@ -56,6 +56,9 @@ export const config: PluginConfigDescriptor<TelemetryConfigType> = {
hidePrivacyStatement: true,
labels: true,
},
dynamicConfig: {
labels: true,
},
deprecations: () => [
(cfg) => {
if (cfg.telemetry?.enabled === false) {

View file

@ -27,6 +27,7 @@ export const labelsSchema = schema.object(
testBuildId: schema.maybe(schema.string()),
testJobId: schema.maybe(schema.string()),
ciBuildName: schema.maybe(schema.string()),
performancePhase: schema.maybe(schema.string()),
/**
* The serverless project type.
* Flagging it as maybe because these settings should never affect how Kibana runs.

View file

@ -40,6 +40,7 @@ import type {
import type { SecurityPluginStart } from '@kbn/security-plugin/server';
import { SavedObjectsClient } from '@kbn/core/server';
import apm from 'elastic-apm-node';
import {
type TelemetrySavedObject,
getTelemetrySavedObject,
@ -173,12 +174,18 @@ export class TelemetryPlugin implements Plugin<TelemetryPluginSetup, TelemetryPl
analytics.registerContextProvider<{ labels: TelemetryConfigLabels }>({
name: 'telemetry labels',
context$: this.config$.pipe(map(({ labels }) => ({ labels }))),
context$: this.config$.pipe(
map(({ labels }) => ({ labels })),
tap(({ labels }) =>
Object.entries(labels).forEach(([key, value]) => apm.setGlobalLabel(key, value))
)
),
schema: {
labels: {
type: 'pass_through',
_meta: {
description: 'Custom labels added to the telemetry.labels config in the kibana.yml',
description:
'Custom labels added to the telemetry.labels config in the kibana.yml. Validated and limited to a known set of labels.',
},
},
},

View file

@ -10,6 +10,7 @@ import { type Observable, firstValueFrom } from 'rxjs';
import type { IRouter, SavedObjectsClient } from '@kbn/core/server';
import { schema } from '@kbn/config-schema';
import { RequestHandler } from '@kbn/core-http-server';
import { labelsSchema } from '../config/telemetry_labels';
import type { TelemetryConfigType } from '../config';
import { v2 } from '../../common/types';
import {
@ -70,6 +71,7 @@ export function registerTelemetryConfigRoutes({
optIn,
sendUsageFrom,
telemetryNotifyUserAboutOptInDefault,
labels: config.labels,
};
return res.ok({ body });
@ -83,6 +85,7 @@ export function registerTelemetryConfigRoutes({
optIn: schema.oneOf([schema.boolean(), schema.literal(null)]),
sendUsageFrom: schema.oneOf([schema.literal('server'), schema.literal('browser')]),
telemetryNotifyUserAboutOptInDefault: schema.boolean(),
labels: labelsSchema,
}),
},
},
@ -90,7 +93,11 @@ export function registerTelemetryConfigRoutes({
// Register the internal versioned API
router.versioned
.get({ access: 'internal', path: FetchTelemetryConfigRoute })
.get({
access: 'internal',
path: FetchTelemetryConfigRoute,
options: { authRequired: 'optional' },
})
// Just because it used to be /v2/, we are creating identical v1 and v2.
.addVersion({ version: '1', validate: v2Validations }, v2Handler)
.addVersion({ version: '2', validate: v2Validations }, v2Handler);

View file

@ -46,6 +46,7 @@ export default function telemetryConfigTest({ getService }: FtrProviderContext)
optIn: null, // the config.js for this FTR sets it to `false`, we are bound to ask again.
sendUsageFrom: 'server',
telemetryNotifyUserAboutOptInDefault: false, // it's not opted-in by default (that's what this flag is about)
labels: {},
});
});
@ -69,6 +70,7 @@ export default function telemetryConfigTest({ getService }: FtrProviderContext)
optIn: true,
sendUsageFrom: 'server',
telemetryNotifyUserAboutOptInDefault: false,
labels: {},
});
});
@ -92,6 +94,7 @@ export default function telemetryConfigTest({ getService }: FtrProviderContext)
optIn: false,
sendUsageFrom: 'server',
telemetryNotifyUserAboutOptInDefault: false,
labels: {},
});
});
@ -136,6 +139,7 @@ export default function telemetryConfigTest({ getService }: FtrProviderContext)
optIn: true,
sendUsageFrom: 'server',
telemetryNotifyUserAboutOptInDefault: false,
labels: {},
});
});
@ -158,6 +162,7 @@ export default function telemetryConfigTest({ getService }: FtrProviderContext)
optIn: null,
sendUsageFrom: 'server',
telemetryNotifyUserAboutOptInDefault: false,
labels: {},
});
});
});

View file

@ -161,7 +161,8 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
'telemetry.labels.testBuildId (string)',
'telemetry.labels.testJobId (string)',
'telemetry.labels.ciBuildName (string)',
'telemetry.labels.serverless (any)',
'telemetry.labels.performancePhase (string)',
'telemetry.labels.serverless (any)', // It's the project type (string), claims any because schema.conditional. Can only be set on Serverless.
'telemetry.hidePrivacyStatement (boolean)',
'telemetry.optIn (boolean)',
'telemetry.sendUsageFrom (alternatives)',
@ -329,6 +330,30 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
'xpack.security.showInsecureClusterWarning (boolean)',
'xpack.security.showNavLinks (boolean)',
'xpack.security.ui (any)',
'telemetry.allowChangingOptInStatus (boolean)',
'telemetry.appendServerlessChannelsSuffix (any)', // It's a boolean (any because schema.conditional)
'telemetry.banner (boolean)',
'telemetry.labels.branch (string)',
'telemetry.labels.ciBuildId (string)',
'telemetry.labels.ciBuildJobId (string)',
'telemetry.labels.ciBuildNumber (number)',
'telemetry.labels.ftrConfig (string)',
'telemetry.labels.gitRev (string)',
'telemetry.labels.isPr (boolean)',
'telemetry.labels.journeyName (string)',
'telemetry.labels.prId (number)',
'telemetry.labels.testBuildId (string)',
'telemetry.labels.testJobId (string)',
'telemetry.labels.ciBuildName (string)',
'telemetry.labels.performancePhase (string)',
'telemetry.labels.serverless (any)', // It's the project type (string), claims any because schema.conditional. Can only be set on Serverless.
'telemetry.hidePrivacyStatement (boolean)',
'telemetry.optIn (boolean)',
'telemetry.sendUsageFrom (alternatives)',
'telemetry.sendUsageTo (any)',
'usageCollection.uiCounters.debug (boolean)',
'usageCollection.uiCounters.enabled (boolean)',
];
// We don't assert that actualExposedConfigKeys and expectedExposedConfigKeys are equal, because test failure messages with large
// arrays are hard to grok. Instead, we take the difference between the two arrays and assert them separately, that way it's

View file

@ -11,6 +11,7 @@ export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless observability API', function () {
loadTestFile(require.resolve('./fleet/fleet'));
loadTestFile(require.resolve('./telemetry/snapshot_telemetry'));
loadTestFile(require.resolve('./telemetry/telemetry_config'));
loadTestFile(require.resolve('./apm_api_integration/feature_flags.ts'));
loadTestFile(require.resolve('./cases'));
});

View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function telemetryConfigTest({ getService }: FtrProviderContext) {
const svlCommonApi = getService('svlCommonApi');
const supertest = getService('supertest');
describe('/api/telemetry/v2/config API Telemetry config', () => {
const baseConfig = {
allowChangingOptInStatus: false,
optIn: true,
sendUsageFrom: 'server',
telemetryNotifyUserAboutOptInDefault: false,
labels: {
serverless: 'observability',
},
};
it('GET should get the default config', async () => {
await supertest
.get('/api/telemetry/v2/config')
.set(svlCommonApi.getInternalRequestHeader())
.expect(200, baseConfig);
});
it('GET should get updated labels after dynamically updating them', async () => {
await supertest
.put('/internal/core/_settings')
.set(svlCommonApi.getInternalRequestHeader())
.set('elastic-api-version', '1')
.send({ 'telemetry.labels.journeyName': 'my-ftr-test' })
.expect(200, { ok: true });
await supertest
.get('/api/telemetry/v2/config')
.set(svlCommonApi.getInternalRequestHeader())
.expect(200, {
...baseConfig,
labels: {
...baseConfig.labels,
journeyName: 'my-ftr-test',
},
});
});
});
}

View file

@ -10,6 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless search API', function () {
loadTestFile(require.resolve('./telemetry/snapshot_telemetry'));
loadTestFile(require.resolve('./telemetry/telemetry_config'));
loadTestFile(require.resolve('./cases/find_cases'));
loadTestFile(require.resolve('./cases/post_case'));
});

View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function telemetryConfigTest({ getService }: FtrProviderContext) {
const svlCommonApi = getService('svlCommonApi');
const supertest = getService('supertest');
describe('/api/telemetry/v2/config API Telemetry config', () => {
const baseConfig = {
allowChangingOptInStatus: false,
optIn: true,
sendUsageFrom: 'server',
telemetryNotifyUserAboutOptInDefault: false,
labels: {
serverless: 'search',
},
};
it('GET should get the default config', async () => {
await supertest
.get('/api/telemetry/v2/config')
.set(svlCommonApi.getInternalRequestHeader())
.expect(200, baseConfig);
});
it('GET should get updated labels after dynamically updating them', async () => {
await supertest
.put('/internal/core/_settings')
.set(svlCommonApi.getInternalRequestHeader())
.set('elastic-api-version', '1')
.send({ 'telemetry.labels.journeyName': 'my-ftr-test' })
.expect(200, { ok: true });
await supertest
.get('/api/telemetry/v2/config')
.set(svlCommonApi.getInternalRequestHeader())
.expect(200, {
...baseConfig,
labels: {
...baseConfig.labels,
journeyName: 'my-ftr-test',
},
});
});
});
}

View file

@ -10,6 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless security API', function () {
loadTestFile(require.resolve('./telemetry/snapshot_telemetry'));
loadTestFile(require.resolve('./telemetry/telemetry_config'));
loadTestFile(require.resolve('./fleet/fleet'));
loadTestFile(require.resolve('./cases'));
});

View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function telemetryConfigTest({ getService }: FtrProviderContext) {
const svlCommonApi = getService('svlCommonApi');
const supertest = getService('supertest');
describe('/api/telemetry/v2/config API Telemetry config', () => {
const baseConfig = {
allowChangingOptInStatus: false,
optIn: true,
sendUsageFrom: 'server',
telemetryNotifyUserAboutOptInDefault: false,
labels: {
serverless: 'security',
},
};
it('GET should get the default config', async () => {
await supertest
.get('/api/telemetry/v2/config')
.set(svlCommonApi.getInternalRequestHeader())
.expect(200, baseConfig);
});
it('GET should get updated labels after dynamically updating them', async () => {
await supertest
.put('/internal/core/_settings')
.set(svlCommonApi.getInternalRequestHeader())
.set('elastic-api-version', '1')
.send({ 'telemetry.labels.journeyName': 'my-ftr-test' })
.expect(200, { ok: true });
await supertest
.get('/api/telemetry/v2/config')
.set(svlCommonApi.getInternalRequestHeader())
.expect(200, {
...baseConfig,
labels: {
...baseConfig.labels,
journeyName: 'my-ftr-test',
},
});
});
});
}