github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 10:40:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Profiling-APM] Removing Profiling dependency from APM (#166253)

Browse source 
This PR removes the Profiling dependency from APM, introduced on `8.10`.

- Exposes a new service in profiling-data-access plugin
- Create a new APM API that calls the new service and checks if
Profiling is initialized
- Move Locators from the Profiling plugin to the Observability-shared
plugin
- Move logic to check Profiling status (has_setup/has_data...) from
Profiling server to profiling-data-access plugin
- Create API tests, testing the status services based on different
scenarios:
  - When profiling hasn't been initialized and there's no data
  - When profiling is initialized but has no data
  - When collector integration is not installed
  - When symbolized integration is not installed
  - When APM server integration is not found

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Cauê Marcondes 2023-09-22 09:16:48 +01:00 • committed by GitHub
parent 0eda41a46d
commit 98d2766de8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 1248 additions and 431 deletions
Download patch file Download diff file Expand all files Collapse all files

13
packages/kbn-profiling-utils/common/profiling_status.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export interface ProfilingStatus {
has_setup: boolean;
has_data: boolean;
pre_8_9_1_data: boolean;
unauthorized?: boolean;
}

1
packages/kbn-profiling-utils/index.ts
View file

@ -49,4 +49,5 @@ export type {
StackTrace,
StackTraceID,
} from './common/profiling';
export type { ProfilingStatus } from './common/profiling_status';
export type { TopNFunctions } from './common/functions';

1
x-pack/plugins/apm/kibana.jsonc
View file

@ -50,7 +50,6 @@
"usageCollection",
"customIntegrations", // Move this to requiredPlugins after completely migrating from the Tutorials Home App
"licenseManagement",
"profiling",
"profilingDataAccess"
],
"requiredBundles": [

56
x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.test.tsx
View file

@ -79,7 +79,8 @@ const expectInfraLocatorsToBeCalled = () => {
describe('TransactionActionMenu component', () => {
beforeAll(() => {
jest.spyOn(hooks, 'useFetcher').mockReturnValue({
data: [],
// return as Profiling had been initialized
data: { initialized: true },
status: hooks.FETCH_STATUS.SUCCESS,
refetch: jest.fn(),
});
@ -253,6 +254,27 @@ describe('TransactionActionMenu component', () => {
expect(container).toMatchSnapshot();
});
describe('Profiling items', () => {
it('renders flamegraph item', async () => {
const component = await renderTransaction(
Transactions.transactionWithHostData
);
expectTextsInDocument(component, ['Host flamegraph']);
});
it('renders topN functions item', async () => {
const component = await renderTransaction(
Transactions.transactionWithHostData
);
expectTextsInDocument(component, ['Host topN functions']);
});
it('renders stacktraces item', async () => {
const component = await renderTransaction(
Transactions.transactionWithHostData
);
expectTextsInDocument(component, ['Host stacktraces']);
});
});
describe('Custom links', () => {
beforeAll(() => {
// Mocks callApmAPI because it's going to be used to fecth the transaction in the custom links flyout.
@ -393,3 +415,35 @@ describe('TransactionActionMenu component', () => {
});
});
});
describe('Profiling not initialized', () => {
beforeAll(() => {
jest.spyOn(hooks, 'useFetcher').mockReturnValue({
// return as Profiling had not been initialized
data: { initialized: false },
status: hooks.FETCH_STATUS.SUCCESS,
refetch: jest.fn(),
});
});
afterEach(() => {
jest.clearAllMocks();
});
it('does not render flamegraph item', async () => {
const component = await renderTransaction(
Transactions.transactionWithHostData
);
expectTextsNotInDocument(component, ['Host flamegraph']);
});
it('does not render topN functions item', async () => {
const component = await renderTransaction(
Transactions.transactionWithHostData
);
expectTextsNotInDocument(component, ['Host topN functions']);
});
it('does not render stacktraces item', async () => {
const component = await renderTransaction(
Transactions.transactionWithHostData
);
expectTextsNotInDocument(component, ['Host stacktraces']);
});
});

17
x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx
View file

@ -18,6 +18,7 @@ import { MlLocatorDefinition } from '@kbn/ml-plugin/public';
import { enableComparisonByDefault } from '@kbn/observability-plugin/public';
import { sharePluginMock } from '@kbn/share-plugin/public/mocks';
import type { InfraLocators } from '@kbn/infra-plugin/common/locators';
import { apmEnableProfilingIntegration } from '@kbn/observability-plugin/common';
import { ApmPluginContext, ApmPluginContextValue } from './apm_plugin_context';
import { ConfigSchema } from '../..';
import { createCallApmApi } from '../../services/rest/create_call_apm_api';
@ -57,6 +58,7 @@ const mockCore = merge({}, coreStart, {
value: 100000,
},
[enableComparisonByDefault]: true,
[apmEnableProfilingIntegration]: true,
};
return uiSettings[key];
},
@ -108,6 +110,21 @@ const mockPlugin = {
},
},
},
observabilityShared: {
locators: {
profiling: {
flamegraphLocator: {
getRedirectUrl: () => '/profiling/flamegraphs/flamegraph',
},
topNFunctionsLocator: {
getRedirectUrl: () => '/profiling/functions/topn',
},
stacktracesLocator: {
getRedirectUrl: () => '/profiling/stacktraces/threads',
},
},
},
},
};
export const infraLocatorsMock: InfraLocators = {

25
x-pack/plugins/apm/public/hooks/use_profiling_plugin.tsx
View file

@ -5,9 +5,9 @@
* 2.0.
*/
import { useEffect, useState } from 'react';
import { apmEnableProfilingIntegration } from '@kbn/observability-plugin/common';
import { useApmPluginContext } from '../context/apm_plugin/use_apm_plugin_context';
import { useFetcher } from './use_fetcher';
export function useProfilingPlugin() {
const { plugins, core } = useApmPluginContext();
@ -15,30 +15,19 @@ export function useProfilingPlugin() {
apmEnableProfilingIntegration,
false
);
const [isProfilingPluginInitialized, setIsProfilingPluginInitialized] =
useState<boolean | undefined>();
useEffect(() => {
async function fetchIsProfilingSetup() {
if (!plugins.profiling) {
setIsProfilingPluginInitialized(false);
return;
}
const resp = await plugins.profiling.hasSetup();
setIsProfilingPluginInitialized(resp);
}
fetchIsProfilingSetup();
}, [plugins.profiling]);
const { data } = useFetcher((callApmApi) => {
return callApmApi('GET /internal/apm/profiling/status');
}, []);
const isProfilingAvailable =
isProfilingIntegrationEnabled && isProfilingPluginInitialized;
isProfilingIntegrationEnabled && data?.initialized;
return {
isProfilingPluginInitialized,
profilingLocators: isProfilingAvailable
? plugins.profiling?.locators
? plugins.observabilityShared.locators.profiling
: undefined,
isProfilingPluginInitialized: data?.initialized,
isProfilingIntegrationEnabled,
isProfilingAvailable,
};

31
x-pack/plugins/apm/server/routes/profiling/route.ts
View file

@ -131,7 +131,38 @@ const profilingFunctionsRoute = createApmServerRoute({
},
});
const profilingStatusRoute = createApmServerRoute({
endpoint: 'GET /internal/apm/profiling/status',
options: { tags: ['access:apm'] },
handler: async (resources): Promise<{ initialized: boolean }> => {
const { context, plugins, logger } = resources;
const [esClient, profilingDataAccessStart] = await Promise.all([
(await context.core).elasticsearch.client,
await plugins.profilingDataAccess?.start(),
]);
if (profilingDataAccessStart) {
try {
const response = await profilingDataAccessStart?.services.getStatus({
esClient: esClient.asCurrentUser,
soClient: (await context.core).savedObjects.client,
spaceId: (
await plugins.spaces?.start()
)?.spacesService.getSpaceId(resources.request),
});
return { initialized: response.has_setup };
} catch (e) {
// If any error happens just return as if profiling has not been initialized
logger.warn('Could not check Universal Profiling status');
}
}
return { initialized: false };
},
});
export const profilingRouteRepository = {
...profilingFlamegraphRoute,
...profilingStatusRoute,
...profilingFunctionsRoute,
};

2
x-pack/plugins/observability_shared/kibana.jsonc
View file

@ -7,7 +7,7 @@
"server": false,
"browser": true,
"configPath": ["xpack", "observability_shared"],
"requiredPlugins": ["cases", "guidedOnboarding", "uiActions", "embeddable"],
"requiredPlugins": ["cases", "guidedOnboarding", "uiActions", "embeddable", "share"],
"optionalPlugins": [],
"requiredBundles": ["data", "inspector", "kibanaReact", "kibanaUtils"],
"extraPublicDirs": ["common"]

0
x-pack/plugins/profiling/public/locators/flamegraph_locator.ts → x-pack/plugins/observability_shared/public/locators/profiling/flamegraph_locator.ts
View file

0
x-pack/plugins/profiling/public/locators/stacktraces_locator.ts → x-pack/plugins/observability_shared/public/locators/profiling/stacktraces_locator.ts
View file

0
x-pack/plugins/profiling/public/locators/topn_functions_locator.ts → x-pack/plugins/observability_shared/public/locators/profiling/topn_functions_locator.ts
View file

27
x-pack/plugins/observability_shared/public/plugin.ts
View file

@ -6,15 +6,22 @@
*/
import { BehaviorSubject } from 'rxjs';
import type { CoreStart, Plugin } from '@kbn/core/public';
import type { CoreStart, Plugin, CoreSetup } from '@kbn/core/public';
import type { GuidedOnboardingPluginStart } from '@kbn/guided-onboarding-plugin/public';
import { CasesUiStart } from '@kbn/cases-plugin/public';
import { SpacesPluginStart } from '@kbn/spaces-plugin/public';
import type { EmbeddableStart } from '@kbn/embeddable-plugin/public';
import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public';
import { createNavigationRegistry } from './components/page_template/helpers/navigation_registry';
import { createLazyObservabilityPageTemplate } from './components/page_template';
import { updateGlobalNavigation } from './services/update_global_navigation';
import { FlamegraphLocatorDefinition } from './locators/profiling/flamegraph_locator';
import { TopNFunctionsLocatorDefinition } from './locators/profiling/topn_functions_locator';
import { StacktracesLocatorDefinition } from './locators/profiling/stacktraces_locator';
export interface ObservabilitySharedSetup {
share: SharePluginSetup;
}
export interface ObservabilitySharedStart {
spaces?: SpacesPluginStart;
@ -22,6 +29,7 @@ export interface ObservabilitySharedStart {
guidedOnboarding: GuidedOnboardingPluginStart;
setIsSidebarEnabled: (isEnabled: boolean) => void;
embeddable: EmbeddableStart;
share: SharePluginStart;
}
export type ObservabilitySharedPluginSetup = ReturnType<ObservabilitySharedPlugin['setup']>;
@ -35,8 +43,21 @@ export class ObservabilitySharedPlugin implements Plugin {
this.isSidebarEnabled$ = new BehaviorSubject<boolean>(true);
}
public setup() {
public setup(coreSetup: CoreSetup, pluginsSetup: ObservabilitySharedSetup) {
return {
locators: {
profiling: {
flamegraphLocator: pluginsSetup.share.url.locators.create(
new FlamegraphLocatorDefinition()
),
topNFunctionsLocator: pluginsSetup.share.url.locators.create(
new TopNFunctionsLocatorDefinition()
),
stacktracesLocator: pluginsSetup.share.url.locators.create(
new StacktracesLocatorDefinition()
),
},
},
navigation: {
registerSections: this.navigationRegistry.registerSections,
},

4
x-pack/plugins/observability_shared/tsconfig.json
View file

@ -33,7 +33,9 @@
"@kbn/kibana-utils-plugin",
"@kbn/shared-ux-router",
"@kbn/embeddable-plugin",
"@kbn/profiling-utils"
"@kbn/profiling-utils",
"@kbn/utility-types",
"@kbn/share-plugin"
],
"exclude": ["target/**/*"]
}

33
x-pack/plugins/profiling/public/plugin.tsx
View file

@ -17,13 +17,10 @@ import type { NavigationSection } from '@kbn/observability-shared-plugin/public'
import type { Location } from 'history';
import { BehaviorSubject, combineLatest, from, map } from 'rxjs';
import { registerEmbeddables } from './embeddables/register_embeddables';
import { FlamegraphLocatorDefinition } from './locators/flamegraph_locator';
import { StacktracesLocatorDefinition } from './locators/stacktraces_locator';
import { TopNFunctionsLocatorDefinition } from './locators/topn_functions_locator';
import { getServices } from './services';
import type { ProfilingPluginPublicSetupDeps, ProfilingPluginPublicStartDeps } from './types';
export type ProfilingPluginSetup = ReturnType<ProfilingPlugin['setup']>;
export type ProfilingPluginSetup = void;
export type ProfilingPluginStart = void;
export class ProfilingPlugin implements Plugin {
@ -133,33 +130,7 @@ export class ProfilingPlugin implements Plugin {
registerEmbeddables(pluginsSetup.embeddable);
return {
locators: {
flamegraphLocator: pluginsSetup.share.url.locators.create(
new FlamegraphLocatorDefinition()
),
topNFunctionsLocator: pluginsSetup.share.url.locators.create(
new TopNFunctionsLocatorDefinition()
),
stacktracesLocator: pluginsSetup.share.url.locators.create(
new StacktracesLocatorDefinition()
),
},
hasSetup: async () => {
try {
const response = (await coreSetup.http.get('/internal/profiling/setup/es_resources')) as {
has_setup: boolean;
has_data: boolean;
unauthorized: boolean;
};
return response.has_setup;
} catch (e) {
// If any error happens while checking return as it has not been set up
return false;
}
},
};
return {};
}
public start(core: CoreStart) {

31
x-pack/plugins/profiling/server/lib/setup/cluster_settings.ts
View file

@ -5,22 +5,8 @@
* 2.0.
*/
import { MAX_BUCKETS } from '@kbn/profiling-data-access-plugin/common';
import { ProfilingSetupOptions } from './types';
import { PartialSetupState } from '../../../common/setup';
const MAX_BUCKETS = 150000;
export async function validateMaximumBuckets({
client,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const settings = await client.getEsClient().cluster.getSettings({});
const maxBuckets = settings.persistent.search?.max_buckets;
return {
settings: {
configured: maxBuckets === MAX_BUCKETS.toString(),
},
};
}
export async function setMaximumBuckets({ client }: ProfilingSetupOptions) {
await client.getEsClient().cluster.putSettings({
@ -32,21 +18,6 @@ export async function setMaximumBuckets({ client }: ProfilingSetupOptions) {
});
}
export async function validateResourceManagement({
client,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const statusResponse = await client.profilingStatus();
return {
resource_management: {
enabled: statusResponse.resource_management.enabled,
},
resources: {
created: statusResponse.resources.created,
pre_8_9_1_data: statusResponse.resources.pre_8_9_1_data,
},
};
}
export async function enableResourceManagement({ client }: ProfilingSetupOptions) {
await client.getEsClient().cluster.putSettings({
persistent: {

92
x-pack/plugins/profiling/server/lib/setup/fleet_policies.ts
View file

@ -5,53 +5,18 @@
* 2.0.
*/
import { SavedObjectsClientContract } from '@kbn/core/server';
import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
import { fetchFindLatestPackageOrThrow } from '@kbn/fleet-plugin/server/services/epm/registry';
import {
COLLECTOR_PACKAGE_POLICY_NAME,
ELASTIC_CLOUD_APM_POLICY,
SYMBOLIZER_PACKAGE_POLICY_NAME,
getApmPolicy,
} from '@kbn/profiling-data-access-plugin/common';
import { omit } from 'lodash';
import { PackageInputType } from '../..';
import { PartialSetupState } from '../../../common/setup';
import { ELASTIC_CLOUD_APM_POLICY, getApmPolicy } from './get_apm_policy';
import { ProfilingSetupOptions } from './types';
const CLOUD_AGENT_POLICY_ID = 'policy-elastic-agent-on-cloud';
const COLLECTOR_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-collector';
const SYMBOLIZER_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-symbolizer';
async function getPackagePolicy({
soClient,
packagePolicyClient,
packageName,
}: {
packagePolicyClient: PackagePolicyClient;
soClient: SavedObjectsClientContract;
packageName: string;
}) {
const packagePolicies = await packagePolicyClient.list(soClient, {});
return packagePolicies.items.find((pkg) => pkg.name === packageName);
}
export async function getCollectorPolicy({
soClient,
packagePolicyClient,
}: {
packagePolicyClient: PackagePolicyClient;
soClient: SavedObjectsClientContract;
}) {
return getPackagePolicy({
soClient,
packagePolicyClient,
packageName: COLLECTOR_PACKAGE_POLICY_NAME,
});
}
export async function validateCollectorPackagePolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const collectorPolicy = await getCollectorPolicy({ soClient, packagePolicyClient });
return { policies: { collector: { installed: !!collectorPolicy } } };
}
export function generateSecretToken() {
let result = '';
@ -126,28 +91,6 @@ export async function createCollectorPackagePolicy({
});
}
export async function getSymbolizerPolicy({
soClient,
packagePolicyClient,
}: {
packagePolicyClient: PackagePolicyClient;
soClient: SavedObjectsClientContract;
}) {
return getPackagePolicy({
soClient,
packagePolicyClient,
packageName: SYMBOLIZER_PACKAGE_POLICY_NAME,
});
}
export async function validateSymbolizerPackagePolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const symbolizerPackagePolicy = await getSymbolizerPolicy({ soClient, packagePolicyClient });
return { policies: { symbolizer: { installed: !!symbolizerPackagePolicy } } };
}
export async function createSymbolizerPackagePolicy({
client,
soClient,
@ -185,29 +128,6 @@ export async function createSymbolizerPackagePolicy({
});
}
export async function validateProfilingInApmPackagePolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
try {
const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
return {
policies: {
apm: {
profilingEnabled: !!(
apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value?.profiling
),
},
},
};
} catch (e) {
// In case apm integration is not available ignore the error and return as profiling is not enabled on the integration
return {
policies: { apm: { profilingEnabled: false } },
};
}
}
export async function removeProfilingFromApmPackagePolicy({
client,
soClient,

2
x-pack/plugins/profiling/server/lib/setup/get_setup_instructions.ts
View file

@ -8,7 +8,7 @@
import { SavedObjectsClientContract } from '@kbn/core/server';
import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
import { fetchFindLatestPackageOrThrow } from '@kbn/fleet-plugin/server/services/epm/registry';
import { getCollectorPolicy, getSymbolizerPolicy } from './fleet_policies';
import { getCollectorPolicy, getSymbolizerPolicy } from '@kbn/profiling-data-access-plugin/common';
export interface SetupDataCollectionInstructions {
collector: {

21
x-pack/plugins/profiling/server/lib/setup/security_role.ts
View file

@ -5,24 +5,11 @@
* 2.0.
*/
import {
METADATA_VERSION,
PROFILING_READER_ROLE_NAME,
} from '@kbn/profiling-data-access-plugin/common';
import { ProfilingSetupOptions } from './types';
import { PartialSetupState } from '../../../common/setup';
const PROFILING_READER_ROLE_NAME = 'profiling-reader';
const METADATA_VERSION = 1;
export async function validateSecurityRole({
client,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const esClient = client.getEsClient();
const roles = await esClient.security.getRole();
const profilingRole = roles[PROFILING_READER_ROLE_NAME];
return {
permissions: {
configured: !!profilingRole && profilingRole.metadata.version === METADATA_VERSION,
},
};
}
export async function setSecurityRole({ client }: ProfilingSetupOptions) {
const esClient = client.getEsClient();

175
x-pack/plugins/profiling/server/routes/setup.ts
View file

@ -8,29 +8,14 @@
import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common';
import { RouteRegisterParameters } from '.';
import { getRoutePaths } from '../../common';
import {
areResourcesSetup,
createDefaultSetupState,
mergePartialSetupStates,
} from '../../common/setup';
import {
enableResourceManagement,
setMaximumBuckets,
validateMaximumBuckets,
validateResourceManagement,
} from '../lib/setup/cluster_settings';
import { enableResourceManagement, setMaximumBuckets } from '../lib/setup/cluster_settings';
import {
createCollectorPackagePolicy,
createSymbolizerPackagePolicy,
removeProfilingFromApmPackagePolicy,
validateCollectorPackagePolicy,
validateProfilingInApmPackagePolicy,
validateSymbolizerPackagePolicy,
} from '../lib/setup/fleet_policies';
import { getSetupInstructions } from '../lib/setup/get_setup_instructions';
import { hasProfilingData } from '../lib/setup/has_profiling_data';
import { setSecurityRole, validateSecurityRole } from '../lib/setup/security_role';
import { ProfilingSetupOptions } from '../lib/setup/types';
import { setSecurityRole } from '../lib/setup/security_role';
import { handleRouteHandlerError } from '../utils/handle_route_error_handler';
import { getClient } from './compat';
@ -52,82 +37,15 @@ export function registerSetupRoute({
try {
const esClient = await getClient(context);
const core = await context.core;
const clientWithDefaultAuth = createProfilingEsClient({
esClient,
request,
useDefaultAuth: true,
});
const clientWithProfilingAuth = createProfilingEsClient({
esClient,
request,
useDefaultAuth: false,
});
const setupOptions: ProfilingSetupOptions = {
client: clientWithDefaultAuth,
logger,
packagePolicyClient: dependencies.start.fleet.packagePolicyService,
const profilingStatus = await dependencies.start.profilingDataAccess.services.getStatus({
esClient,
soClient: core.savedObjects.client,
spaceId:
dependencies.setup.spaces?.spacesService?.getSpaceId(request) ?? DEFAULT_SPACE_ID,
isCloudEnabled: dependencies.setup.cloud.isCloudEnabled,
config: dependencies.config,
};
const state = createDefaultSetupState();
state.cloud.available = dependencies.setup.cloud.isCloudEnabled;
if (!state.cloud.available) {
const msg = `Elastic Cloud is required to set up Elasticsearch and Fleet for Universal Profiling`;
logger.error(msg);
return response.custom({
statusCode: 500,
body: {
message: msg,
},
});
}
const verifyFunctions = [
validateMaximumBuckets,
validateResourceManagement,
validateSecurityRole,
validateCollectorPackagePolicy,
validateSymbolizerPackagePolicy,
validateProfilingInApmPackagePolicy,
];
const partialStates = await Promise.all([
...verifyFunctions.map((fn) => fn(setupOptions)),
hasProfilingData({
...setupOptions,
client: clientWithProfilingAuth,
}),
]);
const mergedState = mergePartialSetupStates(state, partialStates);
return response.ok({
body: {
has_setup: areResourcesSetup(mergedState),
has_data: mergedState.data.available,
pre_8_9_1_data: mergedState.resources.pre_8_9_1_data,
},
spaceId: dependencies.setup.spaces?.spacesService?.getSpaceId(request),
});
return response.ok({ body: profilingStatus });
} catch (error) {
// We cannot fully check the status of all resources
// to make sure Profiling has been set up and has data
// for users with monitor privileges. This privileges
// is needed to call the profiling ES plugin for example.
if (error?.meta?.statusCode === 403) {
return response.ok({
body: {
has_setup: true,
pre_8_9_1_data: false,
has_data: true,
unauthorized: true,
},
});
}
return handleRouteHandlerError({
error,
logger,
@ -146,28 +64,9 @@ export function registerSetupRoute({
},
async (context, request, response) => {
try {
const esClient = await getClient(context);
const core = await context.core;
const clientWithDefaultAuth = createProfilingEsClient({
esClient,
request,
useDefaultAuth: true,
});
const setupOptions: ProfilingSetupOptions = {
client: clientWithDefaultAuth,
logger,
packagePolicyClient: dependencies.start.fleet.packagePolicyService,
soClient: core.savedObjects.client,
spaceId:
dependencies.setup.spaces?.spacesService?.getSpaceId(request) ?? DEFAULT_SPACE_ID,
isCloudEnabled: dependencies.setup.cloud.isCloudEnabled,
config: dependencies.config,
};
const isCloudEnabled = dependencies.setup.cloud.isCloudEnabled;
const state = createDefaultSetupState();
state.cloud.available = dependencies.setup.cloud.isCloudEnabled;
if (!state.cloud.available) {
if (!isCloudEnabled) {
const msg = `Elastic Cloud is required to set up Elasticsearch and Fleet for Universal Profiling`;
logger.error(msg);
return response.custom({
@ -178,28 +77,44 @@ export function registerSetupRoute({
});
}
const partialStates = await Promise.all(
[
validateResourceManagement,
validateSecurityRole,
validateMaximumBuckets,
validateCollectorPackagePolicy,
validateSymbolizerPackagePolicy,
validateProfilingInApmPackagePolicy,
].map((fn) => fn(setupOptions))
const esClient = await getClient(context);
const core = await context.core;
const clientWithDefaultAuth = createProfilingEsClient({
esClient,
request,
useDefaultAuth: true,
});
const clientWithProfilingAuth = createProfilingEsClient({
esClient,
request,
useDefaultAuth: false,
});
const commonParams = {
client: clientWithDefaultAuth,
logger,
packagePolicyClient: dependencies.start.fleet.packagePolicyService,
soClient: core.savedObjects.client,
spaceId:
dependencies.setup.spaces?.spacesService?.getSpaceId(request) ?? DEFAULT_SPACE_ID,
isCloudEnabled,
};
const setupState = await dependencies.start.profilingDataAccess.services.getSetupState(
commonParams,
clientWithProfilingAuth
);
const mergedState = mergePartialSetupStates(state, partialStates);
const executeAdminFunctions = [
...(mergedState.resource_management.enabled ? [] : [enableResourceManagement]),
...(mergedState.permissions.configured ? [] : [setSecurityRole]),
...(mergedState.settings.configured ? [] : [setMaximumBuckets]),
...(setupState.resource_management.enabled ? [] : [enableResourceManagement]),
...(setupState.permissions.configured ? [] : [setSecurityRole]),
...(setupState.settings.configured ? [] : [setMaximumBuckets]),
];
const executeViewerFunctions = [
...(mergedState.policies.collector.installed ? [] : [createCollectorPackagePolicy]),
...(mergedState.policies.symbolizer.installed ? [] : [createSymbolizerPackagePolicy]),
...(mergedState.policies.apm.profilingEnabled
...(setupState.policies.collector.installed ? [] : [createCollectorPackagePolicy]),
...(setupState.policies.symbolizer.installed ? [] : [createSymbolizerPackagePolicy]),
...(setupState.policies.apm.profilingEnabled
? [removeProfilingFromApmPackagePolicy]
: []),
];
@ -208,8 +123,12 @@ export function registerSetupRoute({
return response.ok();
}
await Promise.all(executeAdminFunctions.map((fn) => fn(setupOptions)));
await Promise.all(executeViewerFunctions.map((fn) => fn(setupOptions)));
const setupParams = {
...commonParams,
config: dependencies.config,
};
await Promise.all(executeAdminFunctions.map((fn) => fn(setupParams)));
await Promise.all(executeViewerFunctions.map((fn) => fn(setupParams)));
if (dependencies.telemetryUsageCounter) {
dependencies.telemetryUsageCounter.incrementCounter({

1
x-pack/plugins/profiling/tsconfig.json
View file

@ -45,7 +45,6 @@
"@kbn/share-plugin",
"@kbn/observability-shared-plugin",
"@kbn/licensing-plugin",
"@kbn/utility-types",
"@kbn/usage-collection-plugin",
"@kbn/observability-ai-assistant-plugin",
"@kbn/profiling-data-access-plugin",

37
x-pack/plugins/profiling_data_access/common/cluster_settings.ts Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { PartialSetupState, ProfilingSetupOptions } from './setup';
export const MAX_BUCKETS = 150000;
export async function validateMaximumBuckets({
client,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const settings = await client.getEsClient().cluster.getSettings({});
const maxBuckets = settings.persistent.search?.max_buckets;
return {
settings: {
configured: maxBuckets === MAX_BUCKETS.toString(),
},
};
}
export async function validateResourceManagement({
client,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const statusResponse = await client.profilingStatus();
return {
resource_management: {
enabled: statusResponse.resource_management.enabled,
},
resources: {
created: statusResponse.resources.created,
pre_8_9_1_data: statusResponse.resources.pre_8_9_1_data,
},
};
}

106
x-pack/plugins/profiling_data_access/common/fleet_policies.ts Normal file
View file

@ -0,0 +1,106 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract } from '@kbn/core/server';
import type { PackagePolicyClient } from '@kbn/fleet-plugin/server';
import { getApmPolicy } from './get_apm_policy';
import { PartialSetupState, ProfilingSetupOptions } from './setup';
export const COLLECTOR_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-collector';
export const SYMBOLIZER_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-symbolizer';
async function getPackagePolicy({
soClient,
packagePolicyClient,
packageName,
}: {
packagePolicyClient: PackagePolicyClient;
soClient: SavedObjectsClientContract;
packageName: string;
}) {
const packagePolicies = await packagePolicyClient.list(soClient, {});
return packagePolicies.items.find((pkg) => pkg.name === packageName);
}
export async function getCollectorPolicy({
soClient,
packagePolicyClient,
}: {
packagePolicyClient: PackagePolicyClient;
soClient: SavedObjectsClientContract;
}) {
return getPackagePolicy({
soClient,
packagePolicyClient,
packageName: COLLECTOR_PACKAGE_POLICY_NAME,
});
}
export async function validateCollectorPackagePolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const collectorPolicy = await getCollectorPolicy({ soClient, packagePolicyClient });
return { policies: { collector: { installed: !!collectorPolicy } } };
}
export function generateSecretToken() {
let result = '';
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
for (let i = 0; i < 16; i++) {
const randomIndex = Math.floor(Math.random() * characters.length);
result += characters.charAt(randomIndex);
}
return result;
}
export async function getSymbolizerPolicy({
soClient,
packagePolicyClient,
}: {
packagePolicyClient: PackagePolicyClient;
soClient: SavedObjectsClientContract;
}) {
return getPackagePolicy({
soClient,
packagePolicyClient,
packageName: SYMBOLIZER_PACKAGE_POLICY_NAME,
});
}
export async function validateSymbolizerPackagePolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const symbolizerPackagePolicy = await getSymbolizerPolicy({ soClient, packagePolicyClient });
return { policies: { symbolizer: { installed: !!symbolizerPackagePolicy } } };
}
export async function validateProfilingInApmPackagePolicy({
soClient,
packagePolicyClient,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
try {
const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
return {
policies: {
apm: {
profilingEnabled: !!(
apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value?.profiling
),
},
},
};
} catch (e) {
// In case apm integration is not available ignore the error and return as profiling is not enabled on the integration
return {
policies: { apm: { profilingEnabled: false } },
};
}
}

2
x-pack/plugins/profiling/server/lib/setup/get_apm_policy.ts → x-pack/plugins/profiling_data_access/common/get_apm_policy.ts
View file

@ -6,7 +6,7 @@
*/
import { SavedObjectsClientContract } from '@kbn/core/server';
import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
import type { PackagePolicyClient } from '@kbn/fleet-plugin/server';
export const ELASTIC_CLOUD_APM_POLICY = 'elastic-cloud-apm';

3
x-pack/plugins/profiling/server/lib/setup/has_profiling_data.ts → x-pack/plugins/profiling_data_access/common/has_profiling_data.ts
View file

@ -5,8 +5,7 @@
* 2.0.
*/
import { PartialSetupState } from '../../../common/setup';
import { ProfilingSetupOptions } from './types';
import { PartialSetupState, ProfilingSetupOptions } from './setup';
export async function hasProfilingData({
client,

16
x-pack/plugins/profiling_data_access/common/index.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { getApmPolicy, ELASTIC_CLOUD_APM_POLICY } from './get_apm_policy';
export { MAX_BUCKETS } from './cluster_settings';
export { METADATA_VERSION, PROFILING_READER_ROLE_NAME } from './security_role';
export {
getCollectorPolicy,
getSymbolizerPolicy,
COLLECTOR_PACKAGE_POLICY_NAME,
SYMBOLIZER_PACKAGE_POLICY_NAME,
} from './fleet_policies';

24
x-pack/plugins/profiling_data_access/common/profiling_es_client.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ElasticsearchClient } from '@kbn/core/server';
import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
import type { ProfilingStatusResponse, StackTraceResponse } from '@kbn/profiling-utils';
export interface ProfilingESClient {
search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
operationName: string,
searchRequest: TSearchRequest
): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
profilingStacktraces({}: {
query: QueryDslQueryContainer;
sampleSize: number;
}): Promise<StackTraceResponse>;
profilingStatus(): Promise<ProfilingStatusResponse>;
getEsClient(): ElasticsearchClient;
}

24
x-pack/plugins/profiling_data_access/common/security_role.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { PartialSetupState, ProfilingSetupOptions } from './setup';
export const PROFILING_READER_ROLE_NAME = 'profiling-reader';
export const METADATA_VERSION = 1;
export async function validateSecurityRole({
client,
}: ProfilingSetupOptions): Promise<PartialSetupState> {
const esClient = client.getEsClient();
const roles = await esClient.security.getRole();
const profilingRole = roles[PROFILING_READER_ROLE_NAME];
return {
permissions: {
configured: !!profilingRole && profilingRole.metadata.version === METADATA_VERSION,
},
};
}

0
x-pack/plugins/profiling/common/setup.test.ts → x-pack/plugins/profiling_data_access/common/setup.test.ts
View file

15
x-pack/plugins/profiling/common/setup.ts → x-pack/plugins/profiling_data_access/common/setup.ts
View file

@ -4,9 +4,20 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { merge } from 'lodash';
import type { RecursivePartial } from '@elastic/eui';
import { Logger, SavedObjectsClientContract } from '@kbn/core/server';
import type { PackagePolicyClient } from '@kbn/fleet-plugin/server';
import { merge } from 'lodash';
import { ProfilingESClient } from './profiling_es_client';
export interface ProfilingSetupOptions {
client: ProfilingESClient;
soClient: SavedObjectsClientContract;
packagePolicyClient: PackagePolicyClient;
logger: Logger;
spaceId: string;
isCloudEnabled: boolean;
}
export interface SetupState {
cloud: {

6
x-pack/plugins/profiling_data_access/kibana.jsonc
View file

@ -7,7 +7,11 @@
"server": true,
"browser": false,
"configPath": ["xpack", "profiling"],
"requiredPlugins": ["data"],
"requiredPlugins": [
"data",
"fleet",
"cloud"
],
"optionalPlugins": [],
"requiredBundles": []
}

22
x-pack/plugins/profiling_data_access/server/plugin.ts
View file

@ -5,19 +5,30 @@
* 2.0.
*/
import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server';
import type {
CoreSetup,
CoreStart,
Logger,
Plugin,
PluginInitializerContext,
} from '@kbn/core/server';
import { ProfilingConfig } from '.';
import { registerServices } from './services/register_services';
import { createProfilingEsClient } from './utils/create_profiling_es_client';
import { ProfilingPluginStartDeps } from './types';
export type ProfilingDataAccessPluginSetup = ReturnType<ProfilingDataAccessPlugin['setup']>;
export type ProfilingDataAccessPluginStart = ReturnType<ProfilingDataAccessPlugin['start']>;
export class ProfilingDataAccessPlugin implements Plugin {
constructor(private readonly initializerContext: PluginInitializerContext<ProfilingConfig>) {}
private readonly logger: Logger;
constructor(private readonly initializerContext: PluginInitializerContext<ProfilingConfig>) {
this.logger = initializerContext.logger.get();
}
public setup(core: CoreSetup) {}
public start(core: CoreStart) {
public start(core: CoreStart, plugins: ProfilingPluginStartDeps) {
const config = this.initializerContext.config.get();
const profilingSpecificEsClient = config.elasticsearch
@ -37,6 +48,11 @@ export class ProfilingDataAccessPlugin implements Plugin {
return createProfilingEsClient({ esClient });
},
logger: this.logger,
deps: {
fleet: plugins.fleet,
cloud: plugins.cloud,
},
});
// called after all plugins are set up

56
x-pack/plugins/profiling_data_access/server/services/get_setup_state/index.ts Normal file
View file

@ -0,0 +1,56 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
validateMaximumBuckets,
validateResourceManagement,
} from '../../../common/cluster_settings';
import {
validateCollectorPackagePolicy,
validateProfilingInApmPackagePolicy,
validateSymbolizerPackagePolicy,
} from '../../../common/fleet_policies';
import { hasProfilingData } from '../../../common/has_profiling_data';
import { ProfilingESClient } from '../../../common/profiling_es_client';
import { validateSecurityRole } from '../../../common/security_role';
import {
ProfilingSetupOptions,
createDefaultSetupState,
mergePartialSetupStates,
} from '../../../common/setup';
import { RegisterServicesParams } from '../register_services';
export async function getSetupState(
options: ProfilingSetupOptions,
clientWithProfilingAuth: ProfilingESClient
) {
const state = createDefaultSetupState();
state.cloud.available = options.isCloudEnabled;
const verifyFunctions = [
validateMaximumBuckets,
validateResourceManagement,
validateSecurityRole,
validateCollectorPackagePolicy,
validateSymbolizerPackagePolicy,
validateProfilingInApmPackagePolicy,
];
const partialStates = await Promise.all([
...verifyFunctions.map((fn) => fn(options)),
hasProfilingData({
...options,
client: clientWithProfilingAuth,
}),
]);
return mergePartialSetupStates(state, partialStates);
}
export function createGetSetupState(params: RegisterServicesParams) {
return getSetupState;
}

15
x-pack/plugins/profiling_data_access/server/services/register_services.ts
View file

@ -5,9 +5,13 @@
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core/server';
import { CloudStart } from '@kbn/cloud-plugin/server';
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { FleetStartContract } from '@kbn/fleet-plugin/server';
import { createFetchFlamechart } from './fetch_flamechart';
import { ProfilingESClient } from '../utils/create_profiling_es_client';
import { createGetStatusService } from './status';
import { createGetSetupState } from './get_setup_state';
import { ProfilingESClient } from '../../common/profiling_es_client';
import { createFetchFunctions } from './functions';
export interface RegisterServicesParams {
@ -15,11 +19,18 @@ export interface RegisterServicesParams {
esClient: ElasticsearchClient;
useDefaultAuth?: boolean;
}) => ProfilingESClient;
logger: Logger;
deps: {
fleet: FleetStartContract;
cloud: CloudStart;
};
}
export function registerServices(params: RegisterServicesParams) {
return {
fetchFlamechartData: createFetchFlamechart(params),
getStatus: createGetStatusService(params),
getSetupState: createGetSetupState(params),
fetchFunction: createFetchFunctions(params),
};
}

2
x-pack/plugins/profiling_data_access/server/services/search_stack_traces/index.ts
View file

@ -6,7 +6,7 @@
*/
import { decodeStackTraceResponse } from '@kbn/profiling-utils';
import { ProfilingESClient } from '../../utils/create_profiling_es_client';
import { ProfilingESClient } from '../../../common/profiling_es_client';
import { kqlQuery } from '../../utils/query';
export async function searchStackTraces({

80
x-pack/plugins/profiling_data_access/server/services/status/index.ts Normal file
View file

@ -0,0 +1,80 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
import { ProfilingStatus } from '@kbn/profiling-utils';
import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common';
import { getSetupState } from '../get_setup_state';
import { RegisterServicesParams } from '../register_services';
import { ProfilingSetupOptions, areResourcesSetup } from '../../../common/setup';
interface HasSetupParams {
soClient: SavedObjectsClientContract;
esClient: ElasticsearchClient;
spaceId?: string;
}
export function createGetStatusService({
createProfilingEsClient,
deps,
logger,
}: RegisterServicesParams) {
return async ({ esClient, soClient, spaceId }: HasSetupParams): Promise<ProfilingStatus> => {
try {
const isCloudEnabled = deps.cloud.isCloudEnabled;
if (!isCloudEnabled) {
// When not on cloud just return that is has not set up and has no data
return {
has_setup: false,
has_data: false,
pre_8_9_1_data: false,
};
}
const clientWithDefaultAuth = createProfilingEsClient({
esClient,
useDefaultAuth: true,
});
const clientWithProfilingAuth = createProfilingEsClient({
esClient,
useDefaultAuth: false,
});
const setupOptions: ProfilingSetupOptions = {
client: clientWithDefaultAuth,
logger,
packagePolicyClient: deps.fleet.packagePolicyService,
soClient,
spaceId: spaceId ?? DEFAULT_SPACE_ID,
isCloudEnabled,
};
const setupState = await getSetupState(setupOptions, clientWithProfilingAuth);
return {
has_setup: areResourcesSetup(setupState),
has_data: setupState.data.available,
pre_8_9_1_data: setupState.resources.pre_8_9_1_data,
};
} catch (error) {
// We cannot fully check the status of all resources
// to make sure Profiling has been set up and has data
// for users with monitor privileges. This privileges
// is needed to call the profiling ES plugin for example.
if (error?.meta?.statusCode === 403 || error?.originalError?.meta?.statusCode === 403) {
return {
has_setup: true,
pre_8_9_1_data: false,
has_data: true,
unauthorized: true,
};
}
throw error;
}
};
}

13
x-pack/plugins/profiling_data_access/server/types.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { CloudStart } from '@kbn/cloud-plugin/server';
import { FleetStartContract } from '@kbn/fleet-plugin/server';
export interface ProfilingPluginStartDeps {
fleet: FleetStartContract;
cloud: CloudStart;
}

15
x-pack/plugins/profiling_data_access/server/utils/create_profiling_es_client.ts
View file

@ -5,26 +5,13 @@
* 2.0.
*/
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ElasticsearchClient } from '@kbn/core/server';
import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
import type { ProfilingStatusResponse, StackTraceResponse } from '@kbn/profiling-utils';
import { ProfilingESClient } from '../../common/profiling_es_client';
import { unwrapEsResponse } from './unwrap_es_response';
import { withProfilingSpan } from './with_profiling_span';
export interface ProfilingESClient {
search<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
operationName: string,
searchRequest: TSearchRequest
): Promise<InferSearchResponseOf<TDocument, TSearchRequest>>;
profilingStacktraces({}: {
query: QueryDslQueryContainer;
sampleSize: number;
}): Promise<StackTraceResponse>;
profilingStatus(): Promise<ProfilingStatusResponse>;
getEsClient(): ElasticsearchClient;
}
export function createProfilingEsClient({
esClient,
}: {

6
x-pack/plugins/profiling_data_access/tsconfig.json
View file

@ -4,6 +4,7 @@
"outDir": "target/types"
},
"include": [
"common/**/*",
"server/**/*",
"jest.config.js"
],
@ -16,6 +17,9 @@
"@kbn/es-query",
"@kbn/es-types",
"@kbn/apm-utils",
"@kbn/profiling-utils"
"@kbn/profiling-utils",
"@kbn/fleet-plugin",
"@kbn/cloud-plugin",
"@kbn/spaces-plugin",
]
}

70
x-pack/test/profiling_api_integration/common/bettertest.ts Normal file
View file

@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { format } from 'url';
import supertest from 'supertest';
import request from 'superagent';
type HttpMethod = 'get' | 'post' | 'put' | 'delete';
export type BetterTest = <T extends any>(options: {
pathname: string;
query?: Record<string, any>;
method?: HttpMethod;
body?: any;
}) => Promise<{ status: number; body: T }>;
/*
* This is a wrapper around supertest that throws an error if the response status is not 200.
* This is useful for tests that expect a 200 response
* It also makes it easier to debug tests that fail because of a 500 response.
*/
export function getBettertest(st: supertest.SuperTest<supertest.Test>): BetterTest {
return async ({ pathname, method = 'get', query, body }) => {
const url = format({ pathname, query });
let res: request.Response;
if (body) {
res = await st[method](url).send(body).set('kbn-xsrf', 'true');
} else {
res = await st[method](url).set('kbn-xsrf', 'true');
}
// supertest doesn't throw on http errors
if (res?.status !== 200 && res?.status !== 202) {
throw new BetterTestError(res);
}
return res;
};
}
type ErrorResponse = Omit<request.Response, 'body'> & {
body: {
statusCode: number;
error: string;
message: string;
attributes: object;
};
};
export class BetterTestError extends Error {
res: ErrorResponse;
constructor(res: request.Response) {
// @ts-expect-error
const req = res.req as any;
super(
`Unhandled BetterTestError:
Status: "${res.status}"
Path: "${req.method} ${req.path}"
Body: ${JSON.stringify(res.body)}`
);
this.res = res;
}
}

29
x-pack/test/profiling_api_integration/common/config.ts
View file

@ -5,26 +5,24 @@
* 2.0.
*/
import { format, UrlObject } from 'url';
import { FtrConfigProviderContext } from '@kbn/test';
import supertest from 'supertest';
import { getRoutePaths } from '@kbn/profiling-plugin/common';
import { format, UrlObject } from 'url';
import { ProfilingFtrConfigName } from '../configs';
import { createProfilingApiClient } from './api_supertest';
import { createProfilingUsers } from './create_profiling_users';
import {
PROFILING_TEST_PASSWORD,
ProfilingUsername,
} from './create_profiling_users/authentication';
import {
FtrProviderContext,
InheritedFtrProviderContext,
InheritedServices,
} from './ftr_provider_context';
import { RegistryProvider } from './registry';
import { createProfilingApiClient } from './api_supertest';
import {
ProfilingUsername,
PROFILING_TEST_PASSWORD,
} from './create_profiling_users/authentication';
import { createProfilingUsers } from './create_profiling_users';
export type CreateTestConfig = ReturnType<typeof createTestConfig>;
const profilingRoutePaths = getRoutePaths();
export async function getProfilingApiClient({
kibanaServer,
@ -112,19 +110,6 @@ export function createTestConfig(
await supertest(kibanaServerUrl).post('/api/fleet/setup').set('kbn-xsrf', 'foo');
const result = await adminUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
if (!result.body.has_setup) {
// eslint-disable-next-line no-console
console.log('Setting up Universal Profiling');
await adminUser({
endpoint: `POST ${profilingRoutePaths.HasSetupESResources}`,
});
// eslint-disable-next-line no-console
console.log('Universal Profiling set up');
}
return {
noAccessUser: await getProfilingApiClient({
kibanaServer,

46
x-pack/test/profiling_api_integration/common/registry.ts
View file

@ -9,12 +9,10 @@ import { joinByKey } from '@kbn/apm-plugin/common/utils/join_by_key';
import { maybe } from '@kbn/apm-plugin/common/utils/maybe';
import callsites from 'callsites';
import { castArray, groupBy } from 'lodash';
import Path from 'path';
import fs from 'fs';
import { ProfilingFtrConfigName } from '../configs';
import { getBettertest } from './bettertest';
import { FtrProviderContext } from './ftr_provider_context';
const esArchiversPath = Path.posix.join(__dirname, 'fixtures', 'es_archiver', 'profiling');
import { cleanUpProfilingData } from '../utils/profiling_data';
interface RunCondition {
config: ProfilingFtrConfigName;
@ -22,6 +20,9 @@ interface RunCondition {
export function RegistryProvider({ getService }: FtrProviderContext) {
const profilingFtrConfig = getService('profilingFtrConfig');
const supertest = getService('supertest');
const bettertest = getBettertest(supertest);
const es = getService('es');
const callbacks: Array<
@ -97,16 +98,6 @@ export function RegistryProvider({ getService }: FtrProviderContext) {
const logger = getService('log');
const logWithTimer = () => {
const start = process.hrtime();
return (message: string) => {
const diff = process.hrtime(start);
const time = `${Math.round(diff[0] * 1000 + diff[1] / 1e6)}ms`;
logger.info(`(${time}) ${message}`);
};
};
const groups = joinByKey(callbacks, ['config'], (a, b) => ({
...a,
...b,
@ -126,36 +117,11 @@ export function RegistryProvider({ getService }: FtrProviderContext) {
groupsForConfig.forEach((group) => {
const { runs } = group;
const runBefore = async () => {
const log = logWithTimer();
const content = fs.readFileSync(`${esArchiversPath}/data.json`, 'utf8');
log(`Loading profiling data`);
await es.bulk({ operations: content.split('\n'), refresh: 'wait_for' });
log('Loaded profiling data');
};
const runAfter = async () => {
const log = logWithTimer();
log(`Unloading Profiling data`);
const indices = await es.cat.indices({ format: 'json' });
const profilingIndices = indices
.filter((index) => index.index !== undefined)
.map((index) => index.index)
.filter((index) => {
return index!.startsWith('profiling') || index!.startsWith('.profiling');
}) as string[];
await Promise.all([
...profilingIndices.map((index) => es.indices.delete({ index })),
es.indices.deleteDataStream({
name: 'profiling-events*',
}),
]);
log('Unloaded Profiling data');
await cleanUpProfilingData({ es, bettertest, logger });
};
describe('Loading profiling data', () => {
before(runBefore);
runs.forEach((run) => {
run.cb();
});

2
x-pack/test/profiling_api_integration/tests/__snapshots__/functions.spec.snap generated
View file

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`Profiling API tests functions.spec.ts cloud Loading profiling data Functions api returns correct result 1`] = `
exports[`Profiling API tests functions.spec.ts cloud Loading profiling data Functions api With data returns correct result 1`] = `
Object {
"SamplingRate": 1,
"TopN": Array [

9
x-pack/test/profiling_api_integration/tests/feature_controls.spec.ts
View file

@ -10,6 +10,8 @@ import { getRoutePaths } from '@kbn/profiling-plugin/common';
import { ProfilingApiError } from '../common/api_supertest';
import { getProfilingApiClient } from '../common/config';
import { FtrProviderContext } from '../common/ftr_provider_context';
import { setupProfiling } from '../utils/profiling_data';
import { getBettertest } from '../common/bettertest';
const profilingRoutePaths = getRoutePaths();
@ -17,7 +19,8 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
const registry = getService('registry');
const profilingApiClient = getService('profilingApiClient');
const log = getService('log');
const supertest = getService('supertest');
const bettertest = getBettertest(supertest);
const start = encodeURIComponent(new Date(Date.now() - 10000).valueOf());
const end = encodeURIComponent(new Date().valueOf());
@ -111,7 +114,9 @@ export default function featureControlsTests({ getService }: FtrProviderContext)
}
registry.when('Profiling feature controls', { config: 'cloud' }, () => {
before(async () => {});
before(async () => {
await setupProfiling(bettertest, log);
});
it(`returns forbidden for users with no access to profiling APIs`, async () => {
await executeRequests({
runAsUser: profilingApiClient.noAccessUser,

56
x-pack/test/profiling_api_integration/tests/functions.spec.ts
View file

@ -9,39 +9,55 @@ import { getRoutePaths } from '@kbn/profiling-plugin/common';
import { TopNFunctions } from '@kbn/profiling-utils';
import expect from '@kbn/expect';
import { FtrProviderContext } from '../common/ftr_provider_context';
import { loadProfilingData, setupProfiling } from '../utils/profiling_data';
import { getBettertest } from '../common/bettertest';
const profilingRoutePaths = getRoutePaths();
export default function featureControlsTests({ getService }: FtrProviderContext) {
const registry = getService('registry');
const profilingApiClient = getService('profilingApiClient');
const log = getService('log');
const supertest = getService('supertest');
const bettertest = getBettertest(supertest);
const es = getService('es');
const start = new Date('2023-03-17T01:00:00.000Z').getTime();
const end = new Date('2023-03-17T01:05:00.000Z').getTime();
registry.when('Functions api', { config: 'cloud' }, () => {
let functions: TopNFunctions;
before(async () => {
const response = await profilingApiClient.adminUser({
endpoint: `GET ${profilingRoutePaths.TopNFunctions}`,
params: {
query: {
timeFrom: start,
timeTo: end,
kuery: '',
startIndex: 0,
endIndex: 5,
},
},
});
functions = response.body as TopNFunctions;
await setupProfiling(bettertest, log);
await loadProfilingData(es, log);
});
it(`returns correct result`, async () => {
expect(functions.TopN.length).to.equal(5);
expect(functions.TotalCount).to.equal(3599);
expect(functions.selfCPU).to.equal(397);
expect(functions.totalCPU).to.equal(399);
expectSnapshot(functions).toMatch();
describe('With data', () => {
let functions: TopNFunctions;
before(async () => {
await setupProfiling(bettertest, log);
await loadProfilingData(es, log);
const response = await profilingApiClient.adminUser({
endpoint: `GET ${profilingRoutePaths.TopNFunctions}`,
params: {
query: {
timeFrom: start,
timeTo: end,
kuery: '',
startIndex: 0,
endIndex: 5,
},
},
});
functions = response.body as TopNFunctions;
});
it(`returns correct result`, async () => {
expect(functions.TopN.length).to.equal(5);
expect(functions.TotalCount).to.equal(3599);
expect(functions.selfCPU).to.equal(397);
expect(functions.totalCPU).to.equal(399);
expectSnapshot(functions).toMatch();
});
});
});
}

357
x-pack/test/profiling_api_integration/tests/has_setup.spec.ts Normal file
View file

@ -0,0 +1,357 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { getRoutePaths } from '@kbn/profiling-plugin/common';
import { ProfilingStatus } from '@kbn/profiling-utils';
import { getBettertest } from '../common/bettertest';
import { FtrProviderContext } from '../common/ftr_provider_context';
import { deletePackagePolicy, getProfilingPackagePolicyIds } from '../utils/fleet';
import { loadProfilingData, setupProfiling } from '../utils/profiling_data';
const profilingRoutePaths = getRoutePaths();
export default function featureControlsTests({ getService }: FtrProviderContext) {
const registry = getService('registry');
const profilingApiClient = getService('profilingApiClient');
const supertest = getService('supertest');
const bettertest = getBettertest(supertest);
const logger = getService('log');
const es = getService('es');
registry.when('Profiling status check', { config: 'cloud' }, () => {
describe('Profiling is not set up and no data is loaded', () => {
describe('Admin user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.adminUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has not been set up`, async () => {
expect(statusCheck.has_setup).to.be(false);
});
it(`does not have data`, async () => {
expect(statusCheck.has_data).to.be(false);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
});
describe('Viewer user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.readUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`has data`, async () => {
expect(statusCheck.has_data).to.be(true);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
it(`is unauthorized to fully check profiling status `, async () => {
expect(statusCheck.unauthorized).to.be(true);
});
});
});
describe('Collector integration is not installed', () => {
let collectorId: string | undefined;
before(async () => {
await setupProfiling(bettertest, logger);
const response = await getProfilingPackagePolicyIds(bettertest);
collectorId = response.collectorId;
if (collectorId) {
await deletePackagePolicy(bettertest, collectorId);
}
});
it('expectes a collector integration to exist', () => {
expect(collectorId).not.to.be(undefined);
});
describe('Admin user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.adminUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has not been set up`, async () => {
expect(statusCheck.has_setup).to.be(false);
});
it(`does not have data`, async () => {
expect(statusCheck.has_data).to.be(false);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
});
describe('Viewer user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.readUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`has data`, async () => {
expect(statusCheck.has_data).to.be(true);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
it(`is unauthorized to fully check profiling status `, async () => {
expect(statusCheck.unauthorized).to.be(true);
});
});
});
describe('Symbolizer integration is not installed', () => {
let symbolizerId: string | undefined;
before(async () => {
await setupProfiling(bettertest, logger);
const response = await getProfilingPackagePolicyIds(bettertest);
symbolizerId = response.symbolizerId;
if (symbolizerId) {
await deletePackagePolicy(bettertest, symbolizerId);
}
});
it('expectes a symbolizer integration to exist', () => {
expect(symbolizerId).not.to.be(undefined);
});
describe('Admin user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.adminUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has not been set up`, async () => {
expect(statusCheck.has_setup).to.be(false);
});
it(`does not have data`, async () => {
expect(statusCheck.has_data).to.be(false);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
});
describe('Viewer user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.readUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`has data`, async () => {
expect(statusCheck.has_data).to.be(true);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
it(`is unauthorized to fully check profiling status `, async () => {
expect(statusCheck.unauthorized).to.be(true);
});
});
});
describe('APM integration is not installed', () => {
before(async () => {
await setupProfiling(bettertest, logger);
await deletePackagePolicy(bettertest, 'elastic-cloud-apm');
});
describe('Admin user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.adminUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`does not have data`, async () => {
expect(statusCheck.has_data).to.be(false);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
});
describe('Viewer user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.readUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`has data`, async () => {
expect(statusCheck.has_data).to.be(true);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
it(`is unauthorized to fully check profiling status `, async () => {
expect(statusCheck.unauthorized).to.be(true);
});
});
});
describe('Profiling is set up', () => {
before(async () => {
await setupProfiling(bettertest, logger);
});
describe('without data', () => {
describe('Admin user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.adminUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`does not have data`, async () => {
expect(statusCheck.has_data).to.be(false);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
});
describe('Viewer user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.readUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`has data`, async () => {
expect(statusCheck.has_data).to.be(true);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
it(`is unauthorized to fully check profiling status `, async () => {
expect(statusCheck.unauthorized).to.be(true);
});
});
});
describe('with data', () => {
before(async () => {
await loadProfilingData(es, logger);
});
describe('Admin user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.adminUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`does not have data`, async () => {
expect(statusCheck.has_data).to.be(true);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
});
describe('Viewer user', () => {
let statusCheck: ProfilingStatus;
before(async () => {
const response = await profilingApiClient.readUser({
endpoint: `GET ${profilingRoutePaths.HasSetupESResources}`,
});
statusCheck = response.body;
});
it(`has been set up`, async () => {
expect(statusCheck.has_setup).to.be(true);
});
it(`has data`, async () => {
expect(statusCheck.has_data).to.be(true);
});
it(`does not have pre 8.9.1 data`, async () => {
expect(statusCheck.pre_8_9_1_data).to.be(false);
});
it(`is unauthorized to fully check profiling status `, async () => {
expect(statusCheck.unauthorized).to.be(true);
});
});
});
});
});
}

38
x-pack/test/profiling_api_integration/utils/fleet.ts Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { PackagePolicy } from '@kbn/fleet-plugin/common';
import {
COLLECTOR_PACKAGE_POLICY_NAME,
SYMBOLIZER_PACKAGE_POLICY_NAME,
} from '@kbn/profiling-data-access-plugin/common';
import { BetterTest } from '../common/bettertest';
export async function deletePackagePolicy(bettertest: BetterTest, packagePolicyId: string) {
return bettertest({
pathname: `/api/fleet/package_policies/delete`,
method: 'post',
body: { packagePolicyIds: [packagePolicyId] },
});
}
export async function getProfilingPackagePolicyIds(bettertest: BetterTest) {
const response = await bettertest<{ items: PackagePolicy[] }>({
pathname: '/api/fleet/package_policies',
method: 'get',
});
const collector = response.body.items.find((item) => item.name === COLLECTOR_PACKAGE_POLICY_NAME);
const symbolizer = response.body.items.find(
(item) => item.name === SYMBOLIZER_PACKAGE_POLICY_NAME
);
return {
collectorId: collector?.id,
symbolizerId: symbolizer?.id,
};
}

97
x-pack/test/profiling_api_integration/utils/profiling_data.ts Normal file
View file

@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Client } from '@elastic/elasticsearch';
import { getRoutePaths } from '@kbn/profiling-plugin/common';
import { ProfilingStatus } from '@kbn/profiling-utils';
import { ToolingLog } from '@kbn/tooling-log';
import fs from 'fs';
import Path from 'path';
import { BetterTest } from '../common/bettertest';
import { deletePackagePolicy, getProfilingPackagePolicyIds } from './fleet';
const profilingRoutePaths = getRoutePaths();
const esArchiversPath = Path.posix.join(
__dirname,
'..',
'common',
'fixtures',
'es_archiver',
'profiling'
);
function logWithTimer(logger: ToolingLog) {
const start = process.hrtime();
return (message: string) => {
const diff = process.hrtime(start);
const time = `${Math.round(diff[0] * 1000 + diff[1] / 1e6)}ms`;
logger.info(`(${time}) ${message}`);
};
}
export async function cleanUpProfilingData({
es,
bettertest,
logger,
}: {
es: Client;
bettertest: BetterTest;
logger: ToolingLog;
}) {
const log = logWithTimer(logger);
log(`Unloading Profiling data`);
const [indices, { collectorId, symbolizerId }] = await Promise.all([
es.cat.indices({ format: 'json' }),
getProfilingPackagePolicyIds(bettertest),
]);
const profilingIndices = indices
.filter((index) => index.index !== undefined)
.map((index) => index.index)
.filter((index) => {
return index!.startsWith('profiling') || index!.startsWith('.profiling');
}) as string[];
await Promise.all([
...profilingIndices.map((index) => es.indices.delete({ index })),
es.indices.deleteDataStream({
name: 'profiling-events*',
}),
collectorId ? deletePackagePolicy(bettertest, collectorId) : Promise.resolve(),
symbolizerId ? deletePackagePolicy(bettertest, symbolizerId) : Promise.resolve(),
]);
log('Unloaded Profiling data');
}
export async function setupProfiling(bettertest: BetterTest, logger: ToolingLog) {
const log = logWithTimer(logger);
const response = await bettertest<ProfilingStatus>({
method: 'get',
pathname: profilingRoutePaths.HasSetupESResources,
});
if (response.body.has_setup) {
log(`Skipping Universal Profiling set up, already set up`);
} else {
log(`Setting up Universal Profiling`);
await bettertest<ProfilingStatus>({
method: 'post',
pathname: profilingRoutePaths.HasSetupESResources,
});
log(`Universal Profiling set up`);
}
}
export async function loadProfilingData(es: Client, logger: ToolingLog) {
const log = logWithTimer(logger);
log(`Loading profiling data`);
const content = fs.readFileSync(`${esArchiversPath}/data.json`, 'utf8');
await es.bulk({ operations: content.split('\n'), refresh: 'wait_for' });
log('Loaded profiling data');
}

1
x-pack/test/tsconfig.json
View file

@ -142,5 +142,6 @@
"@kbn/stack-alerts-plugin",
"@kbn/apm-data-access-plugin",
"@kbn/profiling-utils",
"@kbn/profiling-data-access-plugin",
]
}