github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Profling] Single-click setup from Kibana (#148959)

Browse source 
Co-authored-by: inge4pres <francesco.gualazzi@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tim Rühsen <tim.ruehsen@gmx.de>
Co-authored-by: Francesco Gualazzi <inge4pres@users.noreply.github.com>
Closes https://github.com/elastic/prodfiler/issues/2884
This commit is contained in:
Dario Gieselaar 2023-02-07 14:10:30 +01:00 • committed by GitHub
parent 969f7b575d
commit 807b402f0b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 2213 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

40
x-pack/plugins/profiling/server/lib/setup/get_setup_instructions.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract } from '@kbn/core/server';
import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
import { getApmPolicy } from './steps/get_apm_policy';
export interface SetupDataCollectionInstructions {
variables: {
apmServerUrl: string;
secretToken: string;
};
}
export async function getSetupInstructions({
packagePolicyClient,
soClient,
}: {
packagePolicyClient: PackagePolicyClient;
soClient: SavedObjectsClientContract;
}): Promise<SetupDataCollectionInstructions> {
const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
if (!apmPolicy) {
throw new Error('Could not find APM policy');
}
const apmServerVars = apmPolicy.inputs[0].vars;
return {
variables: {
apmServerUrl: apmServerVars!.url.value!,
secretToken: apmServerVars!.secret_token.value!,
},
};
}

23
x-pack/plugins/profiling/server/lib/setup/has_profiling_data.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProfilingESClient } from '../../utils/create_profiling_es_client';
export async function hasProfilingData({
client,
}: {
client: ProfilingESClient;
}): Promise<boolean> {
const hasProfilingDataResponse = await client.search('has_any_profiling_data', {
index: 'profiling*',
size: 0,
track_total_hits: 1,
terminate_after: 1,
});
return hasProfilingDataResponse.hits.total.value > 0;
}

53
x-pack/plugins/profiling/server/lib/setup/mappings/README.md Normal file
View file

@ -0,0 +1,53 @@
## Universal Profiling mappings
### Server routes
* Check if ES setup is done
curl -H "content-type: application/json" -u <user:pass> \
-XGET "http://localhost:5601/api/profiling/v1/setup/es_resources"
* Apply the ES setup (mappings + Fleet policy)
curl -H "content-type: application/json" -u <user:pass> -H "kbn-xsrf: reporting" \
-XPOST "http://localhost:5601/api/profiling/v1/setup/es_resources"
* check data has been ingested
curl -H "content-type: application/json" -u <user:pass> \
-XGET "http://localhost:5601/api/profiling/v1/setup/has_data"
### Testing in Cloud
Be sure to have configured `EC_API_KEY` env var with an API key for Cloud (ESS).
Build and push a Kibana image with the latest changes.
Choose a unique identifier for the build, then:
```
node scripts/build --docker-images --skip-docker-ubi --skip-docker-ubuntu
docker tag docker.elastic.co/kibana-ci/kibana-cloud:8.7.0-SNAPSHOT docker.elastic.co/observability-ci/kibana:<UNIQUE_IDENTIFIER>
docker push docker.elastic.co/observability-ci/kibana:<UNIQUE_IDENTIFIER>
```
Then, within `apm-server` repo:
```
cd testing/cloud
make
vim docker_image.auto.tfvars
```
Replace the `"kibana"` key in `docker_image_tag_override=` map with your unique identifier tag from previous step.
Now you can run:
```
terraform init
terraform apply -var-file docker_image.auto.tfvars
```
and once completed, you'll see the output with information on how to access the deployment.
When changing code in Kibana, you don't need to tear down the Terraform deployment, simply update the `docker_image.auto.tfvars`
with the new tag and run `terraform apply ...` as above: this will update Kibana.

16
x-pack/plugins/profiling/server/lib/setup/steps/catch_resource_already_exists_exception.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isResponseError } from '@kbn/es-errors';
export function catchResourceAlreadyExistsException(error: any) {
if (isResponseError(error) && error.body?.error?.type === 'resource_already_exists_exception') {
return Promise.resolve();
}
return Promise.reject(error);
}

76
x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_events.json Normal file
View file

@ -0,0 +1,76 @@
{
"settings": {
"index": {
"number_of_shards": "4",
"max_result_window": 150000,
"refresh_interval": "10s",
"sort": {
"field": [
"service.name",
"@timestamp",
"orchestrator.resource.name",
"container.name",
"process.thread.name",
"host.id"
]
}
},
"codec": "best_compression"
},
"mappings": {
"_source": {
"enabled": false
},
"properties": {
"ecs.version": {
"type": "keyword",
"index": true
},
"service.name": {
"type": "keyword"
},
"@timestamp": {
"type": "date",
"format": "epoch_second"
},
"host.id": {
"type": "keyword"
},
"Stacktrace.id": {
"type": "keyword",
"index": false
},
"orchestrator.resource.name": {
"type": "keyword"
},
"container.name": {
"type": "keyword"
},
"process.thread.name": {
"type": "keyword"
},
"Stacktrace.count": {
"type": "short",
"index": false
},
"agent.version": {
"type": "keyword"
},
"host.ip": {
"type": "ip"
},
"host.ipstring": {
"type": "keyword"
},
"host.name": {
"type": "keyword"
},
"os.kernel": {
"type": "keyword"
},
"tags": {
"type": "keyword"
}
}
}
}

35
x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_executables.json Normal file
View file

@ -0,0 +1,35 @@
{
"settings": {
"index": {
"refresh_interval": "10s"
}
},
"mappings": {
"_source": {
"mode": "synthetic"
},
"properties": {
"ecs.version": {
"type": "keyword",
"index": true
},
"Executable.build.id": {
"type": "keyword",
"index": true
},
"Executable.file.name": {
"type": "keyword",
"index": true
},
"@timestamp": {
"type": "date",
"format": "epoch_second"
},
"Symbolization.lastprocessed": {
"type": "date",
"format": "epoch_second",
"index": false
}
}
}
}

9
x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_ilm.json Normal file
View file

@ -0,0 +1,9 @@
{
"settings": {
"index": {
"lifecycle": {
"name": "profiling"
}
}
}
}

43
x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stackframes.json Normal file
View file

@ -0,0 +1,43 @@
{
"settings": {
"index": {
"number_of_shards": 16,
"refresh_interval": "10s"
}
},
"mappings": {
"_source": {
"enabled": true
},
"properties": {
"ecs.version": {
"type": "keyword",
"index": true,
"doc_values": false,
"store": false
},
"Stackframe.line.number": {
"type": "integer",
"index": false,
"doc_values": false,
"store": false
},
"Stackframe.file.name": {
"type": "keyword",
"index": false,
"doc_values": false,
"store": false
},
"Stackframe.function.name": {
"type": "keyword",
"index": false
},
"Stackframe.function.offset": {
"type": "integer",
"index": false,
"doc_values": false,
"store": false
}
}
}
}

27
x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stacktraces.json Normal file
View file

@ -0,0 +1,27 @@
{
"settings": {
"index": {
"number_of_shards": 16,
"refresh_interval": "10s"
}
},
"mappings": {
"_source": {
"mode": "synthetic"
},
"properties": {
"ecs.version": {
"type": "keyword",
"index": true
},
"Stacktrace.frame.ids": {
"type": "keyword",
"index": false
},
"Stacktrace.frame.types": {
"type": "keyword",
"index": false
}
}
}
}

44
x-pack/plugins/profiling/server/lib/setup/steps/get_apm_package_step.ts Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { installPackage, getInstallation } from '@kbn/fleet-plugin/server/services/epm/packages';
import {
fetchFindLatestPackageOrThrow,
pkgToPkgKey,
} from '@kbn/fleet-plugin/server/services/epm/registry';
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
export function getApmPackageStep({
client,
soClient,
spaceId,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
const esClient = client.getEsClient();
return {
name: 'apm_package',
hasCompleted: async () => {
const installation = await getInstallation({
pkgName: 'apm',
savedObjectsClient: soClient,
});
return !!installation;
},
init: async () => {
const { name, version } = await fetchFindLatestPackageOrThrow('apm');
await installPackage({
installSource: 'registry',
esClient,
savedObjectsClient: soClient,
pkgkey: pkgToPkgKey({ name, version }),
spaceId,
force: true,
});
},
};
}

21
x-pack/plugins/profiling/server/lib/setup/steps/get_apm_policy.ts Normal file
View file

@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract } from '@kbn/core/server';
import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
export const ELASTIC_CLOUD_APM_POLICY = 'elastic-cloud-apm';
export async function getApmPolicy({
packagePolicyClient,
soClient,
}: {
packagePolicyClient: PackagePolicyClient;
soClient: SavedObjectsClientContract;
}) {
return await packagePolicyClient.get(soClient, ELASTIC_CLOUD_APM_POLICY);
}

32
x-pack/plugins/profiling/server/lib/setup/steps/get_cluster_settings_step.ts Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
const MAX_BUCKETS = 150000;
export function getClusterSettingsStep({
client,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
return {
name: 'cluster_settings',
hasCompleted: async () => {
const settings = await client.getEsClient().cluster.getSettings({});
return settings.persistent.search?.max_buckets === MAX_BUCKETS.toString();
},
init: async () => {
await client.getEsClient().cluster.putSettings({
persistent: {
search: {
max_buckets: MAX_BUCKETS,
},
},
});
},
};
}

86
x-pack/plugins/profiling/server/lib/setup/steps/get_component_templates_step.ts Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IndicesIndexState } from '@elastic/elasticsearch/lib/api/types';
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
import componentTemplateProfilingIlm from './component_template_profiling_ilm.json';
import componentTemplateProfilingEvents from './component_template_profiling_events.json';
import componentTemplateProfilingExecutables from './component_template_profiling_executables.json';
import componentTemplateProfilingStackframes from './component_template_profiling_stackframes.json';
import componentTemplateProfilingStacktraces from './component_template_profiling_stacktraces.json';
export enum ProfilingComponentTemplateName {
Ilm = 'profiling-ilm',
Events = 'profiling-events',
Executables = 'profiling-executables',
Stackframes = 'profiling-stackframes',
Stacktraces = 'profiling-stacktraces',
}
export function getComponentTemplatesStep({
client,
logger,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
const esClient = client.getEsClient();
return {
name: 'component_templates',
hasCompleted: async () => {
return Promise.all(
[
ProfilingComponentTemplateName.Ilm,
ProfilingComponentTemplateName.Events,
ProfilingComponentTemplateName.Executables,
ProfilingComponentTemplateName.Stackframes,
ProfilingComponentTemplateName.Stacktraces,
].map((componentTemplateName) =>
esClient.cluster.getComponentTemplate({
name: componentTemplateName,
})
)
).then(
() => Promise.resolve(true),
(error) => {
logger.debug('Some component templates could not be fetched');
logger.debug(error);
return Promise.resolve(false);
}
);
},
init: async () => {
await Promise.all([
esClient.cluster.putComponentTemplate({
name: ProfilingComponentTemplateName.Ilm,
create: false,
template: componentTemplateProfilingIlm,
}),
esClient.cluster.putComponentTemplate({
name: ProfilingComponentTemplateName.Events,
create: false,
template: componentTemplateProfilingEvents as IndicesIndexState,
_meta: {
description: 'Mappings for profiling events data stream',
},
}),
esClient.cluster.putComponentTemplate({
name: ProfilingComponentTemplateName.Executables,
create: false,
template: componentTemplateProfilingExecutables as IndicesIndexState,
}),
esClient.cluster.putComponentTemplate({
name: ProfilingComponentTemplateName.Stackframes,
create: false,
template: componentTemplateProfilingStackframes as IndicesIndexState,
}),
esClient.cluster.putComponentTemplate({
name: ProfilingComponentTemplateName.Stacktraces,
create: false,
template: componentTemplateProfilingStacktraces as IndicesIndexState,
}),
]);
},
};
}

63
x-pack/plugins/profiling/server/lib/setup/steps/get_create_events_data_streams.ts Normal file
View file

@ -0,0 +1,63 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception';
function getEventDataStreamNames() {
const subSampledIndicesIdx = Array.from(Array(11).keys(), (item: number) => item + 1);
const subSampledIndexName = (pow: number): string => {
return `profiling-events-5pow${String(pow).padStart(2, '0')}`;
};
// Generate all the possible index template names
const eventsIndices = ['profiling-events-all'].concat(
subSampledIndicesIdx.map((pow) => subSampledIndexName(pow))
);
return eventsIndices;
}
export function getCreateEventsDataStreamsStep({
client,
logger,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
const esClient = client.getEsClient();
const dataStreamNames = getEventDataStreamNames();
return {
name: 'create_events_data_streams',
hasCompleted: async () => {
const dataStreams = await esClient.indices.getDataStream({
name: 'profiling-events*',
});
const allDataStreams = dataStreams.data_streams.map((dataStream) => dataStream.name);
const missingDataStreams = dataStreamNames.filter(
(eventIndex) => !allDataStreams.includes(eventIndex)
);
if (missingDataStreams.length > 0) {
logger.debug(`Missing event indices: ${missingDataStreams.join(', ')}`);
}
return missingDataStreams.length === 0;
},
init: async () => {
await Promise.all(
dataStreamNames.map((dataStreamName) =>
esClient.indices
.createDataStream({
name: dataStreamName,
})
.catch(catchResourceAlreadyExistsException)
)
);
},
};
}

321
x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts Normal file
View file

@ -0,0 +1,321 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { MappingSourceField } from '@elastic/elasticsearch/lib/api/types';
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception';
const SQ_EXECUTABLES_INDEX = 'profiling-sq-executables';
const LEAFFRAMES_INDEX = 'profiling-sq-leafframes';
const SYMBOLS_INDEX = 'profiling-symbols';
const ILM_LOCK_INDEX = '.profiling-ilm-lock';
const getKeyValueIndices = () => {
const kvIndices = ['profiling-stacktraces', 'profiling-stackframes', 'profiling-executables'];
const pairs: Array<{ index: string; alias: string }> = kvIndices.flatMap((index) => {
return [
{ index: `${index}-000001`, alias: index },
{ index: `${index}-000002`, alias: `${index}-next` },
];
});
return pairs;
};
export function getCreateIndicesStep({
client,
logger,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
const esClient = client.getEsClient();
const keyValueIndices = getKeyValueIndices();
return {
name: 'create_indices',
hasCompleted: async () => {
const nonKvIndices = [SQ_EXECUTABLES_INDEX, LEAFFRAMES_INDEX, SYMBOLS_INDEX, ILM_LOCK_INDEX];
const results = await Promise.all([
esClient.cat
.indices({
index: keyValueIndices
.map(({ index }) => index)
.concat(nonKvIndices)
.map((index) => index + '*')
.join(','),
format: 'json',
})
.then((response) => {
const allIndices = response.map((index) => index.index!);
const missingIndices = keyValueIndices
.map(({ index }) => index)
.concat(nonKvIndices)
.filter((index) => !allIndices.includes(index));
if (missingIndices.length) {
logger.debug(`Missing indices: ${missingIndices.join(',')}`);
}
return missingIndices.length === 0;
})
.catch((error) => {
logger.debug(`Failed fetching indices: ${error}`);
return Promise.resolve(false);
}),
esClient.cat
.aliases({
name: keyValueIndices.map(({ alias }) => alias + '*').join(','),
format: 'json',
})
.then((response) => {
const allAliases = response.map((index) => index.alias!);
const missingAliases = keyValueIndices
.map(({ alias }) => alias)
.filter((alias) => !allAliases.includes(alias));
if (missingAliases.length) {
logger.debug(`Missing aliases: ${missingAliases.join(',')}`);
}
return missingAliases.length === 0;
})
.catch((error) => {
logger.debug(`Failed fetching aliases: ${error}`);
return Promise.resolve(false);
}),
]);
return results.every(Boolean);
},
init: async () => {
await Promise.all([
...keyValueIndices.map(({ index, alias }) => {
return esClient.indices
.create({
index,
aliases: {
[alias]: {
is_write_index: true,
},
},
})
.catch(catchResourceAlreadyExistsException);
}),
esClient.indices
.create({
index: SQ_EXECUTABLES_INDEX,
settings: {
index: {
refresh_interval: '10s',
},
},
mappings: {
_source: {
mode: 'synthetic',
} as MappingSourceField,
properties: {
'ecs.version': {
type: 'keyword',
index: true,
},
'Executable.file.id': {
type: 'keyword',
index: false,
},
'Time.created': {
type: 'date',
index: true,
},
'Symbolization.time.next': {
type: 'date',
index: true,
},
'Symbolization.retries': {
type: 'short',
index: true,
},
},
},
})
.catch(catchResourceAlreadyExistsException),
esClient.indices
.create({
index: LEAFFRAMES_INDEX,
settings: {
index: {
refresh_interval: '10s',
},
},
mappings: {
_source: {
mode: 'synthetic',
} as MappingSourceField,
properties: {
'ecs.version': {
type: 'keyword',
index: true,
},
'Stacktrace.frame.id': {
type: 'keyword',
index: false,
},
'Time.created': {
type: 'date',
index: true,
},
'Symbolization.time.next': {
type: 'date',
index: true,
},
'Symbolization.retries': {
type: 'short',
index: true,
},
},
},
})
.catch(catchResourceAlreadyExistsException),
esClient.indices
.create({
index: SYMBOLS_INDEX,
settings: {
index: {
number_of_shards: '16',
refresh_interval: '10s',
},
},
mappings: {
_source: {
enabled: true,
} as MappingSourceField,
properties: {
'ecs.version': {
type: 'keyword',
index: true,
doc_values: false,
store: false,
},
'Symbol.function.name': {
// name of the function
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.file.name': {
// file path
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.call.file.name': {
// (for inlined functions) file path where inline function was called
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.call.line': {
// (for inlined functions) line where inline function was called
type: 'integer',
index: false,
doc_values: false,
store: false,
},
'Symbol.function.line': {
// function start line (only available from DWARF). Currently unused.
type: 'integer',
index: false,
doc_values: false,
store: false,
},
'Symbol.depth': {
// inline depth
type: 'integer',
index: false,
doc_values: false,
store: false,
},
// pairs of (32bit PC offset, 32bit line number) followed by 64bit PC range base at the end.
// To find line number for a given PC: find lowest offset such as offsetBase+PC >= offset, then read corresponding line number.
// offsetBase could seemingly be available from exec_pc_range (it's the first value of the pair), but it's not the case.
// Ranges are stored as points, which cannot be retrieve when disabling _source.
// See https://www.elastic.co/guide/en/elasticsearch/reference/current/point.html .
'Symbol.linetable.base': {
// Linetable: base for offsets (64bit PC range base)
type: 'unsigned_long',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.length': {
// Linetable: length of range (PC range is [base, base+length))
type: 'unsigned_long',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.offsets': {
// Linetable: concatenated offsets (each value is ULEB128encoded)
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.linetable.lines': {
// Linetable: concatenated lines (each value is ULEB128 encoded)
type: 'keyword',
index: false,
doc_values: false,
store: false,
},
'Symbol.file.id': {
// fileID. used for deletion and Symbol.exec.pcrange collision handling on symbolization
type: 'keyword',
index: true,
doc_values: false,
store: false,
},
'Symbol.exec.pcrange': {
// PC ranges [begin, end)
type: 'ip_range',
index: true,
doc_values: false,
store: false,
},
},
},
})
.catch(catchResourceAlreadyExistsException),
esClient.indices
.create({
index: ILM_LOCK_INDEX,
settings: {
index: {
hidden: true,
},
},
mappings: {
properties: {
'@timestamp': {
type: 'date',
format: 'epoch_second',
},
phase: {
type: 'keyword',
},
},
},
})
.catch(catchResourceAlreadyExistsException),
]);
},
};
}

106
x-pack/plugins/profiling/server/lib/setup/steps/get_fleet_policy_step.ts Normal file
View file

@ -0,0 +1,106 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core/server';
import { merge, omit } from 'lodash';
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
import { getApmPolicy } from './get_apm_policy';
async function createIngestAPIKey(esClient: ElasticsearchClient) {
const apiKeyResponse = await esClient.security.createApiKey({
name: 'profiling-manager',
role_descriptors: {
profiling_manager: {
indices: [
{
names: ['profiling-*', '.profiling-*'],
privileges: [
'read',
'create_doc',
'create',
'write',
'index',
'create_index',
'view_index_metadata',
'manage',
],
},
],
cluster: ['monitor'],
},
},
});
return atob(apiKeyResponse.encoded);
}
export function getFleetPolicyStep({
client,
soClient,
logger,
packagePolicyClient,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
const esClient = client.getEsClient();
return {
name: 'fleet_policy',
hasCompleted: async () => {
try {
const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
return apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value.profiling;
} catch (error) {
logger.debug('Could not fetch fleet policy');
logger.debug(error);
return false;
}
},
init: async () => {
const apmPolicyApiKey = await createIngestAPIKey(client.getEsClient());
const profilingApmConfig = {
profiling: {
enabled: true,
elasticsearch: {
api_key: apmPolicyApiKey,
},
metrics: {
elasticsearch: {
hosts: [
'https://1b6c02856ea642a6ac14499b01507233.us-east-2.aws.elastic-cloud.com:443',
],
api_key: 'woq-IoMBRbbiEbPugtWW:_iBmc1PdSout7sf5FCkEpA',
},
},
keyvalue_retention: {
// 60 days
age: '1440h',
// 200 Gib
size_bytes: 200 * 1024 * 1024 * 1024,
execution_interval: '12h',
},
},
};
const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient });
if (!apmPolicy) {
throw new Error(`Could not find APM policy`);
}
const modifiedPolicyInputs = apmPolicy.inputs.map((input) => {
return input.type === 'apm'
? merge({}, input, { config: { 'apm-server': { value: profilingApmConfig } } })
: input;
});
await packagePolicyClient.update(soClient, esClient, apmPolicy.id, {
...omit(apmPolicy, 'id', 'revision', 'updated_at', 'updated_by'),
inputs: modifiedPolicyInputs,
});
},
};
}

43
x-pack/plugins/profiling/server/lib/setup/steps/get_ilm_step.ts Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception';
import ilmProfiling from './ilm_profiling.json';
const LIFECYCLE_POLICY_NAME = 'profiling';
export function getIlmStep({
client,
logger,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
const esClient = client.getEsClient();
return {
name: 'ilm',
hasCompleted: () => {
return esClient.ilm.getLifecycle({ name: LIFECYCLE_POLICY_NAME }).then(
() => {
return Promise.resolve(true);
},
(error) => {
logger.debug('ILM policy not installed');
logger.debug(error);
return Promise.resolve(false);
}
);
},
init: async () => {
await esClient.ilm
.putLifecycle({
name: LIFECYCLE_POLICY_NAME,
policy: ilmProfiling,
})
.catch(catchResourceAlreadyExistsException);
},
};
}

80
x-pack/plugins/profiling/server/lib/setup/steps/get_index_templates_step.ts Normal file
View file

@ -0,0 +1,80 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
import { ProfilingComponentTemplateName } from './get_component_templates_step';
enum ProfilingIndexTemplate {
Events = 'profiling-events',
Executables = 'profiling-executables',
Stacktraces = 'profiling-stacktraces',
Stackframes = 'profiling-stackframes',
}
export function getIndexTemplatesStep({
client,
logger,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
const esClient = client.getEsClient();
return {
name: 'index_templates',
hasCompleted: async () => {
return Promise.all(
[
ProfilingIndexTemplate.Events,
ProfilingIndexTemplate.Executables,
ProfilingIndexTemplate.Stacktraces,
ProfilingIndexTemplate.Stackframes,
].map((indexTemplateName) =>
esClient.indices.getIndexTemplate({
name: indexTemplateName,
})
)
).then(
() => Promise.resolve(true),
(error) => {
logger.debug('Some index templates could not be fetched');
logger.debug(error);
return Promise.resolve(false);
}
);
},
init: async () => {
await Promise.all([
esClient.indices.putIndexTemplate({
name: ProfilingIndexTemplate.Events,
create: false,
index_patterns: [ProfilingIndexTemplate.Events + '*'],
data_stream: {
hidden: false,
},
composed_of: [ProfilingComponentTemplateName.Events, ProfilingComponentTemplateName.Ilm],
priority: 100,
_meta: {
description: `Index template for ${ProfilingIndexTemplate.Events}`,
},
}),
...[
ProfilingIndexTemplate.Executables,
ProfilingIndexTemplate.Stacktraces,
ProfilingIndexTemplate.Stackframes,
].map((indexTemplateName) => {
return esClient.indices.putIndexTemplate({
name: indexTemplateName,
// Don't fail if the index template already exists, simply overwrite the format
create: false,
index_patterns: [indexTemplateName + '*'],
composed_of: [indexTemplateName],
_meta: {
description: `Index template for ${indexTemplateName}`,
},
});
}),
]);
},
};
}

24
x-pack/plugins/profiling/server/lib/setup/steps/get_is_cloud_enabled_step.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
export function getIsCloudEnabledStep({
isCloudEnabled,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
return {
name: 'is_cloud',
hasCompleted: async () => {
return isCloudEnabled;
},
init: async () => {
if (!isCloudEnabled) {
throw new Error(`Universal Profiling is only available on Elastic Cloud.`);
}
},
};
}

49
x-pack/plugins/profiling/server/lib/setup/steps/get_security_step.ts Normal file
View file

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
const PROFILING_READER_ROLE_NAME = 'profiling-reader';
export function getSecurityStep({
client,
logger,
}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep {
const esClient = client.getEsClient();
return {
name: 'security',
hasCompleted: () => {
return esClient.security
.getRole({
name: PROFILING_READER_ROLE_NAME,
})
.then(
() => {
return Promise.resolve(true);
},
(error) => {
logger.debug('Could not fetch profiling-reader role');
logger.debug(error);
return Promise.resolve(false);
}
);
},
init: async () => {
await esClient.security.putRole({
name: PROFILING_READER_ROLE_NAME,
indices: [
{
names: ['profiling-*'],
privileges: ['read', 'view_index_metadata'],
},
],
cluster: ['monitor'],
});
},
};
}

35
x-pack/plugins/profiling/server/lib/setup/steps/ilm_profiling.json Normal file
View file

@ -0,0 +1,35 @@
{
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_primary_shard_size": "50gb",
"max_age": "7d"
},
"set_priority": {
"priority": 100
}
}
},
"warm": {
"min_age": "30d",
"actions": {
"set_priority": {
"priority": 50
},
"shrink": {
"number_of_shards": 2
}
}
},
"delete": {
"min_age": "60d",
"actions": {
"delete": {
"delete_searchable_snapshot": true
}
}
}
}
}

35
x-pack/plugins/profiling/server/lib/setup/steps/index.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getClusterSettingsStep } from './get_cluster_settings_step';
import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types';
import { getComponentTemplatesStep } from './get_component_templates_step';
import { getIlmStep } from './get_ilm_step';
import { getIndexTemplatesStep } from './get_index_templates_step';
import { getFleetPolicyStep } from './get_fleet_policy_step';
import { getSecurityStep } from './get_security_step';
import { getApmPackageStep } from './get_apm_package_step';
import { getCreateEventsDataStreamsStep } from './get_create_events_data_streams';
import { getCreateIndicesStep } from './get_create_indices_step';
import { getIsCloudEnabledStep } from './get_is_cloud_enabled_step';
export function getProfilingSetupSteps(
options: ProfilingSetupStepFactoryOptions
): ProfilingSetupStep[] {
return [
getIsCloudEnabledStep(options),
getApmPackageStep(options),
getClusterSettingsStep(options),
getIlmStep(options),
getComponentTemplatesStep(options),
getIndexTemplatesStep(options),
getCreateEventsDataStreamsStep(options),
getCreateIndicesStep(options),
getSecurityStep(options),
getFleetPolicyStep(options),
];
}

26
x-pack/plugins/profiling/server/lib/setup/types.ts Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract } from '@kbn/core/server';
import { PackagePolicyClient } from '@kbn/fleet-plugin/server';
import { Logger } from '@kbn/logging';
import { ProfilingESClient } from '../../utils/create_profiling_es_client';
export interface ProfilingSetupStep {
name: string;
init: () => Promise<void>;
hasCompleted: () => Promise<boolean>;
}
export interface ProfilingSetupStepFactoryOptions {
client: ProfilingESClient;
soClient: SavedObjectsClientContract;
packagePolicyClient: PackagePolicyClient;
logger: Logger;
spaceId: string;
isCloudEnabled: boolean;
}

13
x-pack/plugins/profiling/server/plugin.ts
View file

@ -59,10 +59,15 @@ export class ProfilingPlugin
setup: deps,
},
services: {
createProfilingEsClient: ({ request, esClient: defaultEsClient }) => {
const esClient = profilingSpecificEsClient
? profilingSpecificEsClient.asScoped(request).asInternalUser
: defaultEsClient;
createProfilingEsClient: ({
request,
esClient: defaultEsClient,
useDefaultAuth = false,
}) => {
const esClient =
profilingSpecificEsClient && !useDefaultAuth
? profilingSpecificEsClient.asScoped(request).asInternalUser
: defaultEsClient;
return createProfilingEsClient({ request, esClient });
},

12
x-pack/plugins/profiling/server/routes/index.ts
View file

@ -5,21 +5,19 @@
* 2.0.
*/
import type { IRouter, Logger } from '@kbn/core/server';
import type { KibanaRequest } from '@kbn/core-http-server';
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import type { KibanaRequest } from '@kbn/core-http-server';
import type { IRouter, Logger } from '@kbn/core/server';
import {
ProfilingPluginSetupDeps,
ProfilingPluginStartDeps,
ProfilingRequestHandlerContext,
} from '../types';
import { ProfilingESClient } from '../utils/create_profiling_es_client';
import { registerCacheExecutablesRoute, registerCacheStackFramesRoute } from './cache';
import { registerFlameChartSearchRoute } from './flamechart';
import { registerTopNFunctionsSearchRoute } from './functions';
import { registerSetupRoute } from './setup';
import {
registerTraceEventsTopNContainersSearchRoute,
registerTraceEventsTopNDeploymentsSearchRoute,
@ -39,6 +37,7 @@ export interface RouteRegisterParameters {
createProfilingEsClient: (params: {
request: KibanaRequest;
esClient: ElasticsearchClient;
useDefaultAuth?: boolean;
}) => ProfilingESClient;
};
}
@ -53,4 +52,7 @@ export function registerRoutes(params: RouteRegisterParameters) {
registerTraceEventsTopNHostsSearchRoute(params);
registerTraceEventsTopNStackTracesSearchRoute(params);
registerTraceEventsTopNThreadsSearchRoute(params);
// Setup of Profiling resources, automates the configuration of Universal Profiling
// and will show instructions on how to add data
registerSetupRoute(params);
}

165
x-pack/plugins/profiling/server/routes/setup.ts Normal file
View file

@ -0,0 +1,165 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { eachSeries } from 'async';
import { Logger } from '@kbn/logging';
import { RouteRegisterParameters } from '.';
import { getRoutePaths } from '../../common';
import { getSetupInstructions } from '../lib/setup/get_setup_instructions';
import { getProfilingSetupSteps } from '../lib/setup/steps';
import { handleRouteHandlerError } from '../utils/handle_route_error_handler';
import { hasProfilingData } from '../lib/setup/has_profiling_data';
import { getClient } from './compat';
import { ProfilingSetupStep } from '../lib/setup/types';
function checkSteps({ steps, logger }: { steps: ProfilingSetupStep[]; logger: Logger }) {
return Promise.all(
steps.map(async (step) => {
try {
return { name: step.name, completed: await step.hasCompleted() };
} catch (error) {
logger.error(error);
return { name: step.name, completed: false, error: error.toString() };
}
})
);
}
export function registerSetupRoute({
router,
logger,
services: { createProfilingEsClient },
dependencies,
}: RouteRegisterParameters) {
const paths = getRoutePaths();
// Check if ES resources needed for Universal Profiling to work exist
router.get(
{
path: paths.HasSetupESResources,
validate: false,
},
async (context, request, response) => {
try {
const esClient = await getClient(context);
logger.debug('checking if profiling ES configurations are installed');
const core = await context.core;
const steps = getProfilingSetupSteps({
client: createProfilingEsClient({
esClient,
request,
useDefaultAuth: true,
}),
logger,
packagePolicyClient: dependencies.start.fleet.packagePolicyService,
soClient: core.savedObjects.client,
spaceId: dependencies.setup.spaces.spacesService.getSpaceId(request),
isCloudEnabled: dependencies.setup.cloud.isCloudEnabled,
});
const hasDataPromise = hasProfilingData({
client: createProfilingEsClient({
esClient,
request,
}),
});
const stepCompletionResultsPromises = checkSteps({ steps, logger });
const hasData = await hasDataPromise;
if (hasData) {
return response.ok({
body: {
has_data: true,
has_setup: true,
steps: [],
},
});
}
const stepCompletionResults = await stepCompletionResultsPromises;
// Reply to clients if we have already created all 12 events template indices.
// This is kind of simplistic but can be a good first step to ensure
// Profiling resources will be created.
return response.ok({
body: {
has_setup: stepCompletionResults.every((step) => step.completed),
has_data: false,
steps: stepCompletionResults,
},
});
} catch (error) {
return handleRouteHandlerError({ error, logger, response });
}
}
);
// Configure ES resources needed by Universal Profiling using the mappings
router.post(
{
path: paths.HasSetupESResources,
validate: {},
},
async (context, request, response) => {
try {
const esClient = await getClient(context);
logger.info('Applying initial setup of Elasticsearch resources');
const steps = getProfilingSetupSteps({
client: createProfilingEsClient({ esClient, request, useDefaultAuth: true }),
logger,
packagePolicyClient: dependencies.start.fleet.packagePolicyService,
soClient: (await context.core).savedObjects.client,
spaceId: dependencies.setup.spaces.spacesService.getSpaceId(request),
isCloudEnabled: dependencies.setup.cloud.isCloudEnabled,
});
await eachSeries(steps, (step, cb) => {
logger.debug(`Executing step ${step.name}`);
step
.init()
.then(() => cb())
.catch(cb);
});
const checkedSteps = await checkSteps({ steps, logger });
if (checkedSteps.every((step) => step.completed)) {
return response.ok();
}
return response.custom({
statusCode: 500,
body: {
message: `Failed to complete all steps`,
steps: checkedSteps,
},
});
} catch (error) {
return handleRouteHandlerError({ error, logger, response });
}
}
);
// Show users the instructions on how to setup Universal Profiling agents
router.get(
{
path: paths.SetupDataCollectionInstructions,
validate: false,
},
async (context, request, response) => {
try {
const setupInstructions = await getSetupInstructions({
packagePolicyClient: dependencies.start.fleet.packagePolicyService,
soClient: (await context.core).savedObjects.client,
});
return response.ok({ body: setupInstructions });
} catch (error) {
return handleRouteHandlerError({ error, logger, response });
}
}
);
}

1
x-pack/plugins/profiling/server/routes/topn.test.ts
View file

@ -53,6 +53,7 @@ describe('TopN data from Elasticsearch', () => {
},
}) as Promise<any>
),
getEsClient: jest.fn(() => context.elasticsearch.client.asCurrentUser),
};
const logger = loggerMock.create();

23
x-pack/plugins/profiling/server/types.ts
View file

@ -5,21 +5,32 @@
* 2.0.
*/
import { RequestHandlerContext } from '@kbn/core/server';
import { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server';
import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server';
import { CustomRequestHandlerContext } from '@kbn/core-http-request-handler-context-server';
import type { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server';
import type { ObservabilityPluginSetup } from '@kbn/observability-plugin/server';
import { SpacesPluginStart, SpacesPluginSetup } from '@kbn/spaces-plugin/server';
import { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server';
import { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server';
export interface ProfilingPluginSetupDeps {
observability: ObservabilityPluginSetup;
features: FeaturesPluginSetup;
spaces: SpacesPluginSetup;
cloud: CloudSetup;
fleet: FleetSetupContract;
}
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface ProfilingPluginStartDeps {}
export interface ProfilingPluginStartDeps {
observability: {};
features: {};
spaces: SpacesPluginStart;
cloud: CloudStart;
fleet: FleetStartContract;
}
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface ProfilingPluginSetup {}
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface ProfilingPluginStart {}
export type ProfilingRequestHandlerContext = RequestHandlerContext;
export type ProfilingRequestHandlerContext = CustomRequestHandlerContext<{}>;

4
x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts
View file

@ -40,6 +40,7 @@ export interface ProfilingESClient {
query: QueryDslQueryContainer;
sampleSize: number;
}): Promise<StackTraceResponse>;
getEsClient(): ElasticsearchClient;
}
export function createProfilingEsClient({
@ -116,5 +117,8 @@ export function createProfilingEsClient({
return unwrapEsResponse(promise) as Promise<StackTraceResponse>;
},
getEsClient() {
return esClient;
},
};
}