mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[APM] Add an internal endpoint for debugging telemetry (#132511)
* [APM] Add telemetry to service groups quries * Add service groups in telemetry schema * Add an internal route to test apm telemetry * Update endpoint to run telemetry jobs and display data * Update telemetry README * Move service_groups task work to another PR * Clean up * Use versioned link in x-pack/plugins/apm/dev_docs/telemetry.md Co-authored-by: Søren Louv-Jansen <sorenlouv@gmail.com> * Update x-pack/plugins/apm/server/routes/debug_telemetry/route.ts Co-authored-by: Søren Louv-Jansen <sorenlouv@gmail.com> * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' Co-authored-by: Søren Louv-Jansen <sorenlouv@gmail.com> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Katerina Patticha
GitHub
parent
9a8993268d
commit
a487d7c994
8 changed files with 54 additions and 300 deletions
|
|
@ -19,7 +19,7 @@ to the telemetry cluster using the
|
|||
During the APM server-side plugin's setup phase a
|
||||
[Saved Object](https://www.elastic.co/guide/en/kibana/master/managing-saved-objects.html)
|
||||
for APM telemetry is registered and a
|
||||
[task manager](../../task_manager/server/README.md) task is registered and started.
|
||||
[task manager](../../task_manager/README.md) task is registered and started.
|
||||
The task periodically queries the APM indices and saves the results in the Saved
|
||||
Object, and the usage collector periodically gets the data from the saved object
|
||||
and uploads it to the telemetry cluster.
|
||||
|
|
@ -27,23 +27,19 @@ and uploads it to the telemetry cluster.
|
|||
Once uploaded to the telemetry cluster, the data telemetry is stored in
|
||||
`stack_stats.kibana.plugins.apm` in the xpack-phone-home index.
|
||||
|
||||
### Generating sample data
|
||||
### Collect a new telemetry field
|
||||
|
||||
The script in `scripts/upload_telemetry_data` can generate sample telemetry data and upload it to a cluster of your choosing.
|
||||
In order to collect a new telemetry field you need to add a task which performs the query that collects the data from the cluster.
|
||||
|
||||
You'll need to set the `GITHUB_TOKEN` environment variable to a token that has `repo` scope so it can read from the
|
||||
[elastic/telemetry](https://github.com/elastic/telemetry) repository. (You probably have a token that works for this in
|
||||
~/.backport/config.json.)
|
||||
All the available tasks are [here](https://github.com/elastic/kibana/blob/ba84602455671f0f6175bbc0fd2e8f302c60bbe6/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts)
|
||||
|
||||
The script will run as the `elastic` user using the elasticsearch hosts and password settings from the config/kibana.yml
|
||||
and/or config/kibana.dev.yml files.
|
||||
### Debug telemetry
|
||||
|
||||
Running the script with `--clear` will delete the index first.
|
||||
The following endpoint will run the `apm-telemetry-task` which is responsible for collecting the telemetry data and once it's completed it will return the telemetry attributes.
|
||||
|
||||
If you're using an Elasticsearch instance without TLS verification (if you have `elasticsearch.ssl.verificationMode: none` set in your kibana.yml)
|
||||
you can run the script with `env NODE_TLS_REJECT_UNAUTHORIZED=0` to avoid TLS connection errors.
|
||||
|
||||
After running the script you should see sample telemetry data in the "xpack-phone-home" index.
|
||||
```
|
||||
GET /internal/apm/debug-telemetry
|
||||
```
|
||||
|
||||
### Updating Data Telemetry Mappings
|
||||
|
||||
|
|
|
|||
|
|
@ -1,126 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { DeepPartial } from 'utility-types';
|
||||
import {
|
||||
merge,
|
||||
omit,
|
||||
defaultsDeep,
|
||||
range,
|
||||
mapValues,
|
||||
isPlainObject,
|
||||
flatten,
|
||||
} from 'lodash';
|
||||
import uuid from 'uuid';
|
||||
import {
|
||||
CollectTelemetryParams,
|
||||
collectDataTelemetry,
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
} from '../../server/lib/apm_telemetry/collect_data_telemetry';
|
||||
|
||||
interface GenerateOptions {
|
||||
days: number;
|
||||
instances: number;
|
||||
variation: {
|
||||
min: number;
|
||||
max: number;
|
||||
};
|
||||
}
|
||||
|
||||
const randomize = (
|
||||
value: unknown,
|
||||
instanceVariation: number,
|
||||
dailyGrowth: number
|
||||
) => {
|
||||
if (typeof value === 'boolean') {
|
||||
return Math.random() > 0.5;
|
||||
}
|
||||
if (typeof value === 'number') {
|
||||
return Math.round(instanceVariation * dailyGrowth * value);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const mapValuesDeep = (
|
||||
obj: Record<string, any>,
|
||||
iterator: (value: unknown, key: string, obj: Record<string, any>) => unknown
|
||||
): Record<string, any> =>
|
||||
mapValues(obj, (val, key) =>
|
||||
isPlainObject(val) ? mapValuesDeep(val, iterator) : iterator(val, key!, obj)
|
||||
);
|
||||
|
||||
export async function generateSampleDocuments(
|
||||
options: DeepPartial<GenerateOptions> & {
|
||||
collectTelemetryParams: CollectTelemetryParams;
|
||||
}
|
||||
) {
|
||||
const { collectTelemetryParams, ...preferredOptions } = options;
|
||||
|
||||
const opts: GenerateOptions = defaultsDeep(
|
||||
{
|
||||
days: 100,
|
||||
instances: 50,
|
||||
variation: {
|
||||
min: 0.1,
|
||||
max: 4,
|
||||
},
|
||||
},
|
||||
preferredOptions
|
||||
);
|
||||
|
||||
const sample = await collectDataTelemetry(collectTelemetryParams);
|
||||
|
||||
console.log('Collected telemetry'); // eslint-disable-line no-console
|
||||
console.log('\n' + JSON.stringify(sample, null, 2)); // eslint-disable-line no-console
|
||||
|
||||
const dateOfScriptExecution = new Date();
|
||||
|
||||
return flatten(
|
||||
range(0, opts.instances).map(() => {
|
||||
const instanceId = uuid.v4();
|
||||
const defaults = {
|
||||
cluster_uuid: instanceId,
|
||||
stack_stats: {
|
||||
kibana: {
|
||||
versions: {
|
||||
version: '8.0.0',
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const instanceVariation =
|
||||
Math.random() * (opts.variation.max - opts.variation.min) +
|
||||
opts.variation.min;
|
||||
|
||||
return range(0, opts.days).map((dayNo) => {
|
||||
const dailyGrowth = Math.pow(1.005, opts.days - 1 - dayNo);
|
||||
|
||||
const timestamp = Date.UTC(
|
||||
dateOfScriptExecution.getFullYear(),
|
||||
dateOfScriptExecution.getMonth(),
|
||||
-dayNo
|
||||
);
|
||||
|
||||
const generated = mapValuesDeep(omit(sample, 'versions'), (value) =>
|
||||
randomize(value, instanceVariation, dailyGrowth)
|
||||
);
|
||||
|
||||
return merge({}, defaults, {
|
||||
timestamp,
|
||||
stack_stats: {
|
||||
kibana: {
|
||||
plugins: {
|
||||
apm: merge({}, sample, generated),
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
})
|
||||
);
|
||||
}
|
||||
|
|
@ -1,156 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
// This script downloads the telemetry mapping, runs the APM telemetry tasks,
|
||||
// generates a bunch of randomized data based on the downloaded sample,
|
||||
// and uploads it to a cluster of your choosing in the same format as it is
|
||||
// stored in the telemetry cluster. Its purpose is twofold:
|
||||
// - Easier testing of the telemetry tasks
|
||||
// - Validate whether we can run the queries we want to on the telemetry data
|
||||
|
||||
import { merge, chunk, flatten, omit } from 'lodash';
|
||||
import { argv } from 'yargs';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import { Logger } from '@kbn/core/server';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import { CollectTelemetryParams } from '../../server/lib/apm_telemetry/collect_data_telemetry';
|
||||
import { downloadTelemetryTemplate } from '../shared/download_telemetry_template';
|
||||
import { mergeApmTelemetryMapping } from '../../common/apm_telemetry';
|
||||
import { generateSampleDocuments } from './generate_sample_documents';
|
||||
import { readKibanaConfig } from '../shared/read_kibana_config';
|
||||
import { getHttpAuth } from '../shared/get_http_auth';
|
||||
import { createOrUpdateIndex } from '../shared/create_or_update_index';
|
||||
import { getEsClient } from '../shared/get_es_client';
|
||||
|
||||
async function uploadData() {
|
||||
const githubToken = process.env.GITHUB_TOKEN;
|
||||
|
||||
if (!githubToken) {
|
||||
throw new Error('GITHUB_TOKEN was not provided.');
|
||||
}
|
||||
|
||||
const xpackTelemetryIndexName = 'xpack-phone-home';
|
||||
const telemetryTemplate = await downloadTelemetryTemplate({
|
||||
githubToken,
|
||||
});
|
||||
|
||||
const config = readKibanaConfig();
|
||||
|
||||
const httpAuth = getHttpAuth(config);
|
||||
|
||||
const client = getEsClient({
|
||||
node: config['elasticsearch.hosts'],
|
||||
...(httpAuth
|
||||
? {
|
||||
auth: { ...httpAuth, username: 'elastic' },
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
|
||||
// The new template is the template downloaded from the telemetry repo, with
|
||||
// our current telemetry mapping merged in, with the "index_patterns" key
|
||||
// (which cannot be used when creating an index) removed.
|
||||
const newTemplate = omit(
|
||||
mergeApmTelemetryMapping(
|
||||
merge(telemetryTemplate, {
|
||||
index_patterns: undefined,
|
||||
settings: {
|
||||
index: { mapping: { total_fields: { limit: 10000 } } },
|
||||
},
|
||||
})
|
||||
),
|
||||
'index_patterns'
|
||||
);
|
||||
|
||||
await createOrUpdateIndex({
|
||||
indexName: xpackTelemetryIndexName,
|
||||
client,
|
||||
template: newTemplate,
|
||||
clear: !!argv.clear,
|
||||
});
|
||||
|
||||
const sampleDocuments = await generateSampleDocuments({
|
||||
collectTelemetryParams: {
|
||||
logger: console as unknown as Logger,
|
||||
indices: {
|
||||
transaction: config['xpack.apm.indices.transaction'],
|
||||
metric: config['xpack.apm.indices.metric'],
|
||||
error: config['xpack.apm.indices.error'],
|
||||
span: config['xpack.apm.indices.span'],
|
||||
onboarding: config['xpack.apm.indices.onboarding'],
|
||||
sourcemap: config['xpack.apm.indices.sourcemap'],
|
||||
apmCustomLinkIndex: '.apm-custom-links',
|
||||
apmAgentConfigurationIndex: '.apm-agent-configuration',
|
||||
},
|
||||
search: (body) => {
|
||||
return client.search(body) as Promise<any>;
|
||||
},
|
||||
indicesStats: (body) => {
|
||||
return client.indices.stats(body);
|
||||
},
|
||||
transportRequest: ((params) => {
|
||||
return;
|
||||
}) as CollectTelemetryParams['transportRequest'],
|
||||
},
|
||||
});
|
||||
|
||||
const chunks = chunk(sampleDocuments, 250);
|
||||
|
||||
await chunks.reduce<Promise<any>>((prev, documents) => {
|
||||
return prev.then(async () => {
|
||||
const body = flatten(
|
||||
documents.map((doc) => [
|
||||
{ index: { _index: xpackTelemetryIndexName } },
|
||||
doc,
|
||||
])
|
||||
);
|
||||
|
||||
return client
|
||||
.bulk({
|
||||
body,
|
||||
refresh: 'wait_for',
|
||||
})
|
||||
.then((response: any) => {
|
||||
if (response.errors) {
|
||||
const firstError = response.items.filter(
|
||||
(item: any) => item.index.status >= 400
|
||||
)[0].index.error;
|
||||
throw new Error(
|
||||
`Failed to upload documents: ${firstError.reason} `
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
uploadData()
|
||||
.catch((e) => {
|
||||
if ('response' in e) {
|
||||
if (typeof e.response === 'string') {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(e.response);
|
||||
} else {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
e.response,
|
||||
['status', 'statusText', 'headers', 'data'],
|
||||
2
|
||||
)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(e);
|
||||
}
|
||||
process.exit(1);
|
||||
})
|
||||
.then(() => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('Finished uploading generated telemetry data');
|
||||
});
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import { merge } from 'lodash';
|
||||
import { Logger } from '@kbn/core/server';
|
||||
import { Logger, SavedObjectsClient } from '@kbn/core/server';
|
||||
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
|
||||
import {
|
||||
ESSearchRequest,
|
||||
|
|
@ -16,6 +16,8 @@ import { ApmIndicesConfig } from '../../../routes/settings/apm_indices/get_apm_i
|
|||
import { tasks } from './tasks';
|
||||
import { APMDataTelemetry } from '../types';
|
||||
|
||||
type ISavedObjectsClient = Pick<SavedObjectsClient, 'find'>;
|
||||
|
||||
type TelemetryTaskExecutor = (params: {
|
||||
indices: ApmIndicesConfig;
|
||||
search<TSearchRequest extends ESSearchRequest>(
|
||||
|
|
@ -37,6 +39,7 @@ type TelemetryTaskExecutor = (params: {
|
|||
path: string;
|
||||
method: 'get';
|
||||
}) => Promise<unknown>;
|
||||
savedObjectsClient: ISavedObjectsClient;
|
||||
}) => Promise<APMDataTelemetry>;
|
||||
|
||||
export interface TelemetryTask {
|
||||
|
|
@ -54,6 +57,7 @@ export function collectDataTelemetry({
|
|||
logger,
|
||||
indicesStats,
|
||||
transportRequest,
|
||||
savedObjectsClient,
|
||||
}: CollectTelemetryParams) {
|
||||
return tasks.reduce((prev, task) => {
|
||||
return prev.then(async (data) => {
|
||||
|
|
@ -65,6 +69,7 @@ export function collectDataTelemetry({
|
|||
indices,
|
||||
indicesStats,
|
||||
transportRequest,
|
||||
savedObjectsClient,
|
||||
});
|
||||
const took = process.hrtime(time);
|
||||
|
||||
|
|
|
|||
|
|
@ -44,7 +44,6 @@ import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
|
|||
import { Span } from '../../../../typings/es_schemas/ui/span';
|
||||
import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
|
||||
import { APMTelemetry } from '../types';
|
||||
|
||||
const TIME_RANGES = ['1d', 'all'] as const;
|
||||
type TimeRange = typeof TIME_RANGES[number];
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@
|
|||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { Observable, firstValueFrom } from 'rxjs';
|
||||
import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
|
||||
import { CoreSetup, Logger, SavedObjectsErrorHelpers } from '@kbn/core/server';
|
||||
|
|
@ -27,7 +26,7 @@ import {
|
|||
import { APMUsage } from './types';
|
||||
import { apmSchema } from './schema';
|
||||
|
||||
const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task';
|
||||
export const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task';
|
||||
|
||||
export async function createApmTelemetry({
|
||||
core,
|
||||
|
|
@ -93,6 +92,7 @@ export async function createApmTelemetry({
|
|||
logger,
|
||||
indicesStats,
|
||||
transportRequest,
|
||||
savedObjectsClient,
|
||||
});
|
||||
|
||||
await savedObjectsClient.create(
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ import { eventMetadataRouteRepository } from '../event_metadata/route';
|
|||
import { suggestionsRouteRepository } from '../suggestions/route';
|
||||
import { agentKeysRouteRepository } from '../agent_keys/route';
|
||||
import { spanLinksRouteRepository } from '../span_links/route';
|
||||
|
||||
import { debugTelemetryRoute } from '../debug_telemetry/route';
|
||||
function getTypedGlobalApmServerRouteRepository() {
|
||||
const repository = {
|
||||
...dataViewRouteRepository,
|
||||
|
|
@ -69,6 +69,7 @@ function getTypedGlobalApmServerRouteRepository() {
|
|||
...eventMetadataRouteRepository,
|
||||
...agentKeysRouteRepository,
|
||||
...spanLinksRouteRepository,
|
||||
...debugTelemetryRoute,
|
||||
};
|
||||
|
||||
return repository;
|
||||
|
|
|
|||
35
x-pack/plugins/apm/server/routes/debug_telemetry/route.ts
Normal file
35
x-pack/plugins/apm/server/routes/debug_telemetry/route.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
|
||||
import { APM_TELEMETRY_TASK_NAME } from '../../lib/apm_telemetry';
|
||||
import { APMTelemetry } from '../../lib/apm_telemetry/types';
|
||||
import {
|
||||
APM_TELEMETRY_SAVED_OBJECT_ID,
|
||||
APM_TELEMETRY_SAVED_OBJECT_TYPE,
|
||||
} from '../../../common/apm_saved_object_constants';
|
||||
export const debugTelemetryRoute = createApmServerRoute({
|
||||
endpoint: 'GET /internal/apm/debug-telemetry',
|
||||
options: {
|
||||
tags: ['access:apm', 'access:apm_write'],
|
||||
},
|
||||
handler: async (resources): Promise<APMTelemetry | unknown> => {
|
||||
const { plugins, context } = resources;
|
||||
const coreContext = await context.core;
|
||||
const taskManagerStart = await plugins.taskManager?.start();
|
||||
const savedObjectsClient = coreContext.savedObjects.client;
|
||||
|
||||
await taskManagerStart?.runNow?.(APM_TELEMETRY_TASK_NAME);
|
||||
|
||||
const apmTelemetryObject = await savedObjectsClient.get(
|
||||
APM_TELEMETRY_SAVED_OBJECT_TYPE,
|
||||
APM_TELEMETRY_SAVED_OBJECT_ID
|
||||
);
|
||||
|
||||
return apmTelemetryObject.attributes;
|
||||
},
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue