[APM] Add description metadata to schema (#157529)

part of: https://github.com/elastic/kibana/issues/157521



#### Feedback 
- Display the index where the fields are store
- Display fields evens if they don't have description field

Notes 
- The issues with the types and the tasks not exist will be addressed in
a separate PR
This commit is contained in:
Katerina 2023-08-04 15:13:19 +02:00 committed by GitHub
parent b7a38984bd
commit 0843781bab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 2435 additions and 492 deletions

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
require('@kbn/babel-register').install();
require('./telemetry/main');

View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import fs from 'fs';
import { apmSchema } from '../../server/lib/apm_telemetry/schema';
const markdownFilePath = 'x-pack/plugins/apm/dev_docs/apm_telemetry_fields.md';
function extractFieldDescriptions(
schema: any,
parentKeys: string[] = []
): string[] {
const fieldDescriptions = [];
let currentKey: string;
for (currentKey in schema) {
if (typeof schema[currentKey] === 'object' && schema[currentKey] !== null) {
const description = schema[currentKey]._meta?.description;
if (description) {
const fullKey = [...parentKeys, currentKey].join('.');
fieldDescriptions.push(`| \`${fullKey}\` | ${description} |`);
}
fieldDescriptions.push(
...extractFieldDescriptions(schema[currentKey], [
...parentKeys,
currentKey,
])
);
}
}
return fieldDescriptions;
}
const metadataTable = extractFieldDescriptions(apmSchema).join('\n');
const markdownTable = `| Field | Description |\n| --- | --- |\n${metadataTable}`;
fs.writeFile(markdownFilePath, markdownTable, (err) => {
if (err) {
console.error('Error writing file:', err);
process.exit(1);
}
console.log(`The "${markdownFilePath}" has been updated successfully.`);
});

View file

@ -41,24 +41,123 @@ const timeframeMapSchema: MakeSchemaFrom<TimeframeMap> = {
const agentSchema: MakeSchemaFrom<APMUsage>['agents'][ElasticAgentName] = {
agent: {
version: { type: 'array', items: { type: 'keyword' } },
activation_method: { type: 'array', items: { type: 'keyword' } },
version: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 agent versions within the last day',
},
},
},
activation_method: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 agent activation methods within the last day',
},
},
},
},
service: {
framework: {
name: { type: 'array', items: { type: 'keyword' } },
version: { type: 'array', items: { type: 'keyword' } },
composite: { type: 'array', items: { type: 'keyword' } },
name: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 service framework name within the last day',
},
},
},
version: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 service framework version within the last day',
},
},
},
composite: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'Composite field containing service framework and version sorted by doc count',
},
},
},
},
language: {
name: { type: 'array', items: { type: 'keyword' } },
version: { type: 'array', items: { type: 'keyword' } },
composite: { type: 'array', items: { type: 'keyword' } },
name: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 service language name within the last day',
},
},
},
version: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 service language version within the last day',
},
},
},
composite: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'Composite field containing service language name and version sorted by doc count.',
},
},
},
},
runtime: {
name: { type: 'array', items: { type: 'keyword' } },
version: { type: 'array', items: { type: 'keyword' } },
composite: { type: 'array', items: { type: 'keyword' } },
name: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 service runtime name within the last day',
},
},
},
version: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 service runtime version within the last day',
},
},
},
composite: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'Composite field containing service runtime name and version sorted by doc count.',
},
},
},
},
},
};
@ -119,44 +218,159 @@ const apmPerAgentSchema: Pick<
};
export const apmPerServiceSchema: MakeSchemaFrom<APMPerService> = {
service_id: keyword,
num_service_nodes: long,
num_transaction_types: long,
timed_out: { type: 'boolean' },
service_id: {
...keyword,
_meta: {
description:
'Unique identifier that combines the SHA256 hashed representation of the service name and environment',
},
},
num_service_nodes: {
...long,
_meta: {
description:
'Total number of the unique service instances that served the transaction within an hour',
},
},
num_transaction_types: {
...long,
_meta: {
description:
'Total number of the unique transaction types within an hour',
},
},
timed_out: {
type: 'boolean',
_meta: {
description: 'Indicates whether the request timed out before completion',
},
},
cloud: {
availability_zones: { type: 'array', items: { type: 'keyword' } },
regions: { type: 'array', items: { type: 'keyword' } },
providers: { type: 'array', items: { type: 'keyword' } },
availability_zones: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 5 cloud availability zones within an hour. Example [ca-central-1a, ca-central-1b]',
},
},
},
regions: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 5 cloud regions within an hour. Example [ca-central-1]',
},
},
},
providers: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 3 cloud provider within an hour. Example [aws]',
},
},
},
},
faas: {
trigger: {
type: { type: 'array', items: { type: 'keyword' } },
type: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 5 faas trigger types within an hour. Example [http, timer, pubsub]',
},
},
},
},
},
agent: {
name: keyword,
version: keyword,
activation_method: keyword,
name: {
...keyword,
_meta: {
description:
'The top value of agent name for the service from transaction documents within an hour. Sorted by _score',
},
},
version: {
...keyword,
_meta: {
description:
'The top value of agent version for the service from transaction documents within an hour. Sorted by _score',
},
},
activation_method: {
...keyword,
_meta: {
description:
'The top value of agent activation method for the service from transaction documents within an hour. Sorted by _score',
},
},
},
service: {
language: {
name: keyword,
version: keyword,
name: {
...keyword,
_meta: {
description:
'The top value of language name for the service from transaction documents within an hour. Sorted by _score',
},
},
version: {
...keyword,
_meta: {
description:
'The top value of language version for the service from transaction documents within an hour. Sorted by _score',
},
},
},
framework: {
name: keyword,
version: keyword,
name: {
...keyword,
_meta: {
description:
'The top value of service framework name from transaction documents within an hour. Sorted by _score. Example AWS Lambda',
},
},
version: {
...keyword,
_meta: {
description:
'The top value of service framework version from transaction documents within an hour. Sorted by _score',
},
},
},
runtime: {
name: keyword,
version: keyword,
name: {
...keyword,
_meta: {
description:
'The top value of service runtime name from transaction documents within an hour. Sorted by _score',
},
},
version: {
...keyword,
_meta: {
description:
'The top value of service runtime version version from transaction documents within an hour. Sorted by _score',
},
},
},
},
// No data found
kubernetes: {
pod: {
name: keyword,
},
},
// No data found
container: {
id: keyword,
},
@ -164,19 +378,62 @@ export const apmPerServiceSchema: MakeSchemaFrom<APMPerService> = {
export const apmSchema: MakeSchemaFrom<APMUsage> = {
...apmPerAgentSchema,
has_any_services: { type: 'boolean' },
has_any_services: {
type: 'boolean',
_meta: {
description:
'Indicates whether any service is being monitored. This is determined by checking all agents within the last day',
},
},
version: {
apm_server: {
major: long,
minor: long,
patch: long,
major: {
...long,
_meta: {
description: 'The major version of the APM server. Example: 7',
},
},
minor: {
...long,
_meta: {
description: 'The minor version of the APM server. Example: 17',
},
},
patch: {
...long,
_meta: {
description: 'The patch version of the APM server. Example 3',
},
},
},
},
environments: {
services_without_environment: long,
services_with_multiple_environments: long,
top_environments: { type: 'array', items: { type: 'keyword' } },
services_without_environment: {
...long,
_meta: {
description:
'Number of services without an assigned environment within the last day. This is determined by checking the "service.environment" field and counting instances where it is null',
},
},
services_with_multiple_environments: {
...long,
_meta: {
description:
'Number of services with more than one assigned environment within the last day',
},
},
top_environments: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 5 environments in terms of document count within tha last day',
},
},
},
},
// #NOTE No task identified for extracting the following information
aggregated_transactions: {
current_implementation: aggregatedTransactionCountSchema,
no_observer_name: aggregatedTransactionCountSchema,
@ -186,21 +443,166 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
only_rum_no_observer_name: aggregatedTransactionCountSchema,
},
cloud: {
availability_zone: { type: 'array', items: { type: 'keyword' } },
provider: { type: 'array', items: { type: 'keyword' } },
region: { type: 'array', items: { type: 'keyword' } },
availability_zone: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 10 cloud availability zones in terms of document count overall. Example: [us-east1-c, us-east1-b]',
},
},
},
provider: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 10 cloud providers in terms of document count overall. Example: [azure]',
},
},
},
region: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 10 cloud regions in terms of document count overall. Example: [us-west1, us-central1]',
},
},
},
},
host: {
os: {
platform: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of the top 10 operating system platforms in terms of document count within an hour. Example: [linux, win32]',
},
},
},
},
},
host: { os: { platform: { type: 'array', items: { type: 'keyword' } } } },
counts: {
transaction: timeframeMapSchema,
span: timeframeMapSchema,
error: timeframeMapSchema,
metric: timeframeMapSchema,
onboarding: timeframeMapSchema,
agent_configuration: timeframeMapAllSchema,
max_transaction_groups_per_service: timeframeMapSchema,
max_error_groups_per_service: timeframeMapSchema,
traces: timeframeMapSchema,
transaction: {
'1d': {
...long,
_meta: {
description:
'Total number of transaction documents within the last day',
},
},
all: {
...long,
_meta: {
description: 'Total number of transaction documents overall',
},
},
},
span: {
'1d': {
...long,
_meta: {
description: 'Total number of span documents within the last day',
},
},
all: {
...long,
_meta: {
description: 'Total number of span documents overall',
},
},
},
error: {
'1d': {
...long,
_meta: {
description: 'Total number of error documents within the last day',
},
},
all: {
...long,
_meta: {
description: 'Total number of error documents overall',
},
},
},
metric: {
'1d': {
...long,
_meta: {
description: 'Total number of metric documents within the last day',
},
},
all: {
...long,
_meta: {
description: 'Total number of metric documents overall',
},
},
},
onboarding: {
'1d': {
...long,
_meta: {
description:
'Total number of onboarding documents within the last day',
},
},
all: {
...long,
_meta: {
description: 'Total number of onboarding documents overall',
},
},
},
agent_configuration: {
all: {
...long,
_meta: {
description:
'Total number of apm-agent-configuration documents overall',
},
},
},
max_transaction_groups_per_service: {
'1d': {
...long,
_meta: {
description:
'Total number of distinct transaction groups for the top service for the last 24 hours',
},
},
},
max_error_groups_per_service: {
'1d': {
...long,
_meta: {
description:
'Total number of distinct error groups for the top service for the last 24 hours',
},
},
},
traces: {
'1d': {
...long,
_meta: {
description: 'Total number of trace documents within the last day',
},
},
all: {
...long,
_meta: {
description: 'Total number of trace documents overall',
},
},
},
// No tasks found
services: timeframeMapSchema,
environments: {
'1d': {
@ -221,31 +623,130 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
},
},
},
// FIXME cardinality types seem to be wrong
cardinality: {
client: { geo: { country_iso_code: { rum: timeframeMap1dSchema } } },
client: {
geo: {
country_iso_code: {
rum: {
'1d': {
...long,
_meta: {
description:
'Unique country iso code captured for the agents js-base, rum-js and opentelemetry/webjs within the last day',
},
},
},
},
},
},
user_agent: {
original: {
all_agents: timeframeMap1dSchema,
rum: timeframeMap1dSchema,
all_agents: {
'1d': {
...long,
_meta: {
description:
'Unique user agent for all agents within the last day',
},
},
},
rum: {
'1d': {
...long,
_meta: {
description:
'Unique user agent for rum agent within the last day',
},
},
},
},
},
transaction: {
name: {
all_agents: timeframeMap1dSchema,
rum: timeframeMap1dSchema,
all_agents: {
'1d': {
...long,
_meta: {
description:
'Unique transaction names for all agents within the last day',
},
},
},
rum: {
'1d': {
...long,
_meta: {
description:
'Unique transaction names for rum agent within the last day',
},
},
},
},
},
},
// Check 1d, and all schema
retainment: {
span: { ms: long },
transaction: { ms: long },
error: { ms: long },
metric: { ms: long },
onboarding: { ms: long },
span: {
ms: {
...long,
_meta: {
description:
'Represent the time difference in milliseconds between the current date and the date when the span document was recorded',
},
},
},
transaction: {
ms: {
...long,
_meta: {
description:
'Represent the time difference in milliseconds between the current date and the date when the transaction document was recorded',
},
},
},
error: {
ms: {
...long,
_meta: {
description:
'Represent the time difference in milliseconds between the current date and the date when the error document was recorded',
},
},
},
metric: {
ms: {
...long,
_meta: {
description:
'Represent the time difference in milliseconds between the current date and the date when the metric document was recorded',
},
},
},
onboarding: {
ms: {
...long,
_meta: {
description:
'Represent the time difference in milliseconds between the current date and the date when the onboarding document was recorded',
},
},
},
},
integrations: {
ml: {
all_jobs_count: {
...long,
_meta: {
description:
'Total number of anomaly detection jobs associated with the jobs apm-*, *-high_mean_response_time',
},
},
},
},
integrations: { ml: { all_jobs_count: long } },
indices: {
// cannot find related data
metric: {
shards: { total: long },
all: {
@ -255,43 +756,249 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
},
},
},
// cannot find related data
traces: {
shards: { total: long },
shards: {
total: {
...long,
_meta: {
description: 'Total number of shards overall',
},
},
},
all: {
total: {
docs: { count: long },
store: { size_in_bytes: long },
docs: {
count: {
...long,
_meta: {
description:
'Total number of transaction and span documents overall',
},
},
},
store: {
size_in_bytes: {
...long,
_meta: {
description: 'Size of the index in byte units overall.',
},
},
},
},
},
},
shards: {
total: {
...long,
_meta: {
description: 'Total number of shards overall',
},
},
},
shards: { total: long },
all: {
total: {
docs: { count: long },
store: { size_in_bytes: long },
docs: {
count: {
...long,
_meta: {
description: 'Total number of all documents overall',
},
},
},
store: {
size_in_bytes: {
...long,
_meta: {
description: 'Size of the index in byte units overall.',
},
},
},
},
},
},
service_groups: {
kuery_fields: { type: 'array', items: { type: 'keyword' } },
total: long,
kuery_fields: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description:
'An array of up to 500 unique fields used to create the service groups across all spaces. Example [service.language.name, service.name] ',
},
},
},
total: {
...long,
_meta: {
description:
'Total number of service groups retrived from the saved object across all spaces',
},
},
},
per_service: { type: 'array', items: { ...apmPerServiceSchema } },
tasks: {
aggregated_transactions: { took: { ms: long } },
cloud: { took: { ms: long } },
host: { took: { ms: long } },
processor_events: { took: { ms: long } },
agent_configuration: { took: { ms: long } },
services: { took: { ms: long } },
versions: { took: { ms: long } },
groupings: { took: { ms: long } },
integrations: { took: { ms: long } },
agents: { took: { ms: long } },
indices_stats: { took: { ms: long } },
cardinality: { took: { ms: long } },
environments: { took: { ms: long } },
service_groups: { took: { ms: long } },
per_service: { took: { ms: long } },
aggregated_transactions: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "aggregated_transactions" task',
},
},
},
},
cloud: {
took: {
ms: {
...long,
_meta: {
description: 'Execution time in milliseconds for the "cloud" task',
},
},
},
},
host: {
took: {
ms: {
...long,
_meta: {
description: 'Execution time in milliseconds for the "host" task',
},
},
},
},
processor_events: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "processor_events" task',
},
},
},
},
agent_configuration: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "agent_configuration" task',
},
},
},
},
services: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "services" task',
},
},
},
},
versions: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "versions" task',
},
},
},
},
groupings: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "groupings" task',
},
},
},
},
integrations: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "integrations" task',
},
},
},
},
agents: {
took: {
ms: {
...long,
_meta: {
description: 'Execution time in milliseconds for the "agents" task',
},
},
},
},
indices_stats: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "indices_stats" task',
},
},
},
},
cardinality: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "cardinality" task',
},
},
},
},
environments: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "environments" task',
},
},
},
},
service_groups: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "service_groups" task',
},
},
},
},
per_service: {
took: {
ms: {
...long,
_meta: {
description:
'Execution time in milliseconds for the "per_service" task',
},
},
},
},
},
};

View file

@ -104,8 +104,8 @@ export interface APMUsage {
metric: TimeframeMap;
onboarding: TimeframeMap;
agent_configuration: TimeframeMapAll;
max_transaction_groups_per_service: TimeframeMap;
max_error_groups_per_service: TimeframeMap;
max_transaction_groups_per_service: TimeframeMap1d;
max_error_groups_per_service: TimeframeMap1d;
traces: TimeframeMap;
services: TimeframeMap;
environments: TimeframeMap1d;