github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[APM][Otel] Use fields instead of _source on APM queries (#195242)

Browse source 
closes https://github.com/elastic/kibana/issues/192606

## Summary

v2 based on the work done in this PR
https://github.com/elastic/kibana/pull/192608 and the suggestion from
Dario https://github.com/elastic/kibana/pull/194424

This PR replaces the _source usage in APM queries with fields to support
Otel data. The idea is to get rid of existing UI errors we have and make
sure that otel data is shown correctly in the UI.

One way to check it is using the [e2e
PoC](https://github.com/elastic/otel-apm-e2e-poc/blob/main/README.md).

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Jenny <dzheni.pavlova@elastic.co>
This commit is contained in:
Carlos Crespo 2024-10-15 11:38:44 +02:00 • committed by GitHub
parent 90b4ba5561
commit 7235ed0425
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
67 changed files with 1612 additions and 385 deletions
Download patch file Download diff file Expand all files Collapse all files

21
packages/kbn-apm-types/src/es_fields/apm.ts
View file

@ -7,7 +7,8 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export const TIMESTAMP = 'timestamp.us';
export const TIMESTAMP_US = 'timestamp.us';
export const AT_TIMESTAMP = '@timestamp';
export const AGENT = 'agent';
export const AGENT_NAME = 'agent.name';
export const AGENT_VERSION = 'agent.version';
@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider';
export const CLOUD_REGION = 'cloud.region';
export const CLOUD_MACHINE_TYPE = 'cloud.machine.type';
export const CLOUD_ACCOUNT_ID = 'cloud.account.id';
export const CLOUD_ACCOUNT_NAME = 'cloud.account.name';
export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
export const CLOUD_SERVICE_NAME = 'cloud.service.name';
export const CLOUD_PROJECT_NAME = 'cloud.project.name';
export const EVENT_SUCCESS_COUNT = 'event.success_count';
@ -48,10 +51,14 @@ export const USER_ID = 'user.id';
export const USER_AGENT_ORIGINAL = 'user_agent.original';
export const USER_AGENT_NAME = 'user_agent.name';
export const OBSERVER_VERSION = 'observer.version';
export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
export const OBSERVER_HOSTNAME = 'observer.hostname';
export const OBSERVER_LISTENING = 'observer.listening';
export const PROCESSOR_EVENT = 'processor.event';
export const PROCESSOR_NAME = 'processor.name';
export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks';
export const TRANSACTION_DURATION = 'transaction.duration.us';
export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram';
export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary';
@ -95,6 +102,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us';
export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy';
export const SPAN_SYNC = 'span.sync';
export const SPAN_STACKTRACE = 'span.stacktrace';
// Parent ID for a transaction or span
export const PARENT_ID = 'parent.id';
@ -110,6 +118,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i
export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array
export const ERROR_EXC_TYPE = 'error.exception.type';
export const ERROR_PAGE_URL = 'error.page.url';
export const ERROR_STACK_TRACE = 'error.stack_trace';
export const ERROR_TYPE = 'error.type';
// METRICS
@ -153,6 +162,12 @@ export const CONTAINER_IMAGE = 'container.image.name';
export const KUBERNETES = 'kubernetes';
export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';
export const KUBERNETES_POD_UID = 'kubernetes.pod.uid';
export const KUBERNETES_NAMESPACE = 'kubernetes.namespace';
export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name';
export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id';
export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name';
export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name';
export const FAAS_ID = 'faas.id';
export const FAAS_NAME = 'faas.name';
@ -198,3 +213,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name';
export const CHILD_ID = 'child.id';
export const LOG_LEVEL = 'log.level';
// Process
export const PROCESS_ARGS = 'process.args';
export const PROCESS_PID = 'process.pid';

6
packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts
View file

@ -14,10 +14,10 @@ export interface APMBaseDoc {
'@timestamp': string;
agent: {
name: string;
version: string;
version?: string;
};
parent?: { id: string }; // parent ID is not available on root transactions
trace?: { id: string };
parent?: { id?: string }; // parent ID is not available on root transactions
trace?: { id?: string };
labels?: {
[key: string]: string | number | boolean;
};

18
packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts
View file

@ -10,26 +10,26 @@
export interface Cloud {
availability_zone?: string;
instance?: {
name: string;
id: string;
name?: string;
id?: string;
};
machine?: {
type: string;
type?: string;
};
project?: {
id: string;
name: string;
id?: string;
name?: string;
};
provider?: string;
region?: string;
account?: {
id: string;
name: string;
id?: string;
name?: string;
};
image?: {
id: string;
id?: string;
};
service?: {
name: string;
name?: string;
};
}

4
packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts
View file

@ -9,5 +9,7 @@
export interface Container {
id?: string | null;
image?: string | null;
image?: {
name?: string;
};
}

4
packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts
View file

@ -8,7 +8,7 @@
*/
export interface Http {
request?: { method: string; [key: string]: unknown };
response?: { status_code: number; [key: string]: unknown };
request?: { method?: string };
response?: { status_code?: number };
version?: string;
}

2
packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts
View file

@ -8,7 +8,7 @@
*/
export interface Kubernetes {
pod?: { uid?: string | null; [key: string]: unknown };
pod?: { uid?: string | null; name?: string };
namespace?: string;
replicaset?: {
name?: string;

4
packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts
View file

@ -13,6 +13,6 @@ export interface Observer {
id?: string;
name?: string;
type?: string;
version: string;
version_major: number;
version?: string;
version_major?: number;
}

2
packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts
View file

@ -9,5 +9,5 @@
// only for RUM agent: shared by error and transaction
export interface Page {
url: string;
url?: string;
}

8
packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts
View file

@ -11,18 +11,18 @@ export interface Service {
name: string;
environment?: string;
framework?: {
name: string;
name?: string;
version?: string;
};
node?: {
name?: string;
};
runtime?: {
name: string;
version: string;
name?: string;
version?: string;
};
language?: {
name: string;
name?: string;
version?: string;
};
version?: string;

2
packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts
View file

@ -9,6 +9,6 @@
export interface Url {
domain?: string;
full: string;
full?: string;
original?: string;
}

2
packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts
View file

@ -8,5 +8,5 @@
*/
export interface User {
id: string;
id?: string;
}

2
packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts
View file

@ -14,5 +14,5 @@ export type { ElasticAgentName, OpenTelemetryAgentName, AgentName } from '@kbn/e
export interface Agent {
ephemeral_id?: string;
name: AgentName;
version: string;
version?: string;
}

12
x-pack/packages/observability/observability_utils/object/flatten_object.test.ts
View file

@ -21,6 +21,18 @@ describe('flattenObject', () => {
});
});
it('flattens arrays', () => {
expect(
flattenObject({
child: {
id: [1, 2],
},
})
).toEqual({
'child.id': [1, 2],
});
});
it('does not flatten arrays', () => {
expect(
flattenObject({

40
x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { unflattenObject } from './unflatten_object';
describe('unflattenObject', () => {
it('unflattens deeply nested objects', () => {
expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({
first: {
second: {
third: 'third',
},
},
});
});
it('does not unflatten arrays', () => {
expect(
unflattenObject({
simpleArray: ['0', '1', '2'],
complexArray: [{ one: 'one', two: 'two', three: 'three' }],
'nested.array': [0, 1, 2],
'complex.nested': [{ one: 'one', two: 'two', 'first.second': 'foo', 'first.third': 'bar' }],
})
).toEqual({
simpleArray: ['0', '1', '2'],
complexArray: [{ one: 'one', two: 'two', three: 'three' }],
nested: {
array: [0, 1, 2],
},
complex: {
nested: [{ one: 'one', two: 'two', first: { second: 'foo', third: 'bar' } }],
},
});
});
});

28
x-pack/packages/observability/observability_utils/object/unflatten_object.ts Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { set } from '@kbn/safer-lodash-set';
export function unflattenObject(source: Record<string, any>, target: Record<string, any> = {}) {
// eslint-disable-next-line guard-for-in
for (const key in source) {
const val = source[key as keyof typeof source];
if (Array.isArray(val)) {
const unflattenedArray = val.map((item) => {
if (item && typeof item === 'object' && !Array.isArray(item)) {
return unflattenObject(item);
}
return item;
});
set(target, key, unflattenedArray);
} else {
set(target, key, val);
}
}
return target;
}

1
x-pack/packages/observability/observability_utils/tsconfig.json
View file

@ -21,5 +21,6 @@
"@kbn/es-types",
"@kbn/apm-utils",
"@kbn/es-query",
"@kbn/safer-lodash-set",
]
}

78
x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap generated
View file

@ -37,6 +37,8 @@ Object {
exports[`Error CLOUD_ACCOUNT_ID 1`] = `undefined`;
exports[`Error CLOUD_ACCOUNT_NAME 1`] = `undefined`;
exports[`Error CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`;
exports[`Error CLOUD_INSTANCE_ID 1`] = `undefined`;
@ -45,6 +47,8 @@ exports[`Error CLOUD_INSTANCE_NAME 1`] = `undefined`;
exports[`Error CLOUD_MACHINE_TYPE 1`] = `undefined`;
exports[`Error CLOUD_PROJECT_NAME 1`] = `undefined`;
exports[`Error CLOUD_PROVIDER 1`] = `"gcp"`;
exports[`Error CLOUD_REGION 1`] = `"europe-west1"`;
@ -94,6 +98,8 @@ exports[`Error ERROR_LOG_MESSAGE 1`] = `undefined`;
exports[`Error ERROR_PAGE_URL 1`] = `undefined`;
exports[`Error ERROR_STACK_TRACE 1`] = `undefined`;
exports[`Error ERROR_TYPE 1`] = `undefined`;
exports[`Error EVENT_NAME 1`] = `undefined`;
@ -140,6 +146,8 @@ exports[`Error INDEX 1`] = `undefined`;
exports[`Error KUBERNETES 1`] = `undefined`;
exports[`Error KUBERNETES_CONTAINER_ID 1`] = `undefined`;
exports[`Error KUBERNETES_CONTAINER_NAME 1`] = `undefined`;
exports[`Error KUBERNETES_DEPLOYMENT 1`] = `undefined`;
@ -150,6 +158,8 @@ exports[`Error KUBERNETES_NAMESPACE 1`] = `undefined`;
exports[`Error KUBERNETES_NAMESPACE_NAME 1`] = `undefined`;
exports[`Error KUBERNETES_NODE_NAME 1`] = `undefined`;
exports[`Error KUBERNETES_POD_NAME 1`] = `undefined`;
exports[`Error KUBERNETES_POD_UID 1`] = `undefined`;
@ -228,10 +238,20 @@ exports[`Error OBSERVER_HOSTNAME 1`] = `undefined`;
exports[`Error OBSERVER_LISTENING 1`] = `undefined`;
exports[`Error OBSERVER_VERSION 1`] = `"whatever"`;
exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`;
exports[`Error PARENT_ID 1`] = `"parentId"`;
exports[`Error PROCESS_ARGS 1`] = `undefined`;
exports[`Error PROCESS_PID 1`] = `undefined`;
exports[`Error PROCESSOR_EVENT 1`] = `"error"`;
exports[`Error PROCESSOR_NAME 1`] = `"error"`;
exports[`Error SERVICE 1`] = `
Object {
"language": Object {
@ -296,6 +316,8 @@ exports[`Error SPAN_NAME 1`] = `undefined`;
exports[`Error SPAN_SELF_TIME_SUM 1`] = `undefined`;
exports[`Error SPAN_STACKTRACE 1`] = `undefined`;
exports[`Error SPAN_SUBTYPE 1`] = `undefined`;
exports[`Error SPAN_SYNC 1`] = `undefined`;
@ -304,10 +326,12 @@ exports[`Error SPAN_TYPE 1`] = `undefined`;
exports[`Error TIER 1`] = `undefined`;
exports[`Error TIMESTAMP 1`] = `1337`;
exports[`Error TIMESTAMP_US 1`] = `1337`;
exports[`Error TRACE_ID 1`] = `"trace id"`;
exports[`Error TRANSACTION_AGENT_MARKS 1`] = `undefined`;
exports[`Error TRANSACTION_DURATION 1`] = `undefined`;
exports[`Error TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`;
@ -385,6 +409,8 @@ Object {
exports[`Span CLOUD_ACCOUNT_ID 1`] = `undefined`;
exports[`Span CLOUD_ACCOUNT_NAME 1`] = `undefined`;
exports[`Span CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`;
exports[`Span CLOUD_INSTANCE_ID 1`] = `undefined`;
@ -393,6 +419,8 @@ exports[`Span CLOUD_INSTANCE_NAME 1`] = `undefined`;
exports[`Span CLOUD_MACHINE_TYPE 1`] = `undefined`;
exports[`Span CLOUD_PROJECT_NAME 1`] = `undefined`;
exports[`Span CLOUD_PROVIDER 1`] = `"gcp"`;
exports[`Span CLOUD_REGION 1`] = `"europe-west1"`;
@ -433,6 +461,8 @@ exports[`Span ERROR_LOG_MESSAGE 1`] = `undefined`;
exports[`Span ERROR_PAGE_URL 1`] = `undefined`;
exports[`Span ERROR_STACK_TRACE 1`] = `undefined`;
exports[`Span ERROR_TYPE 1`] = `undefined`;
exports[`Span EVENT_NAME 1`] = `undefined`;
@ -475,6 +505,8 @@ exports[`Span INDEX 1`] = `undefined`;
exports[`Span KUBERNETES 1`] = `undefined`;
exports[`Span KUBERNETES_CONTAINER_ID 1`] = `undefined`;
exports[`Span KUBERNETES_CONTAINER_NAME 1`] = `undefined`;
exports[`Span KUBERNETES_DEPLOYMENT 1`] = `undefined`;
@ -485,6 +517,8 @@ exports[`Span KUBERNETES_NAMESPACE 1`] = `undefined`;
exports[`Span KUBERNETES_NAMESPACE_NAME 1`] = `undefined`;
exports[`Span KUBERNETES_NODE_NAME 1`] = `undefined`;
exports[`Span KUBERNETES_POD_NAME 1`] = `undefined`;
exports[`Span KUBERNETES_POD_UID 1`] = `undefined`;
@ -563,10 +597,20 @@ exports[`Span OBSERVER_HOSTNAME 1`] = `undefined`;
exports[`Span OBSERVER_LISTENING 1`] = `undefined`;
exports[`Span OBSERVER_VERSION 1`] = `"whatever"`;
exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`;
exports[`Span PARENT_ID 1`] = `"parentId"`;
exports[`Span PROCESS_ARGS 1`] = `undefined`;
exports[`Span PROCESS_PID 1`] = `undefined`;
exports[`Span PROCESSOR_EVENT 1`] = `"span"`;
exports[`Span PROCESSOR_NAME 1`] = `"transaction"`;
exports[`Span SERVICE 1`] = `
Object {
"name": "service name",
@ -627,6 +671,8 @@ exports[`Span SPAN_NAME 1`] = `"span name"`;
exports[`Span SPAN_SELF_TIME_SUM 1`] = `undefined`;
exports[`Span SPAN_STACKTRACE 1`] = `undefined`;
exports[`Span SPAN_SUBTYPE 1`] = `"my subtype"`;
exports[`Span SPAN_SYNC 1`] = `false`;
@ -635,10 +681,12 @@ exports[`Span SPAN_TYPE 1`] = `"span type"`;
exports[`Span TIER 1`] = `undefined`;
exports[`Span TIMESTAMP 1`] = `1337`;
exports[`Span TIMESTAMP_US 1`] = `1337`;
exports[`Span TRACE_ID 1`] = `"trace id"`;
exports[`Span TRANSACTION_AGENT_MARKS 1`] = `undefined`;
exports[`Span TRANSACTION_DURATION 1`] = `undefined`;
exports[`Span TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`;
@ -716,6 +764,8 @@ Object {
exports[`Transaction CLOUD_ACCOUNT_ID 1`] = `undefined`;
exports[`Transaction CLOUD_ACCOUNT_NAME 1`] = `undefined`;
exports[`Transaction CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`;
exports[`Transaction CLOUD_INSTANCE_ID 1`] = `undefined`;
@ -724,6 +774,8 @@ exports[`Transaction CLOUD_INSTANCE_NAME 1`] = `undefined`;
exports[`Transaction CLOUD_MACHINE_TYPE 1`] = `undefined`;
exports[`Transaction CLOUD_PROJECT_NAME 1`] = `undefined`;
exports[`Transaction CLOUD_PROVIDER 1`] = `"gcp"`;
exports[`Transaction CLOUD_REGION 1`] = `"europe-west1"`;
@ -768,6 +820,8 @@ exports[`Transaction ERROR_LOG_MESSAGE 1`] = `undefined`;
exports[`Transaction ERROR_PAGE_URL 1`] = `undefined`;
exports[`Transaction ERROR_STACK_TRACE 1`] = `undefined`;
exports[`Transaction ERROR_TYPE 1`] = `undefined`;
exports[`Transaction EVENT_NAME 1`] = `undefined`;
@ -820,6 +874,8 @@ Object {
}
`;
exports[`Transaction KUBERNETES_CONTAINER_ID 1`] = `undefined`;
exports[`Transaction KUBERNETES_CONTAINER_NAME 1`] = `undefined`;
exports[`Transaction KUBERNETES_DEPLOYMENT 1`] = `undefined`;
@ -830,6 +886,8 @@ exports[`Transaction KUBERNETES_NAMESPACE 1`] = `undefined`;
exports[`Transaction KUBERNETES_NAMESPACE_NAME 1`] = `undefined`;
exports[`Transaction KUBERNETES_NODE_NAME 1`] = `undefined`;
exports[`Transaction KUBERNETES_POD_NAME 1`] = `undefined`;
exports[`Transaction KUBERNETES_POD_UID 1`] = `"pod1234567890abcdef"`;
@ -908,10 +966,20 @@ exports[`Transaction OBSERVER_HOSTNAME 1`] = `undefined`;
exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`;
exports[`Transaction OBSERVER_VERSION 1`] = `"whatever"`;
exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`;
exports[`Transaction PARENT_ID 1`] = `"parentId"`;
exports[`Transaction PROCESS_ARGS 1`] = `undefined`;
exports[`Transaction PROCESS_PID 1`] = `undefined`;
exports[`Transaction PROCESSOR_EVENT 1`] = `"transaction"`;
exports[`Transaction PROCESSOR_NAME 1`] = `"transaction"`;
exports[`Transaction SERVICE 1`] = `
Object {
"language": Object {
@ -976,6 +1044,8 @@ exports[`Transaction SPAN_NAME 1`] = `undefined`;
exports[`Transaction SPAN_SELF_TIME_SUM 1`] = `undefined`;
exports[`Transaction SPAN_STACKTRACE 1`] = `undefined`;
exports[`Transaction SPAN_SUBTYPE 1`] = `undefined`;
exports[`Transaction SPAN_SYNC 1`] = `undefined`;
@ -984,10 +1054,12 @@ exports[`Transaction SPAN_TYPE 1`] = `undefined`;
exports[`Transaction TIER 1`] = `undefined`;
exports[`Transaction TIMESTAMP 1`] = `1337`;
exports[`Transaction TIMESTAMP_US 1`] = `1337`;
exports[`Transaction TRACE_ID 1`] = `"trace id"`;
exports[`Transaction TRANSACTION_AGENT_MARKS 1`] = `undefined`;
exports[`Transaction TRANSACTION_DURATION 1`] = `1337`;
exports[`Transaction TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`;

9
x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts
View file

@ -10,12 +10,13 @@ import { AllowUnknownProperties } from '../../typings/common';
import { APMError } from '../../typings/es_schemas/ui/apm_error';
import { Span } from '../../typings/es_schemas/ui/span';
import { Transaction } from '../../typings/es_schemas/ui/transaction';
import * as apmFieldnames from './apm';
import * as infraMetricsFieldnames from './infra_metrics';
import * as allApmFieldNames from './apm';
import * as infraMetricsFieldNames from './infra_metrics';
const { AT_TIMESTAMP, ...apmFieldNames } = allApmFieldNames;
const fieldnames = {
...apmFieldnames,
...infraMetricsFieldnames,
...apmFieldNames,
...infraMetricsFieldNames,
};
describe('Transaction', () => {

58
x-pack/plugins/observability_solution/apm/common/service_metadata.ts
View file

@ -4,5 +4,63 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
CLOUD_AVAILABILITY_ZONE,
CLOUD_INSTANCE_ID,
CLOUD_INSTANCE_NAME,
CLOUD_MACHINE_TYPE,
CLOUD_PROVIDER,
CONTAINER_ID,
HOST_NAME,
KUBERNETES_CONTAINER_NAME,
KUBERNETES_NAMESPACE,
KUBERNETES_DEPLOYMENT_NAME,
KUBERNETES_POD_NAME,
KUBERNETES_POD_UID,
KUBERNETES_REPLICASET_NAME,
SERVICE_NODE_NAME,
SERVICE_RUNTIME_NAME,
SERVICE_RUNTIME_VERSION,
SERVICE_VERSION,
} from './es_fields/apm';
import { asMutableArray } from './utils/as_mutable_array';
export const SERVICE_METADATA_SERVICE_KEYS = asMutableArray([
SERVICE_NODE_NAME,
SERVICE_VERSION,
SERVICE_RUNTIME_NAME,
SERVICE_RUNTIME_VERSION,
] as const);
export const SERVICE_METADATA_CONTAINER_KEYS = asMutableArray([
CONTAINER_ID,
HOST_NAME,
KUBERNETES_POD_UID,
KUBERNETES_POD_NAME,
] as const);
export const SERVICE_METADATA_INFRA_METRICS_KEYS = asMutableArray([
KUBERNETES_CONTAINER_NAME,
KUBERNETES_NAMESPACE,
KUBERNETES_REPLICASET_NAME,
KUBERNETES_DEPLOYMENT_NAME,
] as const);
export const SERVICE_METADATA_CLOUD_KEYS = asMutableArray([
CLOUD_AVAILABILITY_ZONE,
CLOUD_INSTANCE_ID,
CLOUD_INSTANCE_NAME,
CLOUD_MACHINE_TYPE,
CLOUD_PROVIDER,
] as const);
export const SERVICE_METADATA_KUBERNETES_KEYS = asMutableArray([
KUBERNETES_CONTAINER_NAME,
KUBERNETES_NAMESPACE,
KUBERNETES_DEPLOYMENT_NAME,
KUBERNETES_POD_NAME,
KUBERNETES_POD_UID,
KUBERNETES_REPLICASET_NAME,
] as const);
export type ContainerType = 'Kubernetes' | 'Docker' | undefined;

9
x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts
View file

@ -64,16 +64,17 @@ export interface WaterfallSpan {
links?: SpanLink[];
};
transaction?: {
id: string;
id?: string;
};
child?: { id: string[] };
}
export interface WaterfallError {
timestamp: TimestampUs;
trace?: { id: string };
transaction?: { id: string };
parent?: { id: string };
trace?: { id?: string };
transaction?: { id?: string };
parent?: { id?: string };
span?: { id?: string };
error: {
id: string;
log?: {

24
x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx
View file

@ -8,9 +8,9 @@ import { EuiFlexItem, EuiSpacer } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import type { Message } from '@kbn/observability-ai-assistant-plugin/public';
import React, { useMemo, useState } from 'react';
import { AT_TIMESTAMP } from '@kbn/apm-types';
import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
import { Transaction } from '../../../../../typings/es_schemas/ui/transaction';
import { ErrorSampleDetailTabContent } from './error_sample_detail';
import { exceptionStacktraceTab, logStacktraceTab } from './error_tabs';
@ -18,8 +18,26 @@ export function ErrorSampleContextualInsight({
error,
transaction,
}: {
error: APMError;
transaction?: Transaction;
error: {
[AT_TIMESTAMP]: string;
error: Pick<APMError['error'], 'log' | 'exception' | 'id'>;
service: {
name: string;
environment?: string;
language?: {
name?: string;
};
runtime?: {
name?: string;
version?: string;
};
};
};
transaction?: {
transaction: {
name: string;
};
};
}) {
const { observabilityAIAssistant } = useApmPluginContext();

25
x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx
View file

@ -29,14 +29,14 @@ import { first } from 'lodash';
import React, { useEffect, useState } from 'react';
import { useHistory } from 'react-router-dom';
import useAsync from 'react-use/lib/useAsync';
import { ERROR_GROUP_ID } from '../../../../../common/es_fields/apm';
import { AT_TIMESTAMP, ERROR_GROUP_ID } from '../../../../../common/es_fields/apm';
import { TraceSearchType } from '../../../../../common/trace_explorer';
import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
import { useLegacyUrlParams } from '../../../../context/url_params_context/use_url_params';
import { useAnyOfApmParams } from '../../../../hooks/use_apm_params';
import { useApmRouter } from '../../../../hooks/use_apm_router';
import { FETCH_STATUS, isPending } from '../../../../hooks/use_fetcher';
import { FETCH_STATUS, isPending, isSuccess } from '../../../../hooks/use_fetcher';
import { useTraceExplorerEnabledSetting } from '../../../../hooks/use_trace_explorer_enabled_setting';
import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
import { TransactionDetailLink } from '../../../shared/links/apm/transaction_detail_link';
@ -111,8 +111,7 @@ export function ErrorSampleDetails({
const loadingErrorData = isPending(errorFetchStatus);
const isLoading = loadingErrorSamplesData || loadingErrorData;
const isSucceded =
errorSamplesFetchStatus === FETCH_STATUS.SUCCESS && errorFetchStatus === FETCH_STATUS.SUCCESS;
const isSucceeded = isSuccess(errorSamplesFetchStatus) && isSuccess(errorFetchStatus);
useEffect(() => {
setSampleActivePage(0);
@ -137,7 +136,7 @@ export function ErrorSampleDetails({
});
}, [error, transaction, uiActions]);
if (!error && errorSampleIds?.length === 0 && isSucceded) {
if (!error && errorSampleIds?.length === 0 && isSucceeded) {
return (
<EuiEmptyPrompt
title={
@ -160,7 +159,7 @@ export function ErrorSampleDetails({
const status = error.http?.response?.status_code;
const environment = error.service.environment;
const serviceVersion = error.service.version;
const isUnhandled = error.error.exception?.[0].handled === false;
const isUnhandled = error.error.exception?.[0]?.handled === false;
const traceExplorerLink = router.link('/traces/explorer/waterfall', {
query: {
@ -348,14 +347,22 @@ export function ErrorSampleDetailTabContent({
error,
currentTab,
}: {
error: APMError;
error: {
service: {
language?: {
name?: string;
};
};
[AT_TIMESTAMP]: string;
error: Pick<APMError['error'], 'id' | 'log' | 'stack_trace' | 'exception'>;
};
currentTab: ErrorTab;
}) {
const codeLanguage = error?.service.language?.name;
const exceptions = error?.error.exception || [];
const logStackframes = error?.error.log?.stacktrace;
const isPlaintextException =
!!error?.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace;
!!error.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace;
switch (currentTab.key) {
case ErrorTabKey.LogStackTrace:
return <Stacktrace stackframes={logStackframes} codeLanguage={codeLanguage} />;
@ -363,7 +370,7 @@ export function ErrorSampleDetailTabContent({
return isPlaintextException ? (
<PlaintextStacktrace
message={exceptions[0].message}
type={exceptions[0].type}
type={exceptions[0]?.type}
stacktrace={error?.error.stack_trace}
codeLanguage={codeLanguage}
/>

2
x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx
View file

@ -41,7 +41,7 @@ export const metadataTab: ErrorTab = {
}),
};
export function getTabs(error: APMError) {
export function getTabs(error: { error: { log?: APMError['error']['log'] } }) {
const hasLogStacktrace = !isEmpty(error?.error.log?.stacktrace);
return [...(hasLogStacktrace ? [logStacktraceTab] : []), exceptionStacktraceTab, metadataTab];
}

4
x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx
View file

@ -18,7 +18,9 @@ const Label = euiStyled.div`
`;
interface Props {
error: APMError;
error: {
error: Pick<APMError['error'], 'log' | 'exception' | 'culprit'>;
};
}
export function SampleSummary({ error }: Props) {
const logMessage = error.error.log?.message;

4
x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts
View file

@ -6,7 +6,7 @@
*/
import { format } from 'url';
import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
import type { TransactionDetailRedirectInfo } from '../../../../server/routes/transactions/get_transaction_by_trace';
export const getRedirectToTransactionDetailPageUrl = ({
transaction,
@ -14,7 +14,7 @@ export const getRedirectToTransactionDetailPageUrl = ({
rangeTo,
waterfallItemId,
}: {
transaction: Transaction;
transaction: TransactionDetailRedirectInfo;
rangeFrom?: string;
rangeTo?: string;
waterfallItemId?: string;

14
x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx
View file

@ -7,10 +7,18 @@
import React, { ReactNode } from 'react';
import { ERROR_GROUP_ID, SERVICE_NAME } from '../../../../../common/es_fields/apm';
import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
import { DiscoverLink } from './discover_link';
function getDiscoverQuery(error: APMError, kuery?: string) {
interface ErrorForDiscoverQuery {
service: {
name: string;
};
error: {
grouping_key: string;
};
}
function getDiscoverQuery(error: ErrorForDiscoverQuery, kuery?: string) {
const serviceName = error.service.name;
const groupId = error.error.grouping_key;
let query = `${SERVICE_NAME}:"${serviceName}" AND ${ERROR_GROUP_ID}:"${groupId}"`;
@ -36,7 +44,7 @@ function DiscoverErrorLink({
children,
}: {
children?: ReactNode;
readonly error: APMError;
readonly error: ErrorForDiscoverQuery;
readonly kuery?: string;
}) {
return <DiscoverLink query={getDiscoverQuery(error, kuery)} children={children} />;

11
x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx
View file

@ -7,13 +7,16 @@
import React, { useMemo } from 'react';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
import { APMError, AT_TIMESTAMP } from '@kbn/apm-types';
import { getSectionsFromFields } from '../helper';
import { MetadataTable } from '..';
import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
interface Props {
error: APMError;
error: {
[AT_TIMESTAMP]: string;
error: Pick<APMError['error'], 'id'>;
};
}
export function ErrorMetadata({ error }: Props) {
@ -26,8 +29,8 @@ export function ErrorMetadata({ error }: Props) {
id: error.error.id,
},
query: {
start: error['@timestamp'],
end: error['@timestamp'],
start: error[AT_TIMESTAMP],
end: error[AT_TIMESTAMP],
},
},
});

6
x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts
View file

@ -8,7 +8,7 @@
import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
import { tasks } from './tasks';
import { SERVICE_NAME, SERVICE_ENVIRONMENT } from '../../../../common/es_fields/apm';
import { SERVICE_NAME, SERVICE_ENVIRONMENT, AT_TIMESTAMP } from '../../../../common/es_fields/apm';
import { IndicesStatsResponse } from '../telemetry_client';
describe('data telemetry collection tasks', () => {
@ -101,7 +101,7 @@ describe('data telemetry collection tasks', () => {
// a fixed date range.
.mockReturnValueOnce({
hits: {
hits: [{ _source: { '@timestamp': new Date().toISOString() } }],
hits: [{ fields: { [AT_TIMESTAMP]: [new Date().toISOString()] } }],
},
total: {
value: 1,
@ -314,7 +314,7 @@ describe('data telemetry collection tasks', () => {
? { hits: { total: { value: 1 } } }
: {
hits: {
hits: [{ _source: { '@timestamp': 1 } }],
hits: [{ fields: { [AT_TIMESTAMP]: [1] } }],
},
}
);

32
x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts
View file

@ -11,11 +11,13 @@ import { createHash } from 'crypto';
import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash';
import { SavedObjectsClient } from '@kbn/core/server';
import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name';
import {
AGENT_ACTIVATION_METHOD,
AGENT_NAME,
AGENT_VERSION,
AT_TIMESTAMP,
CLIENT_GEO_COUNTRY_ISO_CODE,
CLOUD_AVAILABILITY_ZONE,
CLOUD_PROVIDER,
@ -29,6 +31,7 @@ import {
METRICSET_INTERVAL,
METRICSET_NAME,
OBSERVER_HOSTNAME,
OBSERVER_VERSION,
PARENT_ID,
PROCESSOR_EVENT,
SERVICE_ENVIRONMENT,
@ -54,10 +57,7 @@ import {
SavedServiceGroup,
} from '../../../../common/service_groups';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { APMError } from '../../../../typings/es_schemas/ui/apm_error';
import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
import { Span } from '../../../../typings/es_schemas/ui/span';
import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
import {
APMDataTelemetry,
APMPerService,
@ -193,17 +193,19 @@ export const tasks: TelemetryTask[] = [
size: 1,
track_total_hits: false,
sort: {
'@timestamp': 'desc' as const,
[AT_TIMESTAMP]: 'desc' as const,
},
fields: [AT_TIMESTAMP],
},
})
).hits.hits[0] as { _source: { '@timestamp': string } };
).hits.hits[0];
if (!lastTransaction) {
return {};
}
const end = new Date(lastTransaction._source['@timestamp']).getTime() - 5 * 60 * 1000;
const end =
new Date(lastTransaction.fields[AT_TIMESTAMP]![0] as string).getTime() - 5 * 60 * 1000;
const start = end - 60 * 1000;
@ -512,16 +514,16 @@ export const tasks: TelemetryTask[] = [
},
},
sort: {
'@timestamp': 'asc',
[AT_TIMESTAMP]: 'asc',
},
_source: ['@timestamp'],
fields: [AT_TIMESTAMP],
},
})
: null;
const event = retainmentResponse?.hits.hits[0]?._source as
const event = retainmentResponse?.hits.hits[0]?.fields as
| {
'@timestamp': number;
[AT_TIMESTAMP]: number[];
}
| undefined;
@ -535,7 +537,7 @@ export const tasks: TelemetryTask[] = [
? {
retainment: {
[processorEvent]: {
ms: new Date().getTime() - new Date(event['@timestamp']).getTime(),
ms: new Date().getTime() - new Date(event[AT_TIMESTAMP][0]).getTime(),
},
},
}
@ -690,16 +692,16 @@ export const tasks: TelemetryTask[] = [
sort: {
'@timestamp': 'desc',
},
fields: asMutableArray([OBSERVER_VERSION] as const),
},
});
const hit = response.hits.hits[0]?._source as Pick<Transaction | Span | APMError, 'observer'>;
if (!hit || !hit.observer?.version) {
const event = unflattenKnownApmEventFields(response.hits.hits[0]?.fields);
if (!event || !event.observer?.version) {
return {};
}
const [major, minor, patch] = hit.observer.version.split('.').map((part) => Number(part));
const [major, minor, patch] = event.observer.version.split('.').map((part) => Number(part));
return {
version: {

1
x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts
View file

@ -186,7 +186,6 @@ export const getDestinationMap = ({
},
size: destinationsBySpanId.size,
fields: asMutableArray([SERVICE_NAME, SERVICE_ENVIRONMENT, AGENT_NAME, PARENT_ID] as const),
_source: false,
},
});

6
x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts
View file

@ -9,6 +9,10 @@ import { NOT_AVAILABLE_LABEL } from '../../../common/i18n';
import { Maybe } from '../../../typings/common';
import { APMError } from '../../../typings/es_schemas/ui/apm_error';
export function getErrorName({ error }: { error: Maybe<APMError['error']> }): string {
export function getErrorName({
error,
}: {
error: Maybe<Pick<APMError['error'], 'exception'>> & { log?: { message?: string } };
}): string {
return error?.log?.message || error?.exception?.[0]?.message || NOT_AVAILABLE_LABEL;
}

1
x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts
View file

@ -184,6 +184,7 @@ export function registerTransactionDurationRuleType({
body: {
track_total_hits: false,
size: 0,
_source: false as const,
query: {
bool: {
filter: [

18
x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts
View file

@ -8,6 +8,9 @@
import datemath from '@elastic/datemath';
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { maybe } from '../../../../common/utils/maybe';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { flattenObject, KeyValuePair } from '../../../../common/utils/flatten_object';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { PROCESSOR_EVENT, TRACE_ID } from '../../../../common/es_fields/apm';
@ -86,6 +89,7 @@ export async function getLogCategories({
const rawSamplingProbability = Math.min(100_000 / totalDocCount, 1);
const samplingProbability = rawSamplingProbability < 0.5 ? rawSamplingProbability : 1;
const fields = asMutableArray(['message', TRACE_ID] as const);
const categorizedLogsRes = await search({
index,
size: 1,
@ -108,7 +112,7 @@ export async function getLogCategories({
top_hits: {
sort: { '@timestamp': 'desc' as const },
size: 1,
_source: ['message', TRACE_ID],
fields,
},
},
},
@ -120,9 +124,11 @@ export async function getLogCategories({
const promises = categorizedLogsRes.aggregations?.sampling.categories?.buckets.map(
async ({ doc_count: docCount, key, sample }) => {
const hit = sample.hits.hits[0]._source as { message: string; trace?: { id: string } };
const sampleMessage = hit?.message;
const sampleTraceId = hit?.trace?.id;
const hit = sample.hits.hits[0];
const event = unflattenKnownApmEventFields(hit?.fields);
const sampleMessage = event.message as string;
const sampleTraceId = event.trace?.id;
const errorCategory = key as string;
if (!sampleTraceId) {
@ -140,7 +146,9 @@ export async function getLogCategories({
}
);
const sampleDoc = categorizedLogsRes.hits.hits?.[0]?._source as Record<string, string>;
const event = unflattenKnownApmEventFields(maybe(categorizedLogsRes.hits.hits[0])?.fields);
const sampleDoc = event as Record<string, string>;
return {
logCategories: await Promise.all(promises ?? []),

17
x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts
View file

@ -13,6 +13,10 @@ import moment from 'moment';
import { ESSearchRequest } from '@kbn/es-types';
import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services';
import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types';
import { CONTAINER_ID } from '@kbn/apm-types';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { maybe } from '../../../../common/utils/maybe';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { ApmDocumentType } from '../../../../common/document_type';
import {
APMEventClient,
@ -79,13 +83,17 @@ async function getContainerIdFromLogs({
esClient: ElasticsearchClient;
logSourcesService: LogSourcesService;
}) {
const requiredFields = asMutableArray([CONTAINER_ID] as const);
const index = await logSourcesService.getFlattenedLogSources();
const res = await typedSearch<{ container: { id: string } }, any>(esClient, {
index,
...params,
fields: requiredFields,
});
return res.hits.hits[0]?._source?.container?.id;
const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields);
return event?.container.id;
}
async function getContainerIdFromTraces({
@ -95,6 +103,7 @@ async function getContainerIdFromTraces({
params: APMEventESSearchRequest['body'];
apmEventClient: APMEventClient;
}) {
const requiredFields = asMutableArray([CONTAINER_ID] as const);
const res = await apmEventClient.search('get_container_id_from_traces', {
apm: {
sources: [
@ -104,8 +113,10 @@ async function getContainerIdFromTraces({
},
],
},
body: params,
body: { ...params, fields: requiredFields },
});
return res.hits.hits[0]?._source.container?.id;
const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields);
return event?.container.id;
}

10
x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts
View file

@ -6,6 +6,9 @@
*/
import { rangeQuery } from '@kbn/observability-plugin/server';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { maybe } from '../../../../common/utils/maybe';
import { ApmDocumentType } from '../../../../common/document_type';
import { termQuery } from '../../../../common/utils/term_query';
import {
@ -27,6 +30,7 @@ export async function getDownstreamServiceResource({
end: number;
apmEventClient: APMEventClient;
}) {
const requiredFields = asMutableArray([SPAN_DESTINATION_SERVICE_RESOURCE] as const);
const response = await apmEventClient.search('get_error_group_main_statistics', {
apm: {
sources: [
@ -50,9 +54,11 @@ export async function getDownstreamServiceResource({
],
},
},
fields: requiredFields,
},
});
const hit = response.hits.hits[0];
return hit?._source?.span.destination?.service.resource;
const event = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredFields);
return event?.span.destination.service.resource;
}

14
x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts
View file

@ -12,6 +12,10 @@ import moment from 'moment';
import { ESSearchRequest } from '@kbn/es-types';
import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services';
import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { SERVICE_NAME } from '@kbn/apm-types';
import { maybe } from '../../../../common/utils/maybe';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { ApmDocumentType } from '../../../../common/document_type';
import {
APMEventClient,
@ -102,6 +106,7 @@ async function getServiceNameFromTraces({
params: APMEventESSearchRequest['body'];
apmEventClient: APMEventClient;
}) {
const requiredFields = asMutableArray([SERVICE_NAME] as const);
const res = await apmEventClient.search('get_service_name_from_traces', {
apm: {
sources: [
@ -111,8 +116,13 @@ async function getServiceNameFromTraces({
},
],
},
body: params,
body: {
...params,
fields: requiredFields,
},
});
return res.hits.hits[0]?._source.service.name;
const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields);
return event?.service.name;
}

16
x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts
View file

@ -7,8 +7,14 @@
import { rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import { maybe } from '../../../common/utils/maybe';
import { SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm';
import {
SPAN_DESTINATION_SERVICE_RESOURCE,
SPAN_SUBTYPE,
SPAN_TYPE,
} from '../../../common/es_fields/apm';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
export interface MetadataForDependencyResponse {
@ -27,6 +33,7 @@ export async function getMetadataForDependency({
start: number;
end: number;
}): Promise<MetadataForDependencyResponse> {
const fields = asMutableArray([SPAN_TYPE, SPAN_SUBTYPE] as const);
const sampleResponse = await apmEventClient.search('get_metadata_for_dependency', {
apm: {
events: [ProcessorEvent.span],
@ -46,16 +53,17 @@ export async function getMetadataForDependency({
],
},
},
fields,
sort: {
'@timestamp': 'desc',
},
},
});
const sample = maybe(sampleResponse.hits.hits[0])?._source;
const sample = unflattenKnownApmEventFields(maybe(sampleResponse.hits.hits[0])?.fields);
return {
spanType: sample?.span.type,
spanSubtype: sample?.span.subtype,
spanType: sample?.span?.type,
spanSubtype: sample?.span?.subtype,
};
}

43
x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts
View file

@ -8,8 +8,11 @@
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { kqlQuery, rangeQuery, termQuery, termsQuery } from '@kbn/observability-plugin/server';
import { keyBy } from 'lodash';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import {
AGENT_NAME,
AT_TIMESTAMP,
EVENT_OUTCOME,
SERVICE_ENVIRONMENT,
SERVICE_NAME,
@ -66,6 +69,19 @@ export async function getTopDependencySpans({
sampleRangeFrom?: number;
sampleRangeTo?: number;
}): Promise<DependencySpan[]> {
const topDedsRequiredFields = asMutableArray([
SPAN_ID,
TRACE_ID,
TRANSACTION_ID,
SPAN_NAME,
SERVICE_NAME,
SERVICE_ENVIRONMENT,
AGENT_NAME,
SPAN_DURATION,
EVENT_OUTCOME,
AT_TIMESTAMP,
] as const);
const spans = (
await apmEventClient.search('get_top_dependency_spans', {
apm: {
@ -98,23 +114,18 @@ export async function getTopDependencySpans({
],
},
},
_source: [
SPAN_ID,
TRACE_ID,
TRANSACTION_ID,
SPAN_NAME,
SERVICE_NAME,
SERVICE_ENVIRONMENT,
AGENT_NAME,
SPAN_DURATION,
EVENT_OUTCOME,
'@timestamp',
],
fields: topDedsRequiredFields,
},
})
).hits.hits.map((hit) => hit._source);
).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, topDedsRequiredFields));
const transactionIds = spans.map((span) => span.transaction!.id);
const transactionIds = spans.map((span) => span.transaction.id);
const txRequiredFields = asMutableArray([
TRANSACTION_ID,
TRANSACTION_TYPE,
TRANSACTION_NAME,
] as const);
const transactions = (
await apmEventClient.search('get_transactions_for_dependency_spans', {
@ -129,13 +140,13 @@ export async function getTopDependencySpans({
filter: [...termsQuery(TRANSACTION_ID, ...transactionIds)],
},
},
_source: [TRANSACTION_ID, TRANSACTION_TYPE, TRANSACTION_NAME],
fields: txRequiredFields,
sort: {
'@timestamp': 'desc',
},
},
})
).hits.hits.map((hit) => hit._source);
).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, txRequiredFields));
const transactionsById = keyBy(transactions, (transaction) => transaction.transaction.id);

61
x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts
View file

@ -7,13 +7,17 @@
import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types';
import { kqlQuery, rangeQuery, termQuery, wildcardQuery } from '@kbn/observability-plugin/server';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import {
AT_TIMESTAMP,
ERROR_CULPRIT,
ERROR_EXC_HANDLED,
ERROR_EXC_MESSAGE,
ERROR_EXC_TYPE,
ERROR_GROUP_ID,
ERROR_GROUP_NAME,
ERROR_ID,
ERROR_LOG_MESSAGE,
SERVICE_NAME,
TRACE_ID,
@ -93,6 +97,21 @@ export async function getErrorGroupMainStatistics({
]
: [];
const requiredFields = asMutableArray([
TRACE_ID,
AT_TIMESTAMP,
ERROR_GROUP_ID,
ERROR_ID,
] as const);
const optionalFields = asMutableArray([
ERROR_CULPRIT,
ERROR_LOG_MESSAGE,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
] as const);
const response = await apmEventClient.search('get_error_group_main_statistics', {
apm: {
sources: [
@ -129,16 +148,8 @@ export async function getErrorGroupMainStatistics({
sample: {
top_hits: {
size: 1,
_source: [
TRACE_ID,
ERROR_LOG_MESSAGE,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
ERROR_CULPRIT,
ERROR_GROUP_ID,
'@timestamp',
],
fields: [...requiredFields, ...optionalFields],
_source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE],
sort: {
'@timestamp': 'desc',
},
@ -157,15 +168,33 @@ export async function getErrorGroupMainStatistics({
const errorGroups =
response.aggregations?.error_groups.buckets.map((bucket) => {
const errorSource =
'error' in bucket.sample.hits.hits[0]._source
? bucket.sample.hits.hits[0]._source
: undefined;
const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields);
const mergedEvent = {
...event,
error: {
...(event.error ?? {}),
exception:
(errorSource?.error.exception?.length ?? 0) > 1
? errorSource?.error.exception
: event?.error.exception && [event.error.exception],
},
};
return {
groupId: bucket.key as string,
name: getErrorName(bucket.sample.hits.hits[0]._source),
lastSeen: new Date(bucket.sample.hits.hits[0]._source['@timestamp']).getTime(),
name: getErrorName(mergedEvent),
lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(),
occurrences: bucket.doc_count,
culprit: bucket.sample.hits.hits[0]._source.error.culprit,
handled: bucket.sample.hits.hits[0]._source.error.exception?.[0].handled,
type: bucket.sample.hits.hits[0]._source.error.exception?.[0].type,
traceId: bucket.sample.hits.hits[0]._source.trace?.id,
culprit: mergedEvent.error.culprit,
handled: mergedEvent.error.exception?.[0].handled,
type: mergedEvent.error.exception?.[0].type,
traceId: mergedEvent.trace?.id,
};
}) ?? [];

8
x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts
View file

@ -6,6 +6,7 @@
*/
import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import {
ERROR_GROUP_ID,
@ -42,6 +43,7 @@ export async function getErrorGroupSampleIds({
start: number;
end: number;
}): Promise<ErrorGroupSampleIdsResponse> {
const requiredFields = asMutableArray([ERROR_ID] as const);
const resp = await apmEventClient.search('get_error_group_sample_ids', {
apm: {
sources: [
@ -66,7 +68,7 @@ export async function getErrorGroupSampleIds({
should: [{ term: { [TRANSACTION_SAMPLED]: true } }], // prefer error samples with related transactions
},
},
_source: [ERROR_ID, 'transaction'],
fields: requiredFields,
sort: asMutableArray([
{ _score: { order: 'desc' } }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top
{ '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error
@ -74,8 +76,8 @@ export async function getErrorGroupSampleIds({
},
});
const errorSampleIds = resp.hits.hits.map((item) => {
const source = item._source;
return source.error.id;
const event = unflattenKnownApmEventFields(item.fields, requiredFields);
return event.error?.id;
});
return {

95
x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts
View file

@ -6,7 +6,28 @@
*/
import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { maybe } from '../../../../common/utils/maybe';
import {
AGENT_NAME,
AGENT_VERSION,
AT_TIMESTAMP,
ERROR_EXCEPTION,
ERROR_GROUP_ID,
ERROR_ID,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
PROCESSOR_EVENT,
PROCESSOR_NAME,
SERVICE_NAME,
TIMESTAMP_US,
TRACE_ID,
TRANSACTION_ID,
ERROR_STACK_TRACE,
SPAN_ID,
} from '../../../../common/es_fields/apm';
import { environmentQuery } from '../../../../common/utils/environment_query';
import { ApmDocumentType } from '../../../../common/document_type';
import { RollupInterval } from '../../../../common/rollup';
@ -17,7 +38,15 @@ import { APMError } from '../../../../typings/es_schemas/ui/apm_error';
export interface ErrorSampleDetailsResponse {
transaction: Transaction | undefined;
error: APMError;
error: Omit<APMError, 'transaction' | 'error'> & {
transaction?: { id?: string; type?: string };
error: {
id: string;
} & Omit<APMError['error'], 'exception' | 'log'> & {
exception?: APMError['error']['exception'];
log?: APMError['error']['log'];
};
};
}
export async function getErrorSampleDetails({
@ -36,7 +65,29 @@ export async function getErrorSampleDetails({
apmEventClient: APMEventClient;
start: number;
end: number;
}): Promise<ErrorSampleDetailsResponse> {
}): Promise<Partial<ErrorSampleDetailsResponse>> {
const requiredFields = asMutableArray([
AGENT_NAME,
PROCESSOR_EVENT,
TRACE_ID,
TIMESTAMP_US,
AT_TIMESTAMP,
SERVICE_NAME,
ERROR_ID,
ERROR_GROUP_ID,
] as const);
const optionalFields = asMutableArray([
TRANSACTION_ID,
SPAN_ID,
AGENT_VERSION,
PROCESSOR_NAME,
ERROR_STACK_TRACE,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
] as const);
const params = {
apm: {
sources: [
@ -60,15 +111,29 @@ export async function getErrorSampleDetails({
],
},
},
fields: [...requiredFields, ...optionalFields],
_source: [ERROR_EXCEPTION, 'error.log'],
},
};
const resp = await apmEventClient.search('get_error_sample_details', params);
const error = resp.hits.hits[0]?._source;
const transactionId = error?.transaction?.id;
const traceId = error?.trace?.id;
const hit = maybe(resp.hits.hits[0]);
let transaction;
if (!hit) {
return {
transaction: undefined,
error: undefined,
};
}
const source = 'error' in hit._source ? hit._source : undefined;
const errorFromFields = unflattenKnownApmEventFields(hit.fields, requiredFields);
const transactionId = errorFromFields.transaction?.id ?? errorFromFields.span?.id;
const traceId = errorFromFields.trace.id;
let transaction: Transaction | undefined;
if (transactionId && traceId) {
transaction = await getTransaction({
transactionId,
@ -81,6 +146,20 @@ export async function getErrorSampleDetails({
return {
transaction,
error,
error: {
...errorFromFields,
processor: {
name: errorFromFields.processor.name as 'error',
event: errorFromFields.processor.event as 'error',
},
error: {
...errorFromFields.error,
exception:
(source?.error.exception?.length ?? 0) > 1
? source?.error.exception
: errorFromFields?.error.exception && [errorFromFields.error.exception],
log: source?.error?.log,
},
},
};
}

9
x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts
View file

@ -7,6 +7,7 @@
import { jsonRt, toNumberRt } from '@kbn/io-ts-utils';
import * as t from 'io-ts';
import { notFound } from '@hapi/boom';
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
import { ErrorDistributionResponse, getErrorDistribution } from './distribution/get_distribution';
import { environmentRt, kueryRt, rangeRt } from '../default_api_types';
@ -205,7 +206,7 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({
const { serviceName, errorId } = params.path;
const { environment, kuery, start, end } = params.query;
return getErrorSampleDetails({
const { transaction, error } = await getErrorSampleDetails({
environment,
errorId,
kuery,
@ -214,6 +215,12 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({
start,
end,
});
if (!error) {
throw notFound();
}
return { error, transaction };
},
});

66
x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts
View file

@ -8,6 +8,8 @@
import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types';
import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../../../common/utils/as_mutable_array';
import {
ERROR_CULPRIT,
ERROR_TYPE,
@ -19,6 +21,7 @@ import {
SERVICE_NAME,
TRANSACTION_NAME,
TRANSACTION_TYPE,
AT_TIMESTAMP,
} from '../../../../../common/es_fields/apm';
import { environmentQuery } from '../../../../../common/utils/environment_query';
import { getErrorName } from '../../../../lib/helpers/get_error_name';
@ -68,6 +71,16 @@ export async function getMobileCrashGroupMainStatistics({
? { [maxTimestampAggKey]: sortDirection }
: { _count: sortDirection };
const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const);
const optionalFields = asMutableArray([
ERROR_CULPRIT,
ERROR_LOG_MESSAGE,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
] as const);
const response = await apmEventClient.search('get_crash_group_main_statistics', {
apm: {
events: [ProcessorEvent.error],
@ -99,22 +112,15 @@ export async function getMobileCrashGroupMainStatistics({
sample: {
top_hits: {
size: 1,
_source: [
ERROR_LOG_MESSAGE,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
ERROR_CULPRIT,
ERROR_GROUP_ID,
'@timestamp',
],
fields: [...requiredFields, ...optionalFields],
_source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE],
sort: {
'@timestamp': 'desc',
[AT_TIMESTAMP]: 'desc',
},
},
},
...(sortByLatestOccurrence
? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } }
? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } }
: {}),
},
},
@ -123,14 +129,34 @@ export async function getMobileCrashGroupMainStatistics({
});
return (
response.aggregations?.crash_groups.buckets.map((bucket) => ({
groupId: bucket.key as string,
name: getErrorName(bucket.sample.hits.hits[0]._source),
lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(),
occurrences: bucket.doc_count,
culprit: bucket.sample.hits.hits[0]?._source.error.culprit,
handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled,
type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type,
})) ?? []
response.aggregations?.crash_groups.buckets.map((bucket) => {
const errorSource =
'error' in bucket.sample.hits.hits[0]._source
? bucket.sample.hits.hits[0]._source
: undefined;
const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields);
const mergedEvent = {
...event,
error: {
...(event.error ?? {}),
exception:
(errorSource?.error.exception?.length ?? 0) > 1
? errorSource?.error.exception
: event?.error.exception && [event.error.exception],
},
};
return {
groupId: event.error?.grouping_key,
name: getErrorName(mergedEvent),
lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(),
occurrences: bucket.doc_count,
culprit: mergedEvent.error.culprit,
handled: mergedEvent.error.exception?.[0].handled,
type: mergedEvent.error.exception?.[0].type,
};
}) ?? []
);
}

66
x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts
View file

@ -8,7 +8,10 @@
import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types';
import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import {
AT_TIMESTAMP,
ERROR_CULPRIT,
ERROR_EXC_HANDLED,
ERROR_EXC_MESSAGE,
@ -67,6 +70,16 @@ export async function getMobileErrorGroupMainStatistics({
? { [maxTimestampAggKey]: sortDirection }
: { _count: sortDirection };
const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const);
const optionalFields = asMutableArray([
ERROR_CULPRIT,
ERROR_LOG_MESSAGE,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
] as const);
const response = await apmEventClient.search('get_error_group_main_statistics', {
apm: {
events: [ProcessorEvent.error],
@ -100,22 +113,15 @@ export async function getMobileErrorGroupMainStatistics({
sample: {
top_hits: {
size: 1,
_source: [
ERROR_LOG_MESSAGE,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
ERROR_CULPRIT,
ERROR_GROUP_ID,
'@timestamp',
],
fields: [...requiredFields, ...optionalFields],
_source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE],
sort: {
'@timestamp': 'desc',
[AT_TIMESTAMP]: 'desc',
},
},
},
...(sortByLatestOccurrence
? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } }
? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } }
: {}),
},
},
@ -124,14 +130,34 @@ export async function getMobileErrorGroupMainStatistics({
});
return (
response.aggregations?.error_groups.buckets.map((bucket) => ({
groupId: bucket.key as string,
name: getErrorName(bucket.sample.hits.hits[0]._source),
lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(),
occurrences: bucket.doc_count,
culprit: bucket.sample.hits.hits[0]?._source.error.culprit,
handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled,
type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type,
})) ?? []
response.aggregations?.error_groups.buckets.map((bucket) => {
const errorSource =
'error' in bucket.sample.hits.hits[0]._source
? bucket.sample.hits.hits[0]._source
: undefined;
const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields);
const mergedEvent = {
...event,
error: {
...(event.error ?? {}),
exception:
(errorSource?.error.exception?.length ?? 0) > 1
? errorSource?.error.exception
: event?.error.exception && [event.error.exception],
},
};
return {
groupId: event.error?.grouping_key,
name: getErrorName(mergedEvent),
lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(),
occurrences: bucket.doc_count,
culprit: mergedEvent.error.culprit,
handled: mergedEvent.error.exception?.[0].handled,
type: mergedEvent.error.exception?.[0].type,
};
}) ?? []
);
}

21
x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts
View file

@ -7,9 +7,12 @@
import type { ESFilter } from '@kbn/es-types';
import { rangeQuery } from '@kbn/observability-plugin/server';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { maybe } from '../../../../common/utils/maybe';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { isFiniteNumber } from '../../../../common/utils/is_finite_number';
import { Annotation, AnnotationType } from '../../../../common/annotations';
import { SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm';
import { AT_TIMESTAMP, SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm';
import { environmentQuery } from '../../../../common/utils/environment_query';
import {
getBackwardCompatibleDocumentTypeFilter,
@ -66,6 +69,8 @@ export async function getDerivedServiceAnnotations({
if (versions.length <= 1) {
return [];
}
const requiredFields = asMutableArray([AT_TIMESTAMP] as const);
const annotations = await Promise.all(
versions.map(async (version) => {
const response = await apmEventClient.search('get_first_seen_of_version', {
@ -83,11 +88,21 @@ export async function getDerivedServiceAnnotations({
sort: {
'@timestamp': 'asc',
},
fields: requiredFields,
},
});
const firstSeen = new Date(response.hits.hits[0]._source['@timestamp']).getTime();
const event = unflattenKnownApmEventFields(
maybe(response.hits.hits[0])?.fields,
requiredFields
);
const timestamp = event?.[AT_TIMESTAMP];
if (!timestamp) {
throw new Error('First seen for version was unexpectedly undefined or null.');
}
const firstSeen = new Date(timestamp).getTime();
if (!isFiniteNumber(firstSeen)) {
throw new Error('First seen for version was unexpectedly undefined or null.');
}
@ -99,7 +114,7 @@ export async function getDerivedServiceAnnotations({
return {
type: AnnotationType.VERSION,
id: version,
'@timestamp': firstSeen,
[AT_TIMESTAMP]: firstSeen,
text: version,
};
})

35
x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts
View file

@ -7,6 +7,8 @@
import { rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import {
AGENT_NAME,
SERVICE_NAME,
@ -16,23 +18,7 @@ import {
} from '../../../common/es_fields/apm';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import { getServerlessTypeFromCloudData, ServerlessType } from '../../../common/serverless';
interface ServiceAgent {
agent?: {
name: string;
};
service?: {
runtime?: {
name?: string;
};
};
cloud?: {
provider?: string;
service?: {
name?: string;
};
};
}
import { maybe } from '../../../common/utils/maybe';
export interface ServiceAgentResponse {
agentName?: string;
@ -51,6 +37,13 @@ export async function getServiceAgent({
start: number;
end: number;
}): Promise<ServiceAgentResponse> {
const fields = asMutableArray([
AGENT_NAME,
SERVICE_RUNTIME_NAME,
CLOUD_PROVIDER,
CLOUD_SERVICE_NAME,
] as const);
const params = {
terminate_after: 1,
apm: {
@ -90,6 +83,7 @@ export async function getServiceAgent({
],
},
},
fields,
sort: {
_score: { order: 'desc' as const },
},
@ -97,11 +91,14 @@ export async function getServiceAgent({
};
const response = await apmEventClient.search('get_service_agent_name', params);
if (response.hits.total.value === 0) {
const hit = maybe(response.hits.hits[0]);
if (!hit) {
return {};
}
const { agent, service, cloud } = response.hits.hits[0]._source as ServiceAgent;
const event = unflattenKnownApmEventFields(hit.fields);
const { agent, service, cloud } = event;
const serverlessType = getServerlessTypeFromCloudData(cloud?.provider, cloud?.service?.name);
return {

23
x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts
View file

@ -6,19 +6,20 @@
*/
import { rangeQuery } from '@kbn/observability-plugin/server';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import {
CONTAINER_ID,
CONTAINER_IMAGE,
KUBERNETES,
KUBERNETES_POD_NAME,
KUBERNETES_POD_UID,
} from '../../../common/es_fields/apm';
import {
KUBERNETES_CONTAINER_NAME,
KUBERNETES_NAMESPACE,
KUBERNETES_REPLICASET_NAME,
KUBERNETES_DEPLOYMENT_NAME,
} from '../../../common/es_fields/infra_metrics';
KUBERNETES_CONTAINER_ID,
KUBERNETES_NAMESPACE,
} from '../../../common/es_fields/apm';
import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes';
import { maybe } from '../../../common/utils/maybe';
import { InfraMetricsClient } from '../../lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client';
@ -51,9 +52,21 @@ export const getServiceInstanceContainerMetadata = async ({
{ exists: { field: KUBERNETES_DEPLOYMENT_NAME } },
];
const fields = asMutableArray([
KUBERNETES_POD_NAME,
KUBERNETES_POD_UID,
KUBERNETES_DEPLOYMENT_NAME,
KUBERNETES_CONTAINER_ID,
KUBERNETES_CONTAINER_NAME,
KUBERNETES_NAMESPACE,
KUBERNETES_REPLICASET_NAME,
KUBERNETES_DEPLOYMENT_NAME,
] as const);
const response = await infraMetricsClient.search({
size: 1,
track_total_hits: false,
fields,
query: {
bool: {
filter: [
@ -69,7 +82,7 @@ export const getServiceInstanceContainerMetadata = async ({
},
});
const sample = maybe(response.hits.hits[0])?._source as ServiceInstanceContainerMetadataDetails;
const sample = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields);
return {
kubernetes: {

44
x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts
View file

@ -7,7 +7,16 @@
import { merge } from 'lodash';
import { rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields';
import {
AGENT_NAME,
AT_TIMESTAMP,
METRICSET_NAME,
SERVICE_ENVIRONMENT,
SERVICE_NAME,
SERVICE_NODE_NAME,
} from '../../../common/es_fields/apm';
import { maybe } from '../../../common/utils/maybe';
import {
getBackwardCompatibleDocumentTypeFilter,
@ -20,6 +29,13 @@ import { Container } from '../../../typings/es_schemas/raw/fields/container';
import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes';
import { Host } from '../../../typings/es_schemas/raw/fields/host';
import { Cloud } from '../../../typings/es_schemas/raw/fields/cloud';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import {
SERVICE_METADATA_CLOUD_KEYS,
SERVICE_METADATA_CONTAINER_KEYS,
SERVICE_METADATA_INFRA_METRICS_KEYS,
SERVICE_METADATA_SERVICE_KEYS,
} from '../../../common/service_metadata';
export interface ServiceInstanceMetadataDetailsResponse {
'@timestamp': string;
@ -50,6 +66,18 @@ export async function getServiceInstanceMetadataDetails({
...rangeQuery(start, end),
];
const requiredKeys = asMutableArray([AT_TIMESTAMP, SERVICE_NAME, AGENT_NAME] as const);
const optionalKeys = asMutableArray([
SERVICE_ENVIRONMENT,
...SERVICE_METADATA_SERVICE_KEYS,
...SERVICE_METADATA_CLOUD_KEYS,
...SERVICE_METADATA_CONTAINER_KEYS,
...SERVICE_METADATA_INFRA_METRICS_KEYS,
] as const);
const fields = [...requiredKeys, ...optionalKeys];
async function getApplicationMetricSample() {
const response = await apmEventClient.search(
'get_service_instance_metadata_details_application_metric',
@ -66,11 +94,12 @@ export async function getServiceInstanceMetadataDetails({
filter: filter.concat({ term: { [METRICSET_NAME]: 'app' } }),
},
},
fields,
},
}
);
return maybe(response.hits.hits[0]?._source);
return unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredKeys);
}
async function getTransactionEventSample() {
@ -85,11 +114,14 @@ export async function getServiceInstanceMetadataDetails({
terminate_after: 1,
size: 1,
query: { bool: { filter } },
fields,
},
}
);
return maybe(response.hits.hits[0]?._source);
return unflattenKnownApmEventFields(
maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent
);
}
async function getTransactionMetricSample() {
@ -108,10 +140,14 @@ export async function getServiceInstanceMetadataDetails({
filter: filter.concat(getBackwardCompatibleDocumentTypeFilter(true)),
},
},
fields,
},
}
);
return maybe(response.hits.hits[0]?._source);
return unflattenKnownApmEventFields(
maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent
);
}
// we can expect the most detail of application metrics,

28
x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts
View file

@ -7,37 +7,26 @@
import { rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields';
import { environmentQuery } from '../../../common/utils/environment_query';
import {
AGENT,
CONTAINER,
CLOUD,
CLOUD_AVAILABILITY_ZONE,
CLOUD_REGION,
CLOUD_MACHINE_TYPE,
CLOUD_SERVICE_NAME,
CONTAINER_ID,
HOST,
KUBERNETES,
SERVICE,
SERVICE_NAME,
SERVICE_NODE_NAME,
SERVICE_VERSION,
FAAS_ID,
FAAS_TRIGGER_TYPE,
LABEL_TELEMETRY_AUTO_VERSION,
} from '../../../common/es_fields/apm';
import { ContainerType } from '../../../common/service_metadata';
import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import { should } from './get_service_metadata_icons';
import { isOpenTelemetryAgentName, hasOpenTelemetryPrefix } from '../../../common/agent_name';
type ServiceMetadataDetailsRaw = Pick<
TransactionRaw,
'service' | 'agent' | 'host' | 'container' | 'kubernetes' | 'cloud' | 'labels'
>;
import { maybe } from '../../../common/utils/maybe';
export interface ServiceMetadataDetails {
service?: {
@ -112,7 +101,6 @@ export async function getServiceMetadataDetails({
body: {
track_total_hits: 1,
size: 1,
_source: [SERVICE, AGENT, HOST, CONTAINER, KUBERNETES, CLOUD, LABEL_TELEMETRY_AUTO_VERSION],
query: { bool: { filter, should } },
aggs: {
serviceVersions: {
@ -166,13 +154,17 @@ export async function getServiceMetadataDetails({
},
totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } },
},
fields: ['*'],
},
};
const response = await apmEventClient.search('get_service_metadata_details', params);
const hit = response.hits.hits[0]?._source as ServiceMetadataDetailsRaw | undefined;
if (!hit) {
const event = unflattenKnownApmEventFields(
maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent
);
if (!event) {
return {
service: undefined,
container: undefined,
@ -180,7 +172,7 @@ export async function getServiceMetadataDetails({
};
}
const { service, agent, host, kubernetes, container, cloud, labels } = hit;
const { service, agent, host, kubernetes, container, cloud, labels } = event;
const serviceMetadataDetails = {
versions: response.aggregations?.serviceVersions.buckets.map((bucket) => bucket.key as string),

26
x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts
View file

@ -7,12 +7,15 @@
import { rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import type { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields';
import { maybe } from '../../../common/utils/maybe';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import {
AGENT_NAME,
CLOUD_PROVIDER,
CLOUD_SERVICE_NAME,
CONTAINER_ID,
KUBERNETES,
SERVICE_NAME,
KUBERNETES_POD_NAME,
HOST_OS_PLATFORM,
@ -20,14 +23,11 @@ import {
AGENT_VERSION,
SERVICE_FRAMEWORK_NAME,
} from '../../../common/es_fields/apm';
import { ContainerType } from '../../../common/service_metadata';
import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
import { ContainerType, SERVICE_METADATA_KUBERNETES_KEYS } from '../../../common/service_metadata';
import { getProcessorEventForTransactions } from '../../lib/helpers/transactions';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless';
type ServiceMetadataIconsRaw = Pick<TransactionRaw, 'kubernetes' | 'cloud' | 'container' | 'agent'>;
export interface ServiceMetadataIcons {
agentName?: string;
containerType?: ContainerType;
@ -61,6 +61,14 @@ export async function getServiceMetadataIcons({
}): Promise<ServiceMetadataIcons> {
const filter = [{ term: { [SERVICE_NAME]: serviceName } }, ...rangeQuery(start, end)];
const fields = asMutableArray([
CLOUD_PROVIDER,
CONTAINER_ID,
AGENT_NAME,
CLOUD_SERVICE_NAME,
...SERVICE_METADATA_KUBERNETES_KEYS,
] as const);
const params = {
apm: {
events: [
@ -72,8 +80,8 @@ export async function getServiceMetadataIcons({
body: {
track_total_hits: 1,
size: 1,
_source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, CLOUD_SERVICE_NAME],
query: { bool: { filter, should } },
fields,
},
};
@ -88,9 +96,11 @@ export async function getServiceMetadataIcons({
};
}
const { kubernetes, cloud, container, agent } = response.hits.hits[0]
._source as ServiceMetadataIconsRaw;
const event = unflattenKnownApmEventFields(
maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent
);
const { kubernetes, cloud, container, agent } = event ?? {};
let containerType: ContainerType;
if (!!kubernetes) {
containerType = 'Kubernetes';

46
x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts
View file

@ -7,6 +7,8 @@
import { rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { isEmpty } from 'lodash';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import {
PROCESSOR_EVENT,
SPAN_ID,
@ -16,8 +18,6 @@ import {
TRACE_ID,
TRANSACTION_ID,
} from '../../../common/es_fields/apm';
import type { SpanRaw } from '../../../typings/es_schemas/raw/span_raw';
import type { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
import { getBufferedTimerange } from './utils';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
@ -39,12 +39,16 @@ async function fetchLinkedChildrenOfSpan({
end,
});
const requiredFields = asMutableArray([TRACE_ID, PROCESSOR_EVENT] as const);
const optionalFields = asMutableArray([SPAN_ID, TRANSACTION_ID] as const);
const response = await apmEventClient.search('fetch_linked_children_of_span', {
apm: {
events: [ProcessorEvent.span, ProcessorEvent.transaction],
},
_source: [SPAN_LINKS, TRACE_ID, SPAN_ID, PROCESSOR_EVENT, TRANSACTION_ID],
_source: [SPAN_LINKS],
body: {
fields: [...requiredFields, ...optionalFields],
track_total_hits: false,
size: 1000,
query: {
@ -58,19 +62,32 @@ async function fetchLinkedChildrenOfSpan({
},
},
});
const linkedChildren = response.hits.hits.map((hit) => {
const source = 'span' in hit._source ? hit._source : undefined;
const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
return {
...event,
span: {
...event.span,
links: source?.span?.links ?? [],
},
};
});
// Filter out documents that don't have any span.links that match the combination of traceId and spanId
return response.hits.hits.filter(({ _source: source }) => {
const spanLinks = source.span?.links?.filter((spanLink) => {
return linkedChildren.filter((linkedChild) => {
const spanLinks = linkedChild?.span?.links?.filter((spanLink) => {
return spanLink.trace.id === traceId && (spanId ? spanLink.span.id === spanId : true);
});
return !isEmpty(spanLinks);
});
}
function getSpanId(source: TransactionRaw | SpanRaw) {
return source.processor.event === ProcessorEvent.span
? (source as SpanRaw).span.id
: (source as TransactionRaw).transaction?.id;
function getSpanId(
linkedChild: Awaited<ReturnType<typeof fetchLinkedChildrenOfSpan>>[number]
): string {
return (linkedChild.span.id ?? linkedChild.transaction?.id) as string;
}
export async function getSpanLinksCountById({
@ -90,8 +107,9 @@ export async function getSpanLinksCountById({
start,
end,
});
return linkedChildren.reduce<Record<string, number>>((acc, { _source: source }) => {
source.span?.links?.forEach((link) => {
return linkedChildren.reduce<Record<string, number>>((acc, item) => {
item.span?.links?.forEach((link) => {
// Ignores span links that don't belong to this trace
if (link.trace.id === traceId) {
acc[link.span.id] = (acc[link.span.id] || 0) + 1;
@ -122,10 +140,10 @@ export async function getLinkedChildrenOfSpan({
end,
});
return linkedChildren.map(({ _source: source }) => {
return linkedChildren.map((item) => {
return {
trace: { id: source.trace.id },
span: { id: getSpanId(source) },
trace: { id: item.trace.id },
span: { id: getSpanId(item) },
};
});
}

2
x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts
View file

@ -56,7 +56,7 @@ export async function getLinkedParentsOfSpan({
},
});
const source = response.hits.hits?.[0]?._source as TransactionRaw | SpanRaw;
const source = response.hits.hits?.[0]?._source as Pick<TransactionRaw | SpanRaw, 'span'>;
return source?.span?.links || [];
}

153
x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts
View file

@ -7,6 +7,8 @@
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { chunk, compact, isEmpty, keyBy } from 'lodash';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import {
SERVICE_NAME,
SPAN_ID,
@ -25,8 +27,6 @@ import {
import { Environment } from '../../../common/environment_rt';
import { SpanLinkDetails } from '../../../common/span_links';
import { SpanLink } from '../../../typings/es_schemas/raw/fields/span_links';
import { SpanRaw } from '../../../typings/es_schemas/raw/span_raw';
import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
import { getBufferedTimerange } from './utils';
import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
@ -48,26 +48,35 @@ async function fetchSpanLinksDetails({
end,
});
const requiredFields = asMutableArray([
TRACE_ID,
SERVICE_NAME,
AGENT_NAME,
PROCESSOR_EVENT,
] as const);
const requiredTxFields = asMutableArray([
TRANSACTION_ID,
TRANSACTION_NAME,
TRANSACTION_DURATION,
] as const);
const requiredSpanFields = asMutableArray([
SPAN_ID,
SPAN_NAME,
SPAN_DURATION,
SPAN_SUBTYPE,
SPAN_TYPE,
] as const);
const optionalFields = asMutableArray([SERVICE_ENVIRONMENT] as const);
const response = await apmEventClient.search('get_span_links_details', {
apm: {
events: [ProcessorEvent.span, ProcessorEvent.transaction],
},
_source: [
TRACE_ID,
SPAN_ID,
TRANSACTION_ID,
SERVICE_NAME,
SPAN_NAME,
TRANSACTION_NAME,
TRANSACTION_DURATION,
SPAN_DURATION,
PROCESSOR_EVENT,
SPAN_SUBTYPE,
SPAN_TYPE,
AGENT_NAME,
SERVICE_ENVIRONMENT,
],
body: {
fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields],
track_total_hits: false,
size: 1000,
query: {
@ -106,16 +115,67 @@ async function fetchSpanLinksDetails({
const spanIdsMap = keyBy(spanLinks, 'span.id');
return response.hits.hits.filter(({ _source: source }) => {
// The above query might return other spans from the same transaction because siblings spans share the same transaction.id
// so, if it is a span we need to guarantee that the span.id is the same as the span links ids
if (source.processor.event === ProcessorEvent.span) {
const span = source as SpanRaw;
const hasSpanId = spanIdsMap[span.span.id] || false;
return hasSpanId;
}
return true;
});
return response.hits.hits
.filter((hit) => {
// The above query might return other spans from the same transaction because siblings spans share the same transaction.id
// so, if it is a span we need to guarantee that the span.id is the same as the span links ids
if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) {
const spanLink = unflattenKnownApmEventFields(hit.fields, [
...requiredFields,
...requiredSpanFields,
]);
const hasSpanId = Boolean(spanIdsMap[spanLink.span.id] || false);
return hasSpanId;
}
return true;
})
.map((hit) => {
const commonEvent = unflattenKnownApmEventFields(hit.fields, requiredFields);
const commonDetails = {
serviceName: commonEvent.service.name,
agentName: commonEvent.agent.name,
environment: commonEvent.service.environment as Environment,
transactionId: commonEvent.transaction?.id,
};
if (commonEvent.processor.event === ProcessorEvent.transaction) {
const event = unflattenKnownApmEventFields(hit.fields, [
...requiredFields,
...requiredTxFields,
]);
return {
traceId: event.trace.id,
spanId: event.transaction.id,
processorEvent: commonEvent.processor.event,
transactionId: event.transaction.id,
details: {
...commonDetails,
spanName: event.transaction.name,
duration: event.transaction.duration.us,
},
};
} else {
const event = unflattenKnownApmEventFields(hit.fields, [
...requiredFields,
...requiredSpanFields,
]);
return {
traceId: event.trace.id,
spanId: event.span.id,
processorEvent: commonEvent.processor.event,
details: {
...commonDetails,
spanName: event.span.name,
duration: event.span.duration.us,
spanSubtype: event.span.subtype,
spanType: event.span.type,
},
};
}
});
}
export async function getSpanLinksDetails({
@ -153,39 +213,20 @@ export async function getSpanLinksDetails({
// Creates a map for all span links details found
const spanLinksDetailsMap = linkedSpans.reduce<Record<string, SpanLinkDetails>>(
(acc, { _source: source }) => {
const commonDetails = {
serviceName: source.service.name,
agentName: source.agent.name,
environment: source.service.environment as Environment,
transactionId: source.transaction?.id,
};
if (source.processor.event === ProcessorEvent.transaction) {
const transaction = source as TransactionRaw;
const key = `${transaction.trace.id}:${transaction.transaction.id}`;
(acc, spanLink) => {
if (spanLink.processorEvent === ProcessorEvent.transaction) {
const key = `${spanLink.traceId}:${spanLink.transactionId}`;
acc[key] = {
traceId: source.trace.id,
spanId: transaction.transaction.id,
details: {
...commonDetails,
spanName: transaction.transaction.name,
duration: transaction.transaction.duration.us,
},
traceId: spanLink.traceId,
spanId: spanLink.transactionId,
details: spanLink.details,
};
} else {
const span = source as SpanRaw;
const key = `${span.trace.id}:${span.span.id}`;
const key = `${spanLink.traceId}:${spanLink.spanId}`;
acc[key] = {
traceId: source.trace.id,
spanId: span.span.id,
details: {
...commonDetails,
spanName: span.span.name,
duration: span.span.duration.us,
spanSubtype: span.span.subtype,
spanType: span.span.type,
},
traceId: spanLink.traceId,
spanId: spanLink.spanId,
details: spanLink.details,
};
}

19
x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap generated
View file

@ -12,15 +12,26 @@ Object {
},
"body": Object {
"_source": Array [
"error.log.message",
"error.exception.message",
"error.exception.handled",
"error.exception.type",
],
"fields": Array [
"timestamp.us",
"trace.id",
"transaction.id",
"parent.id",
"service.name",
"error.id",
"error.log.message",
"error.exception",
"error.grouping_key",
"processor.event",
"parent.id",
"transaction.id",
"span.id",
"error.culprit",
"error.log.message",
"error.exception.message",
"error.exception.handled",
"error.exception.type",
],
"query": Object {
"bool": Object {

190
x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts
View file

@ -10,12 +10,17 @@ import { SortResults } from '@elastic/elasticsearch/lib/api/types';
import { QueryDslQueryContainer, Sort } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { rangeQuery } from '@kbn/observability-plugin/server';
import { last } from 'lodash';
import { last, omit } from 'lodash';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../common/utils/as_mutable_array';
import { APMConfig } from '../..';
import {
AGENT_NAME,
CHILD_ID,
ERROR_EXCEPTION,
ERROR_CULPRIT,
ERROR_EXC_HANDLED,
ERROR_EXC_MESSAGE,
ERROR_EXC_TYPE,
ERROR_GROUP_ID,
ERROR_ID,
ERROR_LOG_LEVEL,
@ -37,7 +42,7 @@ import {
SPAN_SUBTYPE,
SPAN_SYNC,
SPAN_TYPE,
TIMESTAMP,
TIMESTAMP_US,
TRACE_ID,
TRANSACTION_DURATION,
TRANSACTION_ID,
@ -84,6 +89,26 @@ export async function getTraceItems({
const maxTraceItems = maxTraceItemsFromUrlParam ?? config.ui.maxTraceItems;
const excludedLogLevels = ['debug', 'info', 'warning'];
const requiredFields = asMutableArray([
TIMESTAMP_US,
TRACE_ID,
SERVICE_NAME,
ERROR_ID,
ERROR_GROUP_ID,
PROCESSOR_EVENT,
] as const);
const optionalFields = asMutableArray([
PARENT_ID,
TRANSACTION_ID,
SPAN_ID,
ERROR_CULPRIT,
ERROR_LOG_MESSAGE,
ERROR_EXC_MESSAGE,
ERROR_EXC_HANDLED,
ERROR_EXC_TYPE,
] as const);
const errorResponsePromise = apmEventClient.search('get_errors_docs', {
apm: {
sources: [
@ -96,23 +121,14 @@ export async function getTraceItems({
body: {
track_total_hits: false,
size: 1000,
_source: [
TIMESTAMP,
TRACE_ID,
TRANSACTION_ID,
PARENT_ID,
SERVICE_NAME,
ERROR_ID,
ERROR_LOG_MESSAGE,
ERROR_EXCEPTION,
ERROR_GROUP_ID,
],
query: {
bool: {
filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)],
must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } },
},
},
fields: [...requiredFields, ...optionalFields],
_source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE],
},
});
@ -133,8 +149,32 @@ export async function getTraceItems({
const traceDocsTotal = traceResponse.total;
const exceedsMax = traceDocsTotal > maxTraceItems;
const traceDocs = traceResponse.hits.map((hit) => hit._source);
const errorDocs = errorResponse.hits.hits.map((hit) => hit._source);
const traceDocs = traceResponse.hits.map(({ hit }) => hit);
const errorDocs = errorResponse.hits.hits.map((hit) => {
const errorSource = 'error' in hit._source ? hit._source : undefined;
const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
const waterfallErrorEvent: WaterfallError = {
...event,
parent: {
...event?.parent,
id: event?.parent?.id ?? event?.span?.id,
},
error: {
...(event.error ?? {}),
exception:
(errorSource?.error.exception?.length ?? 0) > 1
? errorSource?.error.exception
: event?.error.exception && [event.error.exception],
log: errorSource?.error.log,
},
};
return waterfallErrorEvent;
});
return {
exceedsMax,
@ -220,41 +260,54 @@ async function getTraceDocsPerPage({
start: number;
end: number;
searchAfter?: SortResults;
}) {
}): Promise<{
hits: Array<{ hit: WaterfallTransaction | WaterfallSpan; sort: SortResults | undefined }>;
total: number;
}> {
const size = Math.min(maxTraceItems, MAX_ITEMS_PER_PAGE);
const requiredFields = asMutableArray([
AGENT_NAME,
TIMESTAMP_US,
TRACE_ID,
SERVICE_NAME,
PROCESSOR_EVENT,
] as const);
const requiredTxFields = asMutableArray([
TRANSACTION_ID,
TRANSACTION_DURATION,
TRANSACTION_NAME,
TRANSACTION_TYPE,
] as const);
const requiredSpanFields = asMutableArray([
SPAN_ID,
SPAN_TYPE,
SPAN_NAME,
SPAN_DURATION,
] as const);
const optionalFields = asMutableArray([
PARENT_ID,
SERVICE_ENVIRONMENT,
EVENT_OUTCOME,
TRANSACTION_RESULT,
FAAS_COLDSTART,
SPAN_SUBTYPE,
SPAN_ACTION,
SPAN_COMPOSITE_COUNT,
SPAN_COMPOSITE_COMPRESSION_STRATEGY,
SPAN_COMPOSITE_SUM,
SPAN_SYNC,
CHILD_ID,
] as const);
const body = {
track_total_hits: true,
size,
search_after: searchAfter,
_source: [
TIMESTAMP,
TRACE_ID,
PARENT_ID,
SERVICE_NAME,
SERVICE_ENVIRONMENT,
AGENT_NAME,
EVENT_OUTCOME,
PROCESSOR_EVENT,
TRANSACTION_DURATION,
TRANSACTION_ID,
TRANSACTION_NAME,
TRANSACTION_TYPE,
TRANSACTION_RESULT,
FAAS_COLDSTART,
SPAN_ID,
SPAN_TYPE,
SPAN_SUBTYPE,
SPAN_ACTION,
SPAN_NAME,
SPAN_DURATION,
SPAN_LINKS,
SPAN_COMPOSITE_COUNT,
SPAN_COMPOSITE_COMPRESSION_STRATEGY,
SPAN_COMPOSITE_SUM,
SPAN_SYNC,
CHILD_ID,
],
_source: [SPAN_LINKS],
query: {
bool: {
filter: [
@ -266,6 +319,7 @@ async function getTraceDocsPerPage({
},
},
},
fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields],
sort: [
{ _score: 'asc' },
{
@ -291,7 +345,51 @@ async function getTraceDocsPerPage({
});
return {
hits: res.hits.hits,
hits: res.hits.hits.map((hit) => {
const sort = hit.sort;
const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined;
if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) {
const spanEvent = unflattenKnownApmEventFields(hit.fields, [
...requiredFields,
...requiredSpanFields,
]);
const spanWaterfallEvent: WaterfallSpan = {
...omit(spanEvent, 'child'),
processor: {
event: 'span',
},
span: {
...spanEvent.span,
composite: spanEvent.span.composite
? (spanEvent.span.composite as Required<WaterfallSpan['span']>['composite'])
: undefined,
links: spanLinksSource,
},
...(spanEvent.child ? { child: spanEvent.child as WaterfallSpan['child'] } : {}),
};
return { sort, hit: spanWaterfallEvent };
}
const txEvent = unflattenKnownApmEventFields(hit.fields, [
...requiredFields,
...requiredTxFields,
]);
const txWaterfallEvent: WaterfallTransaction = {
...txEvent,
processor: {
event: 'transaction',
},
span: {
...txEvent.span,
links: spanLinksSource,
},
};
return { hit: txWaterfallEvent, sort };
}),
total: res.hits.total.value,
};
}

4
x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts
View file

@ -89,10 +89,10 @@ export async function getTraceSamplesByQuery({
},
event_category_field: PROCESSOR_EVENT,
query,
filter_path: 'hits.sequences.events._source.trace.id',
fields: [TRACE_ID],
})
).hits?.sequences?.flatMap((sequence) =>
sequence.events.map((event) => (event._source as { trace: { id: string } }).trace.id)
sequence.events.map((event) => (event.fields as { [TRACE_ID]: [string] })[TRACE_ID][0])
) ?? [];
}

13
x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts
View file

@ -12,7 +12,10 @@ import { getSearchTransactionsEvents } from '../../lib/helpers/transactions';
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
import { environmentRt, kueryRt, probabilityRt, rangeRt } from '../default_api_types';
import { getTransaction } from '../transactions/get_transaction';
import { getRootTransactionByTraceId } from '../transactions/get_transaction_by_trace';
import {
type TransactionDetailRedirectInfo,
getRootTransactionByTraceId,
} from '../transactions/get_transaction_by_trace';
import {
getTopTracesPrimaryStats,
TopTracesPrimaryStatsResponse,
@ -128,7 +131,7 @@ const rootTransactionByTraceIdRoute = createApmServerRoute({
handler: async (
resources
): Promise<{
transaction: Transaction;
transaction?: TransactionDetailRedirectInfo;
}> => {
const {
params: {
@ -155,7 +158,7 @@ const transactionByIdRoute = createApmServerRoute({
handler: async (
resources
): Promise<{
transaction: Transaction;
transaction?: Transaction;
}> => {
const {
params: {
@ -191,7 +194,7 @@ const transactionByNameRoute = createApmServerRoute({
handler: async (
resources
): Promise<{
transaction: Transaction;
transaction?: TransactionDetailRedirectInfo;
}> => {
const {
params: {
@ -295,7 +298,7 @@ const transactionFromTraceByIdRoute = createApmServerRoute({
query: rangeRt,
}),
options: { tags: ['access:apm'] },
handler: async (resources): Promise<Transaction> => {
handler: async (resources): Promise<Transaction | undefined> => {
const { params } = resources;
const {
path: { transactionId, traceId },

24
x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap generated
View file

@ -11,6 +11,25 @@ Object {
],
},
"body": Object {
"_source": Array [
"span.links",
"transaction.agent.marks",
],
"fields": Array [
"trace.id",
"agent.name",
"processor.event",
"@timestamp",
"timestamp.us",
"service.name",
"transaction.id",
"transaction.duration.us",
"transaction.name",
"transaction.sampled",
"transaction.type",
"processor.name",
"service.language.name",
],
"query": Object {
"bool": Object {
"filter": Array [
@ -311,6 +330,11 @@ Object {
],
},
"body": Object {
"fields": Array [
"transaction.id",
"trace.id",
"@timestamp",
],
"query": Object {
"bool": Object {
"filter": Array [

22
x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts
View file

@ -7,7 +7,11 @@
import { rangeQuery, termQuery } from '@kbn/observability-plugin/server';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { SPAN_ID, TRACE_ID } from '../../../../common/es_fields/apm';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields';
import { merge, omit } from 'lodash';
import { maybe } from '../../../../common/utils/maybe';
import { SPAN_ID, SPAN_STACKTRACE, TRACE_ID } from '../../../../common/es_fields/apm';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { getTransaction } from '../get_transaction';
@ -38,6 +42,8 @@ export async function getSpan({
track_total_hits: false,
size: 1,
terminate_after: 1,
fields: ['*'],
_source: [SPAN_STACKTRACE],
query: {
bool: {
filter: asMutableArray([
@ -60,5 +66,17 @@ export async function getSpan({
: undefined,
]);
return { span: spanResp.hits.hits[0]?._source, parentTransaction };
const hit = maybe(spanResp.hits.hits[0]);
const spanFromSource = hit && 'span' in hit._source ? hit._source : undefined;
const event = unflattenKnownApmEventFields(hit?.fields as undefined | FlattenedApmEvent);
return {
span: event
? merge({}, omit(event, 'span.links'), spanFromSource, {
processor: { event: 'span' as const, name: 'transaction' as const },
})
: undefined,
parentTransaction,
};
}

70
x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts
View file

@ -6,7 +6,26 @@
*/
import { rangeQuery, termQuery } from '@kbn/observability-plugin/server';
import { TRACE_ID, TRANSACTION_ID } from '../../../../common/es_fields/apm';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import type { Transaction } from '@kbn/apm-types';
import { maybe } from '../../../../common/utils/maybe';
import {
AGENT_NAME,
PROCESSOR_EVENT,
SERVICE_NAME,
TIMESTAMP_US,
TRACE_ID,
TRANSACTION_DURATION,
TRANSACTION_ID,
TRANSACTION_NAME,
TRANSACTION_SAMPLED,
TRANSACTION_TYPE,
AT_TIMESTAMP,
PROCESSOR_NAME,
SPAN_LINKS,
TRANSACTION_AGENT_MARKS,
SERVICE_LANGUAGE_NAME,
} from '../../../../common/es_fields/apm';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { ApmDocumentType } from '../../../../common/document_type';
@ -24,7 +43,23 @@ export async function getTransaction({
apmEventClient: APMEventClient;
start: number;
end: number;
}) {
}): Promise<Transaction | undefined> {
const requiredFields = asMutableArray([
TRACE_ID,
AGENT_NAME,
PROCESSOR_EVENT,
AT_TIMESTAMP,
TIMESTAMP_US,
SERVICE_NAME,
TRANSACTION_ID,
TRANSACTION_DURATION,
TRANSACTION_NAME,
TRANSACTION_SAMPLED,
TRANSACTION_TYPE,
] as const);
const optionalFields = asMutableArray([PROCESSOR_NAME, SERVICE_LANGUAGE_NAME] as const);
const resp = await apmEventClient.search('get_transaction', {
apm: {
sources: [
@ -47,8 +82,37 @@ export async function getTransaction({
]),
},
},
fields: [...requiredFields, ...optionalFields],
_source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS],
},
});
return resp.hits.hits[0]?._source;
const hit = maybe(resp.hits.hits[0]);
if (!hit) {
return undefined;
}
const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
const source =
'span' in hit._source && 'transaction' in hit._source
? (hit._source as {
transaction: Pick<Required<Transaction>['transaction'], 'marks'>;
span?: Pick<Required<Transaction>['span'], 'links'>;
})
: undefined;
return {
...event,
transaction: {
...event.transaction,
marks: source?.transaction.marks,
},
processor: {
name: 'transaction',
event: 'transaction',
},
span: source?.span,
};
}

28
x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts
View file

@ -6,11 +6,22 @@
*/
import { rangeQuery } from '@kbn/observability-plugin/server';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { maybe } from '../../../../common/utils/maybe';
import { ApmDocumentType } from '../../../../common/document_type';
import { SERVICE_NAME, TRANSACTION_NAME } from '../../../../common/es_fields/apm';
import {
AT_TIMESTAMP,
SERVICE_NAME,
TRACE_ID,
TRANSACTION_DURATION,
TRANSACTION_ID,
TRANSACTION_NAME,
TRANSACTION_TYPE,
} from '../../../../common/es_fields/apm';
import { RollupInterval } from '../../../../common/rollup';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
import { TransactionDetailRedirectInfo } from '../get_transaction_by_trace';
export async function getTransactionByName({
transactionName,
@ -24,7 +35,17 @@ export async function getTransactionByName({
apmEventClient: APMEventClient;
start: number;
end: number;
}) {
}): Promise<TransactionDetailRedirectInfo | undefined> {
const requiredFields = asMutableArray([
AT_TIMESTAMP,
TRACE_ID,
TRANSACTION_ID,
TRANSACTION_TYPE,
TRANSACTION_NAME,
TRANSACTION_DURATION,
SERVICE_NAME,
] as const);
const resp = await apmEventClient.search('get_transaction', {
apm: {
sources: [
@ -47,8 +68,9 @@ export async function getTransactionByName({
]),
},
},
fields: requiredFields,
},
});
return resp.hits.hits[0]?._source;
return unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields);
}

53
x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts
View file

@ -7,9 +7,40 @@
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { rangeQuery } from '@kbn/observability-plugin/server';
import { TRACE_ID, PARENT_ID } from '../../../../common/es_fields/apm';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { maybe } from '../../../../common/utils/maybe';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import {
TRACE_ID,
PARENT_ID,
AT_TIMESTAMP,
TRANSACTION_DURATION,
TRANSACTION_ID,
TRANSACTION_NAME,
TRANSACTION_TYPE,
SERVICE_NAME,
} from '../../../../common/es_fields/apm';
import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
export interface TransactionDetailRedirectInfo {
[AT_TIMESTAMP]: string;
trace: {
id: string;
};
transaction: {
id: string;
type: string;
name: string;
duration: {
us: number;
};
};
service: {
name: string;
};
}
export async function getRootTransactionByTraceId({
traceId,
apmEventClient,
@ -20,7 +51,19 @@ export async function getRootTransactionByTraceId({
apmEventClient: APMEventClient;
start: number;
end: number;
}) {
}): Promise<{
transaction: TransactionDetailRedirectInfo | undefined;
}> {
const requiredFields = asMutableArray([
TRACE_ID,
TRANSACTION_ID,
TRANSACTION_NAME,
AT_TIMESTAMP,
TRANSACTION_TYPE,
TRANSACTION_DURATION,
SERVICE_NAME,
] as const);
const params = {
apm: {
events: [ProcessorEvent.transaction as const],
@ -45,11 +88,15 @@ export async function getRootTransactionByTraceId({
filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)],
},
},
fields: requiredFields,
},
};
const resp = await apmEventClient.search('get_root_transaction_by_trace_id', params);
const event = unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields);
return {
transaction: resp.hits.hits[0]?._source,
transaction: event,
};
}

23
x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts
View file

@ -7,7 +7,10 @@
import { Sort, QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
import { asMutableArray } from '../../../../common/utils/as_mutable_array';
import {
AT_TIMESTAMP,
SERVICE_NAME,
TRACE_ID,
TRANSACTION_ID,
@ -77,6 +80,8 @@ export async function getTraceSamples({
});
}
const requiredFields = asMutableArray([TRANSACTION_ID, TRACE_ID, AT_TIMESTAMP] as const);
const response = await apmEventClient.search('get_trace_samples_hits', {
apm: {
events: [ProcessorEvent.transaction],
@ -94,6 +99,7 @@ export async function getTraceSamples({
},
},
size: TRACE_SAMPLES_SIZE,
fields: requiredFields,
sort: [
{
_score: {
@ -101,7 +107,7 @@ export async function getTraceSamples({
},
},
{
'@timestamp': {
[AT_TIMESTAMP]: {
order: 'desc',
},
},
@ -109,12 +115,15 @@ export async function getTraceSamples({
},
});
const traceSamples = response.hits.hits.map((hit) => ({
score: hit._score,
timestamp: hit._source['@timestamp'],
transactionId: hit._source.transaction.id,
traceId: hit._source.trace.id,
}));
const traceSamples = response.hits.hits.map((hit) => {
const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
return {
score: hit._score,
timestamp: event[AT_TIMESTAMP],
transactionId: event.transaction.id,
traceId: event.trace.id,
};
});
return { traceSamples };
});

1
x-pack/plugins/observability_solution/apm_data_access/server/utils.ts
View file

@ -15,3 +15,4 @@ export {
} from './lib/helpers';
export { withApmSpan } from './utils/with_apm_span';
export { unflattenKnownApmEventFields } from './utils/unflatten_known_fields';

137
x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts Normal file
View file

@ -0,0 +1,137 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { unflattenKnownApmEventFields } from './unflatten_known_fields';
describe('unflattenKnownApmEventFields', () => {
it('should return an empty object when input is empty', () => {
const input = {};
const expectedOutput = {};
expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
});
it('should correctly unflatten a simple flat input', () => {
const input = {
'@timestamp': '2024-10-10T10:10:10.000Z',
};
const expectedOutput = {
'@timestamp': '2024-10-10T10:10:10.000Z',
};
expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
});
it('should override unknown fields', () => {
const input = {
'service.name': 'node-svc',
'service.name.text': 'node-svc',
};
const expectedOutput = {
service: {
name: 'node-svc',
},
};
expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
});
it('should correctly unflatten multiple nested fields', () => {
const input = {
'service.name': 'node-svc',
'service.version': '1.0.0',
'service.environment': 'production',
'agent.name': 'nodejs',
};
const expectedOutput = {
service: {
name: 'node-svc',
version: '1.0.0',
environment: 'production',
},
agent: {
name: 'nodejs',
},
};
expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
});
it('should handle multiple values for multi-valued fields', () => {
const input = {
'service.name': 'node-svc',
'service.tags': ['foo', 'bar'],
};
const expectedOutput = {
service: {
name: 'node-svc',
tags: ['foo', 'bar'],
},
};
expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
});
it('should correctly unflatten with empty multi-valued fields', () => {
const input = {
'service.name': 'node-svc',
'service.tags': [],
};
const expectedOutput = {
service: {
name: 'node-svc',
tags: [],
},
};
expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
});
it('should retain unknown fields in the output', () => {
const input = {
'service.name': 'node-svc',
'unknown.texts': ['foo', 'bar'],
'unknown.field': 'foo',
unknonwField: 'bar',
};
const expectedOutput = {
service: {
name: 'node-svc',
},
unknown: {
field: 'foo',
texts: ['foo', 'bar'],
},
unknonwField: 'bar',
};
expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
});
it('should correctly unflatten nested fields with mandatory field', () => {
const input = {
'service.name': 'node-svc',
'service.environment': undefined,
};
const requiredFields: ['service.name'] = ['service.name'];
const expectedOutput = {
service: {
name: 'node-svc',
},
};
expect(unflattenKnownApmEventFields(input, requiredFields)).toEqual(expectedOutput);
});
it('should throw an exception when mandatory field is not in the input', () => {
const input = {
'service.environment': 'PROD',
};
const requiredFields: ['service.name'] = ['service.name'];
// @ts-expect-error
expect(() => unflattenKnownApmEventFields(input, requiredFields)).toThrowError(
'Missing required fields service.name in event'
);
});
});

168
x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts Normal file
View file

@ -0,0 +1,168 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { DedotObject } from '@kbn/utility-types';
import * as APM_EVENT_FIELDS_MAP from '@kbn/apm-types/es_fields';
import type { ValuesType } from 'utility-types';
import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object';
import { mergePlainObjects } from '@kbn/observability-utils/object/merge_plain_objects';
import { castArray, isArray } from 'lodash';
import { AgentName } from '@kbn/elastic-agent-utils';
import { EventOutcome } from '@kbn/apm-types/src/es_schemas/raw/fields';
import { ProcessorEvent } from '@kbn/observability-plugin/common';
const {
CLOUD,
AGENT,
SERVICE,
ERROR_EXCEPTION,
SPAN_LINKS,
HOST,
KUBERNETES,
CONTAINER,
TIER,
INDEX,
DATA_STEAM_TYPE,
VALUE_OTEL_JVM_PROCESS_MEMORY_HEAP,
VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP,
SPAN_LINKS_SPAN_ID,
SPAN_LINKS_TRACE_ID,
SPAN_STACKTRACE,
...CONCRETE_FIELDS
} = APM_EVENT_FIELDS_MAP;
const ALL_FIELDS = Object.values(CONCRETE_FIELDS);
const KNOWN_MULTI_VALUED_FIELDS = [
APM_EVENT_FIELDS_MAP.CHILD_ID,
APM_EVENT_FIELDS_MAP.PROCESS_ARGS,
] as const;
type KnownField = ValuesType<typeof CONCRETE_FIELDS>;
type KnownSingleValuedField = Exclude<KnownField, KnownMultiValuedField>;
type KnownMultiValuedField = ValuesType<typeof KNOWN_MULTI_VALUED_FIELDS>;
const KNOWN_SINGLE_VALUED_FIELDS = ALL_FIELDS.filter(
(field): field is KnownSingleValuedField => !KNOWN_MULTI_VALUED_FIELDS.includes(field as any)
);
interface TypeOverrideMap {
[APM_EVENT_FIELDS_MAP.SPAN_DURATION]: number;
[APM_EVENT_FIELDS_MAP.AGENT_NAME]: AgentName;
[APM_EVENT_FIELDS_MAP.EVENT_OUTCOME]: EventOutcome;
[APM_EVENT_FIELDS_MAP.FAAS_COLDSTART]: true;
[APM_EVENT_FIELDS_MAP.TRANSACTION_DURATION]: number;
[APM_EVENT_FIELDS_MAP.TIMESTAMP_US]: number;
[APM_EVENT_FIELDS_MAP.PROCESSOR_EVENT]: ProcessorEvent;
[APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_COUNT]: number;
[APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_SUM]: number;
[APM_EVENT_FIELDS_MAP.SPAN_SYNC]: boolean;
[APM_EVENT_FIELDS_MAP.TRANSACTION_SAMPLED]: boolean;
[APM_EVENT_FIELDS_MAP.PROCESSOR_NAME]: 'transaction' | 'metric' | 'error';
[APM_EVENT_FIELDS_MAP.HTTP_RESPONSE_STATUS_CODE]: number;
[APM_EVENT_FIELDS_MAP.PROCESS_PID]: number;
[APM_EVENT_FIELDS_MAP.OBSERVER_VERSION_MAJOR]: number;
[APM_EVENT_FIELDS_MAP.ERROR_EXC_HANDLED]: boolean;
}
type MaybeMultiValue<T extends KnownField, U> = T extends KnownMultiValuedField ? U[] : U;
type TypeOfKnownField<T extends KnownField> = MaybeMultiValue<
T,
T extends keyof TypeOverrideMap ? TypeOverrideMap[T] : string
>;
type MapToSingleOrMultiValue<T extends Record<string, any>> = {
[TKey in keyof T]: TKey extends KnownField
? T[TKey] extends undefined
? TypeOfKnownField<TKey> | undefined
: TypeOfKnownField<TKey>
: unknown;
};
type UnflattenedKnownFields<T extends Record<string, any>> = DedotObject<
MapToSingleOrMultiValue<T>
>;
export type FlattenedApmEvent = Record<KnownSingleValuedField | KnownMultiValuedField, unknown[]>;
export type UnflattenedApmEvent = UnflattenedKnownFields<FlattenedApmEvent>;
export function unflattenKnownApmEventFields<T extends Record<string, any> | undefined = undefined>(
fields: T
): T extends Record<string, any> ? UnflattenedKnownFields<T> : undefined;
export function unflattenKnownApmEventFields<
T extends Record<string, any> | undefined,
U extends Array<keyof Exclude<T, undefined>>
>(
fields: T,
required: U
): T extends Record<string, any>
? UnflattenedKnownFields<T> &
(U extends any[]
? UnflattenedKnownFields<{
[TKey in ValuesType<U>]: keyof T extends TKey ? T[TKey] : unknown[];
}>
: {})
: undefined;
export function unflattenKnownApmEventFields(
hitFields?: Record<string, any>,
requiredFields?: string[]
) {
if (!hitFields) {
return undefined;
}
const missingRequiredFields =
requiredFields?.filter((key) => {
const value = hitFields?.[key];
return value === null || value === undefined || (isArray(value) && value.length === 0);
}) ?? [];
if (missingRequiredFields.length > 0) {
throw new Error(`Missing required fields ${missingRequiredFields.join(', ')} in event`);
}
const copy: Record<string, any> = mapToSingleOrMultiValue({
...hitFields,
});
const [knownFields, unknownFields] = Object.entries(copy).reduce(
(prev, [key, value]) => {
if (ALL_FIELDS.includes(key as KnownField)) {
prev[0][key as KnownField] = value;
} else {
prev[1][key] = value;
}
return prev;
},
[{} as Record<KnownField, any>, {} as Record<string, any>]
);
const unflattened = mergePlainObjects(
{},
unflattenObject(unknownFields),
unflattenObject(knownFields)
);
return unflattened;
}
export function mapToSingleOrMultiValue<T extends Record<string, any>>(
fields: T
): MapToSingleOrMultiValue<T> {
KNOWN_SINGLE_VALUED_FIELDS.forEach((field) => {
const value = fields[field];
if (value !== null && value !== undefined) {
fields[field as keyof T] = castArray(value)[0];
}
});
return fields;
}

4
x-pack/plugins/observability_solution/apm_data_access/tsconfig.json
View file

@ -20,6 +20,8 @@
"@kbn/apm-utils",
"@kbn/core-http-server",
"@kbn/security-plugin-types-server",
"@kbn/observability-utils"
"@kbn/observability-utils",
"@kbn/utility-types",
"@kbn/elastic-agent-utils"
]
}