[8.x] [APM][Otel] Use `fields` instead of `_source` on APM queries (#195242) (#196265)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[APM][Otel] Use `fields` instead of `_source` on
APM queries (#195242)](https://github.com/elastic/kibana/pull/195242)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Carlos
Crespo","email":"crespocarlos@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-10-15T09:38:44Z","message":"[APM][Otel]
Use `fields` instead of `_source` on APM queries (#195242)\n\ncloses
https://github.com/elastic/kibana/issues/192606\r\n\r\n##
Summary\r\n\r\nv2 based on the work done in this
PR\r\nhttps://github.com/elastic/kibana/pull/192608 and the suggestion
from\r\nDario https://github.com/elastic/kibana/pull/194424\r\n\r\nThis
PR replaces the _source usage in APM queries with fields to
support\r\nOtel data. The idea is to get rid of existing UI errors we
have and make\r\nsure that otel data is shown correctly in the
UI.\r\n\r\nOne way to check it is using the
[e2e\r\nPoC](https://github.com/elastic/otel-apm-e2e-poc/blob/main/README.md).\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>\r\nCo-authored-by: Jenny
<dzheni.pavlova@elastic.co>","sha":"7235ed0425100bbf04ff157d0af7980875473c99","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","v9.0.0","apm","apm:opentelemetry","backport:prev-minor","ci:project-deploy-observability","Team:obs-ux-infra_services"],"title":"[APM][Otel]
Use `fields` instead of `_source` on APM
queries","number":195242,"url":"https://github.com/elastic/kibana/pull/195242","mergeCommit":{"message":"[APM][Otel]
Use `fields` instead of `_source` on APM queries (#195242)\n\ncloses
https://github.com/elastic/kibana/issues/192606\r\n\r\n##
Summary\r\n\r\nv2 based on the work done in this
PR\r\nhttps://github.com/elastic/kibana/pull/192608 and the suggestion
from\r\nDario https://github.com/elastic/kibana/pull/194424\r\n\r\nThis
PR replaces the _source usage in APM queries with fields to
support\r\nOtel data. The idea is to get rid of existing UI errors we
have and make\r\nsure that otel data is shown correctly in the
UI.\r\n\r\nOne way to check it is using the
[e2e\r\nPoC](https://github.com/elastic/otel-apm-e2e-poc/blob/main/README.md).\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>\r\nCo-authored-by: Jenny
<dzheni.pavlova@elastic.co>","sha":"7235ed0425100bbf04ff157d0af7980875473c99"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195242","number":195242,"mergeCommit":{"message":"[APM][Otel]
Use `fields` instead of `_source` on APM queries (#195242)\n\ncloses
https://github.com/elastic/kibana/issues/192606\r\n\r\n##
Summary\r\n\r\nv2 based on the work done in this
PR\r\nhttps://github.com/elastic/kibana/pull/192608 and the suggestion
from\r\nDario https://github.com/elastic/kibana/pull/194424\r\n\r\nThis
PR replaces the _source usage in APM queries with fields to
support\r\nOtel data. The idea is to get rid of existing UI errors we
have and make\r\nsure that otel data is shown correctly in the
UI.\r\n\r\nOne way to check it is using the
[e2e\r\nPoC](https://github.com/elastic/otel-apm-e2e-poc/blob/main/README.md).\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>\r\nCo-authored-by: Jenny
<dzheni.pavlova@elastic.co>","sha":"7235ed0425100bbf04ff157d0af7980875473c99"}}]}]
BACKPORT-->

Co-authored-by: Carlos Crespo <crespocarlos@users.noreply.github.com>

packages/kbn-apm-types/src/es_fields/apm.ts

@@ -7,7 +7,8 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export const TIMESTAMP = 'timestamp.us';
+export const TIMESTAMP_US = 'timestamp.us';
+export const AT_TIMESTAMP = '@timestamp';
 export const AGENT = 'agent';
 export const AGENT_NAME = 'agent.name';
 export const AGENT_VERSION = 'agent.version';
@@ -21,9 +22,11 @@ export const CLOUD_PROVIDER = 'cloud.provider';
 export const CLOUD_REGION = 'cloud.region';
 export const CLOUD_MACHINE_TYPE = 'cloud.machine.type';
 export const CLOUD_ACCOUNT_ID = 'cloud.account.id';
+export const CLOUD_ACCOUNT_NAME = 'cloud.account.name';
 export const CLOUD_INSTANCE_ID = 'cloud.instance.id';
 export const CLOUD_INSTANCE_NAME = 'cloud.instance.name';
 export const CLOUD_SERVICE_NAME = 'cloud.service.name';
+export const CLOUD_PROJECT_NAME = 'cloud.project.name';
 
 export const EVENT_SUCCESS_COUNT = 'event.success_count';
 
@@ -48,10 +51,14 @@ export const USER_ID = 'user.id';
 export const USER_AGENT_ORIGINAL = 'user_agent.original';
 export const USER_AGENT_NAME = 'user_agent.name';
 
+export const OBSERVER_VERSION = 'observer.version';
+export const OBSERVER_VERSION_MAJOR = 'observer.version_major';
 export const OBSERVER_HOSTNAME = 'observer.hostname';
 export const OBSERVER_LISTENING = 'observer.listening';
 export const PROCESSOR_EVENT = 'processor.event';
+export const PROCESSOR_NAME = 'processor.name';
 
+export const TRANSACTION_AGENT_MARKS = 'transaction.agent.marks';
 export const TRANSACTION_DURATION = 'transaction.duration.us';
 export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram';
 export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary';
@@ -95,6 +102,7 @@ export const SPAN_COMPOSITE_SUM = 'span.composite.sum.us';
 export const SPAN_COMPOSITE_COMPRESSION_STRATEGY = 'span.composite.compression_strategy';
 
 export const SPAN_SYNC = 'span.sync';
+export const SPAN_STACKTRACE = 'span.stacktrace';
 
 // Parent ID for a transaction or span
 export const PARENT_ID = 'parent.id';
@@ -110,6 +118,7 @@ export const ERROR_EXC_MESSAGE = 'error.exception.message'; // only to be used i
 export const ERROR_EXC_HANDLED = 'error.exception.handled'; // only to be used in es queries, since error.exception is now an array
 export const ERROR_EXC_TYPE = 'error.exception.type';
 export const ERROR_PAGE_URL = 'error.page.url';
+export const ERROR_STACK_TRACE = 'error.stack_trace';
 export const ERROR_TYPE = 'error.type';
 
 // METRICS
@@ -153,6 +162,12 @@ export const CONTAINER_IMAGE = 'container.image.name';
 export const KUBERNETES = 'kubernetes';
 export const KUBERNETES_POD_NAME = 'kubernetes.pod.name';
 export const KUBERNETES_POD_UID = 'kubernetes.pod.uid';
+export const KUBERNETES_NAMESPACE = 'kubernetes.namespace';
+export const KUBERNETES_NODE_NAME = 'kubernetes.node.name';
+export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name';
+export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id';
+export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name';
+export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name';
 
 export const FAAS_ID = 'faas.id';
 export const FAAS_NAME = 'faas.name';
@@ -198,3 +213,7 @@ export const CLIENT_GEO_REGION_NAME = 'client.geo.region_name';
 export const CHILD_ID = 'child.id';
 
 export const LOG_LEVEL = 'log.level';
+
+// Process
+export const PROCESS_ARGS = 'process.args';
+export const PROCESS_PID = 'process.pid';

packages/kbn-apm-types/src/es_schemas/raw/apm_base_doc.ts

@@ -14,10 +14,10 @@ export interface APMBaseDoc {
   '@timestamp': string;
   agent: {
     name: string;
-    version: string;
+    version?: string;
   };
-  parent?: { id: string }; // parent ID is not available on root transactions
+  parent?: { id?: string }; // parent ID is not available on root transactions
-  trace?: { id: string };
+  trace?: { id?: string };
   labels?: {
     [key: string]: string | number | boolean;
   };

packages/kbn-apm-types/src/es_schemas/raw/fields/cloud.ts

@@ -10,26 +10,26 @@
 export interface Cloud {
   availability_zone?: string;
   instance?: {
-    name: string;
+    name?: string;
-    id: string;
+    id?: string;
   };
   machine?: {
-    type: string;
+    type?: string;
   };
   project?: {
-    id: string;
+    id?: string;
-    name: string;
+    name?: string;
   };
   provider?: string;
   region?: string;
   account?: {
-    id: string;
+    id?: string;
-    name: string;
+    name?: string;
   };
   image?: {
-    id: string;
+    id?: string;
   };
   service?: {
-    name: string;
+    name?: string;
   };
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/container.ts

@@ -9,5 +9,7 @@
 
 export interface Container {
   id?: string | null;
-  image?: string | null;
+  image?: {
+    name?: string;
+  };
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/http.ts

@@ -8,7 +8,7 @@
  */
 
 export interface Http {
-  request?: { method: string; [key: string]: unknown };
+  request?: { method?: string };
-  response?: { status_code: number; [key: string]: unknown };
+  response?: { status_code?: number };
   version?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/kubernetes.ts

@@ -8,7 +8,7 @@
  */
 
 export interface Kubernetes {
-  pod?: { uid?: string | null; [key: string]: unknown };
+  pod?: { uid?: string | null; name?: string };
   namespace?: string;
   replicaset?: {
     name?: string;

packages/kbn-apm-types/src/es_schemas/raw/fields/observer.ts

@@ -13,6 +13,6 @@ export interface Observer {
   id?: string;
   name?: string;
   type?: string;
-  version: string;
+  version?: string;
-  version_major: number;
+  version_major?: number;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/page.ts

@@ -9,5 +9,5 @@
 
 // only for RUM agent: shared by error and transaction
 export interface Page {
-  url: string;
+  url?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/service.ts

@@ -11,18 +11,18 @@ export interface Service {
   name: string;
   environment?: string;
   framework?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   node?: {
     name?: string;
   };
   runtime?: {
-    name: string;
+    name?: string;
-    version: string;
+    version?: string;
   };
   language?: {
-    name: string;
+    name?: string;
     version?: string;
   };
   version?: string;

packages/kbn-apm-types/src/es_schemas/raw/fields/url.ts

@@ -9,6 +9,6 @@
 
 export interface Url {
   domain?: string;
-  full: string;
+  full?: string;
   original?: string;
 }

packages/kbn-apm-types/src/es_schemas/raw/fields/user.ts

@@ -8,5 +8,5 @@
  */
 
 export interface User {
-  id: string;
+  id?: string;
 }

packages/kbn-apm-types/src/es_schemas/ui/fields/agent.ts

@@ -14,5 +14,5 @@ export type { ElasticAgentName, OpenTelemetryAgentName, AgentName } from '@kbn/e
 export interface Agent {
   ephemeral_id?: string;
   name: AgentName;
-  version: string;
+  version?: string;
 }

x-pack/packages/observability/observability_utils/object/flatten_object.test.ts

@@ -21,6 +21,18 @@ describe('flattenObject', () => {
     });
   });
 
+  it('flattens arrays', () => {
+    expect(
+      flattenObject({
+        child: {
+          id: [1, 2],
+        },
+      })
+    ).toEqual({
+      'child.id': [1, 2],
+    });
+  });
+
   it('does not flatten arrays', () => {
     expect(
       flattenObject({

x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { unflattenObject } from './unflatten_object';
+
+describe('unflattenObject', () => {
+  it('unflattens deeply nested objects', () => {
+    expect(unflattenObject({ 'first.second.third': 'third' })).toEqual({
+      first: {
+        second: {
+          third: 'third',
+        },
+      },
+    });
+  });
+
+  it('does not unflatten arrays', () => {
+    expect(
+      unflattenObject({
+        simpleArray: ['0', '1', '2'],
+        complexArray: [{ one: 'one', two: 'two', three: 'three' }],
+        'nested.array': [0, 1, 2],
+        'complex.nested': [{ one: 'one', two: 'two', 'first.second': 'foo', 'first.third': 'bar' }],
+      })
+    ).toEqual({
+      simpleArray: ['0', '1', '2'],
+      complexArray: [{ one: 'one', two: 'two', three: 'three' }],
+      nested: {
+        array: [0, 1, 2],
+      },
+      complex: {
+        nested: [{ one: 'one', two: 'two', first: { second: 'foo', third: 'bar' } }],
+      },
+    });
+  });
+});

x-pack/packages/observability/observability_utils/object/unflatten_object.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { set } from '@kbn/safer-lodash-set';
+
+export function unflattenObject(source: Record<string, any>, target: Record<string, any> = {}) {
+  // eslint-disable-next-line guard-for-in
+  for (const key in source) {
+    const val = source[key as keyof typeof source];
+
+    if (Array.isArray(val)) {
+      const unflattenedArray = val.map((item) => {
+        if (item && typeof item === 'object' && !Array.isArray(item)) {
+          return unflattenObject(item);
+        }
+        return item;
+      });
+      set(target, key, unflattenedArray);
+    } else {
+      set(target, key, val);
+    }
+  }
+  return target;
+}

x-pack/packages/observability/observability_utils/tsconfig.json

@@ -21,5 +21,6 @@
     "@kbn/es-types",
     "@kbn/apm-utils",
     "@kbn/es-query",
+    "@kbn/safer-lodash-set",
   ]
 }

x-pack/plugins/observability_solution/apm/common/es_fields/__snapshots__/es_fields.test.ts.snap

@@ -37,6 +37,8 @@ Object {
 
 exports[`Error CLOUD_ACCOUNT_ID 1`] = `undefined`;
 
+exports[`Error CLOUD_ACCOUNT_NAME 1`] = `undefined`;
+
 exports[`Error CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`;
 
 exports[`Error CLOUD_INSTANCE_ID 1`] = `undefined`;
@@ -45,6 +47,8 @@ exports[`Error CLOUD_INSTANCE_NAME 1`] = `undefined`;
 
 exports[`Error CLOUD_MACHINE_TYPE 1`] = `undefined`;
 
+exports[`Error CLOUD_PROJECT_NAME 1`] = `undefined`;
+
 exports[`Error CLOUD_PROVIDER 1`] = `"gcp"`;
 
 exports[`Error CLOUD_REGION 1`] = `"europe-west1"`;
@@ -94,6 +98,8 @@ exports[`Error ERROR_LOG_MESSAGE 1`] = `undefined`;
 
 exports[`Error ERROR_PAGE_URL 1`] = `undefined`;
 
+exports[`Error ERROR_STACK_TRACE 1`] = `undefined`;
+
 exports[`Error ERROR_TYPE 1`] = `undefined`;
 
 exports[`Error EVENT_NAME 1`] = `undefined`;
@@ -140,6 +146,8 @@ exports[`Error INDEX 1`] = `undefined`;
 
 exports[`Error KUBERNETES 1`] = `undefined`;
 
+exports[`Error KUBERNETES_CONTAINER_ID 1`] = `undefined`;
+
 exports[`Error KUBERNETES_CONTAINER_NAME 1`] = `undefined`;
 
 exports[`Error KUBERNETES_DEPLOYMENT 1`] = `undefined`;
@@ -150,6 +158,8 @@ exports[`Error KUBERNETES_NAMESPACE 1`] = `undefined`;
 
 exports[`Error KUBERNETES_NAMESPACE_NAME 1`] = `undefined`;
 
+exports[`Error KUBERNETES_NODE_NAME 1`] = `undefined`;
+
 exports[`Error KUBERNETES_POD_NAME 1`] = `undefined`;
 
 exports[`Error KUBERNETES_POD_UID 1`] = `undefined`;
@@ -228,10 +238,20 @@ exports[`Error OBSERVER_HOSTNAME 1`] = `undefined`;
 
 exports[`Error OBSERVER_LISTENING 1`] = `undefined`;
 
+exports[`Error OBSERVER_VERSION 1`] = `"whatever"`;
+
+exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`;
+
 exports[`Error PARENT_ID 1`] = `"parentId"`;
 
+exports[`Error PROCESS_ARGS 1`] = `undefined`;
+
+exports[`Error PROCESS_PID 1`] = `undefined`;
+
 exports[`Error PROCESSOR_EVENT 1`] = `"error"`;
 
+exports[`Error PROCESSOR_NAME 1`] = `"error"`;
+
 exports[`Error SERVICE 1`] = `
 Object {
   "language": Object {
@@ -296,6 +316,8 @@ exports[`Error SPAN_NAME 1`] = `undefined`;
 
 exports[`Error SPAN_SELF_TIME_SUM 1`] = `undefined`;
 
+exports[`Error SPAN_STACKTRACE 1`] = `undefined`;
+
 exports[`Error SPAN_SUBTYPE 1`] = `undefined`;
 
 exports[`Error SPAN_SYNC 1`] = `undefined`;
@@ -304,10 +326,12 @@ exports[`Error SPAN_TYPE 1`] = `undefined`;
 
 exports[`Error TIER 1`] = `undefined`;
 
-exports[`Error TIMESTAMP 1`] = `1337`;
+exports[`Error TIMESTAMP_US 1`] = `1337`;
 
 exports[`Error TRACE_ID 1`] = `"trace id"`;
 
+exports[`Error TRANSACTION_AGENT_MARKS 1`] = `undefined`;
+
 exports[`Error TRANSACTION_DURATION 1`] = `undefined`;
 
 exports[`Error TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`;
@@ -385,6 +409,8 @@ Object {
 
 exports[`Span CLOUD_ACCOUNT_ID 1`] = `undefined`;
 
+exports[`Span CLOUD_ACCOUNT_NAME 1`] = `undefined`;
+
 exports[`Span CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`;
 
 exports[`Span CLOUD_INSTANCE_ID 1`] = `undefined`;
@@ -393,6 +419,8 @@ exports[`Span CLOUD_INSTANCE_NAME 1`] = `undefined`;
 
 exports[`Span CLOUD_MACHINE_TYPE 1`] = `undefined`;
 
+exports[`Span CLOUD_PROJECT_NAME 1`] = `undefined`;
+
 exports[`Span CLOUD_PROVIDER 1`] = `"gcp"`;
 
 exports[`Span CLOUD_REGION 1`] = `"europe-west1"`;
@@ -433,6 +461,8 @@ exports[`Span ERROR_LOG_MESSAGE 1`] = `undefined`;
 
 exports[`Span ERROR_PAGE_URL 1`] = `undefined`;
 
+exports[`Span ERROR_STACK_TRACE 1`] = `undefined`;
+
 exports[`Span ERROR_TYPE 1`] = `undefined`;
 
 exports[`Span EVENT_NAME 1`] = `undefined`;
@@ -475,6 +505,8 @@ exports[`Span INDEX 1`] = `undefined`;
 
 exports[`Span KUBERNETES 1`] = `undefined`;
 
+exports[`Span KUBERNETES_CONTAINER_ID 1`] = `undefined`;
+
 exports[`Span KUBERNETES_CONTAINER_NAME 1`] = `undefined`;
 
 exports[`Span KUBERNETES_DEPLOYMENT 1`] = `undefined`;
@@ -485,6 +517,8 @@ exports[`Span KUBERNETES_NAMESPACE 1`] = `undefined`;
 
 exports[`Span KUBERNETES_NAMESPACE_NAME 1`] = `undefined`;
 
+exports[`Span KUBERNETES_NODE_NAME 1`] = `undefined`;
+
 exports[`Span KUBERNETES_POD_NAME 1`] = `undefined`;
 
 exports[`Span KUBERNETES_POD_UID 1`] = `undefined`;
@@ -563,10 +597,20 @@ exports[`Span OBSERVER_HOSTNAME 1`] = `undefined`;
 
 exports[`Span OBSERVER_LISTENING 1`] = `undefined`;
 
+exports[`Span OBSERVER_VERSION 1`] = `"whatever"`;
+
+exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`;
+
 exports[`Span PARENT_ID 1`] = `"parentId"`;
 
+exports[`Span PROCESS_ARGS 1`] = `undefined`;
+
+exports[`Span PROCESS_PID 1`] = `undefined`;
+
 exports[`Span PROCESSOR_EVENT 1`] = `"span"`;
 
+exports[`Span PROCESSOR_NAME 1`] = `"transaction"`;
+
 exports[`Span SERVICE 1`] = `
 Object {
   "name": "service name",
@@ -627,6 +671,8 @@ exports[`Span SPAN_NAME 1`] = `"span name"`;
 
 exports[`Span SPAN_SELF_TIME_SUM 1`] = `undefined`;
 
+exports[`Span SPAN_STACKTRACE 1`] = `undefined`;
+
 exports[`Span SPAN_SUBTYPE 1`] = `"my subtype"`;
 
 exports[`Span SPAN_SYNC 1`] = `false`;
@@ -635,10 +681,12 @@ exports[`Span SPAN_TYPE 1`] = `"span type"`;
 
 exports[`Span TIER 1`] = `undefined`;
 
-exports[`Span TIMESTAMP 1`] = `1337`;
+exports[`Span TIMESTAMP_US 1`] = `1337`;
 
 exports[`Span TRACE_ID 1`] = `"trace id"`;
 
+exports[`Span TRANSACTION_AGENT_MARKS 1`] = `undefined`;
+
 exports[`Span TRANSACTION_DURATION 1`] = `undefined`;
 
 exports[`Span TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`;
@@ -716,6 +764,8 @@ Object {
 
 exports[`Transaction CLOUD_ACCOUNT_ID 1`] = `undefined`;
 
+exports[`Transaction CLOUD_ACCOUNT_NAME 1`] = `undefined`;
+
 exports[`Transaction CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`;
 
 exports[`Transaction CLOUD_INSTANCE_ID 1`] = `undefined`;
@@ -724,6 +774,8 @@ exports[`Transaction CLOUD_INSTANCE_NAME 1`] = `undefined`;
 
 exports[`Transaction CLOUD_MACHINE_TYPE 1`] = `undefined`;
 
+exports[`Transaction CLOUD_PROJECT_NAME 1`] = `undefined`;
+
 exports[`Transaction CLOUD_PROVIDER 1`] = `"gcp"`;
 
 exports[`Transaction CLOUD_REGION 1`] = `"europe-west1"`;
@@ -768,6 +820,8 @@ exports[`Transaction ERROR_LOG_MESSAGE 1`] = `undefined`;
 
 exports[`Transaction ERROR_PAGE_URL 1`] = `undefined`;
 
+exports[`Transaction ERROR_STACK_TRACE 1`] = `undefined`;
+
 exports[`Transaction ERROR_TYPE 1`] = `undefined`;
 
 exports[`Transaction EVENT_NAME 1`] = `undefined`;
@@ -820,6 +874,8 @@ Object {
 }
 `;
 
+exports[`Transaction KUBERNETES_CONTAINER_ID 1`] = `undefined`;
+
 exports[`Transaction KUBERNETES_CONTAINER_NAME 1`] = `undefined`;
 
 exports[`Transaction KUBERNETES_DEPLOYMENT 1`] = `undefined`;
@@ -830,6 +886,8 @@ exports[`Transaction KUBERNETES_NAMESPACE 1`] = `undefined`;
 
 exports[`Transaction KUBERNETES_NAMESPACE_NAME 1`] = `undefined`;
 
+exports[`Transaction KUBERNETES_NODE_NAME 1`] = `undefined`;
+
 exports[`Transaction KUBERNETES_POD_NAME 1`] = `undefined`;
 
 exports[`Transaction KUBERNETES_POD_UID 1`] = `"pod1234567890abcdef"`;
@@ -908,10 +966,20 @@ exports[`Transaction OBSERVER_HOSTNAME 1`] = `undefined`;
 
 exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`;
 
+exports[`Transaction OBSERVER_VERSION 1`] = `"whatever"`;
+
+exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`;
+
 exports[`Transaction PARENT_ID 1`] = `"parentId"`;
 
+exports[`Transaction PROCESS_ARGS 1`] = `undefined`;
+
+exports[`Transaction PROCESS_PID 1`] = `undefined`;
+
 exports[`Transaction PROCESSOR_EVENT 1`] = `"transaction"`;
 
+exports[`Transaction PROCESSOR_NAME 1`] = `"transaction"`;
+
 exports[`Transaction SERVICE 1`] = `
 Object {
   "language": Object {
@@ -976,6 +1044,8 @@ exports[`Transaction SPAN_NAME 1`] = `undefined`;
 
 exports[`Transaction SPAN_SELF_TIME_SUM 1`] = `undefined`;
 
+exports[`Transaction SPAN_STACKTRACE 1`] = `undefined`;
+
 exports[`Transaction SPAN_SUBTYPE 1`] = `undefined`;
 
 exports[`Transaction SPAN_SYNC 1`] = `undefined`;
@@ -984,10 +1054,12 @@ exports[`Transaction SPAN_TYPE 1`] = `undefined`;
 
 exports[`Transaction TIER 1`] = `undefined`;
 
-exports[`Transaction TIMESTAMP 1`] = `1337`;
+exports[`Transaction TIMESTAMP_US 1`] = `1337`;
 
 exports[`Transaction TRACE_ID 1`] = `"trace id"`;
 
+exports[`Transaction TRANSACTION_AGENT_MARKS 1`] = `undefined`;
+
 exports[`Transaction TRANSACTION_DURATION 1`] = `1337`;
 
 exports[`Transaction TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`;

x-pack/plugins/observability_solution/apm/common/es_fields/es_fields.test.ts

@@ -10,12 +10,13 @@ import { AllowUnknownProperties } from '../../typings/common';
 import { APMError } from '../../typings/es_schemas/ui/apm_error';
 import { Span } from '../../typings/es_schemas/ui/span';
 import { Transaction } from '../../typings/es_schemas/ui/transaction';
-import * as apmFieldnames from './apm';
+import * as allApmFieldNames from './apm';
-import * as infraMetricsFieldnames from './infra_metrics';
+import * as infraMetricsFieldNames from './infra_metrics';
 
+const { AT_TIMESTAMP, ...apmFieldNames } = allApmFieldNames;
 const fieldnames = {
-  ...apmFieldnames,
+  ...apmFieldNames,
-  ...infraMetricsFieldnames,
+  ...infraMetricsFieldNames,
 };
 
 describe('Transaction', () => {

x-pack/plugins/observability_solution/apm/common/service_metadata.ts

@@ -4,5 +4,63 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+import {
+  CLOUD_AVAILABILITY_ZONE,
+  CLOUD_INSTANCE_ID,
+  CLOUD_INSTANCE_NAME,
+  CLOUD_MACHINE_TYPE,
+  CLOUD_PROVIDER,
+  CONTAINER_ID,
+  HOST_NAME,
+  KUBERNETES_CONTAINER_NAME,
+  KUBERNETES_NAMESPACE,
+  KUBERNETES_DEPLOYMENT_NAME,
+  KUBERNETES_POD_NAME,
+  KUBERNETES_POD_UID,
+  KUBERNETES_REPLICASET_NAME,
+  SERVICE_NODE_NAME,
+  SERVICE_RUNTIME_NAME,
+  SERVICE_RUNTIME_VERSION,
+  SERVICE_VERSION,
+} from './es_fields/apm';
+import { asMutableArray } from './utils/as_mutable_array';
+
+export const SERVICE_METADATA_SERVICE_KEYS = asMutableArray([
+  SERVICE_NODE_NAME,
+  SERVICE_VERSION,
+  SERVICE_RUNTIME_NAME,
+  SERVICE_RUNTIME_VERSION,
+] as const);
+
+export const SERVICE_METADATA_CONTAINER_KEYS = asMutableArray([
+  CONTAINER_ID,
+  HOST_NAME,
+  KUBERNETES_POD_UID,
+  KUBERNETES_POD_NAME,
+] as const);
+
+export const SERVICE_METADATA_INFRA_METRICS_KEYS = asMutableArray([
+  KUBERNETES_CONTAINER_NAME,
+  KUBERNETES_NAMESPACE,
+  KUBERNETES_REPLICASET_NAME,
+  KUBERNETES_DEPLOYMENT_NAME,
+] as const);
+
+export const SERVICE_METADATA_CLOUD_KEYS = asMutableArray([
+  CLOUD_AVAILABILITY_ZONE,
+  CLOUD_INSTANCE_ID,
+  CLOUD_INSTANCE_NAME,
+  CLOUD_MACHINE_TYPE,
+  CLOUD_PROVIDER,
+] as const);
+
+export const SERVICE_METADATA_KUBERNETES_KEYS = asMutableArray([
+  KUBERNETES_CONTAINER_NAME,
+  KUBERNETES_NAMESPACE,
+  KUBERNETES_DEPLOYMENT_NAME,
+  KUBERNETES_POD_NAME,
+  KUBERNETES_POD_UID,
+  KUBERNETES_REPLICASET_NAME,
+] as const);
 
 export type ContainerType = 'Kubernetes' | 'Docker' | undefined;

x-pack/plugins/observability_solution/apm/common/waterfall/typings.ts

@@ -64,16 +64,17 @@ export interface WaterfallSpan {
     links?: SpanLink[];
   };
   transaction?: {
-    id: string;
+    id?: string;
   };
   child?: { id: string[] };
 }
 
 export interface WaterfallError {
   timestamp: TimestampUs;
-  trace?: { id: string };
+  trace?: { id?: string };
-  transaction?: { id: string };
+  transaction?: { id?: string };
-  parent?: { id: string };
+  parent?: { id?: string };
+  span?: { id?: string };
   error: {
     id: string;
     log?: {

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_contextual_insight.tsx

@@ -8,9 +8,9 @@ import { EuiFlexItem, EuiSpacer } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import type { Message } from '@kbn/observability-ai-assistant-plugin/public';
 import React, { useMemo, useState } from 'react';
+import { AT_TIMESTAMP } from '@kbn/apm-types';
 import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
 import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
-import { Transaction } from '../../../../../typings/es_schemas/ui/transaction';
 import { ErrorSampleDetailTabContent } from './error_sample_detail';
 import { exceptionStacktraceTab, logStacktraceTab } from './error_tabs';
 
@@ -18,8 +18,26 @@ export function ErrorSampleContextualInsight({
   error,
   transaction,
 }: {
-  error: APMError;
+  error: {
-  transaction?: Transaction;
+    [AT_TIMESTAMP]: string;
+    error: Pick<APMError['error'], 'log' | 'exception' | 'id'>;
+    service: {
+      name: string;
+      environment?: string;
+      language?: {
+        name?: string;
+      };
+      runtime?: {
+        name?: string;
+        version?: string;
+      };
+    };
+  };
+  transaction?: {
+    transaction: {
+      name: string;
+    };
+  };
 }) {
   const { observabilityAIAssistant } = useApmPluginContext();
 

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_sample_detail.tsx

@@ -29,14 +29,14 @@ import { first } from 'lodash';
 import React, { useEffect, useState } from 'react';
 import { useHistory } from 'react-router-dom';
 import useAsync from 'react-use/lib/useAsync';
-import { ERROR_GROUP_ID } from '../../../../../common/es_fields/apm';
+import { AT_TIMESTAMP, ERROR_GROUP_ID } from '../../../../../common/es_fields/apm';
 import { TraceSearchType } from '../../../../../common/trace_explorer';
 import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
 import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
 import { useLegacyUrlParams } from '../../../../context/url_params_context/use_url_params';
 import { useAnyOfApmParams } from '../../../../hooks/use_apm_params';
 import { useApmRouter } from '../../../../hooks/use_apm_router';
-import { FETCH_STATUS, isPending } from '../../../../hooks/use_fetcher';
+import { FETCH_STATUS, isPending, isSuccess } from '../../../../hooks/use_fetcher';
 import { useTraceExplorerEnabledSetting } from '../../../../hooks/use_trace_explorer_enabled_setting';
 import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
 import { TransactionDetailLink } from '../../../shared/links/apm/transaction_detail_link';
@@ -111,8 +111,7 @@ export function ErrorSampleDetails({
   const loadingErrorData = isPending(errorFetchStatus);
   const isLoading = loadingErrorSamplesData || loadingErrorData;
 
-  const isSucceded =
+  const isSucceeded = isSuccess(errorSamplesFetchStatus) && isSuccess(errorFetchStatus);
-    errorSamplesFetchStatus === FETCH_STATUS.SUCCESS && errorFetchStatus === FETCH_STATUS.SUCCESS;
 
   useEffect(() => {
     setSampleActivePage(0);
@@ -137,7 +136,7 @@ export function ErrorSampleDetails({
     });
   }, [error, transaction, uiActions]);
 
-  if (!error && errorSampleIds?.length === 0 && isSucceded) {
+  if (!error && errorSampleIds?.length === 0 && isSucceeded) {
     return (
       <EuiEmptyPrompt
         title={
@@ -160,7 +159,7 @@ export function ErrorSampleDetails({
   const status = error.http?.response?.status_code;
   const environment = error.service.environment;
   const serviceVersion = error.service.version;
-  const isUnhandled = error.error.exception?.[0].handled === false;
+  const isUnhandled = error.error.exception?.[0]?.handled === false;
 
   const traceExplorerLink = router.link('/traces/explorer/waterfall', {
     query: {
@@ -348,14 +347,22 @@ export function ErrorSampleDetailTabContent({
   error,
   currentTab,
 }: {
-  error: APMError;
+  error: {
+    service: {
+      language?: {
+        name?: string;
+      };
+    };
+    [AT_TIMESTAMP]: string;
+    error: Pick<APMError['error'], 'id' | 'log' | 'stack_trace' | 'exception'>;
+  };
   currentTab: ErrorTab;
 }) {
   const codeLanguage = error?.service.language?.name;
   const exceptions = error?.error.exception || [];
   const logStackframes = error?.error.log?.stacktrace;
   const isPlaintextException =
-    !!error?.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace;
+    !!error.error.stack_trace && exceptions.length === 1 && !exceptions[0].stacktrace;
   switch (currentTab.key) {
     case ErrorTabKey.LogStackTrace:
       return <Stacktrace stackframes={logStackframes} codeLanguage={codeLanguage} />;
@@ -363,7 +370,7 @@ export function ErrorSampleDetailTabContent({
       return isPlaintextException ? (
         <PlaintextStacktrace
           message={exceptions[0].message}
-          type={exceptions[0].type}
+          type={exceptions[0]?.type}
           stacktrace={error?.error.stack_trace}
           codeLanguage={codeLanguage}
         />

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/error_tabs.tsx

@@ -41,7 +41,7 @@ export const metadataTab: ErrorTab = {
   }),
 };
 
-export function getTabs(error: APMError) {
+export function getTabs(error: { error: { log?: APMError['error']['log'] } }) {
   const hasLogStacktrace = !isEmpty(error?.error.log?.stacktrace);
   return [...(hasLogStacktrace ? [logStacktraceTab] : []), exceptionStacktraceTab, metadataTab];
 }

x-pack/plugins/observability_solution/apm/public/components/app/error_group_details/error_sampler/sample_summary.tsx

@@ -18,7 +18,9 @@ const Label = euiStyled.div`
 `;
 
 interface Props {
-  error: APMError;
+  error: {
+    error: Pick<APMError['error'], 'log' | 'exception' | 'culprit'>;
+  };
 }
 export function SampleSummary({ error }: Props) {
   const logMessage = error.error.log?.message;

x-pack/plugins/observability_solution/apm/public/components/app/trace_link/get_redirect_to_transaction_detail_page_url.ts

@@ -6,7 +6,7 @@
  */
 
 import { format } from 'url';
-import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
+import type { TransactionDetailRedirectInfo } from '../../../../server/routes/transactions/get_transaction_by_trace';
 
 export const getRedirectToTransactionDetailPageUrl = ({
   transaction,
@@ -14,7 +14,7 @@ export const getRedirectToTransactionDetailPageUrl = ({
   rangeTo,
   waterfallItemId,
 }: {
-  transaction: Transaction;
+  transaction: TransactionDetailRedirectInfo;
   rangeFrom?: string;
   rangeTo?: string;
   waterfallItemId?: string;

x-pack/plugins/observability_solution/apm/public/components/shared/links/discover_links/discover_error_link.tsx

@@ -7,10 +7,18 @@
 
 import React, { ReactNode } from 'react';
 import { ERROR_GROUP_ID, SERVICE_NAME } from '../../../../../common/es_fields/apm';
-import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
 import { DiscoverLink } from './discover_link';
 
-function getDiscoverQuery(error: APMError, kuery?: string) {
+interface ErrorForDiscoverQuery {
+  service: {
+    name: string;
+  };
+  error: {
+    grouping_key: string;
+  };
+}
+
+function getDiscoverQuery(error: ErrorForDiscoverQuery, kuery?: string) {
   const serviceName = error.service.name;
   const groupId = error.error.grouping_key;
   let query = `${SERVICE_NAME}:"${serviceName}" AND ${ERROR_GROUP_ID}:"${groupId}"`;
@@ -36,7 +44,7 @@ function DiscoverErrorLink({
   children,
 }: {
   children?: ReactNode;
-  readonly error: APMError;
+  readonly error: ErrorForDiscoverQuery;
   readonly kuery?: string;
 }) {
   return <DiscoverLink query={getDiscoverQuery(error, kuery)} children={children} />;

x-pack/plugins/observability_solution/apm/public/components/shared/metadata_table/error_metadata/index.tsx

@@ -7,13 +7,16 @@
 
 import React, { useMemo } from 'react';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
-import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
+import { APMError, AT_TIMESTAMP } from '@kbn/apm-types';
 import { getSectionsFromFields } from '../helper';
 import { MetadataTable } from '..';
 import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
 
 interface Props {
-  error: APMError;
+  error: {
+    [AT_TIMESTAMP]: string;
+    error: Pick<APMError['error'], 'id'>;
+  };
 }
 
 export function ErrorMetadata({ error }: Props) {
@@ -26,8 +29,8 @@ export function ErrorMetadata({ error }: Props) {
             id: error.error.id,
           },
           query: {
-            start: error['@timestamp'],
+            start: error[AT_TIMESTAMP],
-            end: error['@timestamp'],
+            end: error[AT_TIMESTAMP],
           },
         },
       });

x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts

@@ -8,7 +8,7 @@
 import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
 import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { tasks } from './tasks';
-import { SERVICE_NAME, SERVICE_ENVIRONMENT } from '../../../../common/es_fields/apm';
+import { SERVICE_NAME, SERVICE_ENVIRONMENT, AT_TIMESTAMP } from '../../../../common/es_fields/apm';
 import { IndicesStatsResponse } from '../telemetry_client';
 
 describe('data telemetry collection tasks', () => {
@@ -101,7 +101,7 @@ describe('data telemetry collection tasks', () => {
         // a fixed date range.
         .mockReturnValueOnce({
           hits: {
-            hits: [{ _source: { '@timestamp': new Date().toISOString() } }],
+            hits: [{ fields: { [AT_TIMESTAMP]: [new Date().toISOString()] } }],
           },
           total: {
             value: 1,
@@ -314,7 +314,7 @@ describe('data telemetry collection tasks', () => {
             ? { hits: { total: { value: 1 } } }
             : {
                 hits: {
-                  hits: [{ _source: { '@timestamp': 1 } }],
+                  hits: [{ fields: { [AT_TIMESTAMP]: [1] } }],
                 },
               }
         );

x-pack/plugins/observability_solution/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -11,11 +11,13 @@ import { createHash } from 'crypto';
 import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash';
 import { SavedObjectsClient } from '@kbn/core/server';
 import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
 import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name';
 import {
   AGENT_ACTIVATION_METHOD,
   AGENT_NAME,
   AGENT_VERSION,
+  AT_TIMESTAMP,
   CLIENT_GEO_COUNTRY_ISO_CODE,
   CLOUD_AVAILABILITY_ZONE,
   CLOUD_PROVIDER,
@@ -29,6 +31,7 @@ import {
   METRICSET_INTERVAL,
   METRICSET_NAME,
   OBSERVER_HOSTNAME,
+  OBSERVER_VERSION,
   PARENT_ID,
   PROCESSOR_EVENT,
   SERVICE_ENVIRONMENT,
@@ -54,10 +57,7 @@ import {
   SavedServiceGroup,
 } from '../../../../common/service_groups';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
-import { APMError } from '../../../../typings/es_schemas/ui/apm_error';
 import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
-import { Span } from '../../../../typings/es_schemas/ui/span';
-import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
 import {
   APMDataTelemetry,
   APMPerService,
@@ -193,17 +193,19 @@ export const tasks: TelemetryTask[] = [
             size: 1,
             track_total_hits: false,
             sort: {
-              '@timestamp': 'desc' as const,
+              [AT_TIMESTAMP]: 'desc' as const,
             },
+            fields: [AT_TIMESTAMP],
           },
         })
-      ).hits.hits[0] as { _source: { '@timestamp': string } };
+      ).hits.hits[0];
 
       if (!lastTransaction) {
         return {};
       }
 
-      const end = new Date(lastTransaction._source['@timestamp']).getTime() - 5 * 60 * 1000;
+      const end =
+        new Date(lastTransaction.fields[AT_TIMESTAMP]![0] as string).getTime() - 5 * 60 * 1000;
 
       const start = end - 60 * 1000;
 
@@ -512,16 +514,16 @@ export const tasks: TelemetryTask[] = [
                       },
                     },
                     sort: {
-                      '@timestamp': 'asc',
+                      [AT_TIMESTAMP]: 'asc',
                     },
-                    _source: ['@timestamp'],
+                    fields: [AT_TIMESTAMP],
                   },
                 })
               : null;
 
-          const event = retainmentResponse?.hits.hits[0]?._source as
+          const event = retainmentResponse?.hits.hits[0]?.fields as
             | {
-                '@timestamp': number;
+                [AT_TIMESTAMP]: number[];
               }
             | undefined;
 
@@ -535,7 +537,7 @@ export const tasks: TelemetryTask[] = [
               ? {
                   retainment: {
                     [processorEvent]: {
-                      ms: new Date().getTime() - new Date(event['@timestamp']).getTime(),
+                      ms: new Date().getTime() - new Date(event[AT_TIMESTAMP][0]).getTime(),
                     },
                   },
                 }
@@ -690,16 +692,16 @@ export const tasks: TelemetryTask[] = [
           sort: {
             '@timestamp': 'desc',
           },
+          fields: asMutableArray([OBSERVER_VERSION] as const),
         },
       });
 
-      const hit = response.hits.hits[0]?._source as Pick<Transaction | Span | APMError, 'observer'>;
+      const event = unflattenKnownApmEventFields(response.hits.hits[0]?.fields);
-
+      if (!event || !event.observer?.version) {
-      if (!hit || !hit.observer?.version) {
         return {};
       }
 
-      const [major, minor, patch] = hit.observer.version.split('.').map((part) => Number(part));
+      const [major, minor, patch] = event.observer.version.split('.').map((part) => Number(part));
 
       return {
         version: {

x-pack/plugins/observability_solution/apm/server/lib/connections/get_connection_stats/get_destination_map.ts

@@ -186,7 +186,6 @@ export const getDestinationMap = ({
         },
         size: destinationsBySpanId.size,
         fields: asMutableArray([SERVICE_NAME, SERVICE_ENVIRONMENT, AGENT_NAME, PARENT_ID] as const),
-        _source: false,
       },
     });
 

x-pack/plugins/observability_solution/apm/server/lib/helpers/get_error_name.ts

@@ -9,6 +9,10 @@ import { NOT_AVAILABLE_LABEL } from '../../../common/i18n';
 import { Maybe } from '../../../typings/common';
 import { APMError } from '../../../typings/es_schemas/ui/apm_error';
 
-export function getErrorName({ error }: { error: Maybe<APMError['error']> }): string {
+export function getErrorName({
+  error,
+}: {
+  error: Maybe<Pick<APMError['error'], 'exception'>> & { log?: { message?: string } };
+}): string {
   return error?.log?.message || error?.exception?.[0]?.message || NOT_AVAILABLE_LABEL;
 }

x-pack/plugins/observability_solution/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts

@@ -184,6 +184,7 @@ export function registerTransactionDurationRuleType({
         body: {
           track_total_hits: false,
           size: 0,
+          _source: false as const,
           query: {
             bool: {
               filter: [

x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts

@@ -8,6 +8,9 @@
 import datemath from '@elastic/datemath';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { maybe } from '../../../../common/utils/maybe';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { flattenObject, KeyValuePair } from '../../../../common/utils/flatten_object';
 import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
 import { PROCESSOR_EVENT, TRACE_ID } from '../../../../common/es_fields/apm';
@@ -86,6 +89,7 @@ export async function getLogCategories({
   const rawSamplingProbability = Math.min(100_000 / totalDocCount, 1);
   const samplingProbability = rawSamplingProbability < 0.5 ? rawSamplingProbability : 1;
 
+  const fields = asMutableArray(['message', TRACE_ID] as const);
   const categorizedLogsRes = await search({
     index,
     size: 1,
@@ -108,7 +112,7 @@ export async function getLogCategories({
                 top_hits: {
                   sort: { '@timestamp': 'desc' as const },
                   size: 1,
-                  _source: ['message', TRACE_ID],
+                  fields,
                 },
               },
             },
@@ -120,9 +124,11 @@ export async function getLogCategories({
 
   const promises = categorizedLogsRes.aggregations?.sampling.categories?.buckets.map(
     async ({ doc_count: docCount, key, sample }) => {
-      const hit = sample.hits.hits[0]._source as { message: string; trace?: { id: string } };
+      const hit = sample.hits.hits[0];
-      const sampleMessage = hit?.message;
+      const event = unflattenKnownApmEventFields(hit?.fields);
-      const sampleTraceId = hit?.trace?.id;
+
+      const sampleMessage = event.message as string;
+      const sampleTraceId = event.trace?.id;
       const errorCategory = key as string;
 
       if (!sampleTraceId) {
@@ -140,7 +146,9 @@ export async function getLogCategories({
     }
   );
 
-  const sampleDoc = categorizedLogsRes.hits.hits?.[0]?._source as Record<string, string>;
+  const event = unflattenKnownApmEventFields(maybe(categorizedLogsRes.hits.hits[0])?.fields);
+
+  const sampleDoc = event as Record<string, string>;
 
   return {
     logCategories: await Promise.all(promises ?? []),

x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts

@@ -13,6 +13,10 @@ import moment from 'moment';
 import { ESSearchRequest } from '@kbn/es-types';
 import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services';
 import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types';
+import { CONTAINER_ID } from '@kbn/apm-types';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { maybe } from '../../../../common/utils/maybe';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { ApmDocumentType } from '../../../../common/document_type';
 import {
   APMEventClient,
@@ -79,13 +83,17 @@ async function getContainerIdFromLogs({
   esClient: ElasticsearchClient;
   logSourcesService: LogSourcesService;
 }) {
+  const requiredFields = asMutableArray([CONTAINER_ID] as const);
   const index = await logSourcesService.getFlattenedLogSources();
   const res = await typedSearch<{ container: { id: string } }, any>(esClient, {
     index,
     ...params,
+    fields: requiredFields,
   });
 
-  return res.hits.hits[0]?._source?.container?.id;
+  const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields);
+
+  return event?.container.id;
 }
 
 async function getContainerIdFromTraces({
@@ -95,6 +103,7 @@ async function getContainerIdFromTraces({
   params: APMEventESSearchRequest['body'];
   apmEventClient: APMEventClient;
 }) {
+  const requiredFields = asMutableArray([CONTAINER_ID] as const);
   const res = await apmEventClient.search('get_container_id_from_traces', {
     apm: {
       sources: [
@@ -104,8 +113,10 @@ async function getContainerIdFromTraces({
         },
       ],
     },
-    body: params,
+    body: { ...params, fields: requiredFields },
   });
 
-  return res.hits.hits[0]?._source.container?.id;
+  const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields);
+
+  return event?.container.id;
 }

x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts

@@ -6,6 +6,9 @@
  */
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
+import { maybe } from '../../../../common/utils/maybe';
 import { ApmDocumentType } from '../../../../common/document_type';
 import { termQuery } from '../../../../common/utils/term_query';
 import {
@@ -27,6 +30,7 @@ export async function getDownstreamServiceResource({
   end: number;
   apmEventClient: APMEventClient;
 }) {
+  const requiredFields = asMutableArray([SPAN_DESTINATION_SERVICE_RESOURCE] as const);
   const response = await apmEventClient.search('get_error_group_main_statistics', {
     apm: {
       sources: [
@@ -50,9 +54,11 @@ export async function getDownstreamServiceResource({
           ],
         },
       },
+      fields: requiredFields,
     },
   });
 
-  const hit = response.hits.hits[0];
+  const event = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredFields);
-  return hit?._source?.span.destination?.service.resource;
+
+  return event?.span.destination.service.resource;
 }

x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts

@@ -12,6 +12,10 @@ import moment from 'moment';
 import { ESSearchRequest } from '@kbn/es-types';
 import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services';
 import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { SERVICE_NAME } from '@kbn/apm-types';
+import { maybe } from '../../../../common/utils/maybe';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { ApmDocumentType } from '../../../../common/document_type';
 import {
   APMEventClient,
@@ -102,6 +106,7 @@ async function getServiceNameFromTraces({
   params: APMEventESSearchRequest['body'];
   apmEventClient: APMEventClient;
 }) {
+  const requiredFields = asMutableArray([SERVICE_NAME] as const);
   const res = await apmEventClient.search('get_service_name_from_traces', {
     apm: {
       sources: [
@@ -111,8 +116,13 @@ async function getServiceNameFromTraces({
         },
       ],
     },
-    body: params,
+    body: {
+      ...params,
+      fields: requiredFields,
+    },
   });
 
-  return res.hits.hits[0]?._source.service.name;
+  const event = unflattenKnownApmEventFields(maybe(res.hits.hits[0])?.fields, requiredFields);
+
+  return event?.service.name;
 }

x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_metadata_for_dependency.ts

@@ -7,8 +7,14 @@
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import { maybe } from '../../../common/utils/maybe';
-import { SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../common/es_fields/apm';
+import {
+  SPAN_DESTINATION_SERVICE_RESOURCE,
+  SPAN_SUBTYPE,
+  SPAN_TYPE,
+} from '../../../common/es_fields/apm';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 
 export interface MetadataForDependencyResponse {
@@ -27,6 +33,7 @@ export async function getMetadataForDependency({
   start: number;
   end: number;
 }): Promise<MetadataForDependencyResponse> {
+  const fields = asMutableArray([SPAN_TYPE, SPAN_SUBTYPE] as const);
   const sampleResponse = await apmEventClient.search('get_metadata_for_dependency', {
     apm: {
       events: [ProcessorEvent.span],
@@ -46,16 +53,17 @@ export async function getMetadataForDependency({
           ],
         },
       },
+      fields,
       sort: {
         '@timestamp': 'desc',
       },
     },
   });
 
-  const sample = maybe(sampleResponse.hits.hits[0])?._source;
+  const sample = unflattenKnownApmEventFields(maybe(sampleResponse.hits.hits[0])?.fields);
 
   return {
-    spanType: sample?.span.type,
+    spanType: sample?.span?.type,
-    spanSubtype: sample?.span.subtype,
+    spanSubtype: sample?.span?.subtype,
   };
 }

x-pack/plugins/observability_solution/apm/server/routes/dependencies/get_top_dependency_spans.ts

@@ -8,8 +8,11 @@
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { kqlQuery, rangeQuery, termQuery, termsQuery } from '@kbn/observability-plugin/server';
 import { keyBy } from 'lodash';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   AGENT_NAME,
+  AT_TIMESTAMP,
   EVENT_OUTCOME,
   SERVICE_ENVIRONMENT,
   SERVICE_NAME,
@@ -66,6 +69,19 @@ export async function getTopDependencySpans({
   sampleRangeFrom?: number;
   sampleRangeTo?: number;
 }): Promise<DependencySpan[]> {
+  const topDedsRequiredFields = asMutableArray([
+    SPAN_ID,
+    TRACE_ID,
+    TRANSACTION_ID,
+    SPAN_NAME,
+    SERVICE_NAME,
+    SERVICE_ENVIRONMENT,
+    AGENT_NAME,
+    SPAN_DURATION,
+    EVENT_OUTCOME,
+    AT_TIMESTAMP,
+  ] as const);
+
   const spans = (
     await apmEventClient.search('get_top_dependency_spans', {
       apm: {
@@ -98,23 +114,18 @@ export async function getTopDependencySpans({
             ],
           },
         },
-        _source: [
+        fields: topDedsRequiredFields,
-          SPAN_ID,
-          TRACE_ID,
-          TRANSACTION_ID,
-          SPAN_NAME,
-          SERVICE_NAME,
-          SERVICE_ENVIRONMENT,
-          AGENT_NAME,
-          SPAN_DURATION,
-          EVENT_OUTCOME,
-          '@timestamp',
-        ],
       },
     })
-  ).hits.hits.map((hit) => hit._source);
+  ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, topDedsRequiredFields));
 
-  const transactionIds = spans.map((span) => span.transaction!.id);
+  const transactionIds = spans.map((span) => span.transaction.id);
+
+  const txRequiredFields = asMutableArray([
+    TRANSACTION_ID,
+    TRANSACTION_TYPE,
+    TRANSACTION_NAME,
+  ] as const);
 
   const transactions = (
     await apmEventClient.search('get_transactions_for_dependency_spans', {
@@ -129,13 +140,13 @@ export async function getTopDependencySpans({
             filter: [...termsQuery(TRANSACTION_ID, ...transactionIds)],
           },
         },
-        _source: [TRANSACTION_ID, TRANSACTION_TYPE, TRANSACTION_NAME],
+        fields: txRequiredFields,
         sort: {
           '@timestamp': 'desc',
         },
       },
     })
-  ).hits.hits.map((hit) => hit._source);
+  ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, txRequiredFields));
 
   const transactionsById = keyBy(transactions, (transaction) => transaction.transaction.id);
 

x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts

@@ -7,13 +7,17 @@
 
 import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types';
 import { kqlQuery, rangeQuery, termQuery, wildcardQuery } from '@kbn/observability-plugin/server';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import {
+  AT_TIMESTAMP,
   ERROR_CULPRIT,
   ERROR_EXC_HANDLED,
   ERROR_EXC_MESSAGE,
   ERROR_EXC_TYPE,
   ERROR_GROUP_ID,
   ERROR_GROUP_NAME,
+  ERROR_ID,
   ERROR_LOG_MESSAGE,
   SERVICE_NAME,
   TRACE_ID,
@@ -93,6 +97,21 @@ export async function getErrorGroupMainStatistics({
       ]
     : [];
 
+  const requiredFields = asMutableArray([
+    TRACE_ID,
+    AT_TIMESTAMP,
+    ERROR_GROUP_ID,
+    ERROR_ID,
+  ] as const);
+
+  const optionalFields = asMutableArray([
+    ERROR_CULPRIT,
+    ERROR_LOG_MESSAGE,
+    ERROR_EXC_MESSAGE,
+    ERROR_EXC_HANDLED,
+    ERROR_EXC_TYPE,
+  ] as const);
+
   const response = await apmEventClient.search('get_error_group_main_statistics', {
     apm: {
       sources: [
@@ -129,16 +148,8 @@ export async function getErrorGroupMainStatistics({
             sample: {
               top_hits: {
                 size: 1,
-                _source: [
+                fields: [...requiredFields, ...optionalFields],
-                  TRACE_ID,
+                _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE],
-                  ERROR_LOG_MESSAGE,
-                  ERROR_EXC_MESSAGE,
-                  ERROR_EXC_HANDLED,
-                  ERROR_EXC_TYPE,
-                  ERROR_CULPRIT,
-                  ERROR_GROUP_ID,
-                  '@timestamp',
-                ],
                 sort: {
                   '@timestamp': 'desc',
                 },
@@ -157,15 +168,33 @@ export async function getErrorGroupMainStatistics({
 
   const errorGroups =
     response.aggregations?.error_groups.buckets.map((bucket) => {
+      const errorSource =
+        'error' in bucket.sample.hits.hits[0]._source
+          ? bucket.sample.hits.hits[0]._source
+          : undefined;
+
+      const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields);
+
+      const mergedEvent = {
+        ...event,
+        error: {
+          ...(event.error ?? {}),
+          exception:
+            (errorSource?.error.exception?.length ?? 0) > 1
+              ? errorSource?.error.exception
+              : event?.error.exception && [event.error.exception],
+        },
+      };
+
       return {
         groupId: bucket.key as string,
-        name: getErrorName(bucket.sample.hits.hits[0]._source),
+        name: getErrorName(mergedEvent),
-        lastSeen: new Date(bucket.sample.hits.hits[0]._source['@timestamp']).getTime(),
+        lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(),
         occurrences: bucket.doc_count,
-        culprit: bucket.sample.hits.hits[0]._source.error.culprit,
+        culprit: mergedEvent.error.culprit,
-        handled: bucket.sample.hits.hits[0]._source.error.exception?.[0].handled,
+        handled: mergedEvent.error.exception?.[0].handled,
-        type: bucket.sample.hits.hits[0]._source.error.exception?.[0].type,
+        type: mergedEvent.error.exception?.[0].type,
-        traceId: bucket.sample.hits.hits[0]._source.trace?.id,
+        traceId: mergedEvent.trace?.id,
       };
     }) ?? [];
 

x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_sample_ids.ts

@@ -6,6 +6,7 @@
  */
 
 import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import {
   ERROR_GROUP_ID,
@@ -42,6 +43,7 @@ export async function getErrorGroupSampleIds({
   start: number;
   end: number;
 }): Promise<ErrorGroupSampleIdsResponse> {
+  const requiredFields = asMutableArray([ERROR_ID] as const);
   const resp = await apmEventClient.search('get_error_group_sample_ids', {
     apm: {
       sources: [
@@ -66,7 +68,7 @@ export async function getErrorGroupSampleIds({
           should: [{ term: { [TRANSACTION_SAMPLED]: true } }], // prefer error samples with related transactions
         },
       },
-      _source: [ERROR_ID, 'transaction'],
+      fields: requiredFields,
       sort: asMutableArray([
         { _score: { order: 'desc' } }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top
         { '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error
@@ -74,8 +76,8 @@ export async function getErrorGroupSampleIds({
     },
   });
   const errorSampleIds = resp.hits.hits.map((item) => {
-    const source = item._source;
+    const event = unflattenKnownApmEventFields(item.fields, requiredFields);
-    return source.error.id;
+    return event.error?.id;
   });
 
   return {

x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts

@@ -6,7 +6,28 @@
  */
 
 import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
-import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
+import { maybe } from '../../../../common/utils/maybe';
+import {
+  AGENT_NAME,
+  AGENT_VERSION,
+  AT_TIMESTAMP,
+  ERROR_EXCEPTION,
+  ERROR_GROUP_ID,
+  ERROR_ID,
+  ERROR_EXC_MESSAGE,
+  ERROR_EXC_HANDLED,
+  ERROR_EXC_TYPE,
+  PROCESSOR_EVENT,
+  PROCESSOR_NAME,
+  SERVICE_NAME,
+  TIMESTAMP_US,
+  TRACE_ID,
+  TRANSACTION_ID,
+  ERROR_STACK_TRACE,
+  SPAN_ID,
+} from '../../../../common/es_fields/apm';
 import { environmentQuery } from '../../../../common/utils/environment_query';
 import { ApmDocumentType } from '../../../../common/document_type';
 import { RollupInterval } from '../../../../common/rollup';
@@ -17,7 +38,15 @@ import { APMError } from '../../../../typings/es_schemas/ui/apm_error';
 
 export interface ErrorSampleDetailsResponse {
   transaction: Transaction | undefined;
-  error: APMError;
+  error: Omit<APMError, 'transaction' | 'error'> & {
+    transaction?: { id?: string; type?: string };
+    error: {
+      id: string;
+    } & Omit<APMError['error'], 'exception' | 'log'> & {
+        exception?: APMError['error']['exception'];
+        log?: APMError['error']['log'];
+      };
+  };
 }
 
 export async function getErrorSampleDetails({
@@ -36,7 +65,29 @@ export async function getErrorSampleDetails({
   apmEventClient: APMEventClient;
   start: number;
   end: number;
-}): Promise<ErrorSampleDetailsResponse> {
+}): Promise<Partial<ErrorSampleDetailsResponse>> {
+  const requiredFields = asMutableArray([
+    AGENT_NAME,
+    PROCESSOR_EVENT,
+    TRACE_ID,
+    TIMESTAMP_US,
+    AT_TIMESTAMP,
+    SERVICE_NAME,
+    ERROR_ID,
+    ERROR_GROUP_ID,
+  ] as const);
+
+  const optionalFields = asMutableArray([
+    TRANSACTION_ID,
+    SPAN_ID,
+    AGENT_VERSION,
+    PROCESSOR_NAME,
+    ERROR_STACK_TRACE,
+    ERROR_EXC_MESSAGE,
+    ERROR_EXC_HANDLED,
+    ERROR_EXC_TYPE,
+  ] as const);
+
   const params = {
     apm: {
       sources: [
@@ -60,15 +111,29 @@ export async function getErrorSampleDetails({
           ],
         },
       },
+      fields: [...requiredFields, ...optionalFields],
+      _source: [ERROR_EXCEPTION, 'error.log'],
     },
   };
 
   const resp = await apmEventClient.search('get_error_sample_details', params);
-  const error = resp.hits.hits[0]?._source;
+  const hit = maybe(resp.hits.hits[0]);
-  const transactionId = error?.transaction?.id;
-  const traceId = error?.trace?.id;
 
-  let transaction;
+  if (!hit) {
+    return {
+      transaction: undefined,
+      error: undefined,
+    };
+  }
+
+  const source = 'error' in hit._source ? hit._source : undefined;
+
+  const errorFromFields = unflattenKnownApmEventFields(hit.fields, requiredFields);
+
+  const transactionId = errorFromFields.transaction?.id ?? errorFromFields.span?.id;
+  const traceId = errorFromFields.trace.id;
+
+  let transaction: Transaction | undefined;
   if (transactionId && traceId) {
     transaction = await getTransaction({
       transactionId,
@@ -81,6 +146,20 @@ export async function getErrorSampleDetails({
 
   return {
     transaction,
-    error,
+    error: {
+      ...errorFromFields,
+      processor: {
+        name: errorFromFields.processor.name as 'error',
+        event: errorFromFields.processor.event as 'error',
+      },
+      error: {
+        ...errorFromFields.error,
+        exception:
+          (source?.error.exception?.length ?? 0) > 1
+            ? source?.error.exception
+            : errorFromFields?.error.exception && [errorFromFields.error.exception],
+        log: source?.error?.log,
+      },
+    },
   };
 }

x-pack/plugins/observability_solution/apm/server/routes/errors/route.ts

@@ -7,6 +7,7 @@
 
 import { jsonRt, toNumberRt } from '@kbn/io-ts-utils';
 import * as t from 'io-ts';
+import { notFound } from '@hapi/boom';
 import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
 import { ErrorDistributionResponse, getErrorDistribution } from './distribution/get_distribution';
 import { environmentRt, kueryRt, rangeRt } from '../default_api_types';
@@ -205,7 +206,7 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({
     const { serviceName, errorId } = params.path;
     const { environment, kuery, start, end } = params.query;
 
-    return getErrorSampleDetails({
+    const { transaction, error } = await getErrorSampleDetails({
       environment,
       errorId,
       kuery,
@@ -214,6 +215,12 @@ const errorGroupSampleDetailsRoute = createApmServerRoute({
       start,
       end,
     });
+
+    if (!error) {
+      throw notFound();
+    }
+
+    return { error, transaction };
   },
 });
 

x-pack/plugins/observability_solution/apm/server/routes/mobile/crashes/get_crash_groups/get_crash_group_main_statistics.ts

@@ -8,6 +8,8 @@
 import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types';
 import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../../../common/utils/as_mutable_array';
 import {
   ERROR_CULPRIT,
   ERROR_TYPE,
@@ -19,6 +21,7 @@ import {
   SERVICE_NAME,
   TRANSACTION_NAME,
   TRANSACTION_TYPE,
+  AT_TIMESTAMP,
 } from '../../../../../common/es_fields/apm';
 import { environmentQuery } from '../../../../../common/utils/environment_query';
 import { getErrorName } from '../../../../lib/helpers/get_error_name';
@@ -68,6 +71,16 @@ export async function getMobileCrashGroupMainStatistics({
     ? { [maxTimestampAggKey]: sortDirection }
     : { _count: sortDirection };
 
+  const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const);
+
+  const optionalFields = asMutableArray([
+    ERROR_CULPRIT,
+    ERROR_LOG_MESSAGE,
+    ERROR_EXC_MESSAGE,
+    ERROR_EXC_HANDLED,
+    ERROR_EXC_TYPE,
+  ] as const);
+
   const response = await apmEventClient.search('get_crash_group_main_statistics', {
     apm: {
       events: [ProcessorEvent.error],
@@ -99,22 +112,15 @@ export async function getMobileCrashGroupMainStatistics({
             sample: {
               top_hits: {
                 size: 1,
-                _source: [
+                fields: [...requiredFields, ...optionalFields],
-                  ERROR_LOG_MESSAGE,
+                _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE],
-                  ERROR_EXC_MESSAGE,
-                  ERROR_EXC_HANDLED,
-                  ERROR_EXC_TYPE,
-                  ERROR_CULPRIT,
-                  ERROR_GROUP_ID,
-                  '@timestamp',
-                ],
                 sort: {
-                  '@timestamp': 'desc',
+                  [AT_TIMESTAMP]: 'desc',
                 },
               },
             },
             ...(sortByLatestOccurrence
-              ? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } }
+              ? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } }
               : {}),
           },
         },
@@ -123,14 +129,34 @@ export async function getMobileCrashGroupMainStatistics({
   });
 
   return (
-    response.aggregations?.crash_groups.buckets.map((bucket) => ({
+    response.aggregations?.crash_groups.buckets.map((bucket) => {
-      groupId: bucket.key as string,
+      const errorSource =
-      name: getErrorName(bucket.sample.hits.hits[0]._source),
+        'error' in bucket.sample.hits.hits[0]._source
-      lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(),
+          ? bucket.sample.hits.hits[0]._source
-      occurrences: bucket.doc_count,
+          : undefined;
-      culprit: bucket.sample.hits.hits[0]?._source.error.culprit,
+
-      handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled,
+      const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields);
-      type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type,
+
-    })) ?? []
+      const mergedEvent = {
+        ...event,
+        error: {
+          ...(event.error ?? {}),
+          exception:
+            (errorSource?.error.exception?.length ?? 0) > 1
+              ? errorSource?.error.exception
+              : event?.error.exception && [event.error.exception],
+        },
+      };
+
+      return {
+        groupId: event.error?.grouping_key,
+        name: getErrorName(mergedEvent),
+        lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(),
+        occurrences: bucket.doc_count,
+        culprit: mergedEvent.error.culprit,
+        handled: mergedEvent.error.exception?.[0].handled,
+        type: mergedEvent.error.exception?.[0].type,
+      };
+    }) ?? []
   );
 }

x-pack/plugins/observability_solution/apm/server/routes/mobile/errors/get_mobile_error_group_main_statistics.ts

@@ -8,7 +8,10 @@
 import { AggregationsAggregateOrder } from '@elastic/elasticsearch/lib/api/types';
 import { kqlQuery, rangeQuery, termQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import {
+  AT_TIMESTAMP,
   ERROR_CULPRIT,
   ERROR_EXC_HANDLED,
   ERROR_EXC_MESSAGE,
@@ -67,6 +70,16 @@ export async function getMobileErrorGroupMainStatistics({
     ? { [maxTimestampAggKey]: sortDirection }
     : { _count: sortDirection };
 
+  const requiredFields = asMutableArray([ERROR_GROUP_ID, AT_TIMESTAMP] as const);
+
+  const optionalFields = asMutableArray([
+    ERROR_CULPRIT,
+    ERROR_LOG_MESSAGE,
+    ERROR_EXC_MESSAGE,
+    ERROR_EXC_HANDLED,
+    ERROR_EXC_TYPE,
+  ] as const);
+
   const response = await apmEventClient.search('get_error_group_main_statistics', {
     apm: {
       events: [ProcessorEvent.error],
@@ -100,22 +113,15 @@ export async function getMobileErrorGroupMainStatistics({
             sample: {
               top_hits: {
                 size: 1,
-                _source: [
+                fields: [...requiredFields, ...optionalFields],
-                  ERROR_LOG_MESSAGE,
+                _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE],
-                  ERROR_EXC_MESSAGE,
-                  ERROR_EXC_HANDLED,
-                  ERROR_EXC_TYPE,
-                  ERROR_CULPRIT,
-                  ERROR_GROUP_ID,
-                  '@timestamp',
-                ],
                 sort: {
-                  '@timestamp': 'desc',
+                  [AT_TIMESTAMP]: 'desc',
                 },
               },
             },
             ...(sortByLatestOccurrence
-              ? { [maxTimestampAggKey]: { max: { field: '@timestamp' } } }
+              ? { [maxTimestampAggKey]: { max: { field: AT_TIMESTAMP } } }
               : {}),
           },
         },
@@ -124,14 +130,34 @@ export async function getMobileErrorGroupMainStatistics({
   });
 
   return (
-    response.aggregations?.error_groups.buckets.map((bucket) => ({
+    response.aggregations?.error_groups.buckets.map((bucket) => {
-      groupId: bucket.key as string,
+      const errorSource =
-      name: getErrorName(bucket.sample.hits.hits[0]._source),
+        'error' in bucket.sample.hits.hits[0]._source
-      lastSeen: new Date(bucket.sample.hits.hits[0]?._source['@timestamp']).getTime(),
+          ? bucket.sample.hits.hits[0]._source
-      occurrences: bucket.doc_count,
+          : undefined;
-      culprit: bucket.sample.hits.hits[0]?._source.error.culprit,
+
-      handled: bucket.sample.hits.hits[0]?._source.error.exception?.[0].handled,
+      const event = unflattenKnownApmEventFields(bucket.sample.hits.hits[0].fields, requiredFields);
-      type: bucket.sample.hits.hits[0]?._source.error.exception?.[0].type,
+
-    })) ?? []
+      const mergedEvent = {
+        ...event,
+        error: {
+          ...(event.error ?? {}),
+          exception:
+            (errorSource?.error.exception?.length ?? 0) > 1
+              ? errorSource?.error.exception
+              : event?.error.exception && [event.error.exception],
+        },
+      };
+
+      return {
+        groupId: event.error?.grouping_key,
+        name: getErrorName(mergedEvent),
+        lastSeen: new Date(mergedEvent[AT_TIMESTAMP]).getTime(),
+        occurrences: bucket.doc_count,
+        culprit: mergedEvent.error.culprit,
+        handled: mergedEvent.error.exception?.[0].handled,
+        type: mergedEvent.error.exception?.[0].type,
+      };
+    }) ?? []
   );
 }

x-pack/plugins/observability_solution/apm/server/routes/services/annotations/get_derived_service_annotations.ts

@@ -7,9 +7,12 @@
 
 import type { ESFilter } from '@kbn/es-types';
 import { rangeQuery } from '@kbn/observability-plugin/server';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { maybe } from '../../../../common/utils/maybe';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { isFiniteNumber } from '../../../../common/utils/is_finite_number';
 import { Annotation, AnnotationType } from '../../../../common/annotations';
-import { SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm';
+import { AT_TIMESTAMP, SERVICE_NAME, SERVICE_VERSION } from '../../../../common/es_fields/apm';
 import { environmentQuery } from '../../../../common/utils/environment_query';
 import {
   getBackwardCompatibleDocumentTypeFilter,
@@ -66,6 +69,8 @@ export async function getDerivedServiceAnnotations({
   if (versions.length <= 1) {
     return [];
   }
+
+  const requiredFields = asMutableArray([AT_TIMESTAMP] as const);
   const annotations = await Promise.all(
     versions.map(async (version) => {
       const response = await apmEventClient.search('get_first_seen_of_version', {
@@ -83,11 +88,21 @@ export async function getDerivedServiceAnnotations({
           sort: {
             '@timestamp': 'asc',
           },
+          fields: requiredFields,
         },
       });
 
-      const firstSeen = new Date(response.hits.hits[0]._source['@timestamp']).getTime();
+      const event = unflattenKnownApmEventFields(
+        maybe(response.hits.hits[0])?.fields,
+        requiredFields
+      );
 
+      const timestamp = event?.[AT_TIMESTAMP];
+      if (!timestamp) {
+        throw new Error('First seen for version was unexpectedly undefined or null.');
+      }
+
+      const firstSeen = new Date(timestamp).getTime();
       if (!isFiniteNumber(firstSeen)) {
         throw new Error('First seen for version was unexpectedly undefined or null.');
       }
@@ -99,7 +114,7 @@ export async function getDerivedServiceAnnotations({
       return {
         type: AnnotationType.VERSION,
         id: version,
-        '@timestamp': firstSeen,
+        [AT_TIMESTAMP]: firstSeen,
         text: version,
       };
     })

x-pack/plugins/observability_solution/apm/server/routes/services/get_service_agent.ts

@@ -7,6 +7,8 @@
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   AGENT_NAME,
   SERVICE_NAME,
@@ -16,23 +18,7 @@ import {
 } from '../../../common/es_fields/apm';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 import { getServerlessTypeFromCloudData, ServerlessType } from '../../../common/serverless';
-
+import { maybe } from '../../../common/utils/maybe';
-interface ServiceAgent {
-  agent?: {
-    name: string;
-  };
-  service?: {
-    runtime?: {
-      name?: string;
-    };
-  };
-  cloud?: {
-    provider?: string;
-    service?: {
-      name?: string;
-    };
-  };
-}
 
 export interface ServiceAgentResponse {
   agentName?: string;
@@ -51,6 +37,13 @@ export async function getServiceAgent({
   start: number;
   end: number;
 }): Promise<ServiceAgentResponse> {
+  const fields = asMutableArray([
+    AGENT_NAME,
+    SERVICE_RUNTIME_NAME,
+    CLOUD_PROVIDER,
+    CLOUD_SERVICE_NAME,
+  ] as const);
+
   const params = {
     terminate_after: 1,
     apm: {
@@ -90,6 +83,7 @@ export async function getServiceAgent({
           ],
         },
       },
+      fields,
       sort: {
         _score: { order: 'desc' as const },
       },
@@ -97,11 +91,14 @@ export async function getServiceAgent({
   };
 
   const response = await apmEventClient.search('get_service_agent_name', params);
-  if (response.hits.total.value === 0) {
+  const hit = maybe(response.hits.hits[0]);
+  if (!hit) {
     return {};
   }
 
-  const { agent, service, cloud } = response.hits.hits[0]._source as ServiceAgent;
+  const event = unflattenKnownApmEventFields(hit.fields);
+
+  const { agent, service, cloud } = event;
   const serverlessType = getServerlessTypeFromCloudData(cloud?.provider, cloud?.service?.name);
 
   return {

x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_container_metadata.ts

@@ -6,19 +6,20 @@
  */
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   CONTAINER_ID,
   CONTAINER_IMAGE,
   KUBERNETES,
   KUBERNETES_POD_NAME,
   KUBERNETES_POD_UID,
-} from '../../../common/es_fields/apm';
-import {
   KUBERNETES_CONTAINER_NAME,
-  KUBERNETES_NAMESPACE,
   KUBERNETES_REPLICASET_NAME,
   KUBERNETES_DEPLOYMENT_NAME,
-} from '../../../common/es_fields/infra_metrics';
+  KUBERNETES_CONTAINER_ID,
+  KUBERNETES_NAMESPACE,
+} from '../../../common/es_fields/apm';
 import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes';
 import { maybe } from '../../../common/utils/maybe';
 import { InfraMetricsClient } from '../../lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client';
@@ -51,9 +52,21 @@ export const getServiceInstanceContainerMetadata = async ({
     { exists: { field: KUBERNETES_DEPLOYMENT_NAME } },
   ];
 
+  const fields = asMutableArray([
+    KUBERNETES_POD_NAME,
+    KUBERNETES_POD_UID,
+    KUBERNETES_DEPLOYMENT_NAME,
+    KUBERNETES_CONTAINER_ID,
+    KUBERNETES_CONTAINER_NAME,
+    KUBERNETES_NAMESPACE,
+    KUBERNETES_REPLICASET_NAME,
+    KUBERNETES_DEPLOYMENT_NAME,
+  ] as const);
+
   const response = await infraMetricsClient.search({
     size: 1,
     track_total_hits: false,
+    fields,
     query: {
       bool: {
         filter: [
@@ -69,7 +82,7 @@ export const getServiceInstanceContainerMetadata = async ({
     },
   });
 
-  const sample = maybe(response.hits.hits[0])?._source as ServiceInstanceContainerMetadataDetails;
+  const sample = unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields);
 
   return {
     kubernetes: {

x-pack/plugins/observability_solution/apm/server/routes/services/get_service_instance_metadata_details.ts

@@ -7,7 +7,16 @@
 import { merge } from 'lodash';
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
-import { METRICSET_NAME, SERVICE_NAME, SERVICE_NODE_NAME } from '../../../common/es_fields/apm';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields';
+import {
+  AGENT_NAME,
+  AT_TIMESTAMP,
+  METRICSET_NAME,
+  SERVICE_ENVIRONMENT,
+  SERVICE_NAME,
+  SERVICE_NODE_NAME,
+} from '../../../common/es_fields/apm';
 import { maybe } from '../../../common/utils/maybe';
 import {
   getBackwardCompatibleDocumentTypeFilter,
@@ -20,6 +29,13 @@ import { Container } from '../../../typings/es_schemas/raw/fields/container';
 import { Kubernetes } from '../../../typings/es_schemas/raw/fields/kubernetes';
 import { Host } from '../../../typings/es_schemas/raw/fields/host';
 import { Cloud } from '../../../typings/es_schemas/raw/fields/cloud';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
+import {
+  SERVICE_METADATA_CLOUD_KEYS,
+  SERVICE_METADATA_CONTAINER_KEYS,
+  SERVICE_METADATA_INFRA_METRICS_KEYS,
+  SERVICE_METADATA_SERVICE_KEYS,
+} from '../../../common/service_metadata';
 
 export interface ServiceInstanceMetadataDetailsResponse {
   '@timestamp': string;
@@ -50,6 +66,18 @@ export async function getServiceInstanceMetadataDetails({
     ...rangeQuery(start, end),
   ];
 
+  const requiredKeys = asMutableArray([AT_TIMESTAMP, SERVICE_NAME, AGENT_NAME] as const);
+
+  const optionalKeys = asMutableArray([
+    SERVICE_ENVIRONMENT,
+    ...SERVICE_METADATA_SERVICE_KEYS,
+    ...SERVICE_METADATA_CLOUD_KEYS,
+    ...SERVICE_METADATA_CONTAINER_KEYS,
+    ...SERVICE_METADATA_INFRA_METRICS_KEYS,
+  ] as const);
+
+  const fields = [...requiredKeys, ...optionalKeys];
+
   async function getApplicationMetricSample() {
     const response = await apmEventClient.search(
       'get_service_instance_metadata_details_application_metric',
@@ -66,11 +94,12 @@ export async function getServiceInstanceMetadataDetails({
               filter: filter.concat({ term: { [METRICSET_NAME]: 'app' } }),
             },
           },
+          fields,
         },
       }
     );
 
-    return maybe(response.hits.hits[0]?._source);
+    return unflattenKnownApmEventFields(maybe(response.hits.hits[0])?.fields, requiredKeys);
   }
 
   async function getTransactionEventSample() {
@@ -85,11 +114,14 @@ export async function getServiceInstanceMetadataDetails({
           terminate_after: 1,
           size: 1,
           query: { bool: { filter } },
+          fields,
         },
       }
     );
 
-    return maybe(response.hits.hits[0]?._source);
+    return unflattenKnownApmEventFields(
+      maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent
+    );
   }
 
   async function getTransactionMetricSample() {
@@ -108,10 +140,14 @@ export async function getServiceInstanceMetadataDetails({
               filter: filter.concat(getBackwardCompatibleDocumentTypeFilter(true)),
             },
           },
+          fields,
         },
       }
     );
-    return maybe(response.hits.hits[0]?._source);
+
+    return unflattenKnownApmEventFields(
+      maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent
+    );
   }
 
   // we can expect the most detail of application metrics,

x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts

@@ -7,37 +7,26 @@
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields';
 import { environmentQuery } from '../../../common/utils/environment_query';
 import {
-  AGENT,
-  CONTAINER,
-  CLOUD,
   CLOUD_AVAILABILITY_ZONE,
   CLOUD_REGION,
   CLOUD_MACHINE_TYPE,
   CLOUD_SERVICE_NAME,
   CONTAINER_ID,
-  HOST,
-  KUBERNETES,
-  SERVICE,
   SERVICE_NAME,
   SERVICE_NODE_NAME,
   SERVICE_VERSION,
   FAAS_ID,
   FAAS_TRIGGER_TYPE,
-  LABEL_TELEMETRY_AUTO_VERSION,
 } from '../../../common/es_fields/apm';
-
 import { ContainerType } from '../../../common/service_metadata';
-import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 import { should } from './get_service_metadata_icons';
 import { isOpenTelemetryAgentName, hasOpenTelemetryPrefix } from '../../../common/agent_name';
-
+import { maybe } from '../../../common/utils/maybe';
-type ServiceMetadataDetailsRaw = Pick<
-  TransactionRaw,
-  'service' | 'agent' | 'host' | 'container' | 'kubernetes' | 'cloud' | 'labels'
->;
 
 export interface ServiceMetadataDetails {
   service?: {
@@ -112,7 +101,6 @@ export async function getServiceMetadataDetails({
     body: {
       track_total_hits: 1,
       size: 1,
-      _source: [SERVICE, AGENT, HOST, CONTAINER, KUBERNETES, CLOUD, LABEL_TELEMETRY_AUTO_VERSION],
       query: { bool: { filter, should } },
       aggs: {
         serviceVersions: {
@@ -166,13 +154,17 @@ export async function getServiceMetadataDetails({
         },
         totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } },
       },
+      fields: ['*'],
     },
   };
 
   const response = await apmEventClient.search('get_service_metadata_details', params);
 
-  const hit = response.hits.hits[0]?._source as ServiceMetadataDetailsRaw | undefined;
+  const event = unflattenKnownApmEventFields(
-  if (!hit) {
+    maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent
+  );
+
+  if (!event) {
     return {
       service: undefined,
       container: undefined,
@@ -180,7 +172,7 @@ export async function getServiceMetadataDetails({
     };
   }
 
-  const { service, agent, host, kubernetes, container, cloud, labels } = hit;
+  const { service, agent, host, kubernetes, container, cloud, labels } = event;
 
   const serviceMetadataDetails = {
     versions: response.aggregations?.serviceVersions.buckets.map((bucket) => bucket.key as string),

x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts

@@ -7,12 +7,15 @@
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import type { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields';
+import { maybe } from '../../../common/utils/maybe';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   AGENT_NAME,
   CLOUD_PROVIDER,
   CLOUD_SERVICE_NAME,
   CONTAINER_ID,
-  KUBERNETES,
   SERVICE_NAME,
   KUBERNETES_POD_NAME,
   HOST_OS_PLATFORM,
@@ -20,14 +23,11 @@ import {
   AGENT_VERSION,
   SERVICE_FRAMEWORK_NAME,
 } from '../../../common/es_fields/apm';
-import { ContainerType } from '../../../common/service_metadata';
+import { ContainerType, SERVICE_METADATA_KUBERNETES_KEYS } from '../../../common/service_metadata';
-import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
 import { getProcessorEventForTransactions } from '../../lib/helpers/transactions';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless';
 
-type ServiceMetadataIconsRaw = Pick<TransactionRaw, 'kubernetes' | 'cloud' | 'container' | 'agent'>;
-
 export interface ServiceMetadataIcons {
   agentName?: string;
   containerType?: ContainerType;
@@ -61,6 +61,14 @@ export async function getServiceMetadataIcons({
 }): Promise<ServiceMetadataIcons> {
   const filter = [{ term: { [SERVICE_NAME]: serviceName } }, ...rangeQuery(start, end)];
 
+  const fields = asMutableArray([
+    CLOUD_PROVIDER,
+    CONTAINER_ID,
+    AGENT_NAME,
+    CLOUD_SERVICE_NAME,
+    ...SERVICE_METADATA_KUBERNETES_KEYS,
+  ] as const);
+
   const params = {
     apm: {
       events: [
@@ -72,8 +80,8 @@ export async function getServiceMetadataIcons({
     body: {
       track_total_hits: 1,
       size: 1,
-      _source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, CLOUD_SERVICE_NAME],
       query: { bool: { filter, should } },
+      fields,
     },
   };
 
@@ -88,9 +96,11 @@ export async function getServiceMetadataIcons({
     };
   }
 
-  const { kubernetes, cloud, container, agent } = response.hits.hits[0]
+  const event = unflattenKnownApmEventFields(
-    ._source as ServiceMetadataIconsRaw;
+    maybe(response.hits.hits[0])?.fields as undefined | FlattenedApmEvent
+  );
 
+  const { kubernetes, cloud, container, agent } = event ?? {};
   let containerType: ContainerType;
   if (!!kubernetes) {
     containerType = 'Kubernetes';

x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_children.ts

@@ -7,6 +7,8 @@
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { isEmpty } from 'lodash';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   PROCESSOR_EVENT,
   SPAN_ID,
@@ -16,8 +18,6 @@ import {
   TRACE_ID,
   TRANSACTION_ID,
 } from '../../../common/es_fields/apm';
-import type { SpanRaw } from '../../../typings/es_schemas/raw/span_raw';
-import type { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
 import { getBufferedTimerange } from './utils';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 
@@ -39,12 +39,16 @@ async function fetchLinkedChildrenOfSpan({
     end,
   });
 
+  const requiredFields = asMutableArray([TRACE_ID, PROCESSOR_EVENT] as const);
+  const optionalFields = asMutableArray([SPAN_ID, TRANSACTION_ID] as const);
+
   const response = await apmEventClient.search('fetch_linked_children_of_span', {
     apm: {
       events: [ProcessorEvent.span, ProcessorEvent.transaction],
     },
-    _source: [SPAN_LINKS, TRACE_ID, SPAN_ID, PROCESSOR_EVENT, TRANSACTION_ID],
+    _source: [SPAN_LINKS],
     body: {
+      fields: [...requiredFields, ...optionalFields],
       track_total_hits: false,
       size: 1000,
       query: {
@@ -58,19 +62,32 @@ async function fetchLinkedChildrenOfSpan({
       },
     },
   });
+
+  const linkedChildren = response.hits.hits.map((hit) => {
+    const source = 'span' in hit._source ? hit._source : undefined;
+    const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
+
+    return {
+      ...event,
+      span: {
+        ...event.span,
+        links: source?.span?.links ?? [],
+      },
+    };
+  });
   // Filter out documents that don't have any span.links that match the combination of traceId and spanId
-  return response.hits.hits.filter(({ _source: source }) => {
+  return linkedChildren.filter((linkedChild) => {
-    const spanLinks = source.span?.links?.filter((spanLink) => {
+    const spanLinks = linkedChild?.span?.links?.filter((spanLink) => {
       return spanLink.trace.id === traceId && (spanId ? spanLink.span.id === spanId : true);
     });
     return !isEmpty(spanLinks);
   });
 }
 
-function getSpanId(source: TransactionRaw | SpanRaw) {
+function getSpanId(
-  return source.processor.event === ProcessorEvent.span
+  linkedChild: Awaited<ReturnType<typeof fetchLinkedChildrenOfSpan>>[number]
-    ? (source as SpanRaw).span.id
+): string {
-    : (source as TransactionRaw).transaction?.id;
+  return (linkedChild.span.id ?? linkedChild.transaction?.id) as string;
 }
 
 export async function getSpanLinksCountById({
@@ -90,8 +107,9 @@ export async function getSpanLinksCountById({
     start,
     end,
   });
-  return linkedChildren.reduce<Record<string, number>>((acc, { _source: source }) => {
+
-    source.span?.links?.forEach((link) => {
+  return linkedChildren.reduce<Record<string, number>>((acc, item) => {
+    item.span?.links?.forEach((link) => {
       // Ignores span links that don't belong to this trace
       if (link.trace.id === traceId) {
         acc[link.span.id] = (acc[link.span.id] || 0) + 1;
@@ -122,10 +140,10 @@ export async function getLinkedChildrenOfSpan({
     end,
   });
 
-  return linkedChildren.map(({ _source: source }) => {
+  return linkedChildren.map((item) => {
     return {
-      trace: { id: source.trace.id },
+      trace: { id: item.trace.id },
-      span: { id: getSpanId(source) },
+      span: { id: getSpanId(item) },
     };
   });
 }

x-pack/plugins/observability_solution/apm/server/routes/span_links/get_linked_parents.ts

@@ -56,7 +56,7 @@ export async function getLinkedParentsOfSpan({
     },
   });
 
-  const source = response.hits.hits?.[0]?._source as TransactionRaw | SpanRaw;
+  const source = response.hits.hits?.[0]?._source as Pick<TransactionRaw | SpanRaw, 'span'>;
 
   return source?.span?.links || [];
 }

x-pack/plugins/observability_solution/apm/server/routes/span_links/get_span_links_details.ts

@@ -7,6 +7,8 @@
 import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { chunk, compact, isEmpty, keyBy } from 'lodash';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   SERVICE_NAME,
   SPAN_ID,
@@ -25,8 +27,6 @@ import {
 import { Environment } from '../../../common/environment_rt';
 import { SpanLinkDetails } from '../../../common/span_links';
 import { SpanLink } from '../../../typings/es_schemas/raw/fields/span_links';
-import { SpanRaw } from '../../../typings/es_schemas/raw/span_raw';
-import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw';
 import { getBufferedTimerange } from './utils';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 
@@ -48,26 +48,35 @@ async function fetchSpanLinksDetails({
     end,
   });
 
+  const requiredFields = asMutableArray([
+    TRACE_ID,
+    SERVICE_NAME,
+    AGENT_NAME,
+    PROCESSOR_EVENT,
+  ] as const);
+
+  const requiredTxFields = asMutableArray([
+    TRANSACTION_ID,
+    TRANSACTION_NAME,
+    TRANSACTION_DURATION,
+  ] as const);
+
+  const requiredSpanFields = asMutableArray([
+    SPAN_ID,
+    SPAN_NAME,
+    SPAN_DURATION,
+    SPAN_SUBTYPE,
+    SPAN_TYPE,
+  ] as const);
+
+  const optionalFields = asMutableArray([SERVICE_ENVIRONMENT] as const);
+
   const response = await apmEventClient.search('get_span_links_details', {
     apm: {
       events: [ProcessorEvent.span, ProcessorEvent.transaction],
     },
-    _source: [
-      TRACE_ID,
-      SPAN_ID,
-      TRANSACTION_ID,
-      SERVICE_NAME,
-      SPAN_NAME,
-      TRANSACTION_NAME,
-      TRANSACTION_DURATION,
-      SPAN_DURATION,
-      PROCESSOR_EVENT,
-      SPAN_SUBTYPE,
-      SPAN_TYPE,
-      AGENT_NAME,
-      SERVICE_ENVIRONMENT,
-    ],
     body: {
+      fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields],
       track_total_hits: false,
       size: 1000,
       query: {
@@ -106,16 +115,67 @@ async function fetchSpanLinksDetails({
 
   const spanIdsMap = keyBy(spanLinks, 'span.id');
 
-  return response.hits.hits.filter(({ _source: source }) => {
+  return response.hits.hits
-    // The above query might return other spans from the same transaction because siblings spans share the same transaction.id
+    .filter((hit) => {
-    // so, if it is a span we need to guarantee that the span.id is the same as the span links ids
+      // The above query might return other spans from the same transaction because siblings spans share the same transaction.id
-    if (source.processor.event === ProcessorEvent.span) {
+      // so, if it is a span we need to guarantee that the span.id is the same as the span links ids
-      const span = source as SpanRaw;
+      if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) {
-      const hasSpanId = spanIdsMap[span.span.id] || false;
+        const spanLink = unflattenKnownApmEventFields(hit.fields, [
-      return hasSpanId;
+          ...requiredFields,
-    }
+          ...requiredSpanFields,
-    return true;
+        ]);
-  });
+
+        const hasSpanId = Boolean(spanIdsMap[spanLink.span.id] || false);
+        return hasSpanId;
+      }
+      return true;
+    })
+    .map((hit) => {
+      const commonEvent = unflattenKnownApmEventFields(hit.fields, requiredFields);
+
+      const commonDetails = {
+        serviceName: commonEvent.service.name,
+        agentName: commonEvent.agent.name,
+        environment: commonEvent.service.environment as Environment,
+        transactionId: commonEvent.transaction?.id,
+      };
+
+      if (commonEvent.processor.event === ProcessorEvent.transaction) {
+        const event = unflattenKnownApmEventFields(hit.fields, [
+          ...requiredFields,
+          ...requiredTxFields,
+        ]);
+        return {
+          traceId: event.trace.id,
+          spanId: event.transaction.id,
+          processorEvent: commonEvent.processor.event,
+          transactionId: event.transaction.id,
+          details: {
+            ...commonDetails,
+            spanName: event.transaction.name,
+            duration: event.transaction.duration.us,
+          },
+        };
+      } else {
+        const event = unflattenKnownApmEventFields(hit.fields, [
+          ...requiredFields,
+          ...requiredSpanFields,
+        ]);
+
+        return {
+          traceId: event.trace.id,
+          spanId: event.span.id,
+          processorEvent: commonEvent.processor.event,
+          details: {
+            ...commonDetails,
+            spanName: event.span.name,
+            duration: event.span.duration.us,
+            spanSubtype: event.span.subtype,
+            spanType: event.span.type,
+          },
+        };
+      }
+    });
 }
 
 export async function getSpanLinksDetails({
@@ -153,39 +213,20 @@ export async function getSpanLinksDetails({
 
   // Creates a map for all span links details found
   const spanLinksDetailsMap = linkedSpans.reduce<Record<string, SpanLinkDetails>>(
-    (acc, { _source: source }) => {
+    (acc, spanLink) => {
-      const commonDetails = {
+      if (spanLink.processorEvent === ProcessorEvent.transaction) {
-        serviceName: source.service.name,
+        const key = `${spanLink.traceId}:${spanLink.transactionId}`;
-        agentName: source.agent.name,
-        environment: source.service.environment as Environment,
-        transactionId: source.transaction?.id,
-      };
-
-      if (source.processor.event === ProcessorEvent.transaction) {
-        const transaction = source as TransactionRaw;
-        const key = `${transaction.trace.id}:${transaction.transaction.id}`;
         acc[key] = {
-          traceId: source.trace.id,
+          traceId: spanLink.traceId,
-          spanId: transaction.transaction.id,
+          spanId: spanLink.transactionId,
-          details: {
+          details: spanLink.details,
-            ...commonDetails,
-            spanName: transaction.transaction.name,
-            duration: transaction.transaction.duration.us,
-          },
         };
       } else {
-        const span = source as SpanRaw;
+        const key = `${spanLink.traceId}:${spanLink.spanId}`;
-        const key = `${span.trace.id}:${span.span.id}`;
         acc[key] = {
-          traceId: source.trace.id,
+          traceId: spanLink.traceId,
-          spanId: span.span.id,
+          spanId: spanLink.spanId,
-          details: {
+          details: spanLink.details,
-            ...commonDetails,
-            spanName: span.span.name,
-            duration: span.span.duration.us,
-            spanSubtype: span.span.subtype,
-            spanType: span.span.type,
-          },
         };
       }
 

x-pack/plugins/observability_solution/apm/server/routes/traces/__snapshots__/queries.test.ts.snap

@@ -12,15 +12,26 @@ Object {
   },
   "body": Object {
     "_source": Array [
+      "error.log.message",
+      "error.exception.message",
+      "error.exception.handled",
+      "error.exception.type",
+    ],
+    "fields": Array [
       "timestamp.us",
       "trace.id",
-      "transaction.id",
-      "parent.id",
       "service.name",
       "error.id",
-      "error.log.message",
-      "error.exception",
       "error.grouping_key",
+      "processor.event",
+      "parent.id",
+      "transaction.id",
+      "span.id",
+      "error.culprit",
+      "error.log.message",
+      "error.exception.message",
+      "error.exception.handled",
+      "error.exception.type",
     ],
     "query": Object {
       "bool": Object {

x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_items.ts

@@ -10,12 +10,17 @@ import { SortResults } from '@elastic/elasticsearch/lib/api/types';
 import { QueryDslQueryContainer, Sort } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { rangeQuery } from '@kbn/observability-plugin/server';
-import { last } from 'lodash';
+import { last, omit } from 'lodash';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import { APMConfig } from '../..';
 import {
   AGENT_NAME,
   CHILD_ID,
-  ERROR_EXCEPTION,
+  ERROR_CULPRIT,
+  ERROR_EXC_HANDLED,
+  ERROR_EXC_MESSAGE,
+  ERROR_EXC_TYPE,
   ERROR_GROUP_ID,
   ERROR_ID,
   ERROR_LOG_LEVEL,
@@ -37,7 +42,7 @@ import {
   SPAN_SUBTYPE,
   SPAN_SYNC,
   SPAN_TYPE,
-  TIMESTAMP,
+  TIMESTAMP_US,
   TRACE_ID,
   TRANSACTION_DURATION,
   TRANSACTION_ID,
@@ -84,6 +89,26 @@ export async function getTraceItems({
   const maxTraceItems = maxTraceItemsFromUrlParam ?? config.ui.maxTraceItems;
   const excludedLogLevels = ['debug', 'info', 'warning'];
 
+  const requiredFields = asMutableArray([
+    TIMESTAMP_US,
+    TRACE_ID,
+    SERVICE_NAME,
+    ERROR_ID,
+    ERROR_GROUP_ID,
+    PROCESSOR_EVENT,
+  ] as const);
+
+  const optionalFields = asMutableArray([
+    PARENT_ID,
+    TRANSACTION_ID,
+    SPAN_ID,
+    ERROR_CULPRIT,
+    ERROR_LOG_MESSAGE,
+    ERROR_EXC_MESSAGE,
+    ERROR_EXC_HANDLED,
+    ERROR_EXC_TYPE,
+  ] as const);
+
   const errorResponsePromise = apmEventClient.search('get_errors_docs', {
     apm: {
       sources: [
@@ -96,23 +121,14 @@ export async function getTraceItems({
     body: {
       track_total_hits: false,
       size: 1000,
-      _source: [
-        TIMESTAMP,
-        TRACE_ID,
-        TRANSACTION_ID,
-        PARENT_ID,
-        SERVICE_NAME,
-        ERROR_ID,
-        ERROR_LOG_MESSAGE,
-        ERROR_EXCEPTION,
-        ERROR_GROUP_ID,
-      ],
       query: {
         bool: {
           filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)],
           must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } },
         },
       },
+      fields: [...requiredFields, ...optionalFields],
+      _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, ERROR_EXC_HANDLED, ERROR_EXC_TYPE],
     },
   });
 
@@ -133,8 +149,32 @@ export async function getTraceItems({
 
   const traceDocsTotal = traceResponse.total;
   const exceedsMax = traceDocsTotal > maxTraceItems;
-  const traceDocs = traceResponse.hits.map((hit) => hit._source);
+
-  const errorDocs = errorResponse.hits.hits.map((hit) => hit._source);
+  const traceDocs = traceResponse.hits.map(({ hit }) => hit);
+
+  const errorDocs = errorResponse.hits.hits.map((hit) => {
+    const errorSource = 'error' in hit._source ? hit._source : undefined;
+
+    const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
+
+    const waterfallErrorEvent: WaterfallError = {
+      ...event,
+      parent: {
+        ...event?.parent,
+        id: event?.parent?.id ?? event?.span?.id,
+      },
+      error: {
+        ...(event.error ?? {}),
+        exception:
+          (errorSource?.error.exception?.length ?? 0) > 1
+            ? errorSource?.error.exception
+            : event?.error.exception && [event.error.exception],
+        log: errorSource?.error.log,
+      },
+    };
+
+    return waterfallErrorEvent;
+  });
 
   return {
     exceedsMax,
@@ -220,41 +260,54 @@ async function getTraceDocsPerPage({
   start: number;
   end: number;
   searchAfter?: SortResults;
-}) {
+}): Promise<{
+  hits: Array<{ hit: WaterfallTransaction | WaterfallSpan; sort: SortResults | undefined }>;
+  total: number;
+}> {
   const size = Math.min(maxTraceItems, MAX_ITEMS_PER_PAGE);
 
+  const requiredFields = asMutableArray([
+    AGENT_NAME,
+    TIMESTAMP_US,
+    TRACE_ID,
+    SERVICE_NAME,
+    PROCESSOR_EVENT,
+  ] as const);
+
+  const requiredTxFields = asMutableArray([
+    TRANSACTION_ID,
+    TRANSACTION_DURATION,
+    TRANSACTION_NAME,
+    TRANSACTION_TYPE,
+  ] as const);
+
+  const requiredSpanFields = asMutableArray([
+    SPAN_ID,
+    SPAN_TYPE,
+    SPAN_NAME,
+    SPAN_DURATION,
+  ] as const);
+
+  const optionalFields = asMutableArray([
+    PARENT_ID,
+    SERVICE_ENVIRONMENT,
+    EVENT_OUTCOME,
+    TRANSACTION_RESULT,
+    FAAS_COLDSTART,
+    SPAN_SUBTYPE,
+    SPAN_ACTION,
+    SPAN_COMPOSITE_COUNT,
+    SPAN_COMPOSITE_COMPRESSION_STRATEGY,
+    SPAN_COMPOSITE_SUM,
+    SPAN_SYNC,
+    CHILD_ID,
+  ] as const);
+
   const body = {
     track_total_hits: true,
     size,
     search_after: searchAfter,
-    _source: [
+    _source: [SPAN_LINKS],
-      TIMESTAMP,
-      TRACE_ID,
-      PARENT_ID,
-      SERVICE_NAME,
-      SERVICE_ENVIRONMENT,
-      AGENT_NAME,
-      EVENT_OUTCOME,
-      PROCESSOR_EVENT,
-      TRANSACTION_DURATION,
-      TRANSACTION_ID,
-      TRANSACTION_NAME,
-      TRANSACTION_TYPE,
-      TRANSACTION_RESULT,
-      FAAS_COLDSTART,
-      SPAN_ID,
-      SPAN_TYPE,
-      SPAN_SUBTYPE,
-      SPAN_ACTION,
-      SPAN_NAME,
-      SPAN_DURATION,
-      SPAN_LINKS,
-      SPAN_COMPOSITE_COUNT,
-      SPAN_COMPOSITE_COMPRESSION_STRATEGY,
-      SPAN_COMPOSITE_SUM,
-      SPAN_SYNC,
-      CHILD_ID,
-    ],
     query: {
       bool: {
         filter: [
@@ -266,6 +319,7 @@ async function getTraceDocsPerPage({
         },
       },
     },
+    fields: [...requiredFields, ...requiredTxFields, ...requiredSpanFields, ...optionalFields],
     sort: [
       { _score: 'asc' },
       {
@@ -291,7 +345,51 @@ async function getTraceDocsPerPage({
   });
 
   return {
-    hits: res.hits.hits,
+    hits: res.hits.hits.map((hit) => {
+      const sort = hit.sort;
+      const spanLinksSource = 'span' in hit._source ? hit._source.span?.links : undefined;
+
+      if (hit.fields[PROCESSOR_EVENT]?.[0] === ProcessorEvent.span) {
+        const spanEvent = unflattenKnownApmEventFields(hit.fields, [
+          ...requiredFields,
+          ...requiredSpanFields,
+        ]);
+
+        const spanWaterfallEvent: WaterfallSpan = {
+          ...omit(spanEvent, 'child'),
+          processor: {
+            event: 'span',
+          },
+          span: {
+            ...spanEvent.span,
+            composite: spanEvent.span.composite
+              ? (spanEvent.span.composite as Required<WaterfallSpan['span']>['composite'])
+              : undefined,
+            links: spanLinksSource,
+          },
+          ...(spanEvent.child ? { child: spanEvent.child as WaterfallSpan['child'] } : {}),
+        };
+
+        return { sort, hit: spanWaterfallEvent };
+      }
+
+      const txEvent = unflattenKnownApmEventFields(hit.fields, [
+        ...requiredFields,
+        ...requiredTxFields,
+      ]);
+      const txWaterfallEvent: WaterfallTransaction = {
+        ...txEvent,
+        processor: {
+          event: 'transaction',
+        },
+        span: {
+          ...txEvent.span,
+          links: spanLinksSource,
+        },
+      };
+
+      return { hit: txWaterfallEvent, sort };
+    }),
     total: res.hits.total.value,
   };
 }

x-pack/plugins/observability_solution/apm/server/routes/traces/get_trace_samples_by_query.ts

@@ -89,10 +89,10 @@ export async function getTraceSamplesByQuery({
           },
           event_category_field: PROCESSOR_EVENT,
           query,
-          filter_path: 'hits.sequences.events._source.trace.id',
+          fields: [TRACE_ID],
         })
       ).hits?.sequences?.flatMap((sequence) =>
-        sequence.events.map((event) => (event._source as { trace: { id: string } }).trace.id)
+        sequence.events.map((event) => (event.fields as { [TRACE_ID]: [string] })[TRACE_ID][0])
       ) ?? [];
   }
 

x-pack/plugins/observability_solution/apm/server/routes/traces/route.ts

@@ -12,7 +12,10 @@ import { getSearchTransactionsEvents } from '../../lib/helpers/transactions';
 import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
 import { environmentRt, kueryRt, probabilityRt, rangeRt } from '../default_api_types';
 import { getTransaction } from '../transactions/get_transaction';
-import { getRootTransactionByTraceId } from '../transactions/get_transaction_by_trace';
+import {
+  type TransactionDetailRedirectInfo,
+  getRootTransactionByTraceId,
+} from '../transactions/get_transaction_by_trace';
 import {
   getTopTracesPrimaryStats,
   TopTracesPrimaryStatsResponse,
@@ -128,7 +131,7 @@ const rootTransactionByTraceIdRoute = createApmServerRoute({
   handler: async (
     resources
   ): Promise<{
-    transaction: Transaction;
+    transaction?: TransactionDetailRedirectInfo;
   }> => {
     const {
       params: {
@@ -155,7 +158,7 @@ const transactionByIdRoute = createApmServerRoute({
   handler: async (
     resources
   ): Promise<{
-    transaction: Transaction;
+    transaction?: Transaction;
   }> => {
     const {
       params: {
@@ -191,7 +194,7 @@ const transactionByNameRoute = createApmServerRoute({
   handler: async (
     resources
   ): Promise<{
-    transaction: Transaction;
+    transaction?: TransactionDetailRedirectInfo;
   }> => {
     const {
       params: {
@@ -295,7 +298,7 @@ const transactionFromTraceByIdRoute = createApmServerRoute({
     query: rangeRt,
   }),
   options: { tags: ['access:apm'] },
-  handler: async (resources): Promise<Transaction> => {
+  handler: async (resources): Promise<Transaction | undefined> => {
     const { params } = resources;
     const {
       path: { transactionId, traceId },

x-pack/plugins/observability_solution/apm/server/routes/transactions/__snapshots__/queries.test.ts.snap

@@ -11,6 +11,25 @@ Object {
     ],
   },
   "body": Object {
+    "_source": Array [
+      "span.links",
+      "transaction.agent.marks",
+    ],
+    "fields": Array [
+      "trace.id",
+      "agent.name",
+      "processor.event",
+      "@timestamp",
+      "timestamp.us",
+      "service.name",
+      "transaction.id",
+      "transaction.duration.us",
+      "transaction.name",
+      "transaction.sampled",
+      "transaction.type",
+      "processor.name",
+      "service.language.name",
+    ],
     "query": Object {
       "bool": Object {
         "filter": Array [
@@ -311,6 +330,11 @@ Object {
     ],
   },
   "body": Object {
+    "fields": Array [
+      "transaction.id",
+      "trace.id",
+      "@timestamp",
+    ],
     "query": Object {
       "bool": Object {
         "filter": Array [

x-pack/plugins/observability_solution/apm/server/routes/transactions/get_span/index.ts

@@ -7,7 +7,11 @@
 
 import { rangeQuery, termQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
-import { SPAN_ID, TRACE_ID } from '../../../../common/es_fields/apm';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { FlattenedApmEvent } from '@kbn/apm-data-access-plugin/server/utils/unflatten_known_fields';
+import { merge, omit } from 'lodash';
+import { maybe } from '../../../../common/utils/maybe';
+import { SPAN_ID, SPAN_STACKTRACE, TRACE_ID } from '../../../../common/es_fields/apm';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
 import { getTransaction } from '../get_transaction';
@@ -38,6 +42,8 @@ export async function getSpan({
         track_total_hits: false,
         size: 1,
         terminate_after: 1,
+        fields: ['*'],
+        _source: [SPAN_STACKTRACE],
         query: {
           bool: {
             filter: asMutableArray([
@@ -60,5 +66,17 @@ export async function getSpan({
       : undefined,
   ]);
 
-  return { span: spanResp.hits.hits[0]?._source, parentTransaction };
+  const hit = maybe(spanResp.hits.hits[0]);
+  const spanFromSource = hit && 'span' in hit._source ? hit._source : undefined;
+
+  const event = unflattenKnownApmEventFields(hit?.fields as undefined | FlattenedApmEvent);
+
+  return {
+    span: event
+      ? merge({}, omit(event, 'span.links'), spanFromSource, {
+          processor: { event: 'span' as const, name: 'transaction' as const },
+        })
+      : undefined,
+    parentTransaction,
+  };
 }

x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts

@@ -6,7 +6,26 @@
  */
 
 import { rangeQuery, termQuery } from '@kbn/observability-plugin/server';
-import { TRACE_ID, TRANSACTION_ID } from '../../../../common/es_fields/apm';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import type { Transaction } from '@kbn/apm-types';
+import { maybe } from '../../../../common/utils/maybe';
+import {
+  AGENT_NAME,
+  PROCESSOR_EVENT,
+  SERVICE_NAME,
+  TIMESTAMP_US,
+  TRACE_ID,
+  TRANSACTION_DURATION,
+  TRANSACTION_ID,
+  TRANSACTION_NAME,
+  TRANSACTION_SAMPLED,
+  TRANSACTION_TYPE,
+  AT_TIMESTAMP,
+  PROCESSOR_NAME,
+  SPAN_LINKS,
+  TRANSACTION_AGENT_MARKS,
+  SERVICE_LANGUAGE_NAME,
+} from '../../../../common/es_fields/apm';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
 import { ApmDocumentType } from '../../../../common/document_type';
@@ -24,7 +43,23 @@ export async function getTransaction({
   apmEventClient: APMEventClient;
   start: number;
   end: number;
-}) {
+}): Promise<Transaction | undefined> {
+  const requiredFields = asMutableArray([
+    TRACE_ID,
+    AGENT_NAME,
+    PROCESSOR_EVENT,
+    AT_TIMESTAMP,
+    TIMESTAMP_US,
+    SERVICE_NAME,
+    TRANSACTION_ID,
+    TRANSACTION_DURATION,
+    TRANSACTION_NAME,
+    TRANSACTION_SAMPLED,
+    TRANSACTION_TYPE,
+  ] as const);
+
+  const optionalFields = asMutableArray([PROCESSOR_NAME, SERVICE_LANGUAGE_NAME] as const);
+
   const resp = await apmEventClient.search('get_transaction', {
     apm: {
       sources: [
@@ -47,8 +82,37 @@ export async function getTransaction({
           ]),
         },
       },
+      fields: [...requiredFields, ...optionalFields],
+      _source: [SPAN_LINKS, TRANSACTION_AGENT_MARKS],
     },
   });
 
-  return resp.hits.hits[0]?._source;
+  const hit = maybe(resp.hits.hits[0]);
+
+  if (!hit) {
+    return undefined;
+  }
+
+  const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
+
+  const source =
+    'span' in hit._source && 'transaction' in hit._source
+      ? (hit._source as {
+          transaction: Pick<Required<Transaction>['transaction'], 'marks'>;
+          span?: Pick<Required<Transaction>['span'], 'links'>;
+        })
+      : undefined;
+
+  return {
+    ...event,
+    transaction: {
+      ...event.transaction,
+      marks: source?.transaction.marks,
+    },
+    processor: {
+      name: 'transaction',
+      event: 'transaction',
+    },
+    span: source?.span,
+  };
 }

x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_name/index.ts

@@ -6,11 +6,22 @@
  */
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { maybe } from '../../../../common/utils/maybe';
 import { ApmDocumentType } from '../../../../common/document_type';
-import { SERVICE_NAME, TRANSACTION_NAME } from '../../../../common/es_fields/apm';
+import {
+  AT_TIMESTAMP,
+  SERVICE_NAME,
+  TRACE_ID,
+  TRANSACTION_DURATION,
+  TRANSACTION_ID,
+  TRANSACTION_NAME,
+  TRANSACTION_TYPE,
+} from '../../../../common/es_fields/apm';
 import { RollupInterval } from '../../../../common/rollup';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
+import { TransactionDetailRedirectInfo } from '../get_transaction_by_trace';
 
 export async function getTransactionByName({
   transactionName,
@@ -24,7 +35,17 @@ export async function getTransactionByName({
   apmEventClient: APMEventClient;
   start: number;
   end: number;
-}) {
+}): Promise<TransactionDetailRedirectInfo | undefined> {
+  const requiredFields = asMutableArray([
+    AT_TIMESTAMP,
+    TRACE_ID,
+    TRANSACTION_ID,
+    TRANSACTION_TYPE,
+    TRANSACTION_NAME,
+    TRANSACTION_DURATION,
+    SERVICE_NAME,
+  ] as const);
+
   const resp = await apmEventClient.search('get_transaction', {
     apm: {
       sources: [
@@ -47,8 +68,9 @@ export async function getTransactionByName({
           ]),
         },
       },
+      fields: requiredFields,
     },
   });
 
-  return resp.hits.hits[0]?._source;
+  return unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields);
 }

x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction_by_trace/index.ts

@@ -7,9 +7,40 @@
 
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { rangeQuery } from '@kbn/observability-plugin/server';
-import { TRACE_ID, PARENT_ID } from '../../../../common/es_fields/apm';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { maybe } from '../../../../common/utils/maybe';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
+import {
+  TRACE_ID,
+  PARENT_ID,
+  AT_TIMESTAMP,
+  TRANSACTION_DURATION,
+  TRANSACTION_ID,
+  TRANSACTION_NAME,
+  TRANSACTION_TYPE,
+  SERVICE_NAME,
+} from '../../../../common/es_fields/apm';
 import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
 
+export interface TransactionDetailRedirectInfo {
+  [AT_TIMESTAMP]: string;
+  trace: {
+    id: string;
+  };
+  transaction: {
+    id: string;
+    type: string;
+    name: string;
+
+    duration: {
+      us: number;
+    };
+  };
+  service: {
+    name: string;
+  };
+}
+
 export async function getRootTransactionByTraceId({
   traceId,
   apmEventClient,
@@ -20,7 +51,19 @@ export async function getRootTransactionByTraceId({
   apmEventClient: APMEventClient;
   start: number;
   end: number;
-}) {
+}): Promise<{
+  transaction: TransactionDetailRedirectInfo | undefined;
+}> {
+  const requiredFields = asMutableArray([
+    TRACE_ID,
+    TRANSACTION_ID,
+    TRANSACTION_NAME,
+    AT_TIMESTAMP,
+    TRANSACTION_TYPE,
+    TRANSACTION_DURATION,
+    SERVICE_NAME,
+  ] as const);
+
   const params = {
     apm: {
       events: [ProcessorEvent.transaction as const],
@@ -45,11 +88,15 @@ export async function getRootTransactionByTraceId({
           filter: [{ term: { [TRACE_ID]: traceId } }, ...rangeQuery(start, end)],
         },
       },
+      fields: requiredFields,
     },
   };
 
   const resp = await apmEventClient.search('get_root_transaction_by_trace_id', params);
+
+  const event = unflattenKnownApmEventFields(maybe(resp.hits.hits[0])?.fields, requiredFields);
+
   return {
-    transaction: resp.hits.hits[0]?._source,
+    transaction: event,
   };
 }

x-pack/plugins/observability_solution/apm/server/routes/transactions/trace_samples/index.ts

@@ -7,7 +7,10 @@
 import { Sort, QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server';
+import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import {
+  AT_TIMESTAMP,
   SERVICE_NAME,
   TRACE_ID,
   TRANSACTION_ID,
@@ -77,6 +80,8 @@ export async function getTraceSamples({
       });
     }
 
+    const requiredFields = asMutableArray([TRANSACTION_ID, TRACE_ID, AT_TIMESTAMP] as const);
+
     const response = await apmEventClient.search('get_trace_samples_hits', {
       apm: {
         events: [ProcessorEvent.transaction],
@@ -94,6 +99,7 @@ export async function getTraceSamples({
           },
         },
         size: TRACE_SAMPLES_SIZE,
+        fields: requiredFields,
         sort: [
           {
             _score: {
@@ -101,7 +107,7 @@ export async function getTraceSamples({
             },
           },
           {
-            '@timestamp': {
+            [AT_TIMESTAMP]: {
               order: 'desc',
             },
           },
@@ -109,12 +115,15 @@ export async function getTraceSamples({
       },
     });
 
-    const traceSamples = response.hits.hits.map((hit) => ({
+    const traceSamples = response.hits.hits.map((hit) => {
-      score: hit._score,
+      const event = unflattenKnownApmEventFields(hit.fields, requiredFields);
-      timestamp: hit._source['@timestamp'],
+      return {
-      transactionId: hit._source.transaction.id,
+        score: hit._score,
-      traceId: hit._source.trace.id,
+        timestamp: event[AT_TIMESTAMP],
-    }));
+        transactionId: event.transaction.id,
+        traceId: event.trace.id,
+      };
+    });
 
     return { traceSamples };
   });

x-pack/plugins/observability_solution/apm_data_access/server/utils.ts

@@ -15,3 +15,4 @@ export {
 } from './lib/helpers';
 
 export { withApmSpan } from './utils/with_apm_span';
+export { unflattenKnownApmEventFields } from './utils/unflatten_known_fields';

x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.test.ts

@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { unflattenKnownApmEventFields } from './unflatten_known_fields';
+
+describe('unflattenKnownApmEventFields', () => {
+  it('should return an empty object when input is empty', () => {
+    const input = {};
+    const expectedOutput = {};
+    expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
+  });
+
+  it('should correctly unflatten a simple flat input', () => {
+    const input = {
+      '@timestamp': '2024-10-10T10:10:10.000Z',
+    };
+    const expectedOutput = {
+      '@timestamp': '2024-10-10T10:10:10.000Z',
+    };
+    expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
+  });
+
+  it('should override unknown fields', () => {
+    const input = {
+      'service.name': 'node-svc',
+      'service.name.text': 'node-svc',
+    };
+    const expectedOutput = {
+      service: {
+        name: 'node-svc',
+      },
+    };
+
+    expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
+  });
+
+  it('should correctly unflatten multiple nested fields', () => {
+    const input = {
+      'service.name': 'node-svc',
+      'service.version': '1.0.0',
+      'service.environment': 'production',
+      'agent.name': 'nodejs',
+    };
+    const expectedOutput = {
+      service: {
+        name: 'node-svc',
+        version: '1.0.0',
+        environment: 'production',
+      },
+      agent: {
+        name: 'nodejs',
+      },
+    };
+    expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
+  });
+
+  it('should handle multiple values for multi-valued fields', () => {
+    const input = {
+      'service.name': 'node-svc',
+      'service.tags': ['foo', 'bar'],
+    };
+    const expectedOutput = {
+      service: {
+        name: 'node-svc',
+        tags: ['foo', 'bar'],
+      },
+    };
+    expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
+  });
+
+  it('should correctly unflatten with empty multi-valued fields', () => {
+    const input = {
+      'service.name': 'node-svc',
+      'service.tags': [],
+    };
+    const expectedOutput = {
+      service: {
+        name: 'node-svc',
+        tags: [],
+      },
+    };
+    expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
+  });
+
+  it('should retain unknown fields in the output', () => {
+    const input = {
+      'service.name': 'node-svc',
+      'unknown.texts': ['foo', 'bar'],
+      'unknown.field': 'foo',
+      unknonwField: 'bar',
+    };
+    const expectedOutput = {
+      service: {
+        name: 'node-svc',
+      },
+      unknown: {
+        field: 'foo',
+        texts: ['foo', 'bar'],
+      },
+      unknonwField: 'bar',
+    };
+    expect(unflattenKnownApmEventFields(input)).toEqual(expectedOutput);
+  });
+
+  it('should correctly unflatten nested fields with mandatory field', () => {
+    const input = {
+      'service.name': 'node-svc',
+      'service.environment': undefined,
+    };
+
+    const requiredFields: ['service.name'] = ['service.name'];
+
+    const expectedOutput = {
+      service: {
+        name: 'node-svc',
+      },
+    };
+    expect(unflattenKnownApmEventFields(input, requiredFields)).toEqual(expectedOutput);
+  });
+
+  it('should throw an exception when mandatory field is not in the input', () => {
+    const input = {
+      'service.environment': 'PROD',
+    };
+
+    const requiredFields: ['service.name'] = ['service.name'];
+
+    // @ts-expect-error
+    expect(() => unflattenKnownApmEventFields(input, requiredFields)).toThrowError(
+      'Missing required fields service.name in event'
+    );
+  });
+});

x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts

@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DedotObject } from '@kbn/utility-types';
+import * as APM_EVENT_FIELDS_MAP from '@kbn/apm-types/es_fields';
+import type { ValuesType } from 'utility-types';
+import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object';
+import { mergePlainObjects } from '@kbn/observability-utils/object/merge_plain_objects';
+import { castArray, isArray } from 'lodash';
+import { AgentName } from '@kbn/elastic-agent-utils';
+import { EventOutcome } from '@kbn/apm-types/src/es_schemas/raw/fields';
+import { ProcessorEvent } from '@kbn/observability-plugin/common';
+
+const {
+  CLOUD,
+  AGENT,
+  SERVICE,
+  ERROR_EXCEPTION,
+  SPAN_LINKS,
+  HOST,
+  KUBERNETES,
+  CONTAINER,
+  TIER,
+  INDEX,
+  DATA_STEAM_TYPE,
+  VALUE_OTEL_JVM_PROCESS_MEMORY_HEAP,
+  VALUE_OTEL_JVM_PROCESS_MEMORY_NON_HEAP,
+  SPAN_LINKS_SPAN_ID,
+  SPAN_LINKS_TRACE_ID,
+  SPAN_STACKTRACE,
+  ...CONCRETE_FIELDS
+} = APM_EVENT_FIELDS_MAP;
+
+const ALL_FIELDS = Object.values(CONCRETE_FIELDS);
+
+const KNOWN_MULTI_VALUED_FIELDS = [
+  APM_EVENT_FIELDS_MAP.CHILD_ID,
+  APM_EVENT_FIELDS_MAP.PROCESS_ARGS,
+] as const;
+
+type KnownField = ValuesType<typeof CONCRETE_FIELDS>;
+
+type KnownSingleValuedField = Exclude<KnownField, KnownMultiValuedField>;
+type KnownMultiValuedField = ValuesType<typeof KNOWN_MULTI_VALUED_FIELDS>;
+
+const KNOWN_SINGLE_VALUED_FIELDS = ALL_FIELDS.filter(
+  (field): field is KnownSingleValuedField => !KNOWN_MULTI_VALUED_FIELDS.includes(field as any)
+);
+
+interface TypeOverrideMap {
+  [APM_EVENT_FIELDS_MAP.SPAN_DURATION]: number;
+  [APM_EVENT_FIELDS_MAP.AGENT_NAME]: AgentName;
+  [APM_EVENT_FIELDS_MAP.EVENT_OUTCOME]: EventOutcome;
+  [APM_EVENT_FIELDS_MAP.FAAS_COLDSTART]: true;
+  [APM_EVENT_FIELDS_MAP.TRANSACTION_DURATION]: number;
+  [APM_EVENT_FIELDS_MAP.TIMESTAMP_US]: number;
+  [APM_EVENT_FIELDS_MAP.PROCESSOR_EVENT]: ProcessorEvent;
+  [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_COUNT]: number;
+  [APM_EVENT_FIELDS_MAP.SPAN_COMPOSITE_SUM]: number;
+  [APM_EVENT_FIELDS_MAP.SPAN_SYNC]: boolean;
+  [APM_EVENT_FIELDS_MAP.TRANSACTION_SAMPLED]: boolean;
+  [APM_EVENT_FIELDS_MAP.PROCESSOR_NAME]: 'transaction' | 'metric' | 'error';
+  [APM_EVENT_FIELDS_MAP.HTTP_RESPONSE_STATUS_CODE]: number;
+  [APM_EVENT_FIELDS_MAP.PROCESS_PID]: number;
+  [APM_EVENT_FIELDS_MAP.OBSERVER_VERSION_MAJOR]: number;
+  [APM_EVENT_FIELDS_MAP.ERROR_EXC_HANDLED]: boolean;
+}
+
+type MaybeMultiValue<T extends KnownField, U> = T extends KnownMultiValuedField ? U[] : U;
+
+type TypeOfKnownField<T extends KnownField> = MaybeMultiValue<
+  T,
+  T extends keyof TypeOverrideMap ? TypeOverrideMap[T] : string
+>;
+
+type MapToSingleOrMultiValue<T extends Record<string, any>> = {
+  [TKey in keyof T]: TKey extends KnownField
+    ? T[TKey] extends undefined
+      ? TypeOfKnownField<TKey> | undefined
+      : TypeOfKnownField<TKey>
+    : unknown;
+};
+
+type UnflattenedKnownFields<T extends Record<string, any>> = DedotObject<
+  MapToSingleOrMultiValue<T>
+>;
+
+export type FlattenedApmEvent = Record<KnownSingleValuedField | KnownMultiValuedField, unknown[]>;
+
+export type UnflattenedApmEvent = UnflattenedKnownFields<FlattenedApmEvent>;
+
+export function unflattenKnownApmEventFields<T extends Record<string, any> | undefined = undefined>(
+  fields: T
+): T extends Record<string, any> ? UnflattenedKnownFields<T> : undefined;
+
+export function unflattenKnownApmEventFields<
+  T extends Record<string, any> | undefined,
+  U extends Array<keyof Exclude<T, undefined>>
+>(
+  fields: T,
+  required: U
+): T extends Record<string, any>
+  ? UnflattenedKnownFields<T> &
+      (U extends any[]
+        ? UnflattenedKnownFields<{
+            [TKey in ValuesType<U>]: keyof T extends TKey ? T[TKey] : unknown[];
+          }>
+        : {})
+  : undefined;
+
+export function unflattenKnownApmEventFields(
+  hitFields?: Record<string, any>,
+  requiredFields?: string[]
+) {
+  if (!hitFields) {
+    return undefined;
+  }
+  const missingRequiredFields =
+    requiredFields?.filter((key) => {
+      const value = hitFields?.[key];
+      return value === null || value === undefined || (isArray(value) && value.length === 0);
+    }) ?? [];
+
+  if (missingRequiredFields.length > 0) {
+    throw new Error(`Missing required fields ${missingRequiredFields.join(', ')} in event`);
+  }
+
+  const copy: Record<string, any> = mapToSingleOrMultiValue({
+    ...hitFields,
+  });
+
+  const [knownFields, unknownFields] = Object.entries(copy).reduce(
+    (prev, [key, value]) => {
+      if (ALL_FIELDS.includes(key as KnownField)) {
+        prev[0][key as KnownField] = value;
+      } else {
+        prev[1][key] = value;
+      }
+      return prev;
+    },
+    [{} as Record<KnownField, any>, {} as Record<string, any>]
+  );
+
+  const unflattened = mergePlainObjects(
+    {},
+    unflattenObject(unknownFields),
+    unflattenObject(knownFields)
+  );
+
+  return unflattened;
+}
+
+export function mapToSingleOrMultiValue<T extends Record<string, any>>(
+  fields: T
+): MapToSingleOrMultiValue<T> {
+  KNOWN_SINGLE_VALUED_FIELDS.forEach((field) => {
+    const value = fields[field];
+    if (value !== null && value !== undefined) {
+      fields[field as keyof T] = castArray(value)[0];
+    }
+  });
+
+  return fields;
+}

x-pack/plugins/observability_solution/apm_data_access/tsconfig.json

@@ -20,6 +20,8 @@
     "@kbn/apm-utils",
     "@kbn/core-http-server",
     "@kbn/security-plugin-types-server",
-    "@kbn/observability-utils"
+    "@kbn/observability-utils",
+    "@kbn/utility-types",
+    "@kbn/elastic-agent-utils"
   ]
 }