[APM] Wrap Elasticsearch client errors (#91125)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/apm/server/lib/services/annotations/get_stored_annotations.ts

@@ -6,7 +6,11 @@
  */
 
 import { ElasticsearchClient, Logger } from 'kibana/server';
-import { unwrapEsResponse } from '../../../../../observability/server';
+import { ResponseError } from '@elastic/elasticsearch/lib/errors';
+import {
+  unwrapEsResponse,
+  WrappedElasticsearchClientError,
+} from '../../../../../observability/server';
 import { rangeFilter } from '../../../../common/utils/range_filter';
 import { ESSearchResponse } from '../../../../../../typings/elasticsearch';
 import { Annotation as ESAnnotation } from '../../../../../observability/common/annotations';
@@ -72,15 +76,22 @@ export function getStoredAnnotations({
     } catch (error) {
       // index is only created when an annotation has been indexed,
       // so we should handle this error gracefully
-      if (error.body?.error?.type === 'index_not_found_exception') {
-        return [];
-      }
+      if (
+        error instanceof WrappedElasticsearchClientError &&
+        error.originalError instanceof ResponseError
+      ) {
+        const type = error.originalError.body.error.type;
 
-      if (error.body?.error?.type === 'security_exception') {
-        logger.warn(
-          `Unable to get stored annotations due to a security exception. Please make sure that the user has 'indices:data/read/search' permissions for ${annotationsClient.index}`
-        );
-        return [];
+        if (type === 'index_not_found_exception') {
+          return [];
+        }
+
+        if (type === 'security_exception') {
+          logger.warn(
+            `Unable to get stored annotations due to a security exception. Please make sure that the user has 'indices:data/read/search' permissions for ${annotationsClient.index}`
+          );
+          return [];
+        }
       }
 
       throw error;

x-pack/plugins/observability/server/index.ts

@@ -10,7 +10,7 @@ import { PluginInitializerContext } from 'src/core/server';
 import { ObservabilityPlugin, ObservabilityPluginSetup } from './plugin';
 import { createOrUpdateIndex, MappingsDefinition } from './utils/create_or_update_index';
 import { ScopedAnnotationsClient } from './lib/annotations/bootstrap_annotations';
-import { unwrapEsResponse } from './utils/unwrap_es_response';
+import { unwrapEsResponse, WrappedElasticsearchClientError } from './utils/unwrap_es_response';
 
 export const config = {
   schema: schema.object({
@@ -33,4 +33,5 @@ export {
   ObservabilityPluginSetup,
   ScopedAnnotationsClient,
   unwrapEsResponse,
+  WrappedElasticsearchClientError,
 };

x-pack/plugins/observability/server/utils/unwrap_es_response.ts

@@ -4,11 +4,45 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import { ElasticsearchClientError, ResponseError } from '@elastic/elasticsearch/lib/errors';
 import type { UnwrapPromise } from '@kbn/utility-types';
+import { inspect } from 'util';
+
+export class WrappedElasticsearchClientError extends Error {
+  originalError: ElasticsearchClientError;
+  constructor(originalError: ElasticsearchClientError) {
+    super(originalError.message);
+
+    const stack = this.stack;
+
+    this.originalError = originalError;
+
+    if (originalError instanceof ResponseError) {
+      // make sure ES response body is visible when logged to the console
+      // @ts-expect-error
+      this.stack = {
+        valueOf() {
+          const value = stack?.valueOf() ?? '';
+          return value;
+        },
+        toString() {
+          const value =
+            stack?.toString() +
+            `\nResponse: ${inspect(originalError.meta.body, { depth: null })}\n`;
+          return value;
+        },
+      };
+    }
+  }
+}
 
 export function unwrapEsResponse<T extends Promise<{ body: any }>>(
   responsePromise: T
 ): Promise<UnwrapPromise<T>['body']> {
-  return responsePromise.then((res) => res.body);
+  return responsePromise
+    .then((res) => res.body)
+    .catch((err) => {
+      // make sure stacktrace is relative to where client was called
+      throw new WrappedElasticsearchClientError(err);
+    });
 }