[Elasticsearch] Redact logs from known APIs (#153049)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/core/elasticsearch/core-elasticsearch-client-server-internal/index.ts

@@ -10,7 +10,11 @@ export { ScopedClusterClient } from './src/scoped_cluster_client';
 export { ClusterClient } from './src/cluster_client';
 export { configureClient } from './src/configure_client';
 export { type AgentStatsProvider, AgentManager, type NetworkAgent } from './src/agent_manager';
-export { getRequestDebugMeta, getErrorMessage } from './src/log_query_and_deprecation';
+export {
+  type RequestDebugMeta,
+  getRequestDebugMeta,
+  getErrorMessage,
+} from './src/log_query_and_deprecation';
 export {
   PRODUCT_RESPONSE_HEADER,
   DEFAULT_HEADERS,

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/configure_client.test.ts

@@ -212,6 +212,7 @@ describe('configureClient', () => {
       logger,
       client,
       type: 'test',
+      apisToRedactInLogs: [],
     });
   });
 });

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/configure_client.ts

@@ -45,7 +45,8 @@ export const configureClient = (
     ConnectionPool: ClusterConnectionPool,
   });
 
-  instrumentEsQueryAndDeprecationLogger({ logger, client, type });
+  const { apisToRedactInLogs = [] } = config;
+  instrumentEsQueryAndDeprecationLogger({ logger, client, type, apisToRedactInLogs });
 
   return client;
 };

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/log_query_and_deprecation.test.ts

@@ -96,13 +96,23 @@ describe('instrumentQueryAndDeprecationLogger', () => {
   }
 
   it('creates a query logger context based on the `type` parameter', () => {
-    instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test123' });
+    instrumentEsQueryAndDeprecationLogger({
+      logger,
+      client,
+      type: 'test123',
+      apisToRedactInLogs: [],
+    });
     expect(logger.get).toHaveBeenCalledWith('query', 'test123');
   });
 
   describe('logs each query', () => {
     it('when request body is an object', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createResponseWithBody({
         seq_no_primary_term: true,
@@ -120,7 +130,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('when request body is a string', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createResponseWithBody(
         JSON.stringify({
@@ -140,7 +155,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('when request body is a buffer', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createResponseWithBody(
         Buffer.from(
@@ -162,7 +182,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('when request body is a readable stream', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createResponseWithBody(
         Readable.from(
@@ -184,7 +209,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('when request body is not defined', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createResponseWithBody();
 
@@ -196,7 +226,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('properly encode queries', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         body: {},
@@ -217,7 +252,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs queries even in case of errors', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         statusCode: 500,
@@ -248,7 +288,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs debug when the client emits an @elastic/elasticsearch error', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({ body: {} });
       client.diagnostic.emit('response', new errors.TimeoutError('message', response), response);
@@ -259,7 +304,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs debug when the client emits an ResponseError returned by elasticsearch', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         statusCode: 400,
@@ -285,7 +335,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs default error info when the error response body is empty', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       let response: DiagnosticResult<any, any> = createApiResponse({
         statusCode: 400,
@@ -325,7 +380,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('adds meta information to logs', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       let response = createApiResponse({
         statusCode: 400,
@@ -407,7 +467,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs response size', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createResponseWithBody(
         {
@@ -432,7 +497,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
 
   describe('deprecation warnings from response headers', () => {
     it('does not log when no deprecation warning header is returned', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         statusCode: 200,
@@ -458,7 +528,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('does not log when warning header comes from a warn-agent that is not elasticsearch', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         statusCode: 200,
@@ -487,7 +562,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs error when the client receives an Elasticsearch error response for a deprecated request originating from a user', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         statusCode: 400,
@@ -519,7 +599,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs warning when the client receives an Elasticsearch error response for a deprecated request originating from kibana', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         statusCode: 400,
@@ -552,7 +637,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs error when the client receives an Elasticsearch success response for a deprecated request originating from a user', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         statusCode: 200,
@@ -584,7 +674,12 @@ describe('instrumentQueryAndDeprecationLogger', () => {
     });
 
     it('logs warning when the client receives an Elasticsearch success response for a deprecated request originating from kibana', () => {
-      instrumentEsQueryAndDeprecationLogger({ logger, client, type: 'test type' });
+      instrumentEsQueryAndDeprecationLogger({
+        logger,
+        client,
+        type: 'test type',
+        apisToRedactInLogs: [],
+      });
 
       const response = createApiResponse({
         statusCode: 200,
@@ -616,5 +711,287 @@ describe('instrumentQueryAndDeprecationLogger', () => {
         /Query:\n.*200\n.*GET \/_path\?hello\=dolly/
       );
     });
+
+    describe('Request body redaction on some APIs', () => {
+      it('redacts for an API in the extended list (path only)', () => {
+        instrumentEsQueryAndDeprecationLogger({
+          logger,
+          client,
+          type: 'test type',
+          apisToRedactInLogs: [{ path: '/foo' }],
+        });
+
+        const response = createApiResponse({
+          body: {},
+          statusCode: 200,
+          headers: {},
+          params: {
+            method: 'GET',
+            path: '/foo',
+            querystring: { hello: 'dolly' },
+            body: {
+              seq_no_primary_term: true,
+              query: {
+                term: { user: 'kimchy' },
+              },
+            },
+          },
+        });
+
+        client.diagnostic.emit('response', null, response);
+        expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+          "200
+          GET /foo?hello=dolly
+          [redacted]"
+        `);
+      });
+
+      it('redacts for an API that is contained by the declared path (path only)', () => {
+        instrumentEsQueryAndDeprecationLogger({
+          logger,
+          client,
+          type: 'test type',
+          apisToRedactInLogs: [{ path: '/foo' }],
+        });
+
+        const response = createApiResponse({
+          body: {},
+          statusCode: 200,
+          headers: {},
+          params: {
+            method: 'GET',
+            path: '/foo/something/something-else',
+            querystring: { hello: 'dolly' },
+            body: {
+              seq_no_primary_term: true,
+              query: {
+                term: { user: 'kimchy' },
+              },
+            },
+          },
+        });
+
+        client.diagnostic.emit('response', null, response);
+        expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+          "200
+          GET /foo/something/something-else?hello=dolly
+          [redacted]"
+        `);
+      });
+
+      it('redacts for an API in the extended list (method and path)', () => {
+        instrumentEsQueryAndDeprecationLogger({
+          logger,
+          client,
+          type: 'test type',
+          apisToRedactInLogs: [{ method: 'GET', path: '/foo' }],
+        });
+
+        const response = createApiResponse({
+          body: {},
+          statusCode: 200,
+          headers: {},
+          params: {
+            method: 'GET',
+            path: '/foo',
+            querystring: { hello: 'dolly' },
+            body: {
+              seq_no_primary_term: true,
+              query: {
+                term: { user: 'kimchy' },
+              },
+            },
+          },
+        });
+
+        client.diagnostic.emit('response', null, response);
+        expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+          "200
+          GET /foo?hello=dolly
+          [redacted]"
+        `);
+      });
+
+      it('does not redact for an API in the extended list when method does not match', () => {
+        instrumentEsQueryAndDeprecationLogger({
+          logger,
+          client,
+          type: 'test type',
+          apisToRedactInLogs: [{ method: 'PUT', path: '/foo' }],
+        });
+
+        const response = createApiResponse({
+          body: {},
+          statusCode: 200,
+          headers: {},
+          params: {
+            method: 'GET',
+            path: '/foo',
+            querystring: { hello: 'dolly' },
+            body: {
+              seq_no_primary_term: true,
+              query: {
+                term: { user: 'kimchy' },
+              },
+            },
+          },
+        });
+
+        client.diagnostic.emit('response', null, response);
+        expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+          "200
+          GET /foo?hello=dolly
+          {\\"seq_no_primary_term\\":true,\\"query\\":{\\"term\\":{\\"user\\":\\"kimchy\\"}}}"
+        `);
+      });
+
+      it('does not redact for an API in the extended list when path does not match', () => {
+        instrumentEsQueryAndDeprecationLogger({
+          logger,
+          client,
+          type: 'test type',
+          apisToRedactInLogs: [{ path: '/foo' }],
+        });
+
+        const response = createApiResponse({
+          body: {},
+          statusCode: 200,
+          headers: {},
+          params: {
+            method: 'GET',
+            path: '/bar',
+            querystring: { hello: 'dolly' },
+            body: {
+              seq_no_primary_term: true,
+              query: {
+                term: { user: 'kimchy' },
+              },
+            },
+          },
+        });
+
+        client.diagnostic.emit('response', null, response);
+        expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+          "200
+          GET /bar?hello=dolly
+          {\\"seq_no_primary_term\\":true,\\"query\\":{\\"term\\":{\\"user\\":\\"kimchy\\"}}}"
+        `);
+      });
+
+      describe('Known list', () => {
+        beforeEach(() => {
+          instrumentEsQueryAndDeprecationLogger({
+            logger,
+            client,
+            type: 'test type',
+            apisToRedactInLogs: [],
+          });
+        });
+
+        function createResponseWithPath(path: string, method: string = '*') {
+          return createApiResponse({
+            body: {},
+            statusCode: 200,
+            headers: {},
+            params: {
+              method,
+              path,
+              querystring: { hello: 'dolly' },
+              body: { super_secret: 'stuff' },
+            },
+          });
+        }
+
+        it('[*] /_security/', () => {
+          const response = createResponseWithPath('/_security/something');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            * /_security/something?hello=dolly
+            [redacted]"
+          `);
+        });
+
+        it('[*] /_xpack/security/', () => {
+          const response = createResponseWithPath('/_xpack/security/something');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            * /_xpack/security/something?hello=dolly
+            [redacted]"
+          `);
+        });
+
+        it('[POST] /_reindex', () => {
+          const response = createResponseWithPath('/_reindex', 'POST');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            POST /_reindex?hello=dolly
+            [redacted]"
+          `);
+        });
+
+        it('[PUT] /_watcher/watch', () => {
+          const response = createResponseWithPath('/_watcher/watch', 'PUT');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            PUT /_watcher/watch?hello=dolly
+            [redacted]"
+          `);
+        });
+
+        it('[PUT] /_xpack/watcher/watch', () => {
+          const response = createResponseWithPath('/_xpack/watcher/watch', 'PUT');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            PUT /_xpack/watcher/watch?hello=dolly
+            [redacted]"
+          `);
+        });
+
+        it('[PUT] /_snapshot/something', () => {
+          const response = createResponseWithPath('/_snapshot/something', 'PUT');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            PUT /_snapshot/something?hello=dolly
+            [redacted]"
+          `);
+        });
+
+        it('[PUT] /_logstash/pipeline/something', () => {
+          const response = createResponseWithPath('/_logstash/pipeline/something', 'PUT');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            PUT /_logstash/pipeline/something?hello=dolly
+            [redacted]"
+          `);
+        });
+
+        it('[POST] /_nodes/reload_secure_settings', () => {
+          const response = createResponseWithPath('/_nodes/reload_secure_settings', 'POST');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            POST /_nodes/reload_secure_settings?hello=dolly
+            [redacted]"
+          `);
+        });
+
+        it('[POST] /_nodes/*/reload_secure_settings', () => {
+          const response = createResponseWithPath('/_nodes/node-id/reload_secure_settings', 'POST');
+          client.diagnostic.emit('response', null, response);
+          expect(loggingSystemMock.collect(logger).debug[0][0]).toMatchInlineSnapshot(`
+            "200
+            POST /_nodes/node-id/reload_secure_settings?hello=dolly
+            [redacted]"
+          `);
+        });
+      });
+    });
   });
 });

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/log_query_and_deprecation.ts

@@ -13,8 +13,62 @@ import { errors, DiagnosticResult, RequestBody, Client } from '@elastic/elastics
 import numeral from '@elastic/numeral';
 import type { Logger } from '@kbn/logging';
 import type { ElasticsearchErrorDetails } from '@kbn/es-errors';
+import type { ElasticsearchApiToRedactInLogs } from '@kbn/core-elasticsearch-server';
 import { getEcsResponseLog } from './get_ecs_response_log';
 
+/**
+ * The logger-relevant request meta of an ES request
+ */
+export interface RequestDebugMeta {
+  /**
+   * The requested method
+   */
+  method: string;
+  /**
+   * The requested endpoint + querystring
+   */
+  url: string;
+  /**
+   * The request body (it may be redacted)
+   */
+  body: string;
+  /**
+   * The status code of the response
+   */
+  statusCode: number | null;
+}
+
+/**
+ * Known list of APIs that should redact the request body in the logs
+ */
+const APIS_TO_REDACT_IN_LOGS: ElasticsearchApiToRedactInLogs[] = [
+  { path: '/_security/' },
+  { path: '/_xpack/security/' },
+  { method: 'POST', path: '/_reindex' },
+  { method: 'PUT', path: '/_watcher/watch' },
+  { method: 'PUT', path: '/_xpack/watcher/watch' },
+  { method: 'PUT', path: '/_snapshot/' },
+  { method: 'PUT', path: '/_logstash/pipeline/' },
+  { method: 'POST', path: '/_nodes/reload_secure_settings' },
+  { method: 'POST', path: /\/_nodes\/.+\/reload_secure_settings/ },
+];
+
+function shouldRedactBodyInLogs(
+  requestDebugMeta: RequestDebugMeta,
+  extendedList: ElasticsearchApiToRedactInLogs[] = []
+) {
+  return [...APIS_TO_REDACT_IN_LOGS, ...extendedList].some(({ path, method }) => {
+    if (!method || method === requestDebugMeta.method) {
+      if (typeof path === 'string') {
+        return requestDebugMeta.url.includes(path);
+      } else {
+        return path.test(requestDebugMeta.url);
+      }
+    }
+    return false;
+  });
+}
+
 const convertQueryString = (qs: string | Record<string, any> | undefined): string => {
   if (qs === undefined || typeof qs === 'string') {
     return qs ?? '';
@@ -58,35 +112,43 @@ function getContentLength(headers?: IncomingHttpHeaders): number | undefined {
  *
  * so it could be copy-pasted into the Dev console
  */
-function getResponseMessage(event: DiagnosticResult, bytesMsg: string): string {
+function getResponseMessage(
-  const errorMeta = getRequestDebugMeta(event);
+  event: DiagnosticResult,
-  const body = errorMeta.body ? `\n${errorMeta.body}` : '';
+  bytesMsg: string,
-  return `${errorMeta.statusCode}${bytesMsg}\n${errorMeta.method} ${errorMeta.url}${body}`;
+  apisToRedactInLogs: ElasticsearchApiToRedactInLogs[]
+): string {
+  const debugMeta = getRequestDebugMeta(event, apisToRedactInLogs);
+  const body = debugMeta.body ? `\n${debugMeta.body}` : '';
+  return `${debugMeta.statusCode}${bytesMsg}\n${debugMeta.method} ${debugMeta.url}${body}`;
 }
 
 /**
  * Returns stringified debug information from an Elasticsearch request event
  * useful for logging in case of an unexpected failure.
  */
-export function getRequestDebugMeta(event: DiagnosticResult): {
+export function getRequestDebugMeta(
-  url: string;
+  event: DiagnosticResult,
-  body: string;
+  apisToRedactInLogs?: ElasticsearchApiToRedactInLogs[]
-  statusCode: number | null;
+): RequestDebugMeta {
-  method: string;
-} {
   const params = event.meta.request.params;
   // definition is wrong, `params.querystring` can be either a string or an object
   const querystring = convertQueryString(params.querystring);
-  return {
+
+  const debugMeta: RequestDebugMeta = {
     url: `${params.path}${querystring ? `?${querystring}` : ''}`,
     body: params.body ? `${ensureString(params.body)}` : '',
     method: params.method,
     statusCode: event.statusCode!,
   };
+
+  // Some known APIs may contain sensitive information in the request body that we don't want to expose to the logs.
+  return shouldRedactBodyInLogs(debugMeta, apisToRedactInLogs)
+    ? { ...debugMeta, body: '[redacted]' }
+    : debugMeta;
 }
 
 /** HTTP Warning headers have the following syntax:
- * <warn-code> <warn-agent> <warn-text> (where warn-code is a three digit number)
+ * <warn-code> <warn-agent> <warn-text> (where warn-code is a three-digit number)
  * This function tests if a warning comes from an Elasticsearch warn-agent
  * */
 const isEsWarning = (warning: string) => /\d\d\d Elasticsearch-/.test(warning);
@@ -95,10 +157,12 @@ export const instrumentEsQueryAndDeprecationLogger = ({
   logger,
   client,
   type,
+  apisToRedactInLogs,
 }: {
   logger: Logger;
   client: Client;
   type: string;
+  apisToRedactInLogs: ElasticsearchApiToRedactInLogs[];
 }) => {
   const queryLogger = logger.get('query', type);
   const deprecationLogger = logger.get('deprecation');
@@ -111,12 +175,14 @@ export const instrumentEsQueryAndDeprecationLogger = ({
       let queryMsg = '';
       if (error) {
         if (error instanceof errors.ResponseError) {
-          queryMsg = `${getResponseMessage(event, bytesMsg)} ${getErrorMessage(error)}`;
+          queryMsg = `${getResponseMessage(event, bytesMsg, apisToRedactInLogs)} ${getErrorMessage(
+            error
+          )}`;
         } else {
           queryMsg = getErrorMessage(error);
         }
       } else {
-        queryMsg = getResponseMessage(event, bytesMsg);
+        queryMsg = getResponseMessage(event, bytesMsg, apisToRedactInLogs);
       }
 
       queryLogger.debug(queryMsg, meta);
@@ -137,7 +203,7 @@ export const instrumentEsQueryAndDeprecationLogger = ({
             ? 'kibana'
             : 'user';
 
-        // Strip the first 5 stack trace lines as these are irrelavent to finding the call site
+        // Strip the first 5 stack trace lines as these are irrelevant to finding the call site
         const stackTrace = new Error().stack?.split('\n').slice(5).join('\n');
 
         deprecationLogger.debug(

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_config.test.ts

@@ -30,6 +30,7 @@ test('set correct defaults', () => {
   expect(configValue).toMatchInlineSnapshot(`
     ElasticsearchConfig {
       "apiVersion": "master",
+      "apisToRedactInLogs": Array [],
       "compression": false,
       "customHeaders": Object {},
       "healthCheckDelay": "PT2.5S",

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_config.ts

@@ -13,7 +13,11 @@ import { Duration } from 'moment';
 import { readFileSync } from 'fs';
 import type { ServiceConfigDescriptor } from '@kbn/core-base-server-internal';
 import type { ConfigDeprecationProvider } from '@kbn/config';
-import type { IElasticsearchConfig, ElasticsearchSslConfig } from '@kbn/core-elasticsearch-server';
+import type {
+  IElasticsearchConfig,
+  ElasticsearchSslConfig,
+  ElasticsearchApiToRedactInLogs,
+} from '@kbn/core-elasticsearch-server';
 import { getReservedHeaders } from './default_headers';
 
 const hostURISchema = schema.uri({ scheme: ['http', 'https'] });
@@ -169,6 +173,13 @@ export const configSchema = schema.object({
     }),
     schema.boolean({ defaultValue: false })
   ),
+  apisToRedactInLogs: schema.arrayOf(
+    schema.object({
+      path: schema.string(),
+      method: schema.maybe(schema.string()),
+    }),
+    { defaultValue: [] }
+  ),
 });
 
 const deprecations: ConfigDeprecationProvider = () => [
@@ -402,6 +413,11 @@ export class ElasticsearchConfig implements IElasticsearchConfig {
    */
   public readonly customHeaders: ElasticsearchConfigType['customHeaders'];
 
+  /**
+   * Extends the list of APIs that should be redacted in logs.
+   */
+  public readonly apisToRedactInLogs: ElasticsearchApiToRedactInLogs[];
+
   constructor(rawConfig: ElasticsearchConfigType) {
     this.ignoreVersionMismatch = rawConfig.ignoreVersionMismatch;
     this.apiVersion = rawConfig.apiVersion;
@@ -425,6 +441,7 @@ export class ElasticsearchConfig implements IElasticsearchConfig {
     this.idleSocketTimeout = rawConfig.idleSocketTimeout;
     this.compression = rawConfig.compression;
     this.skipStartupConnectionCheck = rawConfig.skipStartupConnectionCheck;
+    this.apisToRedactInLogs = rawConfig.apisToRedactInLogs;
 
     const { alwaysPresentCertificate, verificationMode } = rawConfig.ssl;
     const { key, keyPassphrase, certificate, certificateAuthorities } = readKeyAndCerts(rawConfig);

packages/core/elasticsearch/core-elasticsearch-server/index.ts

@@ -22,6 +22,7 @@ export type {
   FakeRequest,
   ElasticsearchClientSslConfig,
   ElasticsearchClientConfig,
+  ElasticsearchApiToRedactInLogs,
 } from './src/client';
 
 export type {

packages/core/elasticsearch/core-elasticsearch-server/src/client/client_config.ts

@@ -8,6 +8,23 @@
 
 import type { Duration } from 'moment';
 
+/**
+ * Definition of an API that should redact the requested body in the logs
+ */
+export interface ElasticsearchApiToRedactInLogs {
+  /**
+   * The ES path.
+   * - If specified as a string, it'll be checked as `contains`.
+   * - If specified as a RegExp, it'll be tested against the path.
+   */
+  path: string | RegExp;
+  /**
+   * HTTP method.
+   * If not provided, the path will be checked for all methods.
+   */
+  method?: string;
+}
+
 /**
  * Configuration options to be used to create a {@link IClusterClient | cluster client}
  *
@@ -32,6 +49,7 @@ export interface ElasticsearchClientConfig {
   requestTimeout?: Duration | number;
   caFingerprint?: string;
   ssl?: ElasticsearchClientSslConfig;
+  apisToRedactInLogs?: ElasticsearchApiToRedactInLogs[];
 }
 
 /**

packages/core/elasticsearch/core-elasticsearch-server/src/client/index.ts

@@ -19,4 +19,8 @@ export type {
   UnauthorizedErrorHandlerRetryResult,
   UnauthorizedErrorHandlerNotHandledResult,
 } from './unauthorized_error_handler';
-export type { ElasticsearchClientConfig, ElasticsearchClientSslConfig } from './client_config';
+export type {
+  ElasticsearchClientConfig,
+  ElasticsearchClientSslConfig,
+  ElasticsearchApiToRedactInLogs,
+} from './client_config';

packages/core/elasticsearch/core-elasticsearch-server/src/elasticsearch_config.ts

@@ -7,6 +7,7 @@
  */
 
 import type { Duration } from 'moment';
+import type { ElasticsearchApiToRedactInLogs } from './client';
 
 /**
  * @public
@@ -139,6 +140,11 @@ export interface IElasticsearchConfig {
    * either `certificate` or `full`.
    */
   readonly ssl: ElasticsearchSslConfig;
+
+  /**
+   * Extends the list of APIs that should be redacted in logs.
+   */
+  readonly apisToRedactInLogs: ElasticsearchApiToRedactInLogs[];
 }
 
 /**

x-pack/plugins/monitoring/server/config.test.ts

@@ -55,6 +55,7 @@ describe('config schema', () => {
           "debug_mode": false,
           "elasticsearch": Object {
             "apiVersion": "master",
+            "apisToRedactInLogs": Array [],
             "compression": false,
             "customHeaders": Object {},
             "healthCheck": Object {