Http compression config (#50738) (#51984)

HTTP compression for responses may be disabled entirely, or it may
be conditionally enabled based on a whitelist of trusted referrers.

docs/setup/settings.asciidoc

@@ -265,6 +265,12 @@ running behind a proxy. Use the `server.rewriteBasePath` setting to tell Kibana
 if it should remove the basePath from requests it receives, and to prevent a
 deprecation warning at startup. This setting cannot end in a slash (`/`).
 
+[[server-compression]]`server.compression.enabled:`:: *Default: `true`* Set to `false` to disable HTTP compression for all responses.
+
+`server.compression.referrerWhitelist:`:: *Default: none* Specifies an array of trusted hostnames, such as the Kibana host, or a reverse
+proxy sitting in front of it. This determines whether HTTP compression may be used for responses, based on the request's `Referer` header.
+This setting may not be used when `server.compression.enabled` is set to `false`.
+
 [[server-cors]]`server.cors:`:: *Default: `false`* Set to `true` to enable CORS support. This setting is required to configure `server.cors.origin`.
 
 `server.cors.origin:`:: *Default: none* Specifies origins. “origin” must be an array. To use this setting, you must set `server.cors` to `true`. To accept all origins, use `server.cors.origin: ["*"]`.

src/core/server/http/__snapshots__/http_config.test.ts.snap

@@ -1,17 +1,35 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`accepts valid hostnames: valid host names 1`] = `
+exports[`accepts valid hostnames 1`] = `
 Object {
-  "host1": "www.example.com",
-  "host2": "8.8.8.8",
-  "host3": "::1",
-  "host4": "localhost",
+  "host": "www.example.com",
+}
+`;
+
+exports[`accepts valid hostnames 2`] = `
+Object {
+  "host": "8.8.8.8",
+}
+`;
+
+exports[`accepts valid hostnames 3`] = `
+Object {
+  "host": "::1",
+}
+`;
+
+exports[`accepts valid hostnames 4`] = `
+Object {
+  "host": "localhost",
 }
 `;
 
 exports[`has defaults for config 1`] = `
 Object {
   "autoListen": true,
+  "compression": Object {
+    "enabled": true,
+  },
   "cors": false,
   "host": "localhost",
   "keepaliveTimeout": 120000,
@@ -82,3 +100,18 @@ exports[`with TLS throws if TLS is enabled but \`certificate\` is not specified
 exports[`with TLS throws if TLS is enabled but \`key\` is not specified 1`] = `"[ssl]: must specify [certificate] and [key] when ssl is enabled"`;
 
 exports[`with TLS throws if TLS is enabled but \`redirectHttpFromPort\` is equal to \`port\` 1`] = `"Kibana does not accept http traffic to [port] when ssl is enabled (only https is allowed), so [ssl.redirectHttpFromPort] cannot be configured to the same value. Both are [1234]."`;
+
+exports[`with compression accepts valid referrer whitelist 1`] = `
+Array [
+  "www.example.com",
+  "8.8.8.8",
+  "::1",
+  "localhost",
+]
+`;
+
+exports[`with compression throws if invalid referrer whitelist 1`] = `"[compression.referrerWhitelist.0]: value is [asdf$%^] but it must be a valid hostname (see RFC 1123)."`;
+
+exports[`with compression throws if invalid referrer whitelist 2`] = `"[compression.referrerWhitelist]: array size is [0], but cannot be smaller than [1]"`;
+
+exports[`with compression throws if referrer whitelist is specified and compression is disabled 1`] = `"cannot use [compression.referrerWhitelist] when [compression.enabled] is set to false"`;

src/core/server/http/cookie_session_storage.test.ts

@@ -57,6 +57,7 @@ configService.atPath.mockReturnValue(
     ssl: {
       verificationMode: 'none',
     },
+    compression: { enabled: true },
   } as any)
 );
 

src/core/server/http/http_config.test.ts

@@ -21,6 +21,9 @@ import { config, HttpConfig } from '.';
 import { Env } from '../config';
 import { getEnvOptions } from '../config/__mocks__/env';
 
+const validHostnames = ['www.example.com', '8.8.8.8', '::1', 'localhost'];
+const invalidHostname = 'asdf$%^';
+
 test('has defaults for config', () => {
   const httpSchema = config.schema;
   const obj = {};
@@ -28,18 +31,16 @@ test('has defaults for config', () => {
 });
 
 test('accepts valid hostnames', () => {
-  const { host: host1 } = config.schema.validate({ host: 'www.example.com' });
-  const { host: host2 } = config.schema.validate({ host: '8.8.8.8' });
-  const { host: host3 } = config.schema.validate({ host: '::1' });
-  const { host: host4 } = config.schema.validate({ host: 'localhost' });
-
-  expect({ host1, host2, host3, host4 }).toMatchSnapshot('valid host names');
+  for (const val of validHostnames) {
+    const { host } = config.schema.validate({ host: val });
+    expect({ host }).toMatchSnapshot();
+  }
 });
 
 test('throws if invalid hostname', () => {
   const httpSchema = config.schema;
   const obj = {
-    host: 'asdf$%^',
+    host: invalidHostname,
   };
   expect(() => httpSchema.validate(obj)).toThrowErrorMatchingSnapshot();
 });
@@ -296,3 +297,44 @@ describe('with TLS', () => {
     expect(httpConfig.ssl.rejectUnauthorized).toBe(true);
   });
 });
+
+describe('with compression', () => {
+  test('accepts valid referrer whitelist', () => {
+    const {
+      compression: { referrerWhitelist },
+    } = config.schema.validate({
+      compression: {
+        referrerWhitelist: validHostnames,
+      },
+    });
+
+    expect(referrerWhitelist).toMatchSnapshot();
+  });
+
+  test('throws if invalid referrer whitelist', () => {
+    const httpSchema = config.schema;
+    const invalidHostnames = {
+      compression: {
+        referrerWhitelist: [invalidHostname],
+      },
+    };
+    const emptyArray = {
+      compression: {
+        referrerWhitelist: [],
+      },
+    };
+    expect(() => httpSchema.validate(invalidHostnames)).toThrowErrorMatchingSnapshot();
+    expect(() => httpSchema.validate(emptyArray)).toThrowErrorMatchingSnapshot();
+  });
+
+  test('throws if referrer whitelist is specified and compression is disabled', () => {
+    const httpSchema = config.schema;
+    const obj = {
+      compression: {
+        enabled: false,
+        referrerWhitelist: validHostnames,
+      },
+    };
+    expect(() => httpSchema.validate(obj)).toThrowErrorMatchingSnapshot();
+  });
+});

src/core/server/http/http_config.ts

@@ -72,12 +72,26 @@ export const config = {
       socketTimeout: schema.number({
         defaultValue: 120000,
       }),
+      compression: schema.object({
+        enabled: schema.boolean({ defaultValue: true }),
+        referrerWhitelist: schema.maybe(
+          schema.arrayOf(
+            schema.string({
+              hostname: true,
+            }),
+            { minSize: 1 }
+          )
+        ),
+      }),
     },
     {
       validate: rawConfig => {
         if (!rawConfig.basePath && rawConfig.rewriteBasePath) {
           return 'cannot use [rewriteBasePath] when [basePath] is not specified';
         }
+        if (!rawConfig.compression.enabled && rawConfig.compression.referrerWhitelist) {
+          return 'cannot use [compression.referrerWhitelist] when [compression.enabled] is set to false';
+        }
 
         if (
           rawConfig.ssl.enabled &&
@@ -109,6 +123,7 @@ export class HttpConfig {
   public publicDir: string;
   public defaultRoute?: string;
   public ssl: SslConfig;
+  public compression: { enabled: boolean; referrerWhitelist?: string[] };
 
   /**
    * @internal
@@ -126,5 +141,6 @@ export class HttpConfig {
     this.publicDir = env.staticFilesDir;
     this.ssl = new SslConfig(rawConfig.ssl || {});
     this.defaultRoute = rawConfig.defaultRoute;
+    this.compression = rawConfig.compression;
   }
 }

src/core/server/http/http_server.test.ts

@@ -53,6 +53,7 @@ beforeEach(() => {
     maxPayload: new ByteSizeValue(1024),
     port: 10002,
     ssl: { enabled: false },
+    compression: { enabled: true },
   } as HttpConfig;
 
   configWithSSL = {
@@ -578,6 +579,90 @@ test('exposes route details of incoming request to a route handler', async () =>
     });
 });
 
+describe('conditional compression', () => {
+  async function setupServer(innerConfig: HttpConfig) {
+    const { registerRouter, server: innerServer } = await server.setup(innerConfig);
+    const router = new Router('', logger, enhanceWithContext);
+    // we need the large body here so that compression would normally be used
+    const largeRequest = {
+      body: 'hello'.repeat(500),
+      headers: { 'Content-Type': 'text/html; charset=UTF-8' },
+    };
+    router.get({ path: '/', validate: false }, (_context, _req, res) => res.ok(largeRequest));
+    registerRouter(router);
+    await server.start();
+    return innerServer.listener;
+  }
+
+  test('with `compression.enabled: true`', async () => {
+    const listener = await setupServer(config);
+
+    const response = await supertest(listener)
+      .get('/')
+      .set('accept-encoding', 'gzip');
+
+    expect(response.header).toHaveProperty('content-encoding', 'gzip');
+  });
+
+  test('with `compression.enabled: false`', async () => {
+    const listener = await setupServer({
+      ...config,
+      compression: { enabled: false },
+    });
+
+    const response = await supertest(listener)
+      .get('/')
+      .set('accept-encoding', 'gzip');
+
+    expect(response.header).not.toHaveProperty('content-encoding');
+  });
+
+  describe('with defined `compression.referrerWhitelist`', () => {
+    let listener: Server;
+    beforeEach(async () => {
+      listener = await setupServer({
+        ...config,
+        compression: { enabled: true, referrerWhitelist: ['foo'] },
+      });
+    });
+
+    test('enables compression for no referer', async () => {
+      const response = await supertest(listener)
+        .get('/')
+        .set('accept-encoding', 'gzip');
+
+      expect(response.header).toHaveProperty('content-encoding', 'gzip');
+    });
+
+    test('enables compression for whitelisted referer', async () => {
+      const response = await supertest(listener)
+        .get('/')
+        .set('accept-encoding', 'gzip')
+        .set('referer', 'http://foo:1234');
+
+      expect(response.header).toHaveProperty('content-encoding', 'gzip');
+    });
+
+    test('disables compression for non-whitelisted referer', async () => {
+      const response = await supertest(listener)
+        .get('/')
+        .set('accept-encoding', 'gzip')
+        .set('referer', 'http://bar:1234');
+
+      expect(response.header).not.toHaveProperty('content-encoding');
+    });
+
+    test('disables compression for invalid referer', async () => {
+      const response = await supertest(listener)
+        .get('/')
+        .set('accept-encoding', 'gzip')
+        .set('referer', 'http://asdf$%^');
+
+      expect(response.header).not.toHaveProperty('content-encoding');
+    });
+  });
+});
+
 test('exposes route details of incoming request to a route handler (POST + payload options)', async () => {
   const { registerRouter, server: innerServer } = await server.setup(config);
 

src/core/server/http/http_server.ts

@@ -18,6 +18,7 @@
  */
 
 import { Request, Server } from 'hapi';
+import url from 'url';
 
 import { Logger, LoggerFactory } from '../logging';
 import { HttpConfig } from './http_config';
@@ -96,6 +97,7 @@ export class HttpServer {
 
     const basePathService = new BasePath(config.basePath);
     this.setupBasePathRewrite(config, basePathService);
+    this.setupConditionalCompression(config);
 
     return {
       registerRouter: this.registerRouter.bind(this),
@@ -187,6 +189,33 @@ export class HttpServer {
     });
   }
 
+  private setupConditionalCompression(config: HttpConfig) {
+    if (this.server === undefined) {
+      throw new Error('Server is not created yet');
+    }
+
+    const { enabled, referrerWhitelist: list } = config.compression;
+    if (!enabled) {
+      this.log.debug('HTTP compression is disabled');
+      this.server.ext('onRequest', (request, h) => {
+        request.info.acceptEncoding = '';
+        return h.continue;
+      });
+    } else if (list) {
+      this.log.debug(`HTTP compression is only enabled for any referrer in the following: ${list}`);
+      this.server.ext('onRequest', (request, h) => {
+        const { referrer } = request.info;
+        if (referrer !== '') {
+          const { hostname } = url.parse(referrer);
+          if (!hostname || !list.includes(hostname)) {
+            request.info.acceptEncoding = '';
+          }
+        }
+        return h.continue;
+      });
+    }
+  }
+
   private registerOnPostAuth(fn: OnPostAuthHandler) {
     if (this.server === undefined) {
       throw new Error('Server is not created yet');

src/core/server/http/http_tools.test.ts

@@ -90,6 +90,7 @@ describe('timeouts', () => {
       host: '127.0.0.1',
       maxPayload: new ByteSizeValue(1024),
       ssl: {},
+      compression: { enabled: true },
     } as HttpConfig);
     registerRouter(router);
 

src/core/server/http/integration_tests/lifecycle.test.ts

@@ -51,6 +51,7 @@ configService.atPath.mockReturnValue(
     ssl: {
       enabled: false,
     },
+    compression: { enabled: true },
   } as any)
 );
 

src/core/server/http/integration_tests/router.test.ts

@@ -51,6 +51,7 @@ configService.atPath.mockReturnValue(
     ssl: {
       enabled: false,
     },
+    compression: { enabled: true },
   } as any)
 );
 

src/core/server/legacy/config/__snapshots__/legacy_object_to_config_adapter.test.ts.snap

@@ -4,6 +4,9 @@ exports[`#get correctly handles server config.: default 1`] = `
 Object {
   "autoListen": true,
   "basePath": "/abc",
+  "compression": Object {
+    "enabled": true,
+  },
   "cors": false,
   "host": "host",
   "keepaliveTimeout": 5000,
@@ -23,6 +26,9 @@ exports[`#get correctly handles server config.: disabled ssl 1`] = `
 Object {
   "autoListen": true,
   "basePath": "/abc",
+  "compression": Object {
+    "enabled": true,
+  },
   "cors": false,
   "host": "host",
   "keepaliveTimeout": 5000,

src/core/server/legacy/config/legacy_object_to_config_adapter.test.ts

@@ -74,6 +74,7 @@ describe('#get', () => {
         port: 1234,
         rewriteBasePath: false,
         ssl: { enabled: true, keyPassphrase: 'some-phrase', someNewValue: 'new' },
+        compression: { enabled: true },
         someNotSupportedValue: 'val',
       },
     });
@@ -90,6 +91,7 @@ describe('#get', () => {
         port: 1234,
         rewriteBasePath: false,
         ssl: { enabled: false, certificate: 'cert', key: 'key' },
+        compression: { enabled: true },
         someNotSupportedValue: 'val',
       },
     });

src/core/server/legacy/config/legacy_object_to_config_adapter.ts

@@ -70,6 +70,7 @@ export class LegacyObjectToConfigAdapter extends ObjectToConfigAdapter {
       ssl: configValue.ssl,
       keepaliveTimeout: configValue.keepaliveTimeout,
       socketTimeout: configValue.socketTimeout,
+      compression: configValue.compression,
     };
   }
 

src/legacy/server/config/schema.js

@@ -109,6 +109,7 @@ export default () => Joi.object({
     maxPayloadBytes: HANDLED_IN_NEW_PLATFORM,
     socketTimeout: HANDLED_IN_NEW_PLATFORM,
     ssl: HANDLED_IN_NEW_PLATFORM,
+    compression: HANDLED_IN_NEW_PLATFORM,
   }).default(),
 
   uiSettings: HANDLED_IN_NEW_PLATFORM,

test/api_integration/apis/core/index.js

@@ -16,21 +16,55 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+import expect from '@kbn/expect';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
 
-  describe('core request context', () => {
-    it('provides access to elasticsearch', async () => (
-      await supertest
-        .get('/requestcontext/elasticsearch')
-        .expect(200, 'Elasticsearch: true')
-    ));
+  describe('core', () => {
+    describe('request context', () => {
+      it('provides access to elasticsearch', async () => (
+        await supertest
+          .get('/requestcontext/elasticsearch')
+          .expect(200, 'Elasticsearch: true')
+      ));
 
-    it('provides access to SavedObjects client', async () => (
-      await supertest
-        .get('/requestcontext/savedobjectsclient')
-        .expect(200, 'SavedObjects client: {"page":1,"per_page":20,"total":0,"saved_objects":[]}')
-    ));
+      it('provides access to SavedObjects client', async () => (
+        await supertest
+          .get('/requestcontext/savedobjectsclient')
+          .expect(200, 'SavedObjects client: {"page":1,"per_page":20,"total":0,"saved_objects":[]}')
+      ));
+    });
+
+    describe('compression', () => {
+      it(`uses compression when there isn't a referer`, async () => {
+        await supertest
+          .get('/app/kibana')
+          .set('accept-encoding', 'gzip')
+          .then(response => {
+            expect(response.headers).to.have.property('content-encoding', 'gzip');
+          });
+      });
+
+      it(`uses compression when there is a whitelisted referer`, async () => {
+        await supertest
+          .get('/app/kibana')
+          .set('accept-encoding', 'gzip')
+          .set('referer', 'https://some-host.com')
+          .then(response => {
+            expect(response.headers).to.have.property('content-encoding', 'gzip');
+          });
+      });
+
+      it(`doesn't use compression when there is a non-whitelisted referer`, async () => {
+        await supertest
+          .get('/app/kibana')
+          .set('accept-encoding', 'gzip')
+          .set('referer', 'https://other.some-host.com')
+          .then(response => {
+            expect(response.headers).not.to.have.property('content-encoding');
+          });
+      });
+    });
   });
 }

test/api_integration/apis/index.js

@@ -34,5 +34,6 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./status'));
     loadTestFile(require.resolve('./stats'));
     loadTestFile(require.resolve('./ui_metric'));
+    loadTestFile(require.resolve('./core'));
   });
 }

test/api_integration/config.js

@@ -40,6 +40,7 @@ export default async function ({ readConfigFile }) {
         '--optimize.enabled=false',
         '--elasticsearch.healthCheck.delay=3600000',
         '--server.xsrf.disableProtection=true',
+        '--server.compression.referrerWhitelist=["some-host.com"]',
       ],
     },
   };