Fix Health Gateway Checks (#144985)

This PR has a few changes that are needed after learning that the
existing control plane container health check uses `/` as opposed to
`/api/status`:
1. The health gateway server now listens at `/` as opposed to
`/api/status`
2. The health gateway now calls Kibana's `/` not `/api/status`
3. The health gateway will treat a 200-299 or 302 response code OR a 401
response code with a `www-authenticate` response header as healthy

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-health-gateway-server/README.md

@@ -1,7 +1,7 @@
 # @kbn/health-gateway-server
 
-This package runs a small server called the Health Gateway, which exists to query
-the status APIs of multiple Kibana instances and return an aggregated result.
+This package runs a small server called the Health Gateway, which exists to 
+check the health of multiple Kibana instances and return an aggregated result.
 
 This is used by the Elastic Cloud infrastructure to run two different Kibana processes 
 with different `node.roles`: one process for handling UI requests, and one for background
@@ -70,8 +70,8 @@ above (5605-5606).
 
 Once you have your `gateway.yml` and have started docker-compose, you can run the
 server from the `/packages/kbn-health-gateway-server` directory with `yarn start`. Then you should
-be able to make requests to the `/api/status` endpoint:
+be able to make requests to the `/` endpoint:
 
 ```bash
-$ curl "https://localhost:3000/api/status"
+$ curl "https://localhost:3000/"
 ```

packages/kbn-health-gateway-server/src/kibana/kibana_service.test.ts

@@ -43,13 +43,13 @@ describe('KibanaService', () => {
       expect(kibanaStart).toBeUndefined();
     });
 
-    test('registers /api/status route with the server', async () => {
+    test('registers / route with the server', async () => {
       const kibanaService = new KibanaService({ config, logger });
       await kibanaService.start({ server });
       expect(server.addRoute).toHaveBeenCalledWith(
         expect.objectContaining({
           method: 'GET',
-          path: '/api/status',
+          path: '/',
         })
       );
     });

packages/kbn-health-gateway-server/src/kibana/kibana_service.ts

@@ -9,7 +9,7 @@
 import type { IConfigService } from '@kbn/config';
 import type { Logger, LoggerFactory } from '@kbn/logging';
 import { ServerStart } from '../server';
-import { createStatusRoute } from './routes';
+import { createRootRoute } from './routes';
 
 interface KibanaServiceStartDependencies {
   server: ServerStart;
@@ -33,7 +33,7 @@ export class KibanaService {
   }
 
   async start({ server }: KibanaServiceStartDependencies) {
-    server.addRoute(createStatusRoute({ config: this.config, log: this.log }));
+    server.addRoute(createRootRoute({ config: this.config, log: this.log }));
   }
 
   stop() {

packages/kbn-health-gateway-server/src/kibana/routes/index.ts

@@ -6,4 +6,4 @@
  * Side Public License, v 1.
  */
 
-export { createStatusRoute } from './status';
+export { createRootRoute } from './root';

packages/kbn-health-gateway-server/src/kibana/routes/root.ts

@@ -9,7 +9,7 @@
 import https from 'https';
 import { URL } from 'url';
 import type { Request, ResponseToolkit } from '@hapi/hapi';
-import nodeFetch, { RequestInit, Response } from 'node-fetch';
+import nodeFetch, { Headers, RequestInit, Response } from 'node-fetch';
 import type { IConfigService } from '@kbn/config';
 import type { Logger } from '@kbn/logging';
 import type { KibanaConfigType } from '../kibana_config';
@@ -17,32 +17,32 @@ import { KibanaConfig } from '../kibana_config';
 
 const HTTPS = 'https:';
 
-const GATEWAY_STATUS_ROUTE = '/api/status';
-const KIBANA_STATUS_ROUTE = '/api/status';
+const GATEWAY_ROOT_ROUTE = '/';
+const KIBANA_ROOT_ROUTE = '/';
 
-interface StatusRouteDependencies {
+interface RootRouteDependencies {
   log: Logger;
   config: IConfigService;
 }
 
 type Fetch = (path: string) => Promise<Response>;
 
-export function createStatusRoute({ config, log }: StatusRouteDependencies) {
+export function createRootRoute({ config, log }: RootRouteDependencies) {
   const kibanaConfig = new KibanaConfig(config.atPathSync<KibanaConfigType>('kibana'));
   const fetch = configureFetch(kibanaConfig);
 
   return {
     method: 'GET',
-    path: GATEWAY_STATUS_ROUTE,
+    path: GATEWAY_ROOT_ROUTE,
     handler: async (req: Request, h: ResponseToolkit) => {
-      const responses = await fetchKibanaStatuses({ fetch, kibanaConfig, log });
-      const { body, statusCode } = mergeStatusResponses(responses);
+      const responses = await fetchKibanaRoots({ fetch, kibanaConfig, log });
+      const { body, statusCode } = mergeResponses(responses);
       return h.response(body).type('application/json').code(statusCode);
     },
   };
 }
 
-async function fetchKibanaStatuses({
+async function fetchKibanaRoots({
   fetch,
   kibanaConfig,
   log,
@@ -53,19 +53,18 @@ async function fetchKibanaStatuses({
 }) {
   const requests = await Promise.allSettled(
     kibanaConfig.hosts.map(async (host) => {
-      log.debug(`Fetching response from ${host}${KIBANA_STATUS_ROUTE}`);
-      const response = fetch(`${host}${KIBANA_STATUS_ROUTE}`).then((res) => res.json());
-      return response;
+      log.debug(`Fetching response from ${host}${KIBANA_ROOT_ROUTE}`);
+      return fetch(`${host}${KIBANA_ROOT_ROUTE}`);
     })
   );
 
   return requests.map((r, i) => {
     if (r.status === 'rejected') {
-      log.error(`Unable to retrieve status from ${kibanaConfig.hosts[i]}${KIBANA_STATUS_ROUTE}`);
+      log.error(`No response from ${kibanaConfig.hosts[i]}${KIBANA_ROOT_ROUTE}`);
     } else {
       log.info(
-        `Got response from ${kibanaConfig.hosts[i]}${KIBANA_STATUS_ROUTE}: ${JSON.stringify(
-          r.value.status?.overall ? r.value.status.overall : r.value
+        `Got response from ${kibanaConfig.hosts[i]}${KIBANA_ROOT_ROUTE}: ${JSON.stringify(
+          r.value.status
         )}`
       );
     }
@@ -73,22 +72,37 @@ async function fetchKibanaStatuses({
   });
 }
 
-function mergeStatusResponses(
+function mergeResponses(
   responses: Array<PromiseFulfilledResult<Response> | PromiseRejectedResult>
 ) {
   let statusCode = 200;
   for (const response of responses) {
-    if (response.status === 'rejected') {
+    if (
+      response.status === 'rejected' ||
+      !isHealthyResponse(response.value.status, response.value.headers)
+    ) {
       statusCode = 503;
     }
   }
 
   return {
-    body: {}, // Need to determine what response body, if any, we want to include
+    body: {}, // The control plane health check ignores the body, so we do the same
     statusCode,
   };
 }
 
+function isHealthyResponse(statusCode: number, headers: Headers) {
+  return isSuccess(statusCode) || isUnauthorized(statusCode, headers);
+}
+
+function isUnauthorized(statusCode: number, headers: Headers): boolean {
+  return statusCode === 401 && headers.has('www-authenticate');
+}
+
+function isSuccess(statusCode: number): boolean {
+  return (statusCode >= 200 && statusCode <= 299) || statusCode === 302;
+}
+
 function generateAgentConfig(sslConfig: KibanaConfig['ssl']) {
   const options: https.AgentOptions = {
     ca: sslConfig.certificateAuthorities,
@@ -133,6 +147,7 @@ function configureFetch(kibanaConfig: KibanaConfig) {
     const fetchOptions: RequestInit = {
       ...(protocol === HTTPS && { agent }),
       signal: controller.signal,
+      redirect: 'manual',
     };
     try {
       const response = await nodeFetch(url, fetchOptions);