Add timeouts and setup enforcement for custom plugins statuses (#77965) (#103149)

Co-authored-by: Josh Dover <1813008+joshdover@users.noreply.github.com>

docs/development/core/server/kibana-plugin-core-server.statusservicesetup.set.md

@@ -24,5 +24,7 @@ set(status$: Observable<ServiceStatus>): void;
 
 ## Remarks
 
+The first emission from this Observable should occur within 30s, else this plugin's status will fallback to `unavailable` until the first emission.
+
 See the [StatusServiceSetup.derivedStatus$](./kibana-plugin-core-server.statusservicesetup.derivedstatus_.md) API for leveraging the default status calculation that is provided by Core.
 

packages/kbn-pm/dist/index.js

@@ -63827,6 +63827,7 @@ function getProjectPaths({
 
   projectPaths.push(Object(path__WEBPACK_IMPORTED_MODULE_0__["resolve"])(rootPath, 'test/plugin_functional/plugins/*'));
   projectPaths.push(Object(path__WEBPACK_IMPORTED_MODULE_0__["resolve"])(rootPath, 'test/interpreter_functional/plugins/*'));
+  projectPaths.push(Object(path__WEBPACK_IMPORTED_MODULE_0__["resolve"])(rootPath, 'test/server_integration/__fixtures__/plugins/*'));
   projectPaths.push(Object(path__WEBPACK_IMPORTED_MODULE_0__["resolve"])(rootPath, 'examples/*'));
 
   if (!ossOnly) {

packages/kbn-pm/src/config.ts

@@ -31,6 +31,7 @@ export function getProjectPaths({ rootPath, ossOnly, skipKibanaPlugins }: Option
   // correct and the expect behavior.
   projectPaths.push(resolve(rootPath, 'test/plugin_functional/plugins/*'));
   projectPaths.push(resolve(rootPath, 'test/interpreter_functional/plugins/*'));
+  projectPaths.push(resolve(rootPath, 'test/server_integration/__fixtures__/plugins/*'));
   projectPaths.push(resolve(rootPath, 'examples/*'));
 
   if (!ossOnly) {

src/core/server/server.test.ts

@@ -114,6 +114,7 @@ test('runs services on "start"', async () => {
   expect(mockSavedObjectsService.start).not.toHaveBeenCalled();
   expect(mockUiSettingsService.start).not.toHaveBeenCalled();
   expect(mockMetricsService.start).not.toHaveBeenCalled();
+  expect(mockStatusService.start).not.toHaveBeenCalled();
 
   await server.start();
 
@@ -121,6 +122,7 @@ test('runs services on "start"', async () => {
   expect(mockSavedObjectsService.start).toHaveBeenCalledTimes(1);
   expect(mockUiSettingsService.start).toHaveBeenCalledTimes(1);
   expect(mockMetricsService.start).toHaveBeenCalledTimes(1);
+  expect(mockStatusService.start).toHaveBeenCalledTimes(1);
 });
 
 test('does not fail on "setup" if there are unused paths detected', async () => {

src/core/server/server.ts

@@ -248,6 +248,7 @@ export class Server {
       savedObjects: savedObjectsStart,
       exposedConfigsToUsage: this.plugins.getExposedPluginConfigsToUsage(),
     });
+    this.status.start();
 
     this.coreStart = {
       capabilities: capabilitiesStart,
@@ -261,7 +262,6 @@ export class Server {
 
     await this.plugins.start(this.coreStart);
 
-    this.status.start();
     await this.http.start();
 
     startTransaction?.end();

src/core/server/status/plugins_status.test.ts

@@ -8,7 +8,7 @@
 
 import { PluginName } from '../plugins';
 import { PluginsStatusService } from './plugins_status';
-import { of, Observable, BehaviorSubject } from 'rxjs';
+import { of, Observable, BehaviorSubject, ReplaySubject } from 'rxjs';
 import { ServiceStatusLevels, CoreStatus, ServiceStatus } from './types';
 import { first } from 'rxjs/operators';
 import { ServiceStatusLevelSnapshotSerializer } from './test_utils';
@@ -34,6 +34,28 @@ describe('PluginStatusService', () => {
     ['c', ['a', 'b']],
   ]);
 
+  describe('set', () => {
+    it('throws an exception if called after registrations are blocked', () => {
+      const service = new PluginsStatusService({
+        core$: coreAllAvailable$,
+        pluginDependencies,
+      });
+
+      service.blockNewRegistrations();
+      expect(() => {
+        service.set(
+          'a',
+          of({
+            level: ServiceStatusLevels.available,
+            summary: 'fail!',
+          })
+        );
+      }).toThrowErrorMatchingInlineSnapshot(
+        `"Custom statuses cannot be registered after setup, plugin [a] attempted"`
+      );
+    });
+  });
+
   describe('getDerivedStatus$', () => {
     it(`defaults to core's most severe status`, async () => {
       const serviceAvailable = new PluginsStatusService({
@@ -231,6 +253,75 @@ describe('PluginStatusService', () => {
         { a: { level: ServiceStatusLevels.available, summary: 'a available' } },
       ]);
     });
+
+    it('updates when a plugin status observable emits', async () => {
+      const service = new PluginsStatusService({
+        core$: coreAllAvailable$,
+        pluginDependencies: new Map([['a', []]]),
+      });
+      const statusUpdates: Array<Record<PluginName, ServiceStatus>> = [];
+      const subscription = service
+        .getAll$()
+        .subscribe((pluginStatuses) => statusUpdates.push(pluginStatuses));
+
+      const aStatus$ = new BehaviorSubject<ServiceStatus>({
+        level: ServiceStatusLevels.degraded,
+        summary: 'a degraded',
+      });
+      service.set('a', aStatus$);
+      aStatus$.next({ level: ServiceStatusLevels.unavailable, summary: 'a unavailable' });
+      aStatus$.next({ level: ServiceStatusLevels.available, summary: 'a available' });
+      subscription.unsubscribe();
+
+      expect(statusUpdates).toEqual([
+        { a: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' } },
+        { a: { level: ServiceStatusLevels.degraded, summary: 'a degraded' } },
+        { a: { level: ServiceStatusLevels.unavailable, summary: 'a unavailable' } },
+        { a: { level: ServiceStatusLevels.available, summary: 'a available' } },
+      ]);
+    });
+
+    it('emits an unavailable status if first emission times out, then continues future emissions', async () => {
+      jest.useFakeTimers();
+      const service = new PluginsStatusService({
+        core$: coreAllAvailable$,
+        pluginDependencies: new Map([
+          ['a', []],
+          ['b', ['a']],
+        ]),
+      });
+
+      const pluginA$ = new ReplaySubject<ServiceStatus>(1);
+      service.set('a', pluginA$);
+      const firstEmission = service.getAll$().pipe(first()).toPromise();
+      jest.runAllTimers();
+
+      expect(await firstEmission).toEqual({
+        a: { level: ServiceStatusLevels.unavailable, summary: 'Status check timed out after 30s' },
+        b: {
+          level: ServiceStatusLevels.unavailable,
+          summary: '[a]: Status check timed out after 30s',
+          detail: 'See the status page for more information',
+          meta: {
+            affectedServices: {
+              a: {
+                level: ServiceStatusLevels.unavailable,
+                summary: 'Status check timed out after 30s',
+              },
+            },
+          },
+        },
+      });
+
+      pluginA$.next({ level: ServiceStatusLevels.available, summary: 'a available' });
+      const secondEmission = service.getAll$().pipe(first()).toPromise();
+      jest.runAllTimers();
+      expect(await secondEmission).toEqual({
+        a: { level: ServiceStatusLevels.available, summary: 'a available' },
+        b: { level: ServiceStatusLevels.available, summary: 'All dependencies are available' },
+      });
+      jest.useRealTimers();
+    });
   });
 
   describe('getDependenciesStatus$', () => {

src/core/server/status/plugins_status.ts

@@ -7,13 +7,22 @@
  */
 
 import { BehaviorSubject, Observable, combineLatest, of } from 'rxjs';
-import { map, distinctUntilChanged, switchMap, debounceTime } from 'rxjs/operators';
+import {
+  map,
+  distinctUntilChanged,
+  switchMap,
+  debounceTime,
+  timeoutWith,
+  startWith,
+} from 'rxjs/operators';
 import { isDeepStrictEqual } from 'util';
 
 import { PluginName } from '../plugins';
-import { ServiceStatus, CoreStatus } from './types';
+import { ServiceStatus, CoreStatus, ServiceStatusLevels } from './types';
 import { getSummaryStatus } from './get_summary_status';
 
+const STATUS_TIMEOUT_MS = 30 * 1000; // 30 seconds
+
 interface Deps {
   core$: Observable<CoreStatus>;
   pluginDependencies: ReadonlyMap<PluginName, PluginName[]>;
@@ -23,6 +32,7 @@ export class PluginsStatusService {
   private readonly pluginStatuses = new Map<PluginName, Observable<ServiceStatus>>();
   private readonly update$ = new BehaviorSubject(true);
   private readonly defaultInheritedStatus$: Observable<ServiceStatus>;
+  private newRegistrationsAllowed = true;
 
   constructor(private readonly deps: Deps) {
     this.defaultInheritedStatus$ = this.deps.core$.pipe(
@@ -35,10 +45,19 @@ export class PluginsStatusService {
   }
 
   public set(plugin: PluginName, status$: Observable<ServiceStatus>) {
+    if (!this.newRegistrationsAllowed) {
+      throw new Error(
+        `Custom statuses cannot be registered after setup, plugin [${plugin}] attempted`
+      );
+    }
     this.pluginStatuses.set(plugin, status$);
     this.update$.next(true); // trigger all existing Observables to update from the new source Observable
   }
 
+  public blockNewRegistrations() {
+    this.newRegistrationsAllowed = false;
+  }
+
   public getAll$(): Observable<Record<PluginName, ServiceStatus>> {
     return this.getPluginStatuses$([...this.deps.pluginDependencies.keys()]);
   }
@@ -86,13 +105,22 @@ export class PluginsStatusService {
     return this.update$.pipe(
       switchMap(() => {
         const pluginStatuses = plugins
-          .map(
-            (depName) =>
-              [depName, this.pluginStatuses.get(depName) ?? this.getDerivedStatus$(depName)] as [
-                PluginName,
-                Observable<ServiceStatus>
-              ]
-          )
+          .map((depName) => {
+            const pluginStatus = this.pluginStatuses.get(depName)
+              ? this.pluginStatuses.get(depName)!.pipe(
+                  timeoutWith(
+                    STATUS_TIMEOUT_MS,
+                    this.pluginStatuses.get(depName)!.pipe(
+                      startWith({
+                        level: ServiceStatusLevels.unavailable,
+                        summary: `Status check timed out after ${STATUS_TIMEOUT_MS / 1000}s`,
+                      })
+                    )
+                  )
+                )
+              : this.getDerivedStatus$(depName);
+            return [depName, pluginStatus] as [PluginName, Observable<ServiceStatus>];
+          })
           .map(([pName, status$]) =>
             status$.pipe(map((status) => [pName, status] as [PluginName, ServiceStatus]))
           );

src/core/server/status/status_service.ts

@@ -135,9 +135,11 @@ export class StatusService implements CoreService<InternalStatusServiceSetup> {
   }
 
   public start() {
-    if (!this.overall$) {
-      throw new Error('cannot call `start` before `setup`');
+    if (!this.pluginsStatus || !this.overall$) {
+      throw new Error(`StatusService#setup must be called before #start`);
     }
+    this.pluginsStatus.blockNewRegistrations();
+
     getOverallStatusChanges(this.overall$, this.stop$).subscribe((message) => {
       this.logger.info(message);
     });

src/core/server/status/types.ts

@@ -196,6 +196,9 @@ export interface StatusServiceSetup {
    * Completely overrides the default inherited status.
    *
    * @remarks
+   * The first emission from this Observable should occur within 30s, else this plugin's status will fallback to
+   * `unavailable` until the first emission.
+   *
    * See the {@link StatusServiceSetup.derivedStatus$} API for leveraging the default status
    * calculation that is provided by Core.
    */

src/dev/typescript/projects.ts

@@ -58,6 +58,9 @@ export const PROJECTS = [
   ...glob
     .sync('test/interpreter_functional/plugins/*/tsconfig.json', { cwd: REPO_ROOT })
     .map((path) => new Project(resolve(REPO_ROOT, path))),
+  ...glob
+    .sync('test/server_integration/__fixtures__/plugins/*/tsconfig.json', { cwd: REPO_ROOT })
+    .map((path) => new Project(resolve(REPO_ROOT, path))),
 ];
 
 export function filterProjectsByFlag(projectFlag?: string) {

test/plugin_functional/test_suites/core_plugins/status.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import expect from '@kbn/expect';
+import { ServiceStatusLevels } from '../../../../src/core/server';
+import { PluginFunctionalProviderContext } from '../../services';
+
+export default function ({ getService, getPageObjects }: PluginFunctionalProviderContext) {
+  const supertest = getService('supertest');
+  const log = getService('log');
+
+  const delay = (ms: number) => new Promise((r) => setTimeout(r, ms));
+  const getStatus = async (pluginName?: string) => {
+    const resp = await supertest.get('/api/status?v8format=true');
+
+    if (pluginName) {
+      return resp.body.status.plugins[pluginName];
+    } else {
+      return resp.body.status.overall;
+    }
+  };
+
+  const setStatus = async <T extends keyof typeof ServiceStatusLevels>(level: T) =>
+    supertest
+      .post(`/internal/core_plugin_a/status/set?level=${level}`)
+      .set('kbn-xsrf', 'xxx')
+      .expect(200);
+
+  describe('status service', () => {
+    // This test must comes first because the timeout only applies to the initial emission
+    it("returns a timeout for status check that doesn't emit after 30s", async () => {
+      let aStatus = await getStatus('corePluginA');
+      expect(aStatus.level).to.eql('unavailable');
+
+      // Status will remain in unavailable due to core services until custom status timesout
+      // Keep polling until that condition ends, up to a timeout
+      const start = Date.now();
+      while ('elasticsearch' in (aStatus.meta?.affectedServices ?? {})) {
+        aStatus = await getStatus('corePluginA');
+        expect(aStatus.level).to.eql('unavailable');
+
+        // If it's been more than 40s, break out of this loop
+        if (Date.now() - start >= 40_000) {
+          throw new Error(`Timed out waiting for status timeout after 40s`);
+        }
+
+        log.info('Waiting for status check to timeout...');
+        await delay(2000);
+      }
+
+      expect(aStatus.summary).to.eql('Status check timed out after 30s');
+    });
+
+    it('propagates status issues to dependencies', async () => {
+      await setStatus('degraded');
+      await delay(1000);
+      expect((await getStatus('corePluginA')).level).to.eql('degraded');
+      expect((await getStatus('corePluginB')).level).to.eql('degraded');
+
+      await setStatus('available');
+      await delay(1000);
+      expect((await getStatus('corePluginA')).level).to.eql('available');
+      expect((await getStatus('corePluginB')).level).to.eql('available');
+    });
+  });
+}

test/scripts/test/server_integration.sh

@@ -12,3 +12,10 @@ checks-reporter-with-killswitch "Server Integration Tests" \
     --bail \
     --debug \
     --kibana-install-dir $KIBANA_INSTALL_DIR
+
+# Tests that must be run against source in order to build test plugins
+checks-reporter-with-killswitch "Status Integration Tests" \
+  node scripts/functional_tests \
+    --config test/server_integration/http/platform/config.status.ts \
+    --bail \
+    --debug \

test/server_integration/__fixtures__/plugins/status_plugin_a/kibana.json

@@ -0,0 +1,7 @@
+{
+  "id": "statusPluginA",
+  "version": "0.0.1",
+  "kibanaVersion": "kibana",
+  "server": true,
+  "ui": false
+}

test/server_integration/__fixtures__/plugins/status_plugin_a/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "status_plugin_a",
+  "version": "1.0.0",
+  "main": "target/test/server_integration/__fixtures__/plugins/status_plugin_a",
+  "kibana": {
+    "version": "kibana",
+    "templateVersion": "1.0.0"
+  },
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "scripts": {
+    "kbn": "node ../../../../../../scripts/kbn.js",
+    "build": "rm -rf './target' && ../../../../../../node_modules/.bin/tsc"
+  }
+}

test/server_integration/__fixtures__/plugins/status_plugin_a/server/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { StatusPluginAPlugin } from './plugin';
+
+export const plugin = () => new StatusPluginAPlugin();

test/server_integration/__fixtures__/plugins/status_plugin_a/server/plugin.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema } from '@kbn/config-schema';
+import { Subject } from 'rxjs';
+import {
+  Plugin,
+  CoreSetup,
+  ServiceStatus,
+  ServiceStatusLevels,
+} from '../../../../../../src/core/server';
+
+export class StatusPluginAPlugin implements Plugin {
+  private status$ = new Subject<ServiceStatus>();
+
+  public setup(core: CoreSetup, deps: {}) {
+    // Set a custom status that will not emit immediately to force a timeout
+    core.status.set(this.status$);
+
+    const router = core.http.createRouter();
+
+    router.post(
+      {
+        path: '/internal/status_plugin_a/status/set',
+        validate: {
+          query: schema.object({
+            level: schema.oneOf([
+              schema.literal('available'),
+              schema.literal('degraded'),
+              schema.literal('unavailable'),
+              schema.literal('critical'),
+            ]),
+          }),
+        },
+      },
+      (context, req, res) => {
+        const { level } = req.query;
+
+        this.status$.next({
+          level: ServiceStatusLevels[level],
+          summary: `statusPluginA is ${level}`,
+        });
+
+        return res.ok();
+      }
+    );
+  }
+
+  public start() {}
+  public stop() {}
+}

test/server_integration/__fixtures__/plugins/status_plugin_a/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./target",
+    "skipLibCheck": true,
+    "composite": true
+  },
+  "include": [
+    "index.ts",
+    "server/**/*.ts",
+    "../../../../../../typings/**/*",
+  ],
+  "exclude": [],
+  "references": [
+    { "path": "../../../../../src/core/tsconfig.json" }
+  ]
+}

test/server_integration/__fixtures__/plugins/status_plugin_b/kibana.json

@@ -0,0 +1,8 @@
+{
+  "id": "statusPluginB",
+  "version": "0.0.1",
+  "kibanaVersion": "kibana",
+  "server": true,
+  "ui": false,
+  "requiredPlugins": ["statusPluginA"]
+}

test/server_integration/__fixtures__/plugins/status_plugin_b/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "status_plugin_b",
+  "version": "1.0.0",
+  "main": "target/test/server_integration/__fixtures__/plugins/status_plugin_b",
+  "kibana": {
+    "version": "kibana",
+    "templateVersion": "1.0.0"
+  },
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "scripts": {
+    "kbn": "node ../../../../../../scripts/kbn.js",
+    "build": "rm -rf './target' && ../../../../../../node_modules/.bin/tsc"
+  }
+}

test/server_integration/__fixtures__/plugins/status_plugin_b/server/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { StatusPluginBPlugin } from './plugin';
+
+export const plugin = () => new StatusPluginBPlugin();

test/server_integration/__fixtures__/plugins/status_plugin_b/server/plugin.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Plugin } from 'kibana/server';
+
+export class StatusPluginBPlugin implements Plugin {
+  public setup() {}
+  public start() {}
+  public stop() {}
+}

test/server_integration/__fixtures__/plugins/status_plugin_b/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./target",
+    "skipLibCheck": true,
+    "composite": true
+  },
+  "include": [
+    "index.ts",
+    "server/**/*.ts",
+    "../../../../../typings/**/*",
+  ],
+  "exclude": [],
+  "references": [
+    { "path": "../../../../../src/core/tsconfig.json" }
+  ]
+}

test/server_integration/http/platform/config.status.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { FtrConfigProviderContext } from '@kbn/test';
+
+/*
+ * These tests exist in a separate configuration because:
+ * 1) It must run as the first test after Kibana launches to clear the unavailable status. A separate config makes this
+ *    easier to manage and prevent from breaking.
+ * 2) The other server_integration tests run against a built distributable, however the FTR does not support building
+ *    and installing plugins against built Kibana. This test must be run against source only in order to build the
+ *    fixture plugins
+ */
+// eslint-disable-next-line import/no-default-export
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const httpConfig = await readConfigFile(require.resolve('../../config'));
+
+  // Find all folders in __fixtures__/plugins since we treat all them as plugin folder
+  const allFiles = fs.readdirSync(path.resolve(__dirname, '../../__fixtures__/plugins'));
+  const plugins = allFiles.filter((file) =>
+    fs.statSync(path.resolve(__dirname, '../../__fixtures__/plugins', file)).isDirectory()
+  );
+
+  return {
+    testFiles: [
+      // Status test should be first to resolve manually created "unavailable" plugin
+      require.resolve('./status'),
+    ],
+    services: httpConfig.get('services'),
+    servers: httpConfig.get('servers'),
+    junit: {
+      reportName: 'Kibana Platform Status Integration Tests',
+    },
+    esTestCluster: httpConfig.get('esTestCluster'),
+    kbnTestServer: {
+      ...httpConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...httpConfig.get('kbnTestServer.serverArgs'),
+        ...plugins.map(
+          (pluginDir) =>
+            `--plugin-path=${path.resolve(__dirname, '../../__fixtures__/plugins', pluginDir)}`
+        ),
+      ],
+      runOptions: {
+        ...httpConfig.get('kbnTestServer.runOptions'),
+        // Don't wait for Kibana to be completely ready so that we can test the status timeouts
+        wait: /\[Kibana\]\[http\] http server running/,
+      },
+    },
+  };
+}

test/server_integration/http/platform/status.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import expect from '@kbn/expect';
+import type { ServiceStatus, ServiceStatusLevels } from '../../../../src/core/server';
+import { FtrProviderContext } from '../../services/types';
+
+type ServiceStatusSerialized = Omit<ServiceStatus, 'level'> & { level: string };
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ getService }: FtrProviderContext) {
+  const supertest = getService('supertest');
+  const retry = getService('retry');
+
+  const getStatus = async (pluginName: string): Promise<ServiceStatusSerialized> => {
+    const resp = await supertest.get('/api/status?v8format=true');
+
+    return resp.body.status.plugins[pluginName];
+  };
+
+  const setStatus = async <T extends keyof typeof ServiceStatusLevels>(level: T) =>
+    supertest
+      .post(`/internal/status_plugin_a/status/set?level=${level}`)
+      .set('kbn-xsrf', 'xxx')
+      .expect(200);
+
+  describe('status service', () => {
+    // This test must comes first because the timeout only applies to the initial emission
+    it("returns a timeout for status check that doesn't emit after 30s", async () => {
+      let aStatus = await getStatus('statusPluginA');
+      expect(aStatus.level).to.eql('unavailable');
+
+      // Status will remain in unavailable until the custom status check times out
+      // Keep polling until that condition ends, up to a timeout
+      await retry.waitForWithTimeout(`Status check to timeout`, 40_000, async () => {
+        aStatus = await getStatus('statusPluginA');
+        return aStatus.summary === 'Status check timed out after 30s';
+      });
+
+      expect(aStatus.level).to.eql('unavailable');
+      expect(aStatus.summary).to.eql('Status check timed out after 30s');
+    });
+
+    it('propagates status issues to dependencies', async () => {
+      await setStatus('degraded');
+      await retry.waitForWithTimeout(
+        `statusPluginA status to update`,
+        5_000,
+        async () => (await getStatus('statusPluginA')).level === 'degraded'
+      );
+      expect((await getStatus('statusPluginA')).level).to.eql('degraded');
+      expect((await getStatus('statusPluginB')).level).to.eql('degraded');
+
+      await setStatus('available');
+      await retry.waitForWithTimeout(
+        `statusPluginA status to update`,
+        5_000,
+        async () => (await getStatus('statusPluginA')).level === 'available'
+      );
+      expect((await getStatus('statusPluginA')).level).to.eql('available');
+      expect((await getStatus('statusPluginB')).level).to.eql('available');
+    });
+  });
+}

test/tsconfig.json

@@ -17,7 +17,12 @@
     "api_integration/apis/telemetry/fixtures/*.json",
     "api_integration/apis/telemetry/fixtures/*.json",
   ],
-  "exclude": ["target/**/*", "plugin_functional/plugins/**/*", "interpreter_functional/plugins/**/*"],
+  "exclude": [
+    "target/**/*",
+    "interpreter_functional/plugins/**/*",
+    "plugin_functional/plugins/**/*",
+    "server_integration/__fixtures__/plugins/**/*",
+  ],
   "references": [
     { "path": "../src/core/tsconfig.json" },
     { "path": "../src/plugins/telemetry_management_section/tsconfig.json" },
@@ -52,5 +57,7 @@
     { "path": "../src/plugins/visualize/tsconfig.json" },
     { "path": "plugin_functional/plugins/core_app_status/tsconfig.json" },
     { "path": "plugin_functional/plugins/core_provider_plugin/tsconfig.json" },
+    { "path": "server_integration/__fixtures__/plugins/status_plugin_a/tsconfig.json" },
+    { "path": "server_integration/__fixtures__/plugins/status_plugin_b/tsconfig.json" },
   ]
 }