[8.x] [Streams] App plugin (#200060) (#201999)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Streams] App plugin
(#200060)](https://github.com/elastic/kibana/pull/200060)

<!--- Backport version: 7.3.2 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT {commits} BACKPORT-->

---------

Co-authored-by: Caue Marcondes <caue.marcondes@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.eslintrc.js

@@ -952,6 +952,7 @@ module.exports = {
     {
       files: [
         'x-pack/plugins/observability_solution/**/*.{ts,tsx}',
+        'x-pack/plugins/{streams,streams_app}/**/*.{ts,tsx}',
         'x-pack/packages/observability/**/*.{ts,tsx}',
       ],
       rules: {
@@ -959,7 +960,7 @@ module.exports = {
           'error',
           {
             additionalHooks:
-              '^(useAbortableAsync|useMemoWithAbortSignal|useFetcher|useProgressiveFetcher|useBreadcrumb|useAsync|useTimeRangeAsync|useAutoAbortedHttpClient)$',
+              '^(useAbortableAsync|useMemoWithAbortSignal|useFetcher|useProgressiveFetcher|useBreadcrumb|useAsync|useTimeRangeAsync|useAutoAbortedHttpClient|use.*Fetch)$',
           },
         ],
       },
@@ -968,6 +969,7 @@ module.exports = {
       files: [
         'x-pack/plugins/aiops/**/*.tsx',
         'x-pack/plugins/observability_solution/**/*.tsx',
+        'x-pack/plugins/{streams,streams_app}/**/*.{ts,tsx}',
         'src/plugins/ai_assistant_management/**/*.tsx',
         'x-pack/packages/observability/**/*.{ts,tsx}',
       ],
@@ -984,6 +986,7 @@ module.exports = {
     {
       files: [
         'x-pack/plugins/observability_solution/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)',
+        'x-pack/plugins/{streams,streams_app}/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)',
         'src/plugins/ai_assistant_management/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)',
         'x-pack/packages/observability/logs_overview/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)',
       ],

.github/CODEOWNERS

@@ -927,6 +927,7 @@ test/server_integration/plugins/status_plugin_b @elastic/kibana-core
 packages/kbn-std @elastic/kibana-core
 packages/kbn-stdio-dev-helpers @elastic/kibana-operations
 packages/kbn-storybook @elastic/kibana-operations
+x-pack/plugins/streams_app @simianhacker @flash1293 @dgieselaar
 x-pack/plugins/streams @simianhacker @flash1293 @dgieselaar
 x-pack/plugins/observability_solution/synthetics/e2e @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/synthetics @elastic/obs-ux-management-team

docs/developer/plugin-list.asciidoc

@@ -917,6 +917,10 @@ routes, etc.
 |This plugin provides an interface to manage streams
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/streams_app/README.md[streamsApp]
+|Home of the Streams app plugin, which allows users to manage Streams via the UI.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/synthetics/README.md[synthetics]
 |The purpose of this plugin is to provide users of Heartbeat more visibility of what's happening
 in their infrastructure.

package.json

@@ -935,6 +935,7 @@
     "@kbn/status-plugin-a-plugin": "link:test/server_integration/plugins/status_plugin_a",
     "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b",
     "@kbn/std": "link:packages/kbn-std",
+    "@kbn/streams-app-plugin": "link:x-pack/plugins/streams_app",
     "@kbn/streams-plugin": "link:x-pack/plugins/streams",
     "@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics",
     "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location",

packages/deeplinks/observability/constants.ts

@@ -35,3 +35,5 @@ export const OBLT_UX_APP_ID = 'ux';
 export const OBLT_PROFILING_APP_ID = 'profiling';
 
 export const INVENTORY_APP_ID = 'inventory';
+
+export const STREAMS_APP_ID = 'streams';

packages/deeplinks/observability/deep_links.ts

@@ -21,6 +21,7 @@ import {
   OBLT_UX_APP_ID,
   OBLT_PROFILING_APP_ID,
   INVENTORY_APP_ID,
+  STREAMS_APP_ID,
 } from './constants';
 
 type LogsApp = typeof LOGS_APP_ID;
@@ -36,6 +37,7 @@ type AiAssistantApp = typeof AI_ASSISTANT_APP_ID;
 type ObltUxApp = typeof OBLT_UX_APP_ID;
 type ObltProfilingApp = typeof OBLT_PROFILING_APP_ID;
 type InventoryApp = typeof INVENTORY_APP_ID;
+type StreamsApp = typeof STREAMS_APP_ID;
 
 export type AppId =
   | LogsApp
@@ -50,7 +52,8 @@ export type AppId =
   | AiAssistantApp
   | ObltUxApp
   | ObltProfilingApp
-  | InventoryApp;
+  | InventoryApp
+  | StreamsApp;
 
 export type LogsLinkId = 'log-categories' | 'settings' | 'anomalies' | 'stream';
 
@@ -83,13 +86,16 @@ export type SyntheticsLinkId = 'certificates' | 'overview';
 
 export type ProfilingLinkId = 'stacktraces' | 'flamegraphs' | 'functions';
 
+export type StreamsLinkId = 'overview';
+
 export type LinkId =
   | LogsLinkId
   | ObservabilityOverviewLinkId
   | MetricsLinkId
   | ApmLinkId
   | SyntheticsLinkId
-  | ProfilingLinkId;
+  | ProfilingLinkId
+  | StreamsLinkId;
 
 export type DeepLinkId =
   | AppId
@@ -99,4 +105,5 @@ export type DeepLinkId =
   | `${ApmApp}:${ApmLinkId}`
   | `${SyntheticsApp}:${SyntheticsLinkId}`
   | `${ObltProfilingApp}:${ProfilingLinkId}`
-  | `${InventoryApp}:${InventoryLinkId}`;
+  | `${InventoryApp}:${InventoryLinkId}`
+  | `${StreamsApp}:${StreamsLinkId}`;

packages/kbn-es-types/src/search.ts

@@ -695,5 +695,5 @@ export interface ESQLSearchParams {
   locale?: string;
   include_ccs_metadata?: boolean;
   dropNullColumns?: boolean;
-  params?: Array<Record<string, string | undefined>>;
+  params?: estypesWithoutBodyKey.ScalarValue[] | Array<Record<string, string | undefined>>;
 }

packages/kbn-optimizer/limits.yml

@@ -163,6 +163,7 @@ pageLoadAssetSize:
   stackAlerts: 58316
   stackConnectors: 67227
   streams: 16742
+  streamsApp: 20537
   synthetics: 55971
   telemetry: 51957
   telemetryManagementSection: 38586

packages/kbn-server-route-repository-client/src/create_repository_client.ts

@@ -15,12 +15,12 @@ import {
 } from '@kbn/server-route-repository-utils';
 import { httpResponseIntoObservable } from '@kbn/sse-utils-client';
 import { from } from 'rxjs';
-import { HttpFetchOptions, HttpFetchQuery, HttpResponse } from '@kbn/core-http-browser';
+import { HttpFetchQuery, HttpResponse } from '@kbn/core-http-browser';
 import { omit } from 'lodash';
 
 export function createRepositoryClient<
   TRepository extends ServerRouteRepository,
-  TClientOptions extends HttpFetchOptions = {}
+  TClientOptions extends Record<string, any> = {}
 >(core: CoreStart | CoreSetup): RouteRepositoryClient<TRepository, TClientOptions> {
   const fetch = (
     endpoint: string,

packages/kbn-server-route-repository-utils/index.ts

@@ -18,7 +18,6 @@ export type {
   EndpointOf,
   ReturnOf,
   RouteRepositoryClient,
-  RouteState,
   ClientRequestParamsOf,
   DecodedRequestParamsOf,
   ServerRouteRepository,

packages/kbn-server-route-repository-utils/src/parse_endpoint.ts

@@ -7,22 +7,20 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-type Method = 'get' | 'post' | 'put' | 'patch' | 'delete';
+import type { RouteMethod } from '@kbn/core-http-server';
+
+const validMethods: RouteMethod[] = ['delete', 'get', 'patch', 'post', 'put'];
 
 export function parseEndpoint(endpoint: string) {
   const parts = endpoint.split(' ');
 
-  const method = parts[0].trim().toLowerCase() as Method;
+  const method = parts[0].trim().toLowerCase() as Exclude<RouteMethod, 'options'>;
   const pathname = parts[1].trim();
   const version = parts[2]?.trim();
 
-  if (!['get', 'post', 'put', 'patch', 'delete'].includes(method)) {
+  if (!validMethods.includes(method)) {
     throw new Error(`Endpoint ${endpoint} was not prefixed with a valid HTTP method`);
   }
 
-  if (!version && pathname.startsWith('/api')) {
-    throw new Error(`Missing version for public endpoint ${endpoint}`);
-  }
-
   return { method, pathname, version };
 }

packages/kbn-server-route-repository-utils/src/typings.ts

@@ -8,14 +8,13 @@
  */
 
 import type { HttpFetchOptions } from '@kbn/core-http-browser';
-import type { IKibanaResponse } from '@kbn/core-http-server';
+import type { IKibanaResponse, RouteAccess, RouteSecurity } from '@kbn/core-http-server';
 import type {
   KibanaRequest,
   KibanaResponseFactory,
   Logger,
   RequestHandlerContext,
   RouteConfigOptions,
-  RouteSecurity,
   RouteMethod,
 } from '@kbn/core/server';
 import type { ServerSentEvent } from '@kbn/sse-utils';
@@ -23,7 +22,7 @@ import { z } from '@kbn/zod';
 import * as t from 'io-ts';
 import { Observable } from 'rxjs';
 import { Readable } from 'stream';
-import { RequiredKeys, ValuesType } from 'utility-types';
+import { Required, RequiredKeys, ValuesType } from 'utility-types';
 
 type MaybeOptional<T extends { params?: Record<string, any> }> = RequiredKeys<
   T['params']
@@ -51,24 +50,37 @@ export type ZodParamsObject = z.ZodObject<{
 export type IoTsParamsObject = WithoutIncompatibleMethods<t.Type<RouteParams>>;
 
 export type RouteParamsRT = IoTsParamsObject | ZodParamsObject;
+export type ServerRouteHandlerResources = Record<string, any>;
 
-export interface RouteState {
-  [endpoint: string]: ServerRoute<any, any, any, any, any>;
+export interface ServerRouteCreateOptions {
+  [x: string]: any;
 }
 
-export type ServerRouteHandlerResources = Record<string, any>;
-export type ServerRouteCreateOptions = Record<string, any>;
+type RouteMethodOf<TEndpoint extends string> = TEndpoint extends `${infer TRouteMethod} ${string}`
+  ? Lowercase<TRouteMethod> extends RouteMethod
+    ? Lowercase<TRouteMethod>
+    : never
+  : never;
 
-type ValidateEndpoint<TEndpoint extends string> = string extends TEndpoint
+type IsPublicEndpoint<
+  TEndpoint extends string,
+  TRouteAccess extends RouteAccess | undefined
+> = TRouteAccess extends 'public'
   ? true
-  : TEndpoint extends `${string} ${string} ${string}`
+  : TRouteAccess extends 'internal'
+  ? false
+  : TEndpoint extends `${string} /api${string}`
   ? true
-  : TEndpoint extends `${string} ${infer TPathname}`
-  ? TPathname extends `/internal/${string}`
-    ? true
-    : false
   : false;
 
+type IsVersionSpecified<TEndpoint extends string> =
+  TEndpoint extends `${string} ${string} ${string}` ? true : false;
+
+type ValidateEndpoint<
+  TEndpoint extends string,
+  TRouteAccess extends RouteAccess | undefined
+> = IsPublicEndpoint<TEndpoint, TRouteAccess> extends true ? IsVersionSpecified<TEndpoint> : true;
+
 type IsAny<T> = 1 | 0 extends (T extends never ? 1 : 0) ? true : false;
 
 // this ensures only plain objects can be returned, if it's not one
@@ -128,17 +140,27 @@ type ServerRouteHandler<
 
 export type CreateServerRouteFactory<
   TRouteHandlerResources extends ServerRouteHandlerResources,
-  TRouteCreateOptions extends ServerRouteCreateOptions
+  TRouteCreateOptions extends DefaultRouteCreateOptions | undefined
 > = <
   TEndpoint extends string,
   TReturnType extends ServerRouteHandlerReturnType,
-  TRouteParamsRT extends RouteParamsRT | undefined = undefined
+  TRouteParamsRT extends RouteParamsRT | undefined = undefined,
+  TRouteAccess extends RouteAccess | undefined = undefined
 >(
   options: {
-    endpoint: ValidateEndpoint<TEndpoint> extends true ? TEndpoint : never;
+    endpoint: ValidateEndpoint<TEndpoint, TRouteAccess> extends true ? TEndpoint : never;
     handler: ServerRouteHandler<TRouteHandlerResources, TRouteParamsRT, TReturnType>;
     params?: TRouteParamsRT;
-  } & TRouteCreateOptions
+    security?: RouteSecurity;
+  } & Required<
+    {
+      options?: (TRouteCreateOptions extends DefaultRouteCreateOptions ? TRouteCreateOptions : {}) &
+        RouteConfigOptions<RouteMethodOf<TEndpoint>> & {
+          access?: TRouteAccess;
+        };
+    },
+    RequiredKeys<TRouteCreateOptions> extends never ? never : 'options'
+  >
 ) => Record<
   TEndpoint,
   ServerRoute<
@@ -155,16 +177,17 @@ export type ServerRoute<
   TRouteParamsRT extends RouteParamsRT | undefined,
   TRouteHandlerResources extends ServerRouteHandlerResources,
   TReturnType extends ServerRouteHandlerReturnType,
-  TRouteCreateOptions extends ServerRouteCreateOptions
+  TRouteCreateOptions extends DefaultRouteCreateOptions | undefined
 > = {
   endpoint: TEndpoint;
   handler: ServerRouteHandler<TRouteHandlerResources, TRouteParamsRT, TReturnType>;
-} & TRouteCreateOptions &
-  (TRouteParamsRT extends RouteParamsRT ? { params: TRouteParamsRT } : {});
+  security?: RouteSecurity;
+} & (TRouteParamsRT extends RouteParamsRT ? { params: TRouteParamsRT } : {}) &
+  (TRouteCreateOptions extends DefaultRouteCreateOptions ? { options: TRouteCreateOptions } : {});
 
 export type ServerRouteRepository = Record<
   string,
-  ServerRoute<string, RouteParamsRT | undefined, any, any, Record<string, any>>
+  ServerRoute<string, RouteParamsRT | undefined, any, any, ServerRouteCreateOptions | undefined>
 >;
 
 type ClientRequestParamsOfType<TRouteParamsRT extends RouteParamsRT> =
@@ -195,13 +218,7 @@ export type EndpointOf<TServerRouteRepository extends ServerRouteRepository> =
 export type ReturnOf<
   TServerRouteRepository extends ServerRouteRepository,
   TEndpoint extends keyof TServerRouteRepository
-> = TServerRouteRepository[TEndpoint] extends ServerRoute<
-  any,
-  any,
-  any,
-  infer TReturnType,
-  ServerRouteCreateOptions
->
+> = TServerRouteRepository[TEndpoint] extends ServerRoute<any, any, any, infer TReturnType, any>
   ? TReturnType extends IKibanaResponse<infer TWrappedResponseType>
     ? TWrappedResponseType
     : TReturnType
@@ -210,13 +227,7 @@ export type ReturnOf<
 export type DecodedRequestParamsOf<
   TServerRouteRepository extends ServerRouteRepository,
   TEndpoint extends keyof TServerRouteRepository
-> = TServerRouteRepository[TEndpoint] extends ServerRoute<
-  any,
-  infer TRouteParamsRT,
-  any,
-  any,
-  ServerRouteCreateOptions
->
+> = TServerRouteRepository[TEndpoint] extends ServerRoute<any, infer TRouteParamsRT, any, any, any>
   ? TRouteParamsRT extends RouteParamsRT
     ? DecodedRequestParamsOfType<TRouteParamsRT>
     : {}
@@ -230,7 +241,7 @@ export type ClientRequestParamsOf<
   infer TRouteParamsRT,
   any,
   any,
-  ServerRouteCreateOptions
+  ServerRouteCreateOptions | undefined
 >
   ? TRouteParamsRT extends RouteParamsRT
     ? ClientRequestParamsOfType<TRouteParamsRT>
@@ -250,13 +261,17 @@ export interface RouteRepositoryClient<
   fetch<TEndpoint extends Extract<keyof TServerRouteRepository, string>>(
     endpoint: TEndpoint,
     ...args: MaybeOptionalArgs<
-      ClientRequestParamsOf<TServerRouteRepository, TEndpoint> & TAdditionalClientOptions
+      ClientRequestParamsOf<TServerRouteRepository, TEndpoint> &
+        TAdditionalClientOptions &
+        HttpFetchOptions
     >
   ): Promise<ReturnOf<TServerRouteRepository, TEndpoint>>;
   stream<TEndpoint extends Extract<keyof TServerRouteRepository, string>>(
     endpoint: TEndpoint,
     ...args: MaybeOptionalArgs<
-      ClientRequestParamsOf<TServerRouteRepository, TEndpoint> & TAdditionalClientOptions
+      ClientRequestParamsOf<TServerRouteRepository, TEndpoint> &
+        TAdditionalClientOptions &
+        HttpFetchOptions
     >
   ): ReturnOf<TServerRouteRepository, TEndpoint> extends Observable<infer TReturnType>
     ? TReturnType extends ServerSentEvent
@@ -277,6 +292,4 @@ export interface DefaultRouteHandlerResources extends CoreRouteHandlerResources
   logger: Logger;
 }
 
-export interface DefaultRouteCreateOptions {
-  options?: RouteConfigOptions<RouteMethod> & { security?: RouteSecurity };
-}
+export type DefaultRouteCreateOptions = RouteConfigOptions<Exclude<RouteMethod, 'options'>>;

packages/kbn-server-route-repository/index.ts

@@ -23,7 +23,6 @@ export type {
   ServerRouteRepository,
   ServerRoute,
   RouteParamsRT,
-  RouteState,
   DefaultRouteCreateOptions,
   DefaultRouteHandlerResources,
   IoTsParamsObject,

packages/kbn-server-route-repository/src/create_server_route_factory.ts

@@ -8,16 +8,17 @@
  */
 
 import type {
-  DefaultRouteCreateOptions,
   DefaultRouteHandlerResources,
-  ServerRouteCreateOptions,
   ServerRouteHandlerResources,
 } from '@kbn/server-route-repository-utils';
-import type { CreateServerRouteFactory } from '@kbn/server-route-repository-utils/src/typings';
+import type {
+  CreateServerRouteFactory,
+  DefaultRouteCreateOptions,
+} from '@kbn/server-route-repository-utils/src/typings';
 
 export function createServerRouteFactory<
   TRouteHandlerResources extends ServerRouteHandlerResources = DefaultRouteHandlerResources,
-  TRouteCreateOptions extends ServerRouteCreateOptions = DefaultRouteCreateOptions
+  TRouteCreateOptions extends DefaultRouteCreateOptions | undefined = undefined
 >(): CreateServerRouteFactory<TRouteHandlerResources, TRouteCreateOptions> {
   return (route) => ({ [route.endpoint]: route } as any);
 }

packages/kbn-server-route-repository/src/register_routes.test.ts

@@ -15,6 +15,7 @@ import { NEVER } from 'rxjs';
 import * as makeZodValidationObject from './make_zod_validation_object';
 import { registerRoutes } from './register_routes';
 import { passThroughValidationObject, noParamsValidationObject } from './validation_objects';
+import { ServerRouteRepository } from '@kbn/server-route-repository-utils';
 
 describe('registerRoutes', () => {
   const post = jest.fn();
@@ -54,44 +55,82 @@ describe('registerRoutes', () => {
       'POST /internal/route': {
         endpoint: 'POST /internal/route',
         handler: jest.fn(),
-        options: {
-          internal: true,
-        },
       },
       'POST /api/public_route version': {
         endpoint: 'POST /api/public_route version',
         handler: jest.fn(),
+      },
+      'POST /api/internal_but_looks_like_public version': {
+        endpoint: 'POST /api/internal_but_looks_like_public version',
         options: {
-          public: true,
+          access: 'internal',
+        },
+        handler: jest.fn(),
+      },
+      'POST /internal/route_with_security': {
+        endpoint: `POST /internal/route_with_security`,
+        handler: jest.fn(),
+        security: {
+          authz: {
+            enabled: false,
+            reason: 'whatever',
+          },
         },
       },
-    });
+      'POST /api/route_with_security version': {
+        endpoint: `POST /api/route_with_security version`,
+        handler: jest.fn(),
+        security: {
+          authz: {
+            enabled: false,
+            reason: 'whatever',
+          },
+        },
+      },
+    } satisfies ServerRouteRepository);
 
     expect(createRouter).toHaveBeenCalledTimes(1);
 
-    expect(post).toHaveBeenCalledTimes(1);
-
     const [internalRoute] = post.mock.calls[0];
     expect(internalRoute.path).toEqual('/internal/route');
     expect(internalRoute.options).toEqual({
-      internal: true,
+      access: 'internal',
     });
     expect(internalRoute.validate).toEqual(noParamsValidationObject);
 
-    expect(postWithVersion).toHaveBeenCalledTimes(1);
+    const [internalRouteWithSecurity] = post.mock.calls[1];
+
+    expect(internalRouteWithSecurity.path).toEqual('/internal/route_with_security');
+    expect(internalRouteWithSecurity.security).toEqual({
+      authz: {
+        enabled: false,
+        reason: 'whatever',
+      },
+    });
+
     const [publicRoute] = postWithVersion.mock.calls[0];
     expect(publicRoute.path).toEqual('/api/public_route');
-    expect(publicRoute.options).toEqual({
-      public: true,
-    });
     expect(publicRoute.access).toEqual('public');
 
-    expect(postAddVersion).toHaveBeenCalledTimes(1);
+    const [apiInternalRoute] = postWithVersion.mock.calls[1];
+    expect(apiInternalRoute.path).toEqual('/api/internal_but_looks_like_public');
+    expect(apiInternalRoute.access).toEqual('internal');
+
     const [versionedRoute] = postAddVersion.mock.calls[0];
     expect(versionedRoute.version).toEqual('version');
     expect(versionedRoute.validate).toEqual({
       request: noParamsValidationObject,
     });
+
+    const [publicRouteWithSecurity] = postWithVersion.mock.calls[2];
+
+    expect(publicRouteWithSecurity.path).toEqual('/api/route_with_security');
+    expect(publicRouteWithSecurity.security).toEqual({
+      authz: {
+        enabled: false,
+        reason: 'whatever',
+      },
+    });
   });
 
   it('does not allow any params if no schema is provided', () => {

packages/kbn-server-route-repository/src/register_routes.ts

@@ -15,19 +15,20 @@ import { isKibanaResponse } from '@kbn/core-http-server';
 import type { CoreSetup } from '@kbn/core-lifecycle-server';
 import type { Logger } from '@kbn/logging';
 import {
+  DefaultRouteCreateOptions,
+  RouteParamsRT,
   ServerRoute,
-  ServerRouteCreateOptions,
   ZodParamsObject,
   parseEndpoint,
 } from '@kbn/server-route-repository-utils';
+import { ServerSentEvent } from '@kbn/sse-utils';
 import { observableIntoEventSourceStream } from '@kbn/sse-utils-server';
 import { isZod } from '@kbn/zod';
-import { merge } from 'lodash';
+import { merge, omit } from 'lodash';
 import { Observable, isObservable } from 'rxjs';
-import { ServerSentEvent } from '@kbn/sse-utils';
-import { passThroughValidationObject, noParamsValidationObject } from './validation_objects';
-import { validateAndDecodeParams } from './validate_and_decode_params';
 import { makeZodValidationObject } from './make_zod_validation_object';
+import { validateAndDecodeParams } from './validate_and_decode_params';
+import { noParamsValidationObject, passThroughValidationObject } from './validation_objects';
 
 const CLIENT_CLOSED_REQUEST = {
   statusCode: 499,
@@ -43,7 +44,7 @@ export function registerRoutes<TDependencies extends Record<string, any>>({
   dependencies,
 }: {
   core: CoreSetup;
-  repository: Record<string, ServerRoute<string, any, any, any, ServerRouteCreateOptions>>;
+  repository: Record<string, ServerRoute<string, RouteParamsRT | undefined, any, any, any>>;
   logger: Logger;
   dependencies: TDependencies;
 }) {
@@ -52,7 +53,11 @@ export function registerRoutes<TDependencies extends Record<string, any>>({
   const router = core.http.createRouter();
 
   routes.forEach((route) => {
-    const { params, endpoint, options, handler } = route;
+    const { endpoint, handler, security } = route;
+
+    const params = 'params' in route ? route.params : undefined;
+
+    const options: DefaultRouteCreateOptions = 'options' in route ? route.options : {};
 
     const { method, pathname, version } = parseEndpoint(endpoint);
 
@@ -137,14 +142,18 @@ export function registerRoutes<TDependencies extends Record<string, any>>({
       validationObject = passThroughValidationObject;
     }
 
-    const { security, ...restOptions } = options ?? {};
+    const access = options?.access ?? (pathname.startsWith('/internal/') ? 'internal' : 'public');
 
     if (!version) {
       router[method](
         {
           path: pathname,
+          // @ts-expect-error we are essentially calling multiple methods at the same type so TS gets confused
+          options: {
+            ...options,
+            access,
+          },
           security,
-          options: restOptions,
           validate: validationObject,
         },
         wrappedHandler
@@ -152,8 +161,9 @@ export function registerRoutes<TDependencies extends Record<string, any>>({
     } else {
       router.versioned[method]({
         path: pathname,
-        access: pathname.startsWith('/internal/') ? 'internal' : 'public',
-        options: restOptions,
+        access,
+        // @ts-expect-error we are essentially calling multiple methods at the same type so TS gets confused
+        options: omit(options, 'access', 'description', 'summary', 'deprecated', 'discontinued'),
         security,
       }).addVersion(
         {

packages/kbn-server-route-repository/src/test_types.ts

@@ -83,32 +83,44 @@ createServerRouteFactory<{ context: { getSpaceId: () => string } }, {}>()({
 });
 
 // Create options are available when registering a route.
-createServerRouteFactory<{}, { options: { tags: string[] } }>()({
+createServerRouteFactory<{}, {}>()({
   endpoint: 'GET /internal/endpoint_with_params',
   params: t.type({
     path: t.type({
       serviceName: t.string,
     }),
   }),
-  options: {
-    tags: [],
-  },
   handler: async (resources) => {
     assertType<{ params: { path: { serviceName: string } } }>(resources);
   },
 });
 
 // Public APIs should be versioned
-createServerRouteFactory<{}, { options: { tags: string[] } }>()({
+createServerRouteFactory<{}, { tags: string[] }>()({
   // @ts-expect-error
+  endpoint: 'GET /api/endpoint_with_params',
+  tags: [],
+  handler: async (resources) => {},
+});
+
+// `access` is respected
+createServerRouteFactory<{}, { tags: string[] }>()({
   endpoint: 'GET /api/endpoint_with_params',
   options: {
     tags: [],
+    access: 'internal',
   },
   handler: async (resources) => {},
 });
 
-createServerRouteFactory<{}, { options: { tags: string[] } }>()({
+// specifying additional options makes them required
+// @ts-expect-error
+createServerRouteFactory<{}, { tags: string[] }>()({
+  endpoint: 'GET /api/endpoint_with_params 2023-10-31',
+  handler: async (resources) => {},
+});
+
+createServerRouteFactory<{}, { tags: string[] }>()({
   endpoint: 'GET /api/endpoint_with_params 2023-10-31',
   options: {
     tags: [],

packages/kbn-typed-react-router-config/index.ts

@@ -17,3 +17,4 @@ export * from './src/use_match_routes';
 export * from './src/use_params';
 export * from './src/use_router';
 export * from './src/use_route_path';
+export * from './src/breadcrumbs';

packages/kbn-typed-react-router-config/kibana.jsonc

@@ -1,5 +1,5 @@
 {
-  "type": "shared-common",
+  "type": "shared-browser",
   "id": "@kbn/typed-react-router-config",
   "owner": ["@elastic/obs-knowledge-team", "@elastic/obs-ux-management-team"],
   "group": "platform",

packages/kbn-typed-react-router-config/src/breadcrumbs/breadcrumb.tsx

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import React from 'react';
+import { RequiredKeys } from 'utility-types';
+import { useRouterBreadcrumb } from './use_router_breadcrumb';
+import { PathsOf, RouteMap, TypeOf } from '../types';
+
+type AsParamsProps<TObject extends Record<string, any>> = RequiredKeys<TObject> extends never
+  ? {}
+  : { params: TObject };
+
+export type RouterBreadcrumb<TRouteMap extends RouteMap> = <
+  TRoutePath extends PathsOf<TRouteMap>
+>({}: {
+  title: string;
+  children: React.ReactNode;
+  path: TRoutePath;
+} & AsParamsProps<TypeOf<TRouteMap, TRoutePath, false>>) => React.ReactElement;
+
+export function RouterBreadcrumb<
+  TRouteMap extends RouteMap,
+  TRoutePath extends PathsOf<TRouteMap>
+>({
+  title,
+  path,
+  params,
+  children,
+}: {
+  title: string;
+  path: TRoutePath;
+  children: React.ReactElement;
+  params?: Record<string, any>;
+}) {
+  useRouterBreadcrumb(
+    () => ({
+      title,
+      path,
+      params,
+    }),
+    []
+  );
+
+  return children;
+}

packages/kbn-typed-react-router-config/src/breadcrumbs/context.tsx

@@ -0,0 +1,113 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { ChromeBreadcrumb, ScopedHistory } from '@kbn/core/public';
+import { compact, isEqual } from 'lodash';
+import React, { createContext, useMemo, useState } from 'react';
+import { useHistory } from 'react-router-dom';
+import { useBreadcrumbs } from './use_breadcrumbs';
+import {
+  PathsOf,
+  Route,
+  RouteMap,
+  RouteMatch,
+  TypeAsArgs,
+  TypeAsParams,
+  TypeOf,
+  useMatchRoutes,
+  useRouter,
+} from '../..';
+
+export type Breadcrumb<
+  TRouteMap extends RouteMap = RouteMap,
+  TPath extends PathsOf<TRouteMap> = PathsOf<TRouteMap>
+> = {
+  title: string;
+  path: TPath;
+} & TypeAsParams<TypeOf<TRouteMap, TPath, false>>;
+
+interface BreadcrumbApi<TRouteMap extends RouteMap = RouteMap> {
+  set<TPath extends PathsOf<TRouteMap>>(
+    route: Route,
+    breadcrumb: Array<Breadcrumb<TRouteMap, TPath>>
+  ): void;
+  unset(route: Route): void;
+  getBreadcrumbs(matches: RouteMatch[]): Array<Breadcrumb<TRouteMap, PathsOf<TRouteMap>>>;
+}
+
+export const BreadcrumbsContext = createContext<BreadcrumbApi | undefined>(undefined);
+
+export function BreadcrumbsContextProvider<TRouteMap extends RouteMap>({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  const [, forceUpdate] = useState({});
+
+  const breadcrumbs = useMemo(() => {
+    return new Map<Route, Array<Breadcrumb<TRouteMap>>>();
+  }, []);
+
+  const history = useHistory() as ScopedHistory;
+
+  const router = useRouter<TRouteMap>();
+
+  const matches: RouteMatch[] = useMatchRoutes();
+
+  const api = useMemo<BreadcrumbApi<TRouteMap>>(
+    () => ({
+      set(route, breadcrumb) {
+        if (!isEqual(breadcrumbs.get(route), breadcrumb)) {
+          breadcrumbs.set(route, breadcrumb);
+          forceUpdate({});
+        }
+      },
+      unset(route) {
+        if (breadcrumbs.has(route)) {
+          breadcrumbs.delete(route);
+          forceUpdate({});
+        }
+      },
+      getBreadcrumbs(currentMatches: RouteMatch[]) {
+        return compact(
+          currentMatches.flatMap((match) => {
+            const breadcrumb = breadcrumbs.get(match.route);
+
+            return breadcrumb;
+          })
+        );
+      },
+    }),
+    [breadcrumbs]
+  );
+
+  const formattedBreadcrumbs: ChromeBreadcrumb[] = api
+    .getBreadcrumbs(matches)
+    .map((breadcrumb, index, array) => {
+      return {
+        text: breadcrumb.title,
+        ...(index === array.length - 1
+          ? {}
+          : {
+              href: history.createHref({
+                pathname: router.link(
+                  breadcrumb.path,
+                  ...(('params' in breadcrumb ? [breadcrumb.params] : []) as TypeAsArgs<
+                    TypeOf<TRouteMap, PathsOf<TRouteMap>, false>
+                  >)
+                ),
+              }),
+            }),
+      };
+    });
+
+  useBreadcrumbs(formattedBreadcrumbs);
+
+  return <BreadcrumbsContext.Provider value={api}>{children}</BreadcrumbsContext.Provider>;
+}

packages/kbn-typed-react-router-config/src/breadcrumbs/create_router_breadcrumb_component.tsx

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { RouteMap } from '../types';
+import { RouterBreadcrumb } from './breadcrumb';
+
+export function createRouterBreadcrumbComponent<
+  TRouteMap extends RouteMap
+>(): RouterBreadcrumb<TRouteMap> {
+  return RouterBreadcrumb as RouterBreadcrumb<TRouteMap>;
+}

packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export { createRouterBreadcrumbComponent } from './create_router_breadcrumb_component';
+export { createUseBreadcrumbs } from './use_router_breadcrumb';
+export { BreadcrumbsContextProvider } from './context';

packages/kbn-typed-react-router-config/src/breadcrumbs/use_breadcrumbs.ts

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { i18n } from '@kbn/i18n';
+import { ApplicationStart, ChromeBreadcrumb, ChromeStart } from '@kbn/core/public';
+import { MouseEvent, useEffect, useMemo } from 'react';
+import { useKibana } from '@kbn/kibana-react-plugin/public';
+import { ChromeBreadcrumbsAppendExtension } from '@kbn/core-chrome-browser';
+import type { ServerlessPluginStart } from '@kbn/serverless/public';
+
+function addClickHandlers(
+  breadcrumbs: ChromeBreadcrumb[],
+  navigateToHref?: (url: string) => Promise<void>
+) {
+  return breadcrumbs.map((bc) => ({
+    ...bc,
+    ...(bc.href
+      ? {
+          onClick: (event: MouseEvent) => {
+            if (navigateToHref && bc.href) {
+              event.preventDefault();
+              navigateToHref(bc.href);
+            }
+          },
+        }
+      : {}),
+  }));
+}
+
+function getTitleFromBreadCrumbs(breadcrumbs: ChromeBreadcrumb[]) {
+  return breadcrumbs.map(({ text }) => text?.toString() ?? '').reverse();
+}
+
+export const useBreadcrumbs = (
+  extraCrumbs: ChromeBreadcrumb[],
+  options?: {
+    app?: { id: string; label: string };
+    breadcrumbsAppendExtension?: ChromeBreadcrumbsAppendExtension;
+    serverless?: ServerlessPluginStart;
+  }
+) => {
+  const { app, breadcrumbsAppendExtension, serverless } = options ?? {};
+
+  const {
+    services: {
+      chrome: { docTitle, setBreadcrumbs: chromeSetBreadcrumbs, setBreadcrumbsAppendExtension },
+      application: { getUrlForApp, navigateToUrl },
+    },
+  } = useKibana<{
+    application: ApplicationStart;
+    chrome: ChromeStart;
+  }>();
+
+  const setTitle = docTitle.change;
+  const appPath = getUrlForApp(app?.id ?? 'observability-overview') ?? '';
+
+  const setBreadcrumbs = useMemo(
+    () => serverless?.setBreadcrumbs ?? chromeSetBreadcrumbs,
+    [serverless, chromeSetBreadcrumbs]
+  );
+
+  useEffect(() => {
+    if (breadcrumbsAppendExtension) {
+      setBreadcrumbsAppendExtension(breadcrumbsAppendExtension);
+    }
+    return () => {
+      if (breadcrumbsAppendExtension) {
+        setBreadcrumbsAppendExtension(undefined);
+      }
+    };
+  }, [breadcrumbsAppendExtension, setBreadcrumbsAppendExtension]);
+
+  useEffect(() => {
+    const breadcrumbs = serverless
+      ? extraCrumbs
+      : [
+          {
+            text:
+              app?.label ??
+              i18n.translate('xpack.observabilityShared.breadcrumbs.observabilityLinkText', {
+                defaultMessage: 'Observability',
+              }),
+            href: appPath + '/overview',
+          },
+          ...extraCrumbs,
+        ];
+
+    if (setBreadcrumbs) {
+      setBreadcrumbs(addClickHandlers(breadcrumbs, navigateToUrl));
+    }
+    if (setTitle) {
+      setTitle(getTitleFromBreadCrumbs(breadcrumbs));
+    }
+  }, [app?.label, appPath, extraCrumbs, navigateToUrl, serverless, setBreadcrumbs, setTitle]);
+};

packages/kbn-typed-react-router-config/src/breadcrumbs/use_router_breadcrumb.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { useContext, useEffect, useRef } from 'react';
+import { castArray } from 'lodash';
+import { PathsOf, RouteMap, useCurrentRoute } from '../..';
+import { Breadcrumb, BreadcrumbsContext } from './context';
+
+type UseBreadcrumbs<TRouteMap extends RouteMap> = <TPath extends PathsOf<TRouteMap>>(
+  callback: () => Breadcrumb<TRouteMap, TPath> | Array<Breadcrumb<TRouteMap, TPath>>,
+  fnDeps: unknown[]
+) => void;
+
+export function useRouterBreadcrumb(callback: () => Breadcrumb | Breadcrumb[], fnDeps: any[]) {
+  const api = useContext(BreadcrumbsContext);
+
+  if (!api) {
+    throw new Error('Missing Breadcrumb API in context');
+  }
+
+  const { match } = useCurrentRoute();
+
+  const matchedRoute = useRef(match?.route);
+
+  useEffect(() => {
+    if (matchedRoute.current && matchedRoute.current !== match?.route) {
+      api.unset(matchedRoute.current);
+    }
+
+    matchedRoute.current = match?.route;
+
+    if (matchedRoute.current) {
+      api.set(matchedRoute.current, castArray(callback()));
+    }
+
+    return () => {
+      if (matchedRoute.current) {
+        api.unset(matchedRoute.current);
+      }
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [matchedRoute.current, match?.route, ...fnDeps]);
+}
+
+export function createUseBreadcrumbs<TRouteMap extends RouteMap>(): UseBreadcrumbs<TRouteMap> {
+  return useRouterBreadcrumb;
+}

packages/kbn-typed-react-router-config/src/create_router.ts

@@ -11,7 +11,7 @@ import { deepExactRt, mergeRt } from '@kbn/io-ts-utils';
 import { isLeft } from 'fp-ts/lib/Either';
 import { Location } from 'history';
 import { PathReporter } from 'io-ts/lib/PathReporter';
-import { compact, findLastIndex, merge, orderBy } from 'lodash';
+import { compact, findLastIndex, mapValues, merge, orderBy } from 'lodash';
 import qs from 'query-string';
 import {
   MatchedRoute,
@@ -139,7 +139,9 @@ export function createRouter<TRoutes extends RouteMap>(routes: TRoutes): Router<
       if (route?.params) {
         const decoded = deepExactRt(route.params).decode(
           merge({}, route.defaults ?? {}, {
-            path: matchedRoute.match.params,
+            path: mapValues(matchedRoute.match.params, (value) => {
+              return decodeURIComponent(value);
+            }),
             query: qs.parse(location.search, { decode: true }),
           })
         );
@@ -179,7 +181,7 @@ export function createRouter<TRoutes extends RouteMap>(routes: TRoutes): Router<
       .split('/')
       .map((part) => {
         const match = part.match(/(?:{([a-zA-Z]+)})/);
-        return match ? paramsWithBuiltInDefaults.path[match[1]] : part;
+        return match ? encodeURIComponent(paramsWithBuiltInDefaults.path[match[1]]) : part;
       })
       .join('/');
 

packages/kbn-typed-react-router-config/src/types/index.ts

@@ -106,6 +106,12 @@ export type TypeAsArgs<TObject> = keyof TObject extends never
   ? [TObject] | []
   : [TObject];
 
+export type TypeAsParams<TObject> = keyof TObject extends never
+  ? {}
+  : RequiredKeys<TObject> extends never
+  ? never
+  : { params: TObject };
+
 export type FlattenRoutesOf<TRoutes extends RouteMap> = Array<
   ValuesType<{
     [key in keyof MapRoutes<TRoutes>]: ValuesType<MapRoutes<TRoutes>[key]>;

packages/kbn-typed-react-router-config/src/use_router.tsx

@@ -20,7 +20,7 @@ export const RouterContextProvider = ({
   children: React.ReactNode;
 }) => <RouterContext.Provider value={router}>{children}</RouterContext.Provider>;
 
-export function useRouter(): Router<RouteMap> {
+export function useRouter<TRouteMap extends RouteMap = RouteMap>(): Router<TRouteMap> {
   const router = useContext(RouterContext);
 
   if (!router) {

packages/kbn-typed-react-router-config/tsconfig.json

@@ -14,7 +14,12 @@
   ],
   "kbn_references": [
     "@kbn/io-ts-utils",
-    "@kbn/shared-ux-router"
+    "@kbn/shared-ux-router",
+    "@kbn/core",
+    "@kbn/i18n",
+    "@kbn/kibana-react-plugin",
+    "@kbn/core-chrome-browser",
+    "@kbn/serverless"
   ],
   "exclude": [
     "target/**/*",

packages/kbn-utility-types/src/dot.ts

@@ -23,7 +23,9 @@ type DedotKey<
 export type DedotObject<TObject extends Record<string, any>> = UnionToIntersection<
   Exclude<
     ValuesType<{
-      [TKey in keyof TObject]: {} extends Pick<TObject, TKey>
+      [TKey in keyof TObject as string]: string extends TKey
+        ? Record<TKey, TObject[TKey]>
+        : {} extends Pick<TObject, TKey>
         ? DeepPartial<DedotKey<TObject, TKey, Exclude<TObject[TKey], undefined>>>
         : DedotKey<TObject, TKey, TObject[TKey]>;
     }>,

packages/kbn-utility-types/src/tsd_tests/test_d/dot.ts

@@ -7,9 +7,10 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { expectAssignable, expectNotType, expectType } from 'tsd';
 import { DedotObject, DotObject } from '../../dot';
 
+function isAssignable<T>(t: T) {}
+
 interface TestA {
   'my.dotted.key': string;
   'my.dotted.partial.key'?: string;
@@ -60,20 +61,39 @@ const dedotted1 = {} as DedotObject<TestA>;
 
 const dotted1 = {} as DotObject<TestB>;
 
-expectAssignable<DedotObject<TestA>>({} as Dedotted);
-expectAssignable<DotObject<TestB>>({} as Dotted);
-expectAssignable<Dedotted>({} as DedotObject<TestA>);
-expectAssignable<Dotted>({} as DotObject<TestB>);
+isAssignable<DedotObject<TestA>>({} as Dedotted);
 
-expectType<string | undefined>(dedotted1.ym?.dotted?.partial?.key?.toString());
-expectType<string>(dotted1['my.undotted.key'].toString());
-expectNotType<string>(dotted1['my.partial.key']);
-expectType<string | undefined>(dotted1['my.partial.key']?.toString());
-expectNotType<{ baz: string }>({} as DedotObject<TestA>);
-expectNotType<{ baz: string }>({} as DotObject<TestB>);
-expectNotType<{ my: { dotted: { key: string }; partial: { key: number } } }>(
-  {} as DedotObject<TestA>
-);
+isAssignable<DotObject<TestB>>({} as Dotted);
+isAssignable<Dedotted>({} as DedotObject<TestA>);
+isAssignable<Dotted>({} as DotObject<TestB>);
+
+isAssignable<string | undefined>(dedotted1.ym?.dotted?.partial?.key);
+
+isAssignable<string>(dotted1['my.undotted.key'].toString());
+// @ts-expect-error
+isAssignable<string>(dotted1['my.partial.key']);
+
+isAssignable<string | undefined>(dotted1['my.partial.key']?.toString());
+
+// @ts-expect-error
+isAssignable<{ baz: string }>({} as DedotObject<TestA>);
+// @ts-expect-error
+isAssignable<{ baz: string }>({} as DotObject<TestB>);
+
+// @ts-expect-error
+isAssignable<{ my: { dotted: { key: string }; partial: { key: number } } }, DedotObject<TestA>>();
+
+type WithStringKey = {
+  [x: string]: string;
+} & {
+  count: number;
+};
+
+type WithStringKeyDedotted = DedotObject<WithStringKey>;
+
+isAssignable<WithStringKeyDedotted>({} as WithStringKey);
+
+isAssignable<WithStringKey>({} as WithStringKeyDedotted);
 
 interface ObjectWithArray {
   span: {
@@ -88,7 +108,7 @@ interface ObjectWithArray {
   };
 }
 
-expectType<DotObject<ObjectWithArray>>({
+isAssignable<DotObject<ObjectWithArray>>({
   'span.links.span.id': [''],
   'span.links.trace.id': [''],
 });

src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts

@@ -183,6 +183,7 @@ export const applicationUsageSchema = {
    */
   siem: commonSchema,
   space_selector: commonSchema,
+  streams: commonSchema,
   uptime: commonSchema,
   synthetics: commonSchema,
   ux: commonSchema,

src/plugins/telemetry/schema/oss_plugins.json

@@ -7731,6 +7731,137 @@
             }
           }
         },
+        "streams": {
+          "properties": {
+            "appId": {
+              "type": "keyword",
+              "_meta": {
+                "description": "The application being tracked"
+              }
+            },
+            "viewId": {
+              "type": "keyword",
+              "_meta": {
+                "description": "Always `main`"
+              }
+            },
+            "clicks_total": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application since we started counting them"
+              }
+            },
+            "clicks_7_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 7 days"
+              }
+            },
+            "clicks_30_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 30 days"
+              }
+            },
+            "clicks_90_days": {
+              "type": "long",
+              "_meta": {
+                "description": "General number of clicks in the application over the last 90 days"
+              }
+            },
+            "minutes_on_screen_total": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen since we started counting them."
+              }
+            },
+            "minutes_on_screen_7_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 7 days"
+              }
+            },
+            "minutes_on_screen_30_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 30 days"
+              }
+            },
+            "minutes_on_screen_90_days": {
+              "type": "float",
+              "_meta": {
+                "description": "Minutes the application is active and on-screen over the last 90 days"
+              }
+            },
+            "views": {
+              "type": "array",
+              "items": {
+                "properties": {
+                  "appId": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "The application being tracked"
+                    }
+                  },
+                  "viewId": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "The application view being tracked"
+                    }
+                  },
+                  "clicks_total": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the application sub view since we started counting them"
+                    }
+                  },
+                  "clicks_7_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 7 days"
+                    }
+                  },
+                  "clicks_30_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 30 days"
+                    }
+                  },
+                  "clicks_90_days": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "General number of clicks in the active application sub view over the last 90 days"
+                    }
+                  },
+                  "minutes_on_screen_total": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application sub view is active and on-screen since we started counting them."
+                    }
+                  },
+                  "minutes_on_screen_7_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 7 days"
+                    }
+                  },
+                  "minutes_on_screen_30_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 30 days"
+                    }
+                  },
+                  "minutes_on_screen_90_days": {
+                    "type": "float",
+                    "_meta": {
+                      "description": "Minutes the application is active and on-screen active application sub view over the last 90 days"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "uptime": {
           "properties": {
             "appId": {

tsconfig.base.json

@@ -1848,6 +1848,8 @@
       "@kbn/stdio-dev-helpers/*": ["packages/kbn-stdio-dev-helpers/*"],
       "@kbn/storybook": ["packages/kbn-storybook"],
       "@kbn/storybook/*": ["packages/kbn-storybook/*"],
+      "@kbn/streams-app-plugin": ["x-pack/plugins/streams_app"],
+      "@kbn/streams-app-plugin/*": ["x-pack/plugins/streams_app/*"],
       "@kbn/streams-plugin": ["x-pack/plugins/streams"],
       "@kbn/streams-plugin/*": ["x-pack/plugins/streams/*"],
       "@kbn/synthetics-e2e": ["x-pack/plugins/observability_solution/synthetics/e2e"],

x-pack/.i18nrc.json

@@ -112,7 +112,9 @@
     "xpack.observabilityLogsOverview": [
       "packages/observability/logs_overview/src/components"
     ],
-    "xpack.osquery": ["plugins/osquery"],
+    "xpack.osquery": [
+      "plugins/osquery"
+    ],
     "xpack.painlessLab": "plugins/painless_lab",
     "xpack.profiling": [
       "plugins/observability_solution/profiling"
@@ -148,6 +150,9 @@
     "xpack.securitySolutionEss": "plugins/security_solution_ess",
     "xpack.securitySolutionServerless": "plugins/security_solution_serverless",
     "xpack.sessionView": "plugins/session_view",
+    "xpack.streams": [
+      "plugins/streams_app"
+    ],
     "xpack.slo": "plugins/observability_solution/slo",
     "xpack.snapshotRestore": "plugins/snapshot_restore",
     "xpack.spaces": "plugins/spaces",

x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts

@@ -37,4 +37,16 @@ describe('unflattenObject', () => {
       },
     });
   });
+
+  it('handles null values correctly', () => {
+    expect(
+      unflattenObject({
+        'agent.name': null,
+      })
+    ).toEqual({
+      agent: {
+        name: null,
+      },
+    });
+  });
 });

x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts

@@ -6,13 +6,17 @@
  */
 
 import { set } from '@kbn/safer-lodash-set';
+import { DedotObject } from '@kbn/utility-types';
 
-export function unflattenObject(source: Record<string, any>, target: Record<string, any> = {}) {
+export function unflattenObject<T extends Record<string, any>>(
+  source: T,
+  target: Record<string, any> = {}
+): DedotObject<T> {
   // eslint-disable-next-line guard-for-in
   for (const key in source) {
     const val = source[key as keyof typeof source];
     if (Array.isArray(val)) {
-      const unflattenedArray = val.map((item) => {
+      const unflattenedArray = val.map((item: unknown) => {
         if (item && typeof item === 'object' && !Array.isArray(item)) {
           return unflattenObject(item);
         }
@@ -23,5 +27,6 @@ export function unflattenObject(source: Record<string, any>, target: Record<stri
       set(target, key, val);
     }
   }
-  return target;
+
+  return target as DedotObject<T>;
 }

x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json

@@ -19,5 +19,6 @@
     "@kbn/es-query",
     "@kbn/safer-lodash-set",
     "@kbn/inference-common",
+    "@kbn/utility-types",
   ]
 }

x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts

@@ -10,12 +10,15 @@ import type {
   FieldCapsRequest,
   FieldCapsResponse,
   MsearchRequest,
+  ScalarValue,
   SearchResponse,
 } from '@elastic/elasticsearch/lib/api/types';
 import { withSpan } from '@kbn/apm-utils';
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
-import type { ESQLSearchResponse, ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
-import { Required } from 'utility-types';
+import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
+import { Required, ValuesType } from 'utility-types';
+import { DedotObject } from '@kbn/utility-types';
+import { unflattenObject } from '@kbn/task-manager-plugin/server/metrics/lib';
 import { esqlResultToPlainObjects } from '../esql_result_to_plain_objects';
 
 type SearchRequest = ESSearchRequest & {
@@ -24,19 +27,52 @@ type SearchRequest = ESSearchRequest & {
   size: number | boolean;
 };
 
-type EsqlQueryParameters = EsqlQueryRequest & { parseOutput?: boolean };
-type EsqlOutputParameters = Omit<EsqlQueryRequest, 'format' | 'columnar'> & {
-  parseOutput?: true;
-  format?: 'json';
-  columnar?: false;
-};
+export interface EsqlOptions {
+  transform?: 'none' | 'plain' | 'unflatten';
+}
 
-type EsqlParameters = EsqlOutputParameters | EsqlQueryParameters;
+export type EsqlValue = ScalarValue | ScalarValue[];
+
+export type EsqlOutput = Record<string, EsqlValue>;
+
+type MaybeUnflatten<T extends Record<string, any>, TApply> = TApply extends true
+  ? DedotObject<T>
+  : T;
+
+interface UnparsedEsqlResponseOf<TOutput extends EsqlOutput> {
+  columns: Array<{ name: keyof TOutput; type: string }>;
+  values: Array<Array<ValuesType<TOutput>>>;
+}
+
+interface ParsedEsqlResponseOf<
+  TOutput extends EsqlOutput,
+  TOptions extends EsqlOptions | undefined = { transform: 'none' }
+> {
+  hits: Array<
+    MaybeUnflatten<
+      {
+        [key in keyof TOutput]: TOutput[key];
+      },
+      TOptions extends { transform: 'unflatten' } ? true : false
+    >
+  >;
+}
 
 export type InferEsqlResponseOf<
-  TOutput = unknown,
-  TParameters extends EsqlParameters = EsqlParameters
-> = TParameters extends EsqlOutputParameters ? TOutput[] : ESQLSearchResponse;
+  TOutput extends EsqlOutput,
+  TOptions extends EsqlOptions | undefined = { transform: 'none' }
+> = TOptions extends { transform: 'plain' | 'unflatten' }
+  ? ParsedEsqlResponseOf<TOutput, TOptions>
+  : UnparsedEsqlResponseOf<TOutput>;
+
+export type ObservabilityESSearchRequest = SearchRequest;
+
+export type ObservabilityEsQueryRequest = Omit<EsqlQueryRequest, 'format' | 'columnar'>;
+
+export type ParsedEsqlResponse = ParsedEsqlResponseOf<EsqlOutput, EsqlOptions>;
+export type UnparsedEsqlResponse = UnparsedEsqlResponseOf<EsqlOutput>;
+
+export type EsqlQueryResponse = UnparsedEsqlResponse | ParsedEsqlResponse;
 
 /**
  * An Elasticsearch Client with a fully typed `search` method and built-in
@@ -57,14 +93,18 @@ export interface ObservabilityElasticsearchClient {
     operationName: string,
     request: Required<FieldCapsRequest, 'index_filter' | 'fields' | 'index'>
   ): Promise<FieldCapsResponse>;
-  esql<TOutput = unknown, TQueryParams extends EsqlOutputParameters = EsqlOutputParameters>(
+  esql<TOutput extends EsqlOutput = EsqlOutput>(
     operationName: string,
-    parameters: TQueryParams
-  ): Promise<InferEsqlResponseOf<TOutput, TQueryParams>>;
-  esql<TOutput = unknown, TQueryParams extends EsqlQueryParameters = EsqlQueryParameters>(
+    parameters: ObservabilityEsQueryRequest
+  ): Promise<InferEsqlResponseOf<TOutput, { transform: 'none' }>>;
+  esql<
+    TOutput extends EsqlOutput = EsqlOutput,
+    TEsqlOptions extends EsqlOptions = { transform: 'none' }
+  >(
     operationName: string,
-    parameters: TQueryParams
-  ): Promise<InferEsqlResponseOf<TOutput, TQueryParams>>;
+    parameters: ObservabilityEsQueryRequest,
+    options: TEsqlOptions
+  ): Promise<InferEsqlResponseOf<TOutput, TEsqlOptions>>;
   client: ElasticsearchClient;
 }
 
@@ -109,32 +149,41 @@ export function createObservabilityEsClient({
         });
       });
     },
-    esql<TOutput = unknown, TSearchRequest extends EsqlParameters = EsqlParameters>(
+    esql(
       operationName: string,
-      { parseOutput = true, format = 'json', columnar = false, ...parameters }: TSearchRequest
-    ) {
-      logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`);
-      return withSpan({ name: operationName, labels: { plugin } }, () => {
-        return client.esql.query(
-          { ...parameters, format, columnar },
-          {
-            querystring: {
-              drop_null_columns: true,
-            },
-          }
-        );
-      })
-        .then((response) => {
-          logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`);
+      parameters: ObservabilityEsQueryRequest,
+      options?: EsqlOptions
+    ): Promise<InferEsqlResponseOf<EsqlOutput, EsqlOptions>> {
+      return callWithLogger(operationName, parameters, () => {
+        return client.esql
+          .query(
+            { ...parameters },
+            {
+              querystring: {
+                drop_null_columns: true,
+              },
+            }
+          )
+          .then((response) => {
+            const esqlResponse = response as unknown as UnparsedEsqlResponseOf<EsqlOutput>;
 
-          const esqlResponse = response as unknown as ESQLSearchResponse;
+            const transform = options?.transform ?? 'none';
 
-          const shouldParseOutput = parseOutput && !columnar && format === 'json';
-          return shouldParseOutput ? esqlResultToPlainObjects<TOutput>(esqlResponse) : esqlResponse;
-        })
-        .catch((error) => {
-          throw error;
-        });
+            if (transform === 'none') {
+              return esqlResponse;
+            }
+
+            const parsedResponse = { hits: esqlResultToPlainObjects(esqlResponse) };
+
+            if (transform === 'plain') {
+              return parsedResponse;
+            }
+
+            return {
+              hits: parsedResponse.hits.map((hit) => unflattenObject(hit)),
+            };
+          }) as Promise<InferEsqlResponseOf<EsqlOutput, EsqlOptions>>;
+      });
     },
     search<TDocument = unknown, TSearchRequest extends SearchRequest = SearchRequest>(
       operationName: string,

x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts

@@ -27,40 +27,15 @@ describe('esqlResultToPlainObjects', () => {
     expect(output).toEqual([{ name: 'Foo Bar' }]);
   });
 
-  it('should return columns without "text" or "keyword" in their names', () => {
+  it('should not unflatten objects', () => {
     const result: ESQLSearchResponse = {
       columns: [
-        { name: 'name.text', type: 'text' },
-        { name: 'age', type: 'keyword' },
-      ],
-      values: [
-        ['Foo Bar', 30],
-        ['Foo Qux', 25],
+        { name: 'name', type: 'keyword' },
+        { name: 'name.nested', type: 'keyword' },
       ],
+      values: [['Foo Bar', 'Bar Foo']],
     };
     const output = esqlResultToPlainObjects(result);
-    expect(output).toEqual([
-      { name: 'Foo Bar', age: 30 },
-      { name: 'Foo Qux', age: 25 },
-    ]);
-  });
-
-  it('should handle mixed columns correctly', () => {
-    const result: ESQLSearchResponse = {
-      columns: [
-        { name: 'name', type: 'text' },
-        { name: 'name.text', type: 'text' },
-        { name: 'age', type: 'keyword' },
-      ],
-      values: [
-        ['Foo Bar', 'Foo Bar', 30],
-        ['Foo Qux', 'Foo Qux', 25],
-      ],
-    };
-    const output = esqlResultToPlainObjects(result);
-    expect(output).toEqual([
-      { name: 'Foo Bar', age: 30 },
-      { name: 'Foo Qux', age: 25 },
-    ]);
+    expect(output).toEqual([{ name: 'Foo Bar', 'name.nested': 'Bar Foo' }]);
   });
 });

x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts

@@ -6,28 +6,24 @@
  */
 
 import type { ESQLSearchResponse } from '@kbn/es-types';
-import { unflattenObject } from '@kbn/observability-utils-common/object/unflatten_object';
 
-export function esqlResultToPlainObjects<TDocument = unknown>(
-  result: ESQLSearchResponse
-): TDocument[] {
-  return result.values.map((row) => {
-    return unflattenObject(
-      row.reduce<Record<string, any>>((acc, value, index) => {
-        const column = result.columns[index];
-
-        if (!column) {
-          return acc;
-        }
-
-        // Removes the type suffix from the column name
-        const name = column.name.replace(/\.(text|keyword)$/, '');
-        if (!acc[name]) {
-          acc[name] = value;
-        }
+export function esqlResultToPlainObjects<
+  TDocument extends Record<string, any> = Record<string, unknown>
+>(result: ESQLSearchResponse): TDocument[] {
+  return result.values.map((row): TDocument => {
+    return row.reduce<Record<string, unknown>>((acc, value, index) => {
+      const column = result.columns[index];
 
+      if (!column) {
         return acc;
-      }, {})
-    ) as TDocument;
+      }
+
+      const name = column.name;
+      if (!acc[name]) {
+        acc[name] = value;
+      }
+
+      return acc;
+    }, {}) as TDocument;
   });
 }

x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json

@@ -24,5 +24,7 @@
     "@kbn/alerting-plugin",
     "@kbn/rule-registry-plugin",
     "@kbn/rule-data-utils",
+    "@kbn/utility-types",
+    "@kbn/task-manager-plugin",
   ]
 }

x-pack/plugins/entity_manager/server/routes/enablement/check.ts

@@ -45,13 +45,11 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_
  */
 export const checkEntityDiscoveryEnabledRoute = createEntityManagerServerRoute({
   endpoint: 'GET /internal/entities/managed/enablement',
-  options: {
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow',
-      },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow',
     },
   },
   handler: async ({ response, logger, server }) => {

x-pack/plugins/entity_manager/server/routes/enablement/disable.ts

@@ -44,13 +44,11 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_
  */
 export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({
   endpoint: 'DELETE /internal/entities/managed/enablement',
-  options: {
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow',
-      },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow',
     },
   },
   params: z.object({

x-pack/plugins/entity_manager/server/routes/enablement/enable.ts

@@ -63,13 +63,11 @@ import { startTransforms } from '../../lib/entities/start_transforms';
  */
 export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({
   endpoint: 'PUT /internal/entities/managed/enablement',
-  options: {
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow',
-      },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow',
     },
   },
   params: z.object({

x-pack/plugins/entity_manager/server/routes/entities/create.ts

@@ -50,13 +50,11 @@ import { canManageEntityDefinition } from '../../lib/auth';
  */
 export const createEntityDefinitionRoute = createEntityManagerServerRoute({
   endpoint: 'POST /internal/entities/definition',
-  options: {
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
-      },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
     },
   },
   params: z.object({

x-pack/plugins/entity_manager/server/routes/entities/delete.ts

@@ -52,13 +52,11 @@ import { canDeleteEntityDefinition } from '../../lib/auth/privileges';
  */
 export const deleteEntityDefinitionRoute = createEntityManagerServerRoute({
   endpoint: 'DELETE /internal/entities/definition/{id}',
-  options: {
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
-      },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
     },
   },
   params: z.object({

x-pack/plugins/entity_manager/server/routes/entities/get.ts

@@ -50,13 +50,11 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_
  */
 export const getEntityDefinitionRoute = createEntityManagerServerRoute({
   endpoint: 'GET /internal/entities/definition/{id?}',
-  options: {
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
-      },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
     },
   },
   params: z.object({

x-pack/plugins/entity_manager/server/routes/entities/reset.ts

@@ -25,13 +25,11 @@ import { stopTransforms } from '../../lib/entities/stop_transforms';
 
 export const resetEntityDefinitionRoute = createEntityManagerServerRoute({
   endpoint: 'POST /internal/entities/definition/{id}/_reset',
-  options: {
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
-      },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
     },
   },
   params: z.object({

x-pack/plugins/entity_manager/server/routes/entities/update.ts

@@ -54,13 +54,11 @@ import { canManageEntityDefinition } from '../../lib/auth';
  */
 export const updateEntityDefinitionRoute = createEntityManagerServerRoute({
   endpoint: 'PATCH /internal/entities/definition/{id}',
-  options: {
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
-      },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This endpoint mainly manages Elasticsearch resources using the requesting users credentials',
     },
   },
   params: z.object({

x-pack/plugins/observability_solution/apm/server/routes/apm_routes/register_apm_server_routes.ts

@@ -11,6 +11,7 @@ import { Logger, KibanaRequest, KibanaResponseFactory, RouteRegistrar } from '@k
 import { errors } from '@elastic/elasticsearch';
 import agent from 'elastic-apm-node';
 import {
+  DefaultRouteCreateOptions,
   IoTsParamsObject,
   ServerRouteRepository,
   stripNullishRequestParameters,
@@ -30,6 +31,7 @@ import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { ApmFeatureFlags } from '../../../common/apm_feature_flags';
 import type {
   APMCore,
+  APMRouteCreateOptions,
   MinimalApmPluginRequestHandlerContext,
   TelemetryUsageCounter,
 } from '../typings';
@@ -78,7 +80,11 @@ export function registerRoutes({
   const router = core.setup.http.createRouter();
 
   routes.forEach((route) => {
-    const { params, endpoint, options, handler } = route;
+    const { endpoint, handler, security } = route;
+
+    const options = ('options' in route ? route.options : {}) as DefaultRouteCreateOptions &
+      APMRouteCreateOptions;
+    const params = 'params' in route ? route.params : undefined;
 
     const { method, pathname, version } = parseEndpoint(endpoint);
 
@@ -218,6 +224,7 @@ export function registerRoutes({
           path: pathname,
           options,
           validate: passThroughValidationObject,
+          security,
         },
         wrappedHandler
       );
@@ -231,6 +238,7 @@ export function registerRoutes({
         path: pathname,
         access: pathname.includes('/internal/apm') ? 'internal' : 'public',
         options,
+        security,
       }).addVersion(
         {
           version,

x-pack/plugins/observability_solution/apm/server/routes/typings.ts

@@ -9,7 +9,6 @@ import type {
   CoreSetup,
   CustomRequestHandlerContext,
   CoreStart,
-  RouteConfigOptions,
   IScopedClusterClient,
   IUiSettingsClient,
   SavedObjectsClientContract,
@@ -48,21 +47,18 @@ export type MinimalApmPluginRequestHandlerContext = Omit<
 };
 
 export interface APMRouteCreateOptions {
-  options: {
-    tags: Array<
-      | 'access:apm'
-      | 'access:apm_write'
-      | 'access:apm_settings_write'
-      | 'access:ml:canGetJobs'
-      | 'access:ml:canCreateJob'
-      | 'access:ml:canCloseJob'
-      | 'access:ai_assistant'
-      | 'oas-tag:APM agent keys'
-      | 'oas-tag:APM annotations'
-    >;
-    body?: { accepts: Array<'application/json' | 'multipart/form-data'> };
-    disableTelemetry?: boolean;
-  } & RouteConfigOptions<any>;
+  tags: Array<
+    | 'access:apm'
+    | 'access:apm_write'
+    | 'access:apm_settings_write'
+    | 'access:ml:canGetJobs'
+    | 'access:ml:canCreateJob'
+    | 'access:ml:canCloseJob'
+    | 'access:ai_assistant'
+    | 'oas-tag:APM agent keys'
+    | 'oas-tag:APM annotations'
+  >;
+  disableTelemetry?: boolean;
 }
 
 export type TelemetryUsageCounter = ReturnType<UsageCollectionSetup['createUsageCounter']>;

x-pack/plugins/observability_solution/dataset_quality/server/routes/register_routes.ts

@@ -39,14 +39,18 @@ export function registerRoutes({
   const router = core.http.createRouter();
 
   routes.forEach((route) => {
-    const { endpoint, options, handler, params } = route;
+    const { endpoint, handler } = route;
     const { pathname, method } = parseEndpoint(endpoint);
 
+    const params = 'params' in route ? route.params : undefined;
+    const options = 'options' in route ? route.options : {};
+
     (router[method] as RouteRegistrar<typeof method, DatasetQualityRequestHandlerContext>)(
       {
         path: pathname,
         validate: passThroughValidationObject,
         options,
+        security: route.security,
       },
       async (context, request, response) => {
         try {

x-pack/plugins/observability_solution/dataset_quality/server/routes/types.ts

@@ -29,7 +29,5 @@ export interface DatasetQualityRouteHandlerResources {
 }
 
 export interface DatasetQualityRouteCreateOptions {
-  options: {
-    tags: string[];
-  };
+  tags: string[];
 }

x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts

@@ -44,18 +44,25 @@ export async function getLatestEntity({
       return undefined;
     }
 
-    const response = await inventoryEsClient.esql<{
-      source_data_stream?: { type?: string | string[] };
-    }>('get_latest_entities', {
-      query: `FROM ${ENTITIES_LATEST_ALIAS}
+    const response = await inventoryEsClient.esql<
+      {
+        'source_data_stream.type'?: string | string;
+      },
+      { transform: 'plain' }
+    >(
+      'get_latest_entities',
+      {
+        query: `FROM ${ENTITIES_LATEST_ALIAS}
         | WHERE ${ENTITY_TYPE} == ?
         | WHERE ${hostOrContainerIdentityField} == ?
         | KEEP ${SOURCE_DATA_STREAM_TYPE}
       `,
-      params: [entityType, entityId],
-    });
+        params: [entityType, entityId],
+      },
+      { transform: 'plain' }
+    );
 
-    return { sourceDataStreamType: response[0].source_data_stream?.type };
+    return { sourceDataStreamType: response.hits[0]['source_data_stream.type'] };
   } catch (e) {
     logger.error(e);
   }

x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts

@@ -22,7 +22,7 @@ export async function getEntityGroupsBy({
   inventoryEsClient: ObservabilityElasticsearchClient;
   field: string;
   esQuery?: QueryDslQueryContainer;
-}) {
+}): Promise<EntityGroup[]> {
   const from = `FROM ${ENTITIES_LATEST_ALIAS}`;
   const where = [getBuiltinEntityDefinitionIdESQLWhereClause()];
 
@@ -31,8 +31,14 @@ export async function getEntityGroupsBy({
   const limit = `LIMIT ${MAX_NUMBER_OF_ENTITIES}`;
   const query = [from, ...where, group, sort, limit].join(' | ');
 
-  return inventoryEsClient.esql<EntityGroup>('get_entities_groups', {
-    query,
-    filter: esQuery,
-  });
+  const { hits } = await inventoryEsClient.esql<EntityGroup, { transform: 'plain' }>(
+    'get_entities_groups',
+    {
+      query,
+      filter: esQuery,
+    },
+    { transform: 'plain' }
+  );
+
+  return hits;
 }

x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts

@@ -7,7 +7,6 @@
 
 import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client';
 import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common';
-import type { EntityInstance } from '@kbn/entities-schema';
 import { ENTITIES_LATEST_ALIAS } from '../../../common/entities';
 import { getBuiltinEntityDefinitionIdESQLWhereClause } from './query_helper';
 
@@ -16,14 +15,21 @@ export async function getEntityTypes({
 }: {
   inventoryEsClient: ObservabilityElasticsearchClient;
 }) {
-  const entityTypesEsqlResponse = await inventoryEsClient.esql<{
-    entity: Pick<EntityInstance['entity'], 'type'>;
-  }>('get_entity_types', {
-    query: `FROM ${ENTITIES_LATEST_ALIAS}
+  const entityTypesEsqlResponse = await inventoryEsClient.esql<
+    {
+      'entity.type': string;
+    },
+    { transform: 'plain' }
+  >(
+    'get_entity_types',
+    {
+      query: `FROM ${ENTITIES_LATEST_ALIAS}
      | ${getBuiltinEntityDefinitionIdESQLWhereClause()}
      | STATS count = COUNT(${ENTITY_TYPE}) BY ${ENTITY_TYPE}
     `,
-  });
+    },
+    { transform: 'plain' }
+  );
 
-  return entityTypesEsqlResponse.map((response) => response.entity.type);
+  return entityTypesEsqlResponse.hits.map((response) => response['entity.type']);
 }

x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts

@@ -6,13 +6,13 @@
  */
 
 import type { QueryDslQueryContainer, ScalarValue } from '@elastic/elasticsearch/lib/api/types';
-import type { EntityInstance } from '@kbn/entities-schema';
 import {
   ENTITY_DISPLAY_NAME,
   ENTITY_LAST_SEEN,
   ENTITY_TYPE,
 } from '@kbn/observability-shared-plugin/common';
 import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client';
+import { unflattenObject } from '@kbn/observability-utils-common/object/unflatten_object';
 import {
   ENTITIES_LATEST_ALIAS,
   InventoryEntity,
@@ -62,17 +62,38 @@ export async function getLatestEntities({
 
   const query = [from, ...where, sort, limit].join(' | ');
 
-  const latestEntitiesEsqlResponse = await inventoryEsClient.esql<EntityInstance>(
+  const latestEntitiesEsqlResponse = await inventoryEsClient.esql<
+    {
+      'entity.id': string;
+      'entity.type': string;
+      'entity.definition_id': string;
+      'entity.display_name': string;
+      'entity.identity_fields': string | string[];
+      'entity.last_seen_timestamp': string;
+      'entity.definition_version': string;
+      'entity.schema_version': string;
+    } & Record<string, ScalarValue | ScalarValue[]>,
+    { transform: 'plain' }
+  >(
     'get_latest_entities',
     {
       query,
       filter: esQuery,
       params,
-    }
+    },
+    { transform: 'plain' }
   );
 
-  return latestEntitiesEsqlResponse.map((lastestEntity) => {
-    const { entity, ...metadata } = lastestEntity;
+  return latestEntitiesEsqlResponse.hits.map((latestEntity) => {
+    Object.keys(latestEntity).forEach((key) => {
+      const keyOfObject = key as keyof typeof latestEntity;
+      // strip out multi-field aliases
+      if (keyOfObject.endsWith('.text') || keyOfObject.endsWith('.keyword')) {
+        delete latestEntity[keyOfObject];
+      }
+    });
+
+    const { entity, ...metadata } = unflattenObject(latestEntity);
 
     return {
       entityId: entity.id,

x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts

@@ -17,14 +17,18 @@ export async function getHasData({
   logger: Logger;
 }) {
   try {
-    const esqlResults = await inventoryEsClient.esql<{ _count: number }>('get_has_data', {
-      query: `FROM ${ENTITIES_LATEST_ALIAS} 
+    const esqlResults = await inventoryEsClient.esql<{ _count: number }, { transform: 'plain' }>(
+      'get_has_data',
+      {
+        query: `FROM ${ENTITIES_LATEST_ALIAS} 
       | ${getBuiltinEntityDefinitionIdESQLWhereClause()} 
       | STATS _count = COUNT(*)
       | LIMIT 1`,
-    });
+      },
+      { transform: 'plain' }
+    );
 
-    const totalCount = esqlResults[0]._count;
+    const totalCount = esqlResults.hits[0]._count;
 
     return { hasData: totalCount > 0 };
   } catch (e) {

x-pack/plugins/observability_solution/inventory/server/routes/types.ts

@@ -30,10 +30,8 @@ export interface InventoryRouteHandlerResources {
 }
 
 export interface InventoryRouteCreateOptions {
-  options: {
-    timeout?: {
-      idleSocket?: number;
-    };
-    tags: Array<'access:inventory'>;
+  timeout?: {
+    idleSocket?: number;
   };
+  tags: Array<'access:inventory'>;
 }

x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts

@@ -53,10 +53,7 @@ export interface InvestigateAppRouteHandlerResources {
 }
 
 export interface InvestigateAppRouteCreateOptions {
-  options: {
-    timeout?: {
-      idleSocket?: number;
-    };
-    tags: [];
+  timeout?: {
+    idleSocket?: number;
   };
 }

x-pack/plugins/observability_solution/observability/kibana.jsonc

@@ -56,7 +56,8 @@
       "serverless",
       "guidedOnboarding",
       "observabilityAIAssistant",
-      "investigate"
+      "investigate",
+      "streams"
     ],
     "requiredBundles": [
       "data",
@@ -70,4 +71,4 @@
       "common"
     ]
   }
-}
+}

x-pack/plugins/observability_solution/observability/public/navigation_tree.ts

@@ -7,7 +7,7 @@
 import { i18n } from '@kbn/i18n';
 import type { NavigationTreeDefinition } from '@kbn/core-chrome-browser';
 import type { AddSolutionNavigationArg } from '@kbn/navigation-plugin/public';
-import { of } from 'rxjs';
+import { map, of } from 'rxjs';
 import type { ObservabilityPublicPluginsStart } from './plugin';
 
 const title = i18n.translate(
@@ -18,7 +18,7 @@ const title = i18n.translate(
 );
 const icon = 'logoObservability';
 
-export function createNavTree(pluginsStart: ObservabilityPublicPluginsStart) {
+function createNavTree({ streamsAvailable }: { streamsAvailable?: boolean }) {
   const navTree: NavigationTreeDefinition = {
     body: [
       {
@@ -87,6 +87,13 @@ export function createNavTree(pluginsStart: ObservabilityPublicPluginsStart) {
             link: 'inventory',
             spaceBefore: 'm',
           },
+          ...(streamsAvailable
+            ? [
+                {
+                  link: 'streams' as const,
+                },
+              ]
+            : []),
           {
             id: 'apm',
             title: i18n.translate('xpack.observability.obltNav.applications', {
@@ -558,6 +565,8 @@ export const createDefinition = (
   title,
   icon: 'logoObservability',
   homePage: 'observabilityOnboarding',
-  navigationTree$: of(createNavTree(pluginsStart)),
+  navigationTree$: (pluginsStart.streams?.status$ || of({ status: 'disabled' as const })).pipe(
+    map(({ status }) => createNavTree({ streamsAvailable: status === 'enabled' }))
+  ),
   dataTestSubj: 'observabilitySideNav',
 });

x-pack/plugins/observability_solution/observability/public/plugin.ts

@@ -70,6 +70,7 @@ import type {
 import type { UiActionsSetup, UiActionsStart } from '@kbn/ui-actions-plugin/public';
 import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
 import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public';
+import type { StreamsPluginStart, StreamsPluginSetup } from '@kbn/streams-plugin/public';
 import { observabilityAppId, observabilityFeatureId } from '../common';
 import {
   ALERTS_PATH,
@@ -124,6 +125,7 @@ export interface ObservabilityPublicPluginsSetup {
   licensing: LicensingPluginSetup;
   serverless?: ServerlessPluginSetup;
   presentationUtil?: PresentationUtilPluginStart;
+  streams?: StreamsPluginSetup;
 }
 export interface ObservabilityPublicPluginsStart {
   actionTypeRegistry: ActionTypeRegistryContract;
@@ -162,6 +164,7 @@ export interface ObservabilityPublicPluginsStart {
   dataViewFieldEditor: DataViewFieldEditorStart;
   toastNotifications: ToastsStart;
   investigate?: InvestigatePublicStart;
+  streams?: StreamsPluginStart;
 }
 export type ObservabilityPublicStart = ReturnType<Plugin['start']>;
 

x-pack/plugins/observability_solution/observability/server/plugin.ts

@@ -195,7 +195,6 @@ export class ObservabilityPlugin implements Plugin<ObservabilityPluginSetup> {
     void core.getStartServices().then(([coreStart, pluginStart]) => {
       registerRoutes({
         core,
-        config,
         dependencies: {
           pluginsSetup: {
             ...plugins,

x-pack/plugins/observability_solution/observability/server/routes/register_routes.ts

@@ -4,29 +4,16 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { errors } from '@elastic/elasticsearch';
-import Boom from '@hapi/boom';
 import { RulesClientApi } from '@kbn/alerting-plugin/server/types';
-import { CoreSetup, KibanaRequest, Logger, RouteRegistrar } from '@kbn/core/server';
+import { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server';
 import { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server';
 import { RuleDataPluginService } from '@kbn/rule-registry-plugin/server';
-import {
-  IoTsParamsObject,
-  decodeRequestParams,
-  parseEndpoint,
-  passThroughValidationObject,
-  stripNullishRequestParameters,
-} from '@kbn/server-route-repository';
+import { registerRoutes as registerServerRoutes } from '@kbn/server-route-repository';
 import { SpacesPluginStart } from '@kbn/spaces-plugin/server';
-import axios from 'axios';
-import * as t from 'io-ts';
-import { ObservabilityConfig } from '..';
 import { AlertDetailsContextualInsightsService } from '../services';
-import { ObservabilityRequestHandlerContext } from '../types';
 import { AbstractObservabilityServerRouteRepository } from './types';
 
 interface RegisterRoutes {
-  config: ObservabilityConfig;
   core: CoreSetup;
   repository: AbstractObservabilityServerRouteRepository;
   logger: Logger;
@@ -46,81 +33,11 @@ export interface RegisterRoutesDependencies {
   getRulesClientWithRequest: (request: KibanaRequest) => RulesClientApi;
 }
 
-export function registerRoutes({ config, repository, core, logger, dependencies }: RegisterRoutes) {
-  const routes = Object.values(repository);
-
-  const router = core.http.createRouter();
-
-  routes.forEach((route) => {
-    const { endpoint, options, handler, params } = route;
-    const { pathname, method } = parseEndpoint(endpoint);
-
-    (router[method] as RouteRegistrar<typeof method, ObservabilityRequestHandlerContext>)(
-      {
-        path: pathname,
-        validate: passThroughValidationObject,
-        options,
-      },
-      async (context, request, response) => {
-        try {
-          const decodedParams = decodeRequestParams(
-            stripNullishRequestParameters({
-              params: request.params,
-              body: request.body,
-              query: request.query,
-            }),
-            (params as IoTsParamsObject) ?? t.strict({})
-          );
-
-          const data = await handler({
-            config,
-            context,
-            request,
-            logger,
-            params: decodedParams,
-            dependencies,
-          });
-
-          if (data === undefined) {
-            return response.noContent();
-          }
-
-          return response.ok({ body: data });
-        } catch (error) {
-          if (axios.isAxiosError(error)) {
-            logger.error(error);
-            return response.customError({
-              statusCode: error.response?.status || 500,
-              body: {
-                message: error.message,
-              },
-            });
-          }
-
-          if (Boom.isBoom(error)) {
-            logger.error(error.output.payload.message);
-            return response.customError({
-              statusCode: error.output.statusCode,
-              body: { message: error.output.payload.message },
-            });
-          }
-
-          logger.error(error);
-          const opts = {
-            statusCode: 500,
-            body: {
-              message: error.message,
-            },
-          };
-
-          if (error instanceof errors.RequestAbortedError) {
-            opts.statusCode = 499;
-            opts.body.message = 'Client closed request';
-          }
-
-          return response.customError(opts);
-        }
-      }
-    );
+export function registerRoutes({ repository, core, logger, dependencies }: RegisterRoutes) {
+  registerServerRoutes({
+    core,
+    dependencies: { dependencies },
+    logger,
+    repository,
   });
 }

x-pack/plugins/observability_solution/observability/server/routes/types.ts

@@ -13,7 +13,6 @@ import {
 } from './get_global_observability_server_route_repository';
 import { ObservabilityRequestHandlerContext } from '../types';
 import { RegisterRoutesDependencies } from './register_routes';
-import { ObservabilityConfig } from '..';
 
 export type { ObservabilityServerRouteRepository, APIEndpoint };
 
@@ -22,14 +21,11 @@ export interface ObservabilityRouteHandlerResources {
   dependencies: RegisterRoutesDependencies;
   logger: Logger;
   request: KibanaRequest;
-  config: ObservabilityConfig;
 }
 
 export interface ObservabilityRouteCreateOptions {
-  options: {
-    tags: string[];
-    access?: 'public' | 'internal';
-  };
+  tags: string[];
+  access?: 'public' | 'internal';
 }
 
 export type AbstractObservabilityServerRouteRepository = ServerRouteRepository;

x-pack/plugins/observability_solution/observability/tsconfig.json

@@ -111,6 +111,7 @@
     "@kbn/core-ui-settings-server-mocks",
     "@kbn/es-types",
     "@kbn/logging-mocks",
+    "@kbn/streams-plugin",
   ],
   "exclude": ["target/**/*"]
 }

x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts

@@ -34,7 +34,7 @@ export function registerContextFunction({
       visibility: FunctionVisibility.Internal,
     },
     async ({ messages, screenContexts, chat }, signal) => {
-      const { analytics } = (await resources.context.core).coreStart;
+      const { analytics } = await resources.plugins.core.start();
 
       async function getContext() {
         const screenDescription = compact(

x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts

@@ -111,7 +111,18 @@ export class ObservabilityAIAssistantPlugin
             ];
           }),
       };
-    }) as ObservabilityAIAssistantRouteHandlerResources['plugins'];
+    }) as Pick<
+      ObservabilityAIAssistantRouteHandlerResources['plugins'],
+      keyof ObservabilityAIAssistantPluginStartDependencies
+    >;
+
+    const withCore = {
+      ...routeHandlerPlugins,
+      core: {
+        setup: core,
+        start: () => core.getStartServices().then(([coreStart]) => coreStart),
+      },
+    };
 
     const service = (this.service = new ObservabilityAIAssistantService({
       logger: this.logger.get('service'),
@@ -133,7 +144,7 @@ export class ObservabilityAIAssistantPlugin
       core,
       logger: this.logger,
       dependencies: {
-        plugins: routeHandlerPlugins,
+        plugins: withCore,
         service: this.service,
       },
     });

x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts

@@ -194,7 +194,7 @@ const chatRecallRoute = createObservabilityAIAssistantServerRoute({
 
     const response$ = from(
       recallAndScore({
-        analytics: (await resources.context.core).coreStart.analytics,
+        analytics: (await resources.plugins.core.start()).analytics,
         chat: (name, params) =>
           client
             .chat(name, {

x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts

@@ -6,7 +6,7 @@
  */
 import type { CoreSetup } from '@kbn/core/server';
 import type { Logger } from '@kbn/logging';
-import { registerRoutes } from '@kbn/server-route-repository';
+import { DefaultRouteHandlerResources, registerRoutes } from '@kbn/server-route-repository';
 import { getGlobalObservabilityAIAssistantServerRouteRepository } from './get_global_observability_ai_assistant_route_repository';
 import type { ObservabilityAIAssistantRouteHandlerResources } from './types';
 import { ObservabilityAIAssistantPluginStartDependencies } from '../types';
@@ -20,7 +20,7 @@ export function registerServerRoutes({
   logger: Logger;
   dependencies: Omit<
     ObservabilityAIAssistantRouteHandlerResources,
-    'request' | 'context' | 'logger' | 'params'
+    keyof DefaultRouteHandlerResources
   >;
 }) {
   registerRoutes({

x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts

@@ -6,32 +6,36 @@
  */
 
 import type {
+  CoreSetup,
   CoreStart,
   CustomRequestHandlerContext,
   IScopedClusterClient,
   IUiSettingsClient,
-  KibanaRequest,
   SavedObjectsClientContract,
 } from '@kbn/core/server';
-import type { Logger } from '@kbn/logging';
 import type { LicensingApiRequestHandlerContext } from '@kbn/licensing-plugin/server/types';
 import type { RacApiRequestHandlerContext } from '@kbn/rule-registry-plugin/server';
 import type { RulesClientApi } from '@kbn/alerting-plugin/server/types';
+import { DefaultRouteHandlerResources } from '@kbn/server-route-repository-utils';
 import type { ObservabilityAIAssistantService } from '../service';
 import type {
   ObservabilityAIAssistantPluginSetupDependencies,
   ObservabilityAIAssistantPluginStartDependencies,
 } from '../types';
 
+type ObservabilityAIAssistantRequestHandlerContextBase = CustomRequestHandlerContext<{
+  licensing: Pick<LicensingApiRequestHandlerContext, 'license' | 'featureUsage'>;
+  // these two are here for compatibility with APM functions
+  rac: Pick<RacApiRequestHandlerContext, 'getAlertsClient'>;
+  alerting: {
+    getRulesClient: () => RulesClientApi;
+  };
+}>;
+
+// this is the type used across methods, it's stripped down for compatibility
+// with the context that's available when executing as an action
 export type ObservabilityAIAssistantRequestHandlerContext = Omit<
-  CustomRequestHandlerContext<{
-    licensing: Pick<LicensingApiRequestHandlerContext, 'license' | 'featureUsage'>;
-    // these two are here for compatibility with APM functions
-    rac: Pick<RacApiRequestHandlerContext, 'getAlertsClient'>;
-    alerting: {
-      getRulesClient: () => RulesClientApi;
-    };
-  }>,
+  ObservabilityAIAssistantRequestHandlerContextBase,
   'core' | 'resolve'
 > & {
   core: Promise<{
@@ -45,32 +49,41 @@ export type ObservabilityAIAssistantRequestHandlerContext = Omit<
     savedObjects: {
       client: SavedObjectsClientContract;
     };
-    coreStart: CoreStart;
   }>;
 };
 
-export interface ObservabilityAIAssistantRouteHandlerResources {
-  request: KibanaRequest;
-  context: ObservabilityAIAssistantRequestHandlerContext;
-  logger: Logger;
-  service: ObservabilityAIAssistantService;
-  plugins: {
-    [key in keyof ObservabilityAIAssistantPluginSetupDependencies]: {
-      setup: Required<ObservabilityAIAssistantPluginSetupDependencies>[key];
-    };
-  } & {
-    [key in keyof ObservabilityAIAssistantPluginStartDependencies]: {
-      start: () => Promise<Required<ObservabilityAIAssistantPluginStartDependencies>[key]>;
-    };
+interface PluginContractResolveCore {
+  core: {
+    setup: CoreSetup<ObservabilityAIAssistantPluginStartDependencies>;
+    start: () => Promise<CoreStart>;
   };
 }
 
-export interface ObservabilityAIAssistantRouteCreateOptions {
-  options: {
-    timeout?: {
-      payload?: number;
-      idleSocket?: number;
-    };
-    tags: Array<'access:ai_assistant'>;
+type PluginContractResolveDependenciesStart = {
+  [key in keyof ObservabilityAIAssistantPluginStartDependencies]: {
+    start: () => Promise<Required<ObservabilityAIAssistantPluginStartDependencies>[key]>;
   };
+};
+
+type PluginContractResolveDependenciesSetup = {
+  [key in keyof ObservabilityAIAssistantPluginSetupDependencies]: {
+    setup: Required<ObservabilityAIAssistantPluginSetupDependencies>[key];
+  };
+};
+
+export interface ObservabilityAIAssistantRouteHandlerResources
+  extends Omit<DefaultRouteHandlerResources, 'context' | 'response'> {
+  context: ObservabilityAIAssistantRequestHandlerContext;
+  service: ObservabilityAIAssistantService;
+  plugins: PluginContractResolveCore &
+    PluginContractResolveDependenciesSetup &
+    PluginContractResolveDependenciesStart;
+}
+
+export interface ObservabilityAIAssistantRouteCreateOptions {
+  timeout?: {
+    payload?: number;
+    idleSocket?: number;
+  };
+  tags: Array<'access:ai_assistant'>;
 }

x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json

@@ -47,6 +47,7 @@
     "@kbn/ai-assistant-common",
     "@kbn/inference-common",
     "@kbn/core-lifecycle-server",
+    "@kbn/server-route-repository-utils",
   ],
   "exclude": ["target/**/*"]
 }

x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts

@@ -17,7 +17,6 @@ import {
   ObservabilityAIAssistantRequestHandlerContext,
   ObservabilityAIAssistantRouteHandlerResources,
 } from '@kbn/observability-ai-assistant-plugin/server/routes/types';
-import { ObservabilityAIAssistantPluginStartDependencies } from '@kbn/observability-ai-assistant-plugin/server/types';
 import { mapValues } from 'lodash';
 import { firstValueFrom } from 'rxjs';
 import type { ObservabilityAIAssistantAppConfig } from './config';
@@ -59,13 +58,22 @@ export class ObservabilityAIAssistantAppPlugin
         setup: value,
         start: () =>
           core.getStartServices().then((services) => {
-            const [, pluginsStartContracts] = services;
+            const [_, pluginsStartContracts] = services;
+
             return pluginsStartContracts[
-              key as keyof ObservabilityAIAssistantPluginStartDependencies
+              key as keyof ObservabilityAIAssistantAppPluginStartDependencies
             ];
           }),
       };
-    }) as ObservabilityAIAssistantRouteHandlerResources['plugins'];
+    }) as Omit<ObservabilityAIAssistantRouteHandlerResources['plugins'], 'core'>;
+
+    const withCore = {
+      ...routeHandlerPlugins,
+      core: {
+        setup: core,
+        start: () => core.getStartServices().then(([coreStart]) => coreStart),
+      },
+    };
 
     const initResources = async (
       request: KibanaRequest
@@ -90,7 +98,6 @@ export class ObservabilityAIAssistantAppPlugin
           };
         }),
         core: Promise.resolve({
-          coreStart,
           elasticsearch: {
             client: coreStart.elasticsearch.client.asScoped(request),
           },
@@ -110,7 +117,7 @@ export class ObservabilityAIAssistantAppPlugin
         context,
         service: plugins.observabilityAIAssistant.service,
         logger: this.logger.get('connector'),
-        plugins: routeHandlerPlugins,
+        plugins: withCore,
       };
     };
 

x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts

@@ -94,12 +94,14 @@ describe('observabilityAIAssistant rule_connector', () => {
             getAdhocInstructions: () => [],
           }),
         },
-        context: {
-          core: Promise.resolve({
-            coreStart: { http: { basePath: { publicBaseUrl: 'http://kibana.com' } } },
-          }),
-        },
+        context: {},
         plugins: {
+          core: {
+            start: () =>
+              Promise.resolve({
+                http: { basePath: { publicBaseUrl: 'http://kibana.com' } },
+              }),
+          },
           actions: {
             start: async () => {
               return {

x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts

@@ -248,7 +248,7 @@ If available, include the link of the conversation at the end of your answer.`
       isPublic: true,
       connectorId: execOptions.params.connector,
       signal: new AbortController().signal,
-      kibanaPublicUrl: (await resources.context.core).coreStart.http.basePath.publicBaseUrl,
+      kibanaPublicUrl: (await resources.plugins.core.start()).http.basePath.publicBaseUrl,
       instructions: [backgroundInstruction],
       messages: [
         {

x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts

@@ -14,8 +14,8 @@ import type {
 } from '@kbn/core/server';
 import { mapValues } from 'lodash';
 import { i18n } from '@kbn/i18n';
+import { DefaultRouteHandlerResources, registerRoutes } from '@kbn/server-route-repository';
 import { getObservabilityOnboardingServerRouteRepository } from './routes';
-import { registerRoutes } from './routes/register_routes';
 import { ObservabilityOnboardingRouteHandlerResources } from './routes/types';
 import {
   ObservabilityOnboardingPluginSetup,
@@ -71,16 +71,28 @@ export class ObservabilityOnboardingPlugin
     }) as ObservabilityOnboardingRouteHandlerResources['plugins'];
 
     const config = this.initContext.config.get<ObservabilityOnboardingConfig>();
+
+    const dependencies: Omit<
+      ObservabilityOnboardingRouteHandlerResources,
+      keyof DefaultRouteHandlerResources
+    > = {
+      config,
+      kibanaVersion: this.initContext.env.packageInfo.version,
+      plugins: resourcePlugins,
+      services: {
+        esLegacyConfigService: this.esLegacyConfigService,
+      },
+      core: {
+        setup: core,
+        start: () => core.getStartServices().then(([coreStart]) => coreStart),
+      },
+    };
+
     registerRoutes({
       core,
       logger: this.logger,
       repository: getObservabilityOnboardingServerRouteRepository(),
-      plugins: resourcePlugins,
-      config,
-      kibanaVersion: this.initContext.env.packageInfo.version,
-      services: {
-        esLegacyConfigService: this.esLegacyConfigService,
-      },
+      dependencies,
     });
 
     plugins.customIntegrations.registerCustomIntegration({

x-pack/plugins/observability_solution/observability_onboarding/server/routes/register_routes.ts

@@ -1,126 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-import { errors } from '@elastic/elasticsearch';
-import Boom from '@hapi/boom';
-import type { IKibanaResponse } from '@kbn/core/server';
-import { CoreSetup, Logger, RouteRegistrar } from '@kbn/core/server';
-import {
-  IoTsParamsObject,
-  ServerRouteRepository,
-  decodeRequestParams,
-  stripNullishRequestParameters,
-  parseEndpoint,
-  passThroughValidationObject,
-} from '@kbn/server-route-repository';
-import * as t from 'io-ts';
-import { ObservabilityOnboardingConfig } from '..';
-import { EsLegacyConfigService } from '../services/es_legacy_config_service';
-import { ObservabilityOnboardingRequestHandlerContext } from '../types';
-import { ObservabilityOnboardingRouteHandlerResources } from './types';
-
-interface RegisterRoutes {
-  core: CoreSetup;
-  repository: ServerRouteRepository;
-  logger: Logger;
-  plugins: ObservabilityOnboardingRouteHandlerResources['plugins'];
-  config: ObservabilityOnboardingConfig;
-  kibanaVersion: string;
-  services: {
-    esLegacyConfigService: EsLegacyConfigService;
-  };
-}
-
-export function registerRoutes({
-  repository,
-  core,
-  logger,
-  plugins,
-  config,
-  kibanaVersion,
-  services,
-}: RegisterRoutes) {
-  const routes = Object.values(repository);
-
-  const router = core.http.createRouter();
-
-  routes.forEach((route) => {
-    const { endpoint, options, handler, params } = route;
-    const { pathname, method } = parseEndpoint(endpoint);
-
-    (router[method] as RouteRegistrar<typeof method, ObservabilityOnboardingRequestHandlerContext>)(
-      {
-        path: pathname,
-        validate: passThroughValidationObject,
-        options,
-      },
-      async (context, request, response) => {
-        try {
-          const decodedParams = decodeRequestParams(
-            stripNullishRequestParameters({
-              params: request.params,
-              body: request.body,
-              query: request.query,
-            }),
-            (params as IoTsParamsObject) ?? t.strict({})
-          );
-
-          const data = (await handler({
-            context,
-            request,
-            response,
-            logger,
-            params: decodedParams,
-            plugins,
-            core: {
-              setup: core,
-              start: async () => {
-                const [coreStart] = await core.getStartServices();
-                return coreStart;
-              },
-            },
-            config,
-            kibanaVersion,
-            services,
-          })) as any;
-
-          if (data === undefined) {
-            return response.noContent();
-          }
-
-          if (data instanceof response.noContent().constructor) {
-            return data as IKibanaResponse;
-          }
-
-          return response.ok({ body: data });
-        } catch (error) {
-          if (Boom.isBoom(error)) {
-            logger.error(error.output.payload.message);
-            return response.customError({
-              statusCode: error.output.statusCode,
-              body: { message: error.output.payload.message },
-            });
-          }
-
-          logger.error(error);
-          const opts = {
-            statusCode: 500,
-            body: {
-              message: error.message,
-            },
-          };
-
-          if (error instanceof errors.RequestAbortedError) {
-            opts.statusCode = 499;
-            opts.body.message = 'Client closed request';
-          }
-
-          return response.customError(opts);
-        }
-      }
-    );
-  });
-}

x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts

@@ -46,10 +46,8 @@ export interface ObservabilityOnboardingRouteHandlerResources {
 }
 
 export interface ObservabilityOnboardingRouteCreateOptions {
-  options: {
-    tags: string[];
-    xsrfRequired?: boolean;
-  };
+  tags: string[];
+  xsrfRequired?: boolean;
 }
 
 export const IntegrationRT = t.intersection([

x-pack/plugins/observability_solution/observability_shared/kibana.jsonc

@@ -33,4 +33,4 @@
       "common"
     ]
   }
-}
+}

x-pack/plugins/observability_solution/slo/server/routes/register_routes.ts

@@ -6,12 +6,11 @@
  */
 import { CoreSetup, Logger } from '@kbn/core/server';
 import { ServerRoute, registerRoutes } from '@kbn/server-route-repository';
-import { ServerRouteCreateOptions } from '@kbn/server-route-repository-utils';
 import { SLORequestHandlerContext, SLORoutesDependencies } from './types';
 
 interface RegisterRoutes {
   core: CoreSetup;
-  repository: Record<string, ServerRoute<string, any, any, any, ServerRouteCreateOptions>>;
+  repository: Record<string, ServerRoute<string, any, any, any, any>>;
   logger: Logger;
   dependencies: SLORoutesDependencies;
   isServerless: boolean;

x-pack/plugins/observability_solution/slo/tsconfig.json

@@ -99,6 +99,5 @@
     "@kbn/observability-alerting-rule-utils",
     "@kbn/discover-shared-plugin",
     "@kbn/server-route-repository-client",
-    "@kbn/server-route-repository-utils"
   ]
 }

x-pack/plugins/serverless_observability/kibana.jsonc

@@ -25,7 +25,9 @@
       "discover",
       "security"
     ],
-    "optionalPlugins": [],
+    "optionalPlugins": [
+      "streams"
+    ],
     "requiredBundles": []
   }
-}
+}

x-pack/plugins/serverless_observability/public/navigation_tree.ts

@@ -8,374 +8,387 @@
 import { i18n } from '@kbn/i18n';
 import type { NavigationTreeDefinition } from '@kbn/core-chrome-browser';
 
-export const navigationTree: NavigationTreeDefinition = {
-  body: [
-    { type: 'recentlyAccessed' },
-    {
-      type: 'navGroup',
-      id: 'observability_project_nav',
-      title: 'Observability',
-      icon: 'logoObservability',
-      defaultIsCollapsed: false,
-      isCollapsible: false,
-      breadcrumbStatus: 'hidden',
-      children: [
-        {
-          title: i18n.translate('xpack.serverlessObservability.nav.discover', {
-            defaultMessage: 'Discover',
-          }),
-          link: 'last-used-logs-viewer',
-          // avoid duplicate "Discover" breadcrumbs
-          breadcrumbStatus: 'hidden',
-          renderAs: 'item',
-          children: [
-            {
-              link: 'discover',
-              children: [
-                {
-                  link: 'observability-logs-explorer',
-                },
-              ],
-            },
-          ],
-        },
-        {
-          title: i18n.translate('xpack.serverlessObservability.nav.dashboards', {
-            defaultMessage: 'Dashboards',
-          }),
-          link: 'dashboards',
-          getIsActive: ({ pathNameSerialized, prepend }) => {
-            return pathNameSerialized.startsWith(prepend('/app/dashboards'));
+export const createNavigationTree = ({
+  streamsAvailable,
+}: {
+  streamsAvailable?: boolean;
+}): NavigationTreeDefinition => {
+  return {
+    body: [
+      { type: 'recentlyAccessed' },
+      {
+        type: 'navGroup',
+        id: 'observability_project_nav',
+        title: 'Observability',
+        icon: 'logoObservability',
+        defaultIsCollapsed: false,
+        isCollapsible: false,
+        breadcrumbStatus: 'hidden',
+        children: [
+          {
+            title: i18n.translate('xpack.serverlessObservability.nav.discover', {
+              defaultMessage: 'Discover',
+            }),
+            link: 'last-used-logs-viewer',
+            // avoid duplicate "Discover" breadcrumbs
+            breadcrumbStatus: 'hidden',
+            renderAs: 'item',
+            children: [
+              {
+                link: 'discover',
+                children: [
+                  {
+                    link: 'observability-logs-explorer',
+                  },
+                ],
+              },
+            ],
           },
-        },
-        {
-          link: 'observability-overview:alerts',
-        },
-        {
-          link: 'observability-overview:cases',
-          renderAs: 'item',
-          children: [
-            {
-              link: 'observability-overview:cases_configure',
+          {
+            title: i18n.translate('xpack.serverlessObservability.nav.dashboards', {
+              defaultMessage: 'Dashboards',
+            }),
+            link: 'dashboards',
+            getIsActive: ({ pathNameSerialized, prepend }) => {
+              return pathNameSerialized.startsWith(prepend('/app/dashboards'));
             },
-            {
-              link: 'observability-overview:cases_create',
-            },
-          ],
-        },
-        {
-          title: i18n.translate('xpack.serverlessObservability.nav.slo', {
-            defaultMessage: 'SLOs',
-          }),
-          link: 'slo',
-        },
-        {
-          link: 'observabilityAIAssistant',
-          title: i18n.translate('xpack.serverlessObservability.nav.aiAssistant', {
-            defaultMessage: 'AI Assistant',
-          }),
-        },
-        { link: 'inventory', spaceBefore: 'm' },
-        {
-          id: 'apm',
-          title: i18n.translate('xpack.serverlessObservability.nav.applications', {
-            defaultMessage: 'Applications',
-          }),
-          renderAs: 'panelOpener',
-          children: [
-            {
-              children: [
+          },
+          {
+            link: 'observability-overview:alerts',
+          },
+          {
+            link: 'observability-overview:cases',
+            renderAs: 'item',
+            children: [
+              {
+                link: 'observability-overview:cases_configure',
+              },
+              {
+                link: 'observability-overview:cases_create',
+              },
+            ],
+          },
+          {
+            title: i18n.translate('xpack.serverlessObservability.nav.slo', {
+              defaultMessage: 'SLOs',
+            }),
+            link: 'slo',
+          },
+          {
+            link: 'observabilityAIAssistant',
+            title: i18n.translate('xpack.serverlessObservability.nav.aiAssistant', {
+              defaultMessage: 'AI Assistant',
+            }),
+          },
+          { link: 'inventory', spaceBefore: 'm' },
+          ...(streamsAvailable
+            ? [
                 {
-                  link: 'apm:services',
-                  title: i18n.translate('xpack.serverlessObservability.nav.apm.services', {
-                    defaultMessage: 'Service inventory',
-                  }),
+                  link: 'streams' as const,
                 },
-                { link: 'apm:traces' },
-                { link: 'apm:dependencies' },
-                { link: 'apm:settings' },
-                {
-                  id: 'synthetics',
-                  title: i18n.translate('xpack.serverlessObservability.nav.synthetics', {
-                    defaultMessage: 'Synthetics',
-                  }),
-                  children: [
-                    {
-                      title: i18n.translate(
-                        'xpack.serverlessObservability.nav.synthetics.overviewItem',
-                        {
-                          defaultMessage: 'Overview',
-                        }
-                      ),
-                      id: 'synthetics-overview',
-                      link: 'synthetics:overview',
-                      breadcrumbStatus: 'hidden',
-                    },
-                    {
-                      link: 'synthetics:certificates',
-                      title: i18n.translate(
-                        'xpack.serverlessObservability.nav.synthetics.certificatesItem',
-                        {
-                          defaultMessage: 'TLS certificates',
-                        }
-                      ),
-                      id: 'synthetics-certificates',
-                      breadcrumbStatus: 'hidden',
-                    },
-                  ],
-                },
-              ],
-            },
-          ],
-        },
-        {
-          id: 'metrics',
-          title: i18n.translate('xpack.serverlessObservability.nav.infrastructure', {
-            defaultMessage: 'Infrastructure',
-          }),
-          renderAs: 'panelOpener',
-          children: [
-            {
-              children: [
-                {
-                  link: 'metrics:inventory',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.infrastructureInventory',
-                    {
-                      defaultMessage: 'Infrastructure inventory',
-                    }
-                  ),
-                },
-                { link: 'metrics:hosts' },
-                { link: 'metrics:settings' },
-                { link: 'metrics:assetDetails' },
-              ],
-            },
-          ],
-        },
-        {
-          id: 'machine_learning-landing',
-          renderAs: 'panelOpener',
-          title: i18n.translate('xpack.serverlessObservability.nav.machineLearning', {
-            defaultMessage: 'Machine learning',
-          }),
-          children: [
-            {
-              children: [
-                {
-                  link: 'ml:overview',
-                },
-                {
-                  link: 'ml:notifications',
-                },
-                {
-                  link: 'ml:memoryUsage',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.machineLearning.memoryUsage',
-                    {
-                      defaultMessage: 'Memory usage',
-                    }
-                  ),
-                },
-              ],
-            },
-            {
-              id: 'category-anomaly_detection',
-              title: i18n.translate('xpack.serverlessObservability.nav.ml.anomaly_detection', {
-                defaultMessage: 'Anomaly detection',
-              }),
-              breadcrumbStatus: 'hidden',
-              children: [
-                {
-                  link: 'ml:anomalyDetection',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.anomaly_detection.jobs',
-                    {
-                      defaultMessage: 'Jobs',
-                    }
-                  ),
-                },
-                {
-                  link: 'ml:anomalyExplorer',
-                },
-                {
-                  link: 'ml:singleMetricViewer',
-                },
-                {
-                  link: 'ml:settings',
-                },
-                {
-                  link: 'ml:suppliedConfigurations',
-                },
-              ],
-            },
-            {
-              id: 'category-data_frame analytics',
-              title: i18n.translate('xpack.serverlessObservability.nav.ml.data_frame_analytics', {
-                defaultMessage: 'Data frame analytics',
-              }),
-              breadcrumbStatus: 'hidden',
-              children: [
-                {
-                  link: 'ml:dataFrameAnalytics',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.data_frame_analytics.jobs',
-                    {
-                      defaultMessage: 'Jobs',
-                    }
-                  ),
-                },
-                {
-                  link: 'ml:resultExplorer',
-                },
-                {
-                  link: 'ml:analyticsMap',
-                },
-              ],
-            },
-            {
-              id: 'category-model_management',
-              title: i18n.translate('xpack.serverlessObservability.nav.ml.model_management', {
-                defaultMessage: 'Model management',
-              }),
-              breadcrumbStatus: 'hidden',
-              children: [
-                {
-                  link: 'ml:nodesOverview',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.model_management.trainedModels',
-                    {
-                      defaultMessage: 'Trained models',
-                    }
-                  ),
-                },
-              ],
-            },
-            {
-              id: 'category-data_visualizer',
-              title: i18n.translate('xpack.serverlessObservability.nav.ml.data_visualizer', {
-                defaultMessage: 'Data visualizer',
-              }),
-              breadcrumbStatus: 'hidden',
-              children: [
-                {
-                  link: 'ml:fileUpload',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.data_visualizer.file_data_visualizer',
-                    {
-                      defaultMessage: 'File data visualizer',
-                    }
-                  ),
-                },
-                {
-                  link: 'ml:indexDataVisualizer',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.data_visualizer.data_view_data_visualizer',
-                    {
-                      defaultMessage: 'Data view data visualizer',
-                    }
-                  ),
-                },
-                {
-                  link: 'ml:dataDrift',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.data_visualizer.data_drift',
-                    {
-                      defaultMessage: 'Data drift',
-                    }
-                  ),
-                },
-              ],
-            },
-            {
-              id: 'category-aiops_labs',
-              title: i18n.translate('xpack.serverlessObservability.nav.ml.aiops_labs', {
-                defaultMessage: 'Aiops labs',
-              }),
-              breadcrumbStatus: 'hidden',
-              children: [
-                {
-                  link: 'ml:logRateAnalysis',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.aiops_labs.log_rate_analysis',
-                    {
-                      defaultMessage: 'Log rate analysis',
-                    }
-                  ),
-                },
-                {
-                  link: 'ml:logPatternAnalysis',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.aiops_labs.log_pattern_analysis',
-                    {
-                      defaultMessage: 'Log pattern analysis',
-                    }
-                  ),
-                },
-                {
-                  link: 'ml:changePointDetections',
-                  title: i18n.translate(
-                    'xpack.serverlessObservability.nav.ml.aiops_labs.change_point_detection',
-                    {
-                      defaultMessage: 'Change point detection',
-                    }
-                  ),
-                },
-              ],
-            },
-          ],
-        },
-      ],
-    },
-  ],
-  footer: [
-    {
-      type: 'navItem',
-      title: i18n.translate('xpack.serverlessObservability.nav.getStarted', {
-        defaultMessage: 'Add data',
-      }),
-      link: 'observabilityOnboarding',
-      icon: 'launch',
-    },
-    {
-      type: 'navItem',
-      id: 'devTools',
-      title: i18n.translate('xpack.serverlessObservability.nav.devTools', {
-        defaultMessage: 'Developer tools',
-      }),
-      link: 'dev_tools',
-      icon: 'editorCodeBlock',
-    },
-    {
-      type: 'navGroup',
-      id: 'project_settings_project_nav',
-      title: i18n.translate('xpack.serverlessObservability.nav.projectSettings', {
-        defaultMessage: 'Project settings',
-      }),
-      icon: 'gear',
-      breadcrumbStatus: 'hidden',
-      children: [
-        {
-          link: 'management',
-          title: i18n.translate('xpack.serverlessObservability.nav.mngt', {
-            defaultMessage: 'Management',
-          }),
-        },
-        {
-          link: 'integrations',
-        },
-        {
-          link: 'fleet',
-        },
-        {
-          id: 'cloudLinkUserAndRoles',
-          cloudLink: 'userAndRoles',
-        },
-        {
-          id: 'cloudLinkBilling',
-          cloudLink: 'billingAndSub',
-        },
-      ],
-    },
-  ],
+              ]
+            : []),
+          {
+            id: 'apm',
+            title: i18n.translate('xpack.serverlessObservability.nav.applications', {
+              defaultMessage: 'Applications',
+            }),
+            renderAs: 'panelOpener',
+            children: [
+              {
+                children: [
+                  {
+                    link: 'apm:services',
+                    title: i18n.translate('xpack.serverlessObservability.nav.apm.services', {
+                      defaultMessage: 'Service inventory',
+                    }),
+                  },
+                  { link: 'apm:traces' },
+                  { link: 'apm:dependencies' },
+                  { link: 'apm:settings' },
+                  {
+                    id: 'synthetics',
+                    title: i18n.translate('xpack.serverlessObservability.nav.synthetics', {
+                      defaultMessage: 'Synthetics',
+                    }),
+                    children: [
+                      {
+                        title: i18n.translate(
+                          'xpack.serverlessObservability.nav.synthetics.overviewItem',
+                          {
+                            defaultMessage: 'Overview',
+                          }
+                        ),
+                        id: 'synthetics-overview',
+                        link: 'synthetics:overview',
+                        breadcrumbStatus: 'hidden',
+                      },
+                      {
+                        link: 'synthetics:certificates',
+                        title: i18n.translate(
+                          'xpack.serverlessObservability.nav.synthetics.certificatesItem',
+                          {
+                            defaultMessage: 'TLS certificates',
+                          }
+                        ),
+                        id: 'synthetics-certificates',
+                        breadcrumbStatus: 'hidden',
+                      },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+          {
+            id: 'metrics',
+            title: i18n.translate('xpack.serverlessObservability.nav.infrastructure', {
+              defaultMessage: 'Infrastructure',
+            }),
+            renderAs: 'panelOpener',
+            children: [
+              {
+                children: [
+                  {
+                    link: 'metrics:inventory',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.infrastructureInventory',
+                      {
+                        defaultMessage: 'Infrastructure inventory',
+                      }
+                    ),
+                  },
+                  { link: 'metrics:hosts' },
+                  { link: 'metrics:settings' },
+                  { link: 'metrics:assetDetails' },
+                ],
+              },
+            ],
+          },
+          {
+            id: 'machine_learning-landing',
+            renderAs: 'panelOpener',
+            title: i18n.translate('xpack.serverlessObservability.nav.machineLearning', {
+              defaultMessage: 'Machine learning',
+            }),
+            children: [
+              {
+                children: [
+                  {
+                    link: 'ml:overview',
+                  },
+                  {
+                    link: 'ml:notifications',
+                  },
+                  {
+                    link: 'ml:memoryUsage',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.machineLearning.memoryUsage',
+                      {
+                        defaultMessage: 'Memory usage',
+                      }
+                    ),
+                  },
+                ],
+              },
+              {
+                id: 'category-anomaly_detection',
+                title: i18n.translate('xpack.serverlessObservability.nav.ml.anomaly_detection', {
+                  defaultMessage: 'Anomaly detection',
+                }),
+                breadcrumbStatus: 'hidden',
+                children: [
+                  {
+                    link: 'ml:anomalyDetection',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.anomaly_detection.jobs',
+                      {
+                        defaultMessage: 'Jobs',
+                      }
+                    ),
+                  },
+                  {
+                    link: 'ml:anomalyExplorer',
+                  },
+                  {
+                    link: 'ml:singleMetricViewer',
+                  },
+                  {
+                    link: 'ml:settings',
+                  },
+                  {
+                    link: 'ml:suppliedConfigurations',
+                  },
+                ],
+              },
+              {
+                id: 'category-data_frame analytics',
+                title: i18n.translate('xpack.serverlessObservability.nav.ml.data_frame_analytics', {
+                  defaultMessage: 'Data frame analytics',
+                }),
+                breadcrumbStatus: 'hidden',
+                children: [
+                  {
+                    link: 'ml:dataFrameAnalytics',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.data_frame_analytics.jobs',
+                      {
+                        defaultMessage: 'Jobs',
+                      }
+                    ),
+                  },
+                  {
+                    link: 'ml:resultExplorer',
+                  },
+                  {
+                    link: 'ml:analyticsMap',
+                  },
+                ],
+              },
+              {
+                id: 'category-model_management',
+                title: i18n.translate('xpack.serverlessObservability.nav.ml.model_management', {
+                  defaultMessage: 'Model management',
+                }),
+                breadcrumbStatus: 'hidden',
+                children: [
+                  {
+                    link: 'ml:nodesOverview',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.model_management.trainedModels',
+                      {
+                        defaultMessage: 'Trained models',
+                      }
+                    ),
+                  },
+                ],
+              },
+              {
+                id: 'category-data_visualizer',
+                title: i18n.translate('xpack.serverlessObservability.nav.ml.data_visualizer', {
+                  defaultMessage: 'Data visualizer',
+                }),
+                breadcrumbStatus: 'hidden',
+                children: [
+                  {
+                    link: 'ml:fileUpload',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.data_visualizer.file_data_visualizer',
+                      {
+                        defaultMessage: 'File data visualizer',
+                      }
+                    ),
+                  },
+                  {
+                    link: 'ml:indexDataVisualizer',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.data_visualizer.data_view_data_visualizer',
+                      {
+                        defaultMessage: 'Data view data visualizer',
+                      }
+                    ),
+                  },
+                  {
+                    link: 'ml:dataDrift',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.data_visualizer.data_drift',
+                      {
+                        defaultMessage: 'Data drift',
+                      }
+                    ),
+                  },
+                ],
+              },
+              {
+                id: 'category-aiops_labs',
+                title: i18n.translate('xpack.serverlessObservability.nav.ml.aiops_labs', {
+                  defaultMessage: 'Aiops labs',
+                }),
+                breadcrumbStatus: 'hidden',
+                children: [
+                  {
+                    link: 'ml:logRateAnalysis',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.aiops_labs.log_rate_analysis',
+                      {
+                        defaultMessage: 'Log rate analysis',
+                      }
+                    ),
+                  },
+                  {
+                    link: 'ml:logPatternAnalysis',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.aiops_labs.log_pattern_analysis',
+                      {
+                        defaultMessage: 'Log pattern analysis',
+                      }
+                    ),
+                  },
+                  {
+                    link: 'ml:changePointDetections',
+                    title: i18n.translate(
+                      'xpack.serverlessObservability.nav.ml.aiops_labs.change_point_detection',
+                      {
+                        defaultMessage: 'Change point detection',
+                      }
+                    ),
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      },
+    ],
+    footer: [
+      {
+        type: 'navItem',
+        title: i18n.translate('xpack.serverlessObservability.nav.getStarted', {
+          defaultMessage: 'Add data',
+        }),
+        link: 'observabilityOnboarding',
+        icon: 'launch',
+      },
+      {
+        type: 'navItem',
+        id: 'devTools',
+        title: i18n.translate('xpack.serverlessObservability.nav.devTools', {
+          defaultMessage: 'Developer tools',
+        }),
+        link: 'dev_tools',
+        icon: 'editorCodeBlock',
+      },
+      {
+        type: 'navGroup',
+        id: 'project_settings_project_nav',
+        title: i18n.translate('xpack.serverlessObservability.nav.projectSettings', {
+          defaultMessage: 'Project settings',
+        }),
+        icon: 'gear',
+        breadcrumbStatus: 'hidden',
+        children: [
+          {
+            link: 'management',
+            title: i18n.translate('xpack.serverlessObservability.nav.mngt', {
+              defaultMessage: 'Management',
+            }),
+          },
+          {
+            link: 'integrations',
+          },
+          {
+            link: 'fleet',
+          },
+          {
+            id: 'cloudLinkUserAndRoles',
+            cloudLink: 'userAndRoles',
+          },
+          {
+            id: 'cloudLinkBilling',
+            cloudLink: 'billingAndSub',
+          },
+        ],
+      },
+    ],
+  };
 };

x-pack/plugins/serverless_observability/public/plugin.ts

@@ -8,8 +8,8 @@
 import { CoreSetup, CoreStart, Plugin } from '@kbn/core/public';
 import { i18n } from '@kbn/i18n';
 import { appCategories, appIds } from '@kbn/management-cards-navigation';
-import { of } from 'rxjs';
-import { navigationTree } from './navigation_tree';
+import { map, of } from 'rxjs';
+import { createNavigationTree } from './navigation_tree';
 import { createObservabilityDashboardRegistration } from './logs_signal/overview_registration';
 import {
   ServerlessObservabilityPublicSetup,
@@ -50,7 +50,11 @@ export class ServerlessObservabilityPlugin
     setupDeps: ServerlessObservabilityPublicStartDependencies
   ): ServerlessObservabilityPublicStart {
     const { serverless, management, security } = setupDeps;
-    const navigationTree$ = of(navigationTree);
+    const navigationTree$ = (setupDeps.streams?.status$ || of({ status: 'disabled' })).pipe(
+      map(({ status }) => {
+        return createNavigationTree({ streamsAvailable: status === 'enabled' });
+      })
+    );
     serverless.setProjectHome('/app/observability/landing');
     serverless.initNavigation('oblt', navigationTree$, { dataTestSubj: 'svlObservabilitySideNav' });
     const aiAssistantIsEnabled = core.application.capabilities.observabilityAIAssistant?.show;

x-pack/plugins/serverless_observability/public/types.ts

@@ -8,13 +8,14 @@
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { DiscoverSetup } from '@kbn/discover-plugin/public';
 import type { ManagementSetup, ManagementStart } from '@kbn/management-plugin/public';
-import { ObservabilityPublicSetup } from '@kbn/observability-plugin/public';
-import {
+import type { ObservabilityPublicSetup } from '@kbn/observability-plugin/public';
+import type {
   ObservabilitySharedPluginSetup,
   ObservabilitySharedPluginStart,
 } from '@kbn/observability-shared-plugin/public';
-import { SecurityPluginStart } from '@kbn/security-plugin/public';
-import { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/public';
+import type { SecurityPluginStart } from '@kbn/security-plugin/public';
+import type { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/public';
+import type { StreamsPluginStart, StreamsPluginSetup } from '@kbn/streams-plugin/public';
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface ServerlessObservabilityPublicSetup {}
@@ -28,6 +29,7 @@ export interface ServerlessObservabilityPublicSetupDependencies {
   serverless: ServerlessPluginSetup;
   management: ManagementSetup;
   discover: DiscoverSetup;
+  streams?: StreamsPluginSetup;
 }
 
 export interface ServerlessObservabilityPublicStartDependencies {
@@ -36,4 +38,5 @@ export interface ServerlessObservabilityPublicStartDependencies {
   management: ManagementStart;
   data: DataPublicPluginStart;
   security: SecurityPluginStart;
+  streams?: StreamsPluginStart;
 }

x-pack/plugins/serverless_observability/tsconfig.json

@@ -30,5 +30,6 @@
     "@kbn/discover-plugin",
     "@kbn/security-plugin",
     "@kbn/search-types",
+    "@kbn/streams-plugin",
   ]
 }

x-pack/plugins/streams/common/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export type { StreamDefinition } from './types';

x-pack/plugins/streams/public/api/index.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { CoreSetup, CoreStart, HttpFetchOptions } from '@kbn/core/public';
+import type {
+  ClientRequestParamsOf,
+  ReturnOf,
+  RouteRepositoryClient,
+} from '@kbn/server-route-repository';
+import { createRepositoryClient } from '@kbn/server-route-repository-client';
+import type { StreamsRouteRepository } from '../../server';
+
+type FetchOptions = Omit<HttpFetchOptions, 'body'> & {
+  body?: any;
+};
+
+export type StreamsRepositoryClientOptions = Omit<
+  FetchOptions,
+  'query' | 'body' | 'pathname' | 'signal'
+> & {
+  signal: AbortSignal | null;
+};
+
+export type StreamsRepositoryClient = RouteRepositoryClient<
+  StreamsRouteRepository,
+  StreamsRepositoryClientOptions
+>;
+
+export type AutoAbortedStreamsRepositoryClient = RouteRepositoryClient<
+  StreamsRouteRepository,
+  Omit<StreamsRepositoryClientOptions, 'signal'>
+>;
+
+export type StreamsRepositoryEndpoint = keyof StreamsRouteRepository;
+
+export type APIReturnType<TEndpoint extends StreamsRepositoryEndpoint> = ReturnOf<
+  StreamsRouteRepository,
+  TEndpoint
+>;
+
+export type StreamsAPIClientRequestParamsOf<TEndpoint extends StreamsRepositoryEndpoint> =
+  ClientRequestParamsOf<StreamsRouteRepository, TEndpoint>;
+
+export function createStreamsRepositoryClient(
+  core: CoreStart | CoreSetup
+): StreamsRepositoryClient {
+  return createRepositoryClient(core);
+}

x-pack/plugins/streams/public/index.ts

@@ -7,7 +7,12 @@
 
 import { PluginInitializer, PluginInitializerContext } from '@kbn/core/public';
 import { Plugin } from './plugin';
+import { StreamsPluginSetup, StreamsPluginStart } from './types';
 
-export const plugin: PluginInitializer<{}, {}> = (context: PluginInitializerContext) => {
+export type { StreamsPluginSetup, StreamsPluginStart };
+
+export const plugin: PluginInitializer<StreamsPluginSetup, StreamsPluginStart> = (
+  context: PluginInitializerContext
+) => {
   return new Plugin(context);
 };

x-pack/plugins/streams/public/plugin.ts

@@ -8,25 +8,55 @@
 import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public';
 import { Logger } from '@kbn/logging';
 
+import { createRepositoryClient } from '@kbn/server-route-repository-client';
+import { from, shareReplay, startWith } from 'rxjs';
+import { once } from 'lodash';
 import type { StreamsPublicConfig } from '../common/config';
 import { StreamsPluginClass, StreamsPluginSetup, StreamsPluginStart } from './types';
+import { StreamsRepositoryClient } from './api';
 
 export class Plugin implements StreamsPluginClass {
   public config: StreamsPublicConfig;
   public logger: Logger;
 
+  private repositoryClient!: StreamsRepositoryClient;
+
   constructor(context: PluginInitializerContext<{}>) {
     this.config = context.config.get();
     this.logger = context.logger.get();
   }
 
-  setup(core: CoreSetup<StreamsPluginStart>, pluginSetup: StreamsPluginSetup) {
-    return {};
+  setup(core: CoreSetup<{}>, pluginSetup: {}): StreamsPluginSetup {
+    this.repositoryClient = createRepositoryClient(core);
+    return {
+      status$: createStatusObservable(this.logger, this.repositoryClient),
+    };
   }
 
-  start(core: CoreStart) {
-    return {};
+  start(core: CoreStart, pluginsStart: {}): StreamsPluginStart {
+    return {
+      streamsRepositoryClient: this.repositoryClient,
+      status$: createStatusObservable(this.logger, this.repositoryClient),
+    };
   }
 
   stop() {}
 }
+
+const createStatusObservable = once((logger: Logger, repositoryClient: StreamsRepositoryClient) => {
+  return from(
+    repositoryClient
+      .fetch('GET /api/streams/_status', {
+        signal: new AbortController().signal,
+      })
+      .then(
+        (response) => ({
+          status: response.enabled ? ('enabled' as const) : ('disabled' as const),
+        }),
+        (error) => {
+          logger.error(error);
+          return { status: 'unknown' as const };
+        }
+      )
+  ).pipe(startWith({ status: 'unknown' as const }), shareReplay(1));
+});

x-pack/plugins/streams/public/types.ts

@@ -6,11 +6,16 @@
  */
 
 import type { Plugin as PluginClass } from '@kbn/core/public';
+import { Observable } from 'rxjs';
+import type { StreamsRepositoryClient } from './api';
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface StreamsPluginSetup {}
+export interface StreamsPluginSetup {
+  status$: Observable<{ status: 'unknown' | 'enabled' | 'disabled' }>;
+}
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface StreamsPluginStart {}
+export interface StreamsPluginStart {
+  streamsRepositoryClient: StreamsRepositoryClient;
+  status$: Observable<{ status: 'unknown' | 'enabled' | 'disabled' }>;
+}
 
-export type StreamsPluginClass = PluginClass<{}, {}, StreamsPluginSetup, StreamsPluginStart>;
+export type StreamsPluginClass = PluginClass<StreamsPluginSetup, StreamsPluginStart, {}, {}>;

x-pack/plugins/streams/server/index.ts

@@ -17,3 +17,5 @@ export const plugin = async (context: PluginInitializerContext<StreamsConfig>) =
   const { StreamsPlugin } = await import('./plugin');
   return new StreamsPlugin(context);
 };
+
+export type { ListStreamResponse } from './lib/streams/stream_crud';

x-pack/plugins/streams/server/lib/streams/stream_crud.ts

@@ -7,29 +7,29 @@
 
 import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
-import { FieldDefinition, StreamDefinition } from '../../../common/types';
 import { STREAMS_INDEX } from '../../../common/constants';
-import { DefinitionNotFound } from './errors';
-import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates';
+import { FieldDefinition, StreamDefinition } from '../../../common/types';
 import { generateLayer } from './component_templates/generate_layer';
-import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline';
-import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline';
-import { generateIndexTemplate } from './index_templates/generate_index_template';
 import { deleteComponent, upsertComponent } from './component_templates/manage_component_templates';
-import { getIndexTemplateName } from './index_templates/name';
 import { getComponentTemplateName } from './component_templates/name';
-import { getProcessingPipelineName, getReroutePipelineName } from './ingest_pipelines/name';
-import {
-  deleteIngestPipeline,
-  upsertIngestPipeline,
-} from './ingest_pipelines/manage_ingest_pipelines';
-import { getAncestors } from './helpers/hierarchy';
-import { MalformedFields } from './errors/malformed_fields';
 import {
   deleteDataStream,
   rolloverDataStreamIfNecessary,
   upsertDataStream,
 } from './data_streams/manage_data_streams';
+import { DefinitionNotFound } from './errors';
+import { MalformedFields } from './errors/malformed_fields';
+import { getAncestors } from './helpers/hierarchy';
+import { generateIndexTemplate } from './index_templates/generate_index_template';
+import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates';
+import { getIndexTemplateName } from './index_templates/name';
+import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline';
+import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline';
+import {
+  deleteIngestPipeline,
+  upsertIngestPipeline,
+} from './ingest_pipelines/manage_ingest_pipelines';
+import { getProcessingPipelineName, getReroutePipelineName } from './ingest_pipelines/name';
 
 interface BaseParams {
   scopedClusterClient: IScopedClusterClient;
@@ -88,23 +88,40 @@ async function upsertInternalStream({ definition, scopedClusterClient }: BasePar
 
 type ListStreamsParams = BaseParams;
 
-export async function listStreams({ scopedClusterClient }: ListStreamsParams) {
+export interface ListStreamResponse {
+  total: number;
+  definitions: StreamDefinition[];
+}
+
+export async function listStreams({
+  scopedClusterClient,
+}: ListStreamsParams): Promise<ListStreamResponse> {
   const response = await scopedClusterClient.asInternalUser.search<StreamDefinition>({
     index: STREAMS_INDEX,
     size: 10000,
-    fields: ['id'],
-    _source: false,
     sort: [{ id: 'asc' }],
   });
-  const definitions = response.hits.hits.map((hit) => hit.fields as { id: string[] });
-  return definitions;
+  const definitions = response.hits.hits.map((hit) => hit._source!);
+  const total = response.hits.total!;
+
+  return {
+    definitions,
+    total: typeof total === 'number' ? total : total.value,
+  };
 }
 
 interface ReadStreamParams extends BaseParams {
   id: string;
 }
 
-export async function readStream({ id, scopedClusterClient }: ReadStreamParams) {
+export interface ReadStreamResponse {
+  definition: StreamDefinition;
+}
+
+export async function readStream({
+  id,
+  scopedClusterClient,
+}: ReadStreamParams): Promise<ReadStreamResponse> {
   try {
     const response = await scopedClusterClient.asInternalUser.get<StreamDefinition>({
       id,
@@ -126,12 +143,21 @@ interface ReadAncestorsParams extends BaseParams {
   id: string;
 }
 
-export async function readAncestors({ id, scopedClusterClient }: ReadAncestorsParams) {
+export interface ReadAncestorsResponse {
+  ancestors: Array<{ definition: StreamDefinition }>;
+}
+
+export async function readAncestors({
+  id,
+  scopedClusterClient,
+}: ReadAncestorsParams): Promise<ReadAncestorsResponse> {
   const ancestorIds = getAncestors(id);
 
-  return await Promise.all(
-    ancestorIds.map((ancestorId) => readStream({ scopedClusterClient, id: ancestorId }))
-  );
+  return {
+    ancestors: await Promise.all(
+      ancestorIds.map((ancestorId) => readStream({ scopedClusterClient, id: ancestorId }))
+    ),
+  };
 }
 
 interface ReadDescendantsParams extends BaseParams {
@@ -167,7 +193,7 @@ export async function validateAncestorFields(
   id: string,
   fields: FieldDefinition[]
 ) {
-  const ancestors = await readAncestors({
+  const { ancestors } = await readAncestors({
     id,
     scopedClusterClient,
   });

x-pack/plugins/streams/server/plugin.ts

@@ -16,8 +16,7 @@ import {
 } from '@kbn/core/server';
 import { registerRoutes } from '@kbn/server-route-repository';
 import { StreamsConfig, configSchema, exposeToBrowserConfig } from '../common/config';
-import { StreamsRouteRepository } from './routes';
-import { RouteDependencies } from './routes/types';
+import { streamsRouteRepository } from './routes';
 import {
   StreamsPluginSetupDependencies,
   StreamsPluginStartDependencies,
@@ -58,8 +57,8 @@ export class StreamsPlugin
       logger: this.logger,
     } as StreamsServer;
 
-    registerRoutes<RouteDependencies>({
-      repository: StreamsRouteRepository,
+    registerRoutes({
+      repository: streamsRouteRepository,
       dependencies: {
         server: this.server,
         getScopedClients: async ({ request }: { request: KibanaRequest }) => {

x-pack/plugins/streams/server/routes/esql/route.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query';
+import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query';
+import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query';
+import {
+  UnparsedEsqlResponse,
+  createObservabilityEsClient,
+} from '@kbn/observability-utils-server/es/client/create_observability_es_client';
+import { z } from '@kbn/zod';
+import { isNumber } from 'lodash';
+import { createServerRoute } from '../create_server_route';
+
+export const executeEsqlRoute = createServerRoute({
+  endpoint: 'POST /internal/streams/esql',
+  params: z.object({
+    body: z.object({
+      query: z.string(),
+      operationName: z.string(),
+      filter: z.object({}).passthrough().optional(),
+      kuery: z.string().optional(),
+      start: z.number().optional(),
+      end: z.number().optional(),
+    }),
+  }),
+  handler: async ({ params, request, logger, getScopedClients }): Promise<UnparsedEsqlResponse> => {
+    const { scopedClusterClient } = await getScopedClients({ request });
+    const observabilityEsClient = createObservabilityEsClient({
+      client: scopedClusterClient.asCurrentUser,
+      logger,
+      plugin: 'streams',
+    });
+
+    const {
+      body: { operationName, query, filter, kuery, start, end },
+    } = params;
+
+    const response = await observabilityEsClient.esql(
+      operationName,
+      {
+        query,
+        filter: {
+          bool: {
+            filter: [
+              filter || { match_all: {} },
+              ...kqlQuery(kuery),
+              ...excludeFrozenQuery(),
+              ...(isNumber(start) && isNumber(end) ? rangeQuery(start, end) : []),
+            ],
+          },
+        },
+      },
+      { transform: 'none' }
+    );
+
+    return response;
+  },
+});
+
+export const esqlRoutes = {
+  ...executeEsqlRoute,
+};

x-pack/plugins/streams/server/routes/index.ts

@@ -5,15 +5,18 @@
  * 2.0.
  */
 
+import { esqlRoutes } from './esql/route';
 import { deleteStreamRoute } from './streams/delete';
+import { disableStreamsRoute } from './streams/disable';
 import { editStreamRoute } from './streams/edit';
 import { enableStreamsRoute } from './streams/enable';
 import { forkStreamsRoute } from './streams/fork';
 import { listStreamsRoute } from './streams/list';
 import { readStreamRoute } from './streams/read';
 import { resyncStreamsRoute } from './streams/resync';
+import { streamsStatusRoutes } from './streams/settings';
 
-export const StreamsRouteRepository = {
+export const streamsRouteRepository = {
   ...enableStreamsRoute,
   ...resyncStreamsRoute,
   ...forkStreamsRoute,
@@ -21,6 +24,9 @@ export const StreamsRouteRepository = {
   ...editStreamRoute,
   ...deleteStreamRoute,
   ...listStreamsRoute,
+  ...streamsStatusRoutes,
+  ...esqlRoutes,
+  ...disableStreamsRoute,
 };
 
-export type StreamsRouteRepository = typeof StreamsRouteRepository;
+export type StreamsRouteRepository = typeof streamsRouteRepository;

x-pack/plugins/streams/server/routes/streams/delete.ts

@@ -8,6 +8,7 @@
 import { z } from '@kbn/zod';
 import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
+import { badRequest, internal, notFound } from '@hapi/boom';
 import {
   DefinitionNotFound,
   ForkConditionMissing,
@@ -20,18 +21,15 @@ import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'
 import { getParentId } from '../../lib/streams/helpers/hierarchy';
 
 export const deleteStreamRoute = createServerRoute({
-  endpoint: 'DELETE /api/streams/{id} 2023-10-31',
+  endpoint: 'DELETE /api/streams/{id}',
   options: {
-    access: 'public',
-    availability: {
-      stability: 'experimental',
-    },
-    security: {
-      authz: {
-        enabled: false,
-        reason:
-          'This API delegates security to the currently logged in user and their Elasticsearch permissions.',
-      },
+    access: 'internal',
+  },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This API delegates security to the currently logged in user and their Elasticsearch permissions.',
     },
   },
   params: z.object({
@@ -39,7 +37,13 @@ export const deleteStreamRoute = createServerRoute({
       id: z.string(),
     }),
   }),
-  handler: async ({ response, params, logger, request, getScopedClients }) => {
+  handler: async ({
+    response,
+    params,
+    logger,
+    request,
+    getScopedClients,
+  }): Promise<{ acknowledged: true }> => {
     try {
       const { scopedClusterClient } = await getScopedClients({ request });
 
@@ -52,10 +56,10 @@ export const deleteStreamRoute = createServerRoute({
 
       await deleteStream(scopedClusterClient, params.path.id, logger);
 
-      return response.ok({ body: { acknowledged: true } });
+      return { acknowledged: true };
     } catch (e) {
       if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) {
-        return response.notFound({ body: e });
+        throw notFound(e);
       }
 
       if (
@@ -63,15 +67,19 @@ export const deleteStreamRoute = createServerRoute({
         e instanceof ForkConditionMissing ||
         e instanceof MalformedStreamId
       ) {
-        return response.customError({ body: e, statusCode: 400 });
+        throw badRequest(e);
       }
 
-      return response.customError({ body: e, statusCode: 500 });
+      throw internal(e);
     }
   },
 });
 
-async function deleteStream(scopedClusterClient: IScopedClusterClient, id: string, logger: Logger) {
+export async function deleteStream(
+  scopedClusterClient: IScopedClusterClient,
+  id: string,
+  logger: Logger
+) {
   try {
     const { definition } = await readStream({ scopedClusterClient, id });
     for (const child of definition.children) {

x-pack/plugins/streams/server/routes/streams/disable.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { badRequest, internal } from '@hapi/boom';
+import { z } from '@kbn/zod';
+import { SecurityException } from '../../lib/streams/errors';
+import { createServerRoute } from '../create_server_route';
+import { deleteStream } from './delete';
+
+export const disableStreamsRoute = createServerRoute({
+  endpoint: 'POST /api/streams/_disable',
+  params: z.object({}),
+  options: {
+    access: 'internal',
+  },
+  security: {
+    authz: {
+      requiredPrivileges: ['streams_write'],
+    },
+  },
+  handler: async ({
+    request,
+    response,
+    logger,
+    getScopedClients,
+  }): Promise<{ acknowledged: true }> => {
+    try {
+      const { scopedClusterClient } = await getScopedClients({ request });
+
+      await deleteStream(scopedClusterClient, 'logs', logger);
+
+      return { acknowledged: true };
+    } catch (e) {
+      if (e instanceof SecurityException) {
+        throw badRequest(e);
+      }
+      throw internal(e);
+    }
+  },
+});