[8.x] 🌊 Move streams to platform (#212113)

# Backport

This will backport the following commits from `main` to `8.x`:
- 🌊 Move streams to platform
(#211893)](https://github.com/elastic/kibana/pull/211893)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -957,8 +957,9 @@ src/platform/packages/shared/kbn-std @elastic/kibana-core
 packages/kbn-stdio-dev-helpers @elastic/kibana-operations
 src/platform/packages/shared/kbn-storage-adapter @elastic/observability-ui
 packages/kbn-storybook @elastic/kibana-operations
-x-pack/solutions/observability/plugins/streams_app @simianhacker @flash1293 @dgieselaar
-x-pack/solutions/observability/plugins/streams @simianhacker @flash1293 @dgieselaar
+x-pack/platform/plugins/shared/streams_app @elastic/streams-program-team
+x-pack/solutions/observability/plugins/observability_streams_wrapper @elastic/streams-program-team
+x-pack/platform/plugins/shared/streams @simianhacker @flash1293 @dgieselaar
 x-pack/platform/packages/shared/kbn-streams-schema @elastic/streams-program-team
 packages/kbn-styled-components-mapping-cli @elastic/kibana-operations @elastic/eui-team
 x-pack/solutions/observability/plugins/synthetics/e2e @elastic/obs-ux-management-team

docs/developer/plugin-list.asciidoc

@@ -922,14 +922,18 @@ routes, etc.
 |The stack_connectors plugin provides connector types shipped with Kibana, built on top of the framework provided in the actions plugin.
 
 
-|{kib-repo}blob/{branch}/x-pack/solutions/observability/plugins/streams/README.md[streams]
+|{kib-repo}blob/{branch}/x-pack/platform/plugins/shared/streams/README.md[streams]
 |This plugin provides an interface to manage streams
 
 
-|{kib-repo}blob/{branch}/x-pack/solutions/observability/plugins/streams_app/README.md[streamsApp]
+|{kib-repo}blob/{branch}/x-pack/platform/plugins/shared/streams_app/README.md[streamsApp]
 |Home of the Streams app plugin, which allows users to manage Streams via the UI.
 
 
+|{kib-repo}blob/{branch}/x-pack/solutions/observability/plugins/observability_streams_wrapper/README.md[streamsAppWrapper]
+|Observability-specific wrapper for the streams app. This is responsible for actually registering the app and making it accessible in observability contexts
+
+
 |{kib-repo}blob/{branch}/x-pack/solutions/observability/plugins/synthetics/README.md[synthetics]
 |The purpose of this plugin is to provide users of Heartbeat more visibility of what's happening
 in their infrastructure.

package.json

@@ -937,8 +937,9 @@
     "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b",
     "@kbn/std": "link:src/platform/packages/shared/kbn-std",
     "@kbn/storage-adapter": "link:src/platform/packages/shared/kbn-storage-adapter",
-    "@kbn/streams-app-plugin": "link:x-pack/solutions/observability/plugins/streams_app",
-    "@kbn/streams-plugin": "link:x-pack/solutions/observability/plugins/streams",
+    "@kbn/streams-app-plugin": "link:x-pack/platform/plugins/shared/streams_app",
+    "@kbn/streams-app-wrapper-plugin": "link:x-pack/solutions/observability/plugins/observability_streams_wrapper",
+    "@kbn/streams-plugin": "link:x-pack/platform/plugins/shared/streams",
     "@kbn/streams-schema": "link:x-pack/platform/packages/shared/kbn-streams-schema",
     "@kbn/synthetics-plugin": "link:x-pack/solutions/observability/plugins/synthetics",
     "@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture",

packages/kbn-optimizer/limits.yml

@@ -165,6 +165,7 @@ pageLoadAssetSize:
   stackConnectors: 67227
   streams: 16742
   streamsApp: 20537
+  streamsAppWrapper: 20537
   synthetics: 55971
   telemetry: 51957
   telemetryManagementSection: 38586

tsconfig.base.json

@@ -1908,10 +1908,12 @@
       "@kbn/storage-adapter/*": ["src/platform/packages/shared/kbn-storage-adapter/*"],
       "@kbn/storybook": ["packages/kbn-storybook"],
       "@kbn/storybook/*": ["packages/kbn-storybook/*"],
-      "@kbn/streams-app-plugin": ["x-pack/solutions/observability/plugins/streams_app"],
-      "@kbn/streams-app-plugin/*": ["x-pack/solutions/observability/plugins/streams_app/*"],
-      "@kbn/streams-plugin": ["x-pack/solutions/observability/plugins/streams"],
-      "@kbn/streams-plugin/*": ["x-pack/solutions/observability/plugins/streams/*"],
+      "@kbn/streams-app-plugin": ["x-pack/platform/plugins/shared/streams_app"],
+      "@kbn/streams-app-plugin/*": ["x-pack/platform/plugins/shared/streams_app/*"],
+      "@kbn/streams-app-wrapper-plugin": ["x-pack/solutions/observability/plugins/observability_streams_wrapper"],
+      "@kbn/streams-app-wrapper-plugin/*": ["x-pack/solutions/observability/plugins/observability_streams_wrapper/*"],
+      "@kbn/streams-plugin": ["x-pack/platform/plugins/shared/streams"],
+      "@kbn/streams-plugin/*": ["x-pack/platform/plugins/shared/streams/*"],
       "@kbn/streams-schema": ["x-pack/platform/packages/shared/kbn-streams-schema"],
       "@kbn/streams-schema/*": ["x-pack/platform/packages/shared/kbn-streams-schema/*"],
       "@kbn/styled-components-mapping-cli": ["packages/kbn-styled-components-mapping-cli"],

x-pack/platform/plugins/shared/streams/jest.config.js

@@ -8,11 +8,10 @@
 module.exports = {
   preset: '@kbn/test',
   rootDir: '../../../../..',
-  roots: ['<rootDir>/x-pack/solutions/observability/plugins/streams'],
-  coverageDirectory:
-    '<rootDir>/target/kibana-coverage/jest/x-pack/solutions/observability/plugins/streams',
+  roots: ['<rootDir>/x-pack/platform/plugins/shared/streams'],
+  coverageDirectory: '<rootDir>/target/kibana-coverage/jest/x-pack/platform/plugins/shared/streams',
   coverageReporters: ['text', 'html'],
   collectCoverageFrom: [
-    '<rootDir>/x-pack/solutions/observability/plugins/streams/{common,public,server}/**/*.{js,ts,tsx}',
+    '<rootDir>/x-pack/platform/plugins/shared/streams/{common,public,server}/**/*.{js,ts,tsx}',
   ],
 };

x-pack/platform/plugins/shared/streams/kibana.jsonc

@@ -3,8 +3,8 @@
   "id": "@kbn/streams-plugin",
   "owner": "@simianhacker @flash1293 @dgieselaar",
   "description": "A manager for Streams",
-  "group": "observability",
-  "visibility": "private",
+  "group": "platform",
+  "visibility": "shared",
   "plugin": {
     "id": "streams",
     "configPath": ["xpack", "streams"],

x-pack/platform/plugins/shared/streams/server/lib/streams/assets/asset_client.ts

@@ -7,11 +7,11 @@
 import { SanitizedRule } from '@kbn/alerting-plugin/common';
 import { RulesClient } from '@kbn/alerting-plugin/server';
 import { SavedObject, SavedObjectsClientContract } from '@kbn/core/server';
-import { termQuery } from '@kbn/observability-utils-server/es/queries/term_query';
 import { IStorageClient } from '@kbn/storage-adapter';
 import { keyBy } from 'lodash';
 import objectHash from 'object-hash';
 import pLimit from 'p-limit';
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import {
   ASSET_TYPES,
   Asset,
@@ -24,6 +24,22 @@ import {
 import { ASSET_ENTITY_ID, ASSET_ENTITY_TYPE, ASSET_TYPE } from './fields';
 import { AssetStorageSettings } from './storage_settings';
 
+interface TermQueryOpts {
+  queryEmptyString: boolean;
+}
+
+function termQuery<T extends string>(
+  field: T,
+  value: string | boolean | number | undefined | null,
+  opts: TermQueryOpts = { queryEmptyString: true }
+): QueryDslQueryContainer[] {
+  if (value === null || value === undefined || (!opts.queryEmptyString && value === '')) {
+    return [];
+  }
+
+  return [{ term: { [field]: value } }];
+}
+
 function sloSavedObjectToAsset(
   sloId: string,
   savedObject: SavedObject<{ name: string; tags: string[] }>

x-pack/platform/plugins/shared/streams/server/routes/esql/query_helpers.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { estypes } from '@elastic/elasticsearch';
+import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
+
+function excludeTiersQuery(
+  excludedDataTiers: Array<'data_frozen' | 'data_cold' | 'data_warm' | 'data_hot'>
+): estypes.QueryDslQueryContainer[] {
+  return [
+    {
+      bool: {
+        must_not: [
+          {
+            terms: {
+              _tier: excludedDataTiers,
+            },
+          },
+        ],
+      },
+    },
+  ];
+}
+
+export function excludeFrozenQuery(): estypes.QueryDslQueryContainer[] {
+  return excludeTiersQuery(['data_frozen']);
+}
+
+export function kqlQuery(kql?: string): estypes.QueryDslQueryContainer[] {
+  if (!kql) {
+    return [];
+  }
+
+  const ast = fromKueryExpression(kql);
+  return [toElasticsearchQuery(ast)];
+}
+
+export function rangeQuery(
+  start?: number,
+  end?: number,
+  field = '@timestamp'
+): estypes.QueryDslQueryContainer[] {
+  return [
+    {
+      range: {
+        [field]: {
+          gte: start,
+          lte: end,
+          format: 'epoch_millis',
+        },
+      },
+    },
+  ];
+}

x-pack/platform/plugins/shared/streams/server/routes/esql/route.ts

@@ -5,13 +5,11 @@
  * 2.0.
  */
 
-import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query';
-import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query';
-import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query';
 import { UnparsedEsqlResponse, createTracedEsClient } from '@kbn/traced-es-client';
 import { z } from '@kbn/zod';
 import { isNumber } from 'lodash';
 import { createServerRoute } from '../create_server_route';
+import { excludeFrozenQuery, kqlQuery, rangeQuery } from './query_helpers';
 
 export const executeEsqlRoute = createServerRoute({
   endpoint: 'POST /internal/streams/esql',
@@ -37,7 +35,7 @@ export const executeEsqlRoute = createServerRoute({
   }),
   handler: async ({ params, request, logger, getScopedClients }): Promise<UnparsedEsqlResponse> => {
     const { scopedClusterClient } = await getScopedClients({ request });
-    const observabilityEsClient = createTracedEsClient({
+    const tracedEsClient = createTracedEsClient({
       client: scopedClusterClient.asCurrentUser,
       logger,
       plugin: 'streams',
@@ -47,7 +45,7 @@ export const executeEsqlRoute = createServerRoute({
       body: { operationName, query, filter, kuery, start, end },
     } = params;
 
-    const response = await observabilityEsClient.esql(
+    const response = await tracedEsClient.esql(
       operationName,
       {
         query,

x-pack/platform/plugins/shared/streams/tsconfig.json

@@ -29,8 +29,6 @@
     "@kbn/licensing-plugin",
     "@kbn/server-route-repository-client",
     "@kbn/object-utils",
-    "@kbn/observability-utils-server",
-    "@kbn/observability-utils-common",
     "@kbn/alerting-plugin",
     "@kbn/std",
     "@kbn/safer-lodash-set",
@@ -39,6 +37,7 @@
     "@kbn/server-route-repository-utils",
     "@kbn/inference-plugin",
     "@kbn/storage-adapter",
-    "@kbn/traced-es-client"
+    "@kbn/traced-es-client",
+    "@kbn/es-query"
   ]
 }

x-pack/platform/plugins/shared/streams_app/.storybook/get_mock_streams_app_context.tsx

@@ -9,7 +9,6 @@ import { coreMock } from '@kbn/core/public/mocks';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
 import type { StreamsPluginStart } from '@kbn/streams-plugin/public';
-import type { ObservabilitySharedPluginStart } from '@kbn/observability-shared-plugin/public';
 import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
 import type { SharePublicStart } from '@kbn/share-plugin/public/plugin';
 import { NavigationPublicStart } from '@kbn/navigation-plugin/public/types';
@@ -27,7 +26,6 @@ export function getMockStreamsAppContext(): StreamsAppKibanaContext {
     core,
     dependencies: {
       start: {
-        observabilityShared: {} as unknown as ObservabilitySharedPluginStart,
         dataViews: {} as unknown as DataViewsPublicPluginStart,
         data: {} as unknown as DataPublicPluginStart,
         unifiedSearch: {} as unknown as UnifiedSearchPublicPluginStart,
@@ -40,6 +38,7 @@ export function getMockStreamsAppContext(): StreamsAppKibanaContext {
     },
     services: {
       dataStreamsClient: Promise.resolve({} as unknown as DataStreamsStatsClient),
+      PageTemplate: () => null,
     },
     isServerless: false,
   };

x-pack/platform/plugins/shared/streams_app/jest.config.js

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../..',
+  roots: [
+    '<rootDir>/x-pack/platform/plugins/shared/streams_app/public',
+    '<rootDir>/x-pack/platform/plugins/shared/streams_app/common',
+    '<rootDir>/x-pack/platform/plugins/shared/streams_app/server',
+  ],
+  setupFiles: ['<rootDir>/x-pack/platform/plugins/shared/streams_app/.storybook/jest_setup.js'],
+  collectCoverage: true,
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/platform/plugins/shared/streams_app/{public,common,server}/**/*.{js,ts,tsx}',
+  ],
+
+  coverageReporters: ['html'],
+};

x-pack/platform/plugins/shared/streams_app/kibana.jsonc

@@ -1,9 +1,9 @@
 {
   "type": "plugin",
   "id": "@kbn/streams-app-plugin",
-  "owner": "@simianhacker @flash1293 @dgieselaar",
-  "group": "observability",
-  "visibility": "private",
+  "owner": "@elastic/streams-program-team",
+  "group": "platform",
+  "visibility": "shared",
   "plugin": {
     "id": "streamsApp",
     "server": true,
@@ -11,7 +11,6 @@
     "configPath": ["xpack", "streamsApp"],
     "requiredPlugins": [
       "streams",
-      "observabilityShared",
       "data",
       "dataViews",
       "unifiedSearch",

x-pack/platform/plugins/shared/streams_app/public/application.tsx

@@ -5,15 +5,13 @@
  * 2.0.
  */
 import React from 'react';
-import ReactDOM from 'react-dom';
-import { APP_WRAPPER_CLASS, type AppMountParameters, type CoreStart } from '@kbn/core/public';
+import { type AppMountParameters, type CoreStart } from '@kbn/core/public';
 import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
-import { css } from '@emotion/css';
 import type { StreamsAppStartDependencies } from './types';
 import { StreamsAppServices } from './services/types';
 import { AppRoot } from './components/app_root';
 
-export const renderApp = ({
+export const StreamsApplication = ({
   coreStart,
   pluginsStart,
   services,
@@ -25,15 +23,7 @@ export const renderApp = ({
   services: StreamsAppServices;
   isServerless: boolean;
 } & { appMountParameters: AppMountParameters }) => {
-  const { element } = appMountParameters;
-
-  const appWrapperClassName = css`
-    overflow: auto;
-  `;
-  const appWrapperElement = document.getElementsByClassName(APP_WRAPPER_CLASS)[1];
-  appWrapperElement.classList.add(appWrapperClassName);
-
-  ReactDOM.render(
+  return (
     <KibanaRenderContextProvider {...coreStart}>
       <AppRoot
         appMountParameters={appMountParameters}
@@ -42,11 +32,6 @@ export const renderApp = ({
         services={services}
         isServerless={isServerless}
       />
-    </KibanaRenderContextProvider>,
-    element
+    </KibanaRenderContextProvider>
   );
-  return () => {
-    ReactDOM.unmountComponentAtNode(element);
-    appWrapperElement.classList.remove(APP_WRAPPER_CLASS);
-  };
 };

x-pack/platform/plugins/shared/streams_app/public/components/app_root/index.tsx

@@ -13,12 +13,12 @@ import {
   RouteRenderer,
   RouterProvider,
 } from '@kbn/typed-react-router-config';
-import { HeaderMenuPortal } from '@kbn/observability-shared-plugin/public';
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import { StreamsAppContextProvider } from '../streams_app_context_provider';
 import { streamsAppRouter } from '../../routes/config';
 import { StreamsAppStartDependencies } from '../../types';
 import { StreamsAppServices } from '../../services/types';
+import { HeaderMenuPortal } from '../header_menu';
 
 export function AppRoot({
   coreStart,

x-pack/platform/plugins/shared/streams_app/public/components/esql_chart/controlled_esql_chart.tsx

@@ -20,14 +20,24 @@ import {
 } from '@elastic/charts';
 import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiSpacer } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import { getTimeZone } from '@kbn/observability-utils-browser/utils/ui_settings/get_timezone';
 import { css } from '@emotion/css';
 import { AbortableAsyncState } from '@kbn/react-hooks';
 import type { UnparsedEsqlResponse } from '@kbn/traced-es-client';
+import { IUiSettingsClient } from '@kbn/core/public';
+import { UI_SETTINGS } from '@kbn/data-plugin/public';
 import { esqlResultToTimeseries } from '../../util/esql_result_to_timeseries';
 import { useKibana } from '../../hooks/use_kibana';
 import { LoadingPanel } from '../loading_panel';
 
+function getTimeZone(uiSettings?: IUiSettingsClient) {
+  const kibanaTimeZone = uiSettings?.get<'Browser' | string>(UI_SETTINGS.DATEFORMAT_TZ);
+  if (!kibanaTimeZone || kibanaTimeZone === 'Browser') {
+    return 'local';
+  }
+
+  return kibanaTimeZone;
+}
+
 const END_ZONE_LABEL = i18n.translate('xpack.streams.esqlChart.endzone', {
   defaultMessage:
     'The selected time range does not include this entire bucket. It might contain partial data.',

x-pack/platform/plugins/shared/streams_app/public/components/header_menu/header_menu_portal.test.tsx

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { render } from '@testing-library/react';
+import React from 'react';
+import HeaderMenuPortal from './header_menu_portal';
+import { themeServiceMock } from '@kbn/core/public/mocks';
+
+describe('HeaderMenuPortal', () => {
+  describe('when unmounted', () => {
+    it('calls setHeaderActionMenu with undefined', () => {
+      const setHeaderActionMenu = jest.fn();
+      const theme$ = themeServiceMock.createTheme$();
+
+      const { unmount } = render(
+        <HeaderMenuPortal setHeaderActionMenu={setHeaderActionMenu} theme$={theme$}>
+          test
+        </HeaderMenuPortal>
+      );
+
+      unmount();
+
+      expect(setHeaderActionMenu).toHaveBeenCalledWith(undefined);
+    });
+  });
+});