🌊 Move streams to platform (#211893)

This PR moves the `streams` and `streams_app` plugins into platform so
they can be used in other solutions in the future. This PR is not
actually making it available in other solutions yet since we are still
discussing the release plans.

## Inlined helpers

As discussed before, this PR inlines a couple simple helper methods for
query building, time zone normalization, a header portal helper and a
data plugin timefilter state react integration hook as there is no good
place for these outside of the observability solution.

## streams_app plugin

The streams_app plugin is not actually registering anything, instead it
simply exports a component that renders the app which needs to be
consumed by another plugin to turn it into a registered app - for now,
`observability_streams_wrapper` takes over this job.

## observability_streams_wrapper plugin

While 99% of the streams logic is moved into the
`platform/shared/streams_app`, two bits are left behind in
`observability_streams_wrapper`:
* The actual app registration
* Integration with the observability_shared `PageTemplate` component

Once we decide streams should be displayed outside of the observability
solution, it's probably not necessary anymore to decouple app definition
and registration like this because it will always be visible no matter
the solution. Once this is the case, the navigation registration can be
moved into the central `observability` plugin, like it's handled with
other apps like infra.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -919,6 +919,8 @@ x-pack/platform/plugins/shared/serverless @elastic/appex-sharedux
 x-pack/platform/plugins/shared/spaces @elastic/kibana-security
 x-pack/platform/plugins/shared/stack_alerts @elastic/response-ops
 x-pack/platform/plugins/shared/stack_connectors @elastic/response-ops
+x-pack/platform/plugins/shared/streams @elastic/streams-program-team
+x-pack/platform/plugins/shared/streams_app @elastic/streams-program-team
 x-pack/platform/plugins/shared/task_manager @elastic/response-ops
 x-pack/platform/plugins/shared/timelines @elastic/security-threat-hunting-investigations
 x-pack/platform/plugins/shared/triggers_actions_ui @elastic/response-ops
@@ -953,12 +955,11 @@ x-pack/solutions/observability/plugins/observability_ai_assistant_management @el
 x-pack/solutions/observability/plugins/observability_logs_explorer @elastic/obs-ux-logs-team
 x-pack/solutions/observability/plugins/observability_onboarding @elastic/obs-ux-logs-team
 x-pack/solutions/observability/plugins/observability_shared @elastic/observability-ui
+x-pack/solutions/observability/plugins/observability_streams_wrapper @elastic/streams-program-team
 x-pack/solutions/observability/plugins/profiling @elastic/obs-ux-infra_services-team
 x-pack/solutions/observability/plugins/profiling_data_access @elastic/obs-ux-infra_services-team
 x-pack/solutions/observability/plugins/serverless_observability @elastic/obs-ux-management-team
 x-pack/solutions/observability/plugins/slo @elastic/obs-ux-management-team
-x-pack/solutions/observability/plugins/streams @elastic/streams-program-team
-x-pack/solutions/observability/plugins/streams_app @elastic/streams-program-team
 x-pack/solutions/observability/plugins/synthetics @elastic/obs-ux-management-team
 x-pack/solutions/observability/plugins/synthetics/e2e @elastic/obs-ux-management-team
 x-pack/solutions/observability/plugins/uptime @elastic/obs-ux-management-team

package.json

@@ -933,8 +933,9 @@
     "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b",
     "@kbn/std": "link:src/platform/packages/shared/kbn-std",
     "@kbn/storage-adapter": "link:src/platform/packages/shared/kbn-storage-adapter",
-    "@kbn/streams-app-plugin": "link:x-pack/solutions/observability/plugins/streams_app",
+    "@kbn/streams-app-plugin": "link:x-pack/platform/plugins/shared/streams_app",
-    "@kbn/streams-plugin": "link:x-pack/solutions/observability/plugins/streams",
+    "@kbn/streams-app-wrapper-plugin": "link:x-pack/solutions/observability/plugins/observability_streams_wrapper",
+    "@kbn/streams-plugin": "link:x-pack/platform/plugins/shared/streams",
     "@kbn/streams-schema": "link:x-pack/platform/packages/shared/kbn-streams-schema",
     "@kbn/synthetics-plugin": "link:x-pack/solutions/observability/plugins/synthetics",
     "@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture",

packages/kbn-optimizer/limits.yml

@@ -162,6 +162,7 @@ pageLoadAssetSize:
   stackConnectors: 67227
   streams: 16742
   streamsApp: 20537
+  streamsAppWrapper: 20537
   synthetics: 55971
   telemetry: 51957
   telemetryManagementSection: 38586

tsconfig.base.json

@@ -1898,10 +1898,12 @@
       "@kbn/storage-adapter/*": ["src/platform/packages/shared/kbn-storage-adapter/*"],
       "@kbn/storybook": ["packages/kbn-storybook"],
       "@kbn/storybook/*": ["packages/kbn-storybook/*"],
-      "@kbn/streams-app-plugin": ["x-pack/solutions/observability/plugins/streams_app"],
+      "@kbn/streams-app-plugin": ["x-pack/platform/plugins/shared/streams_app"],
-      "@kbn/streams-app-plugin/*": ["x-pack/solutions/observability/plugins/streams_app/*"],
+      "@kbn/streams-app-plugin/*": ["x-pack/platform/plugins/shared/streams_app/*"],
-      "@kbn/streams-plugin": ["x-pack/solutions/observability/plugins/streams"],
+      "@kbn/streams-app-wrapper-plugin": ["x-pack/solutions/observability/plugins/observability_streams_wrapper"],
-      "@kbn/streams-plugin/*": ["x-pack/solutions/observability/plugins/streams/*"],
+      "@kbn/streams-app-wrapper-plugin/*": ["x-pack/solutions/observability/plugins/observability_streams_wrapper/*"],
+      "@kbn/streams-plugin": ["x-pack/platform/plugins/shared/streams"],
+      "@kbn/streams-plugin/*": ["x-pack/platform/plugins/shared/streams/*"],
       "@kbn/streams-schema": ["x-pack/platform/packages/shared/kbn-streams-schema"],
       "@kbn/streams-schema/*": ["x-pack/platform/packages/shared/kbn-streams-schema/*"],
       "@kbn/styled-components-mapping-cli": ["packages/kbn-styled-components-mapping-cli"],

x-pack/platform/plugins/shared/streams/jest.config.js

@@ -8,11 +8,10 @@
 module.exports = {
   preset: '@kbn/test',
   rootDir: '../../../../..',
-  roots: ['<rootDir>/x-pack/solutions/observability/plugins/streams'],
+  roots: ['<rootDir>/x-pack/platform/plugins/shared/streams'],
-  coverageDirectory:
+  coverageDirectory: '<rootDir>/target/kibana-coverage/jest/x-pack/platform/plugins/shared/streams',
-    '<rootDir>/target/kibana-coverage/jest/x-pack/solutions/observability/plugins/streams',
   coverageReporters: ['text', 'html'],
   collectCoverageFrom: [
-    '<rootDir>/x-pack/solutions/observability/plugins/streams/{common,public,server}/**/*.{js,ts,tsx}',
+    '<rootDir>/x-pack/platform/plugins/shared/streams/{common,public,server}/**/*.{js,ts,tsx}',
   ],
 };

x-pack/platform/plugins/shared/streams/kibana.jsonc

@@ -3,8 +3,8 @@
   "id": "@kbn/streams-plugin",
   "owner": "@elastic/streams-program-team",
   "description": "A manager for Streams",
-  "group": "observability",
+  "group": "platform",
-  "visibility": "private",
+  "visibility": "shared",
   "plugin": {
     "id": "streams",
     "configPath": ["xpack", "streams"],

x-pack/platform/plugins/shared/streams/server/lib/streams/assets/asset_client.ts

@@ -7,11 +7,11 @@
 import { SanitizedRule } from '@kbn/alerting-plugin/common';
 import { RulesClient } from '@kbn/alerting-plugin/server';
 import { SavedObject, SavedObjectsClientContract } from '@kbn/core/server';
-import { termQuery } from '@kbn/observability-utils-server/es/queries/term_query';
 import { IStorageClient } from '@kbn/storage-adapter';
 import { keyBy } from 'lodash';
 import objectHash from 'object-hash';
 import pLimit from 'p-limit';
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import {
   ASSET_TYPES,
   Asset,
@@ -24,6 +24,22 @@ import {
 import { ASSET_ENTITY_ID, ASSET_ENTITY_TYPE, ASSET_TYPE } from './fields';
 import { AssetStorageSettings } from './storage_settings';
 
+interface TermQueryOpts {
+  queryEmptyString: boolean;
+}
+
+function termQuery<T extends string>(
+  field: T,
+  value: string | boolean | number | undefined | null,
+  opts: TermQueryOpts = { queryEmptyString: true }
+): QueryDslQueryContainer[] {
+  if (value === null || value === undefined || (!opts.queryEmptyString && value === '')) {
+    return [];
+  }
+
+  return [{ term: { [field]: value } }];
+}
+
 function sloSavedObjectToAsset(
   sloId: string,
   savedObject: SavedObject<{ name: string; tags: string[] }>

x-pack/platform/plugins/shared/streams/server/routes/esql/query_helpers.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { estypes } from '@elastic/elasticsearch';
+import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
+
+function excludeTiersQuery(
+  excludedDataTiers: Array<'data_frozen' | 'data_cold' | 'data_warm' | 'data_hot'>
+): estypes.QueryDslQueryContainer[] {
+  return [
+    {
+      bool: {
+        must_not: [
+          {
+            terms: {
+              _tier: excludedDataTiers,
+            },
+          },
+        ],
+      },
+    },
+  ];
+}
+
+export function excludeFrozenQuery(): estypes.QueryDslQueryContainer[] {
+  return excludeTiersQuery(['data_frozen']);
+}
+
+export function kqlQuery(kql?: string): estypes.QueryDslQueryContainer[] {
+  if (!kql) {
+    return [];
+  }
+
+  const ast = fromKueryExpression(kql);
+  return [toElasticsearchQuery(ast)];
+}
+
+export function rangeQuery(
+  start?: number,
+  end?: number,
+  field = '@timestamp'
+): estypes.QueryDslQueryContainer[] {
+  return [
+    {
+      range: {
+        [field]: {
+          gte: start,
+          lte: end,
+          format: 'epoch_millis',
+        },
+      },
+    },
+  ];
+}

x-pack/platform/plugins/shared/streams/server/routes/esql/route.ts

@@ -5,13 +5,11 @@
  * 2.0.
  */
 
-import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query';
-import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query';
-import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query';
 import { UnparsedEsqlResponse, createTracedEsClient } from '@kbn/traced-es-client';
 import { z } from '@kbn/zod';
 import { isNumber } from 'lodash';
 import { createServerRoute } from '../create_server_route';
+import { excludeFrozenQuery, kqlQuery, rangeQuery } from './query_helpers';
 
 export const executeEsqlRoute = createServerRoute({
   endpoint: 'POST /internal/streams/esql',
@@ -37,7 +35,7 @@ export const executeEsqlRoute = createServerRoute({
   }),
   handler: async ({ params, request, logger, getScopedClients }): Promise<UnparsedEsqlResponse> => {
     const { scopedClusterClient } = await getScopedClients({ request });
-    const observabilityEsClient = createTracedEsClient({
+    const tracedEsClient = createTracedEsClient({
       client: scopedClusterClient.asCurrentUser,
       logger,
       plugin: 'streams',
@@ -47,7 +45,7 @@ export const executeEsqlRoute = createServerRoute({
       body: { operationName, query, filter, kuery, start, end },
     } = params;
 
-    const response = await observabilityEsClient.esql(
+    const response = await tracedEsClient.esql(
       operationName,
       {
         query,

x-pack/platform/plugins/shared/streams/tsconfig.json

@@ -29,8 +29,6 @@
     "@kbn/licensing-plugin",
     "@kbn/server-route-repository-client",
     "@kbn/object-utils",
-    "@kbn/observability-utils-server",
-    "@kbn/observability-utils-common",
     "@kbn/alerting-plugin",
     "@kbn/std",
     "@kbn/safer-lodash-set",
@@ -39,6 +37,7 @@
     "@kbn/server-route-repository-utils",
     "@kbn/inference-plugin",
     "@kbn/storage-adapter",
-    "@kbn/traced-es-client"
+    "@kbn/traced-es-client",
+    "@kbn/es-query"
   ]
 }

x-pack/platform/plugins/shared/streams_app/.storybook/get_mock_streams_app_context.tsx

@@ -9,7 +9,6 @@ import { coreMock } from '@kbn/core/public/mocks';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
 import type { StreamsPluginStart } from '@kbn/streams-plugin/public';
-import type { ObservabilitySharedPluginStart } from '@kbn/observability-shared-plugin/public';
 import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
 import type { SharePublicStart } from '@kbn/share-plugin/public/plugin';
 import { NavigationPublicStart } from '@kbn/navigation-plugin/public/types';
@@ -27,7 +26,6 @@ export function getMockStreamsAppContext(): StreamsAppKibanaContext {
     core,
     dependencies: {
       start: {
-        observabilityShared: {} as unknown as ObservabilitySharedPluginStart,
         dataViews: {} as unknown as DataViewsPublicPluginStart,
         data: {} as unknown as DataPublicPluginStart,
         unifiedSearch: {} as unknown as UnifiedSearchPublicPluginStart,
@@ -40,6 +38,7 @@ export function getMockStreamsAppContext(): StreamsAppKibanaContext {
     },
     services: {
       dataStreamsClient: Promise.resolve({} as unknown as DataStreamsStatsClient),
+      PageTemplate: () => null,
     },
     isServerless: false,
   };

x-pack/platform/plugins/shared/streams_app/jest.config.js

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../..',
+  roots: [
+    '<rootDir>/x-pack/platform/plugins/shared/streams_app/public',
+    '<rootDir>/x-pack/platform/plugins/shared/streams_app/common',
+    '<rootDir>/x-pack/platform/plugins/shared/streams_app/server',
+  ],
+  setupFiles: ['<rootDir>/x-pack/platform/plugins/shared/streams_app/.storybook/jest_setup.js'],
+  collectCoverage: true,
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/platform/plugins/shared/streams_app/{public,common,server}/**/*.{js,ts,tsx}',
+  ],
+
+  coverageReporters: ['html'],
+};

x-pack/platform/plugins/shared/streams_app/kibana.jsonc

@@ -2,8 +2,8 @@
   "type": "plugin",
   "id": "@kbn/streams-app-plugin",
   "owner": "@elastic/streams-program-team",
-  "group": "observability",
+  "group": "platform",
-  "visibility": "private",
+  "visibility": "shared",
   "plugin": {
     "id": "streamsApp",
     "server": true,
@@ -11,7 +11,6 @@
     "configPath": ["xpack", "streamsApp"],
     "requiredPlugins": [
       "streams",
-      "observabilityShared",
       "data",
       "dataViews",
       "unifiedSearch",

x-pack/platform/plugins/shared/streams_app/public/application.tsx

@@ -5,15 +5,13 @@
  * 2.0.
  */
 import React from 'react';
-import ReactDOM from 'react-dom';
+import { type AppMountParameters, type CoreStart } from '@kbn/core/public';
-import { APP_WRAPPER_CLASS, type AppMountParameters, type CoreStart } from '@kbn/core/public';
 import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
-import { css } from '@emotion/css';
 import type { StreamsAppStartDependencies } from './types';
 import { StreamsAppServices } from './services/types';
 import { AppRoot } from './components/app_root';
 
-export const renderApp = ({
+export const StreamsApplication = ({
   coreStart,
   pluginsStart,
   services,
@@ -25,15 +23,7 @@ export const renderApp = ({
   services: StreamsAppServices;
   isServerless: boolean;
 } & { appMountParameters: AppMountParameters }) => {
-  const { element } = appMountParameters;
+  return (
-
-  const appWrapperClassName = css`
-    overflow: auto;
-  `;
-  const appWrapperElement = document.getElementsByClassName(APP_WRAPPER_CLASS)[1];
-  appWrapperElement.classList.add(appWrapperClassName);
-
-  ReactDOM.render(
     <KibanaRenderContextProvider {...coreStart}>
       <AppRoot
         appMountParameters={appMountParameters}
@@ -42,11 +32,6 @@ export const renderApp = ({
         services={services}
         isServerless={isServerless}
       />
-    </KibanaRenderContextProvider>,
+    </KibanaRenderContextProvider>
-    element
   );
-  return () => {
-    ReactDOM.unmountComponentAtNode(element);
-    appWrapperElement.classList.remove(APP_WRAPPER_CLASS);
-  };
 };

x-pack/platform/plugins/shared/streams_app/public/components/app_root/index.tsx

@@ -13,12 +13,12 @@ import {
   RouteRenderer,
   RouterProvider,
 } from '@kbn/typed-react-router-config';
-import { HeaderMenuPortal } from '@kbn/observability-shared-plugin/public';
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import { StreamsAppContextProvider } from '../streams_app_context_provider';
 import { streamsAppRouter } from '../../routes/config';
 import { StreamsAppStartDependencies } from '../../types';
 import { StreamsAppServices } from '../../services/types';
+import { HeaderMenuPortal } from '../header_menu';
 
 export function AppRoot({
   coreStart,

x-pack/platform/plugins/shared/streams_app/public/components/esql_chart/controlled_esql_chart.tsx

@@ -20,14 +20,24 @@ import {
 } from '@elastic/charts';
 import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiSpacer } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import { getTimeZone } from '@kbn/observability-utils-browser/utils/ui_settings/get_timezone';
 import { css } from '@emotion/css';
 import { AbortableAsyncState } from '@kbn/react-hooks';
 import type { UnparsedEsqlResponse } from '@kbn/traced-es-client';
+import { IUiSettingsClient } from '@kbn/core/public';
+import { UI_SETTINGS } from '@kbn/data-plugin/public';
 import { esqlResultToTimeseries } from '../../util/esql_result_to_timeseries';
 import { useKibana } from '../../hooks/use_kibana';
 import { LoadingPanel } from '../loading_panel';
 
+function getTimeZone(uiSettings?: IUiSettingsClient) {
+  const kibanaTimeZone = uiSettings?.get<'Browser' | string>(UI_SETTINGS.DATEFORMAT_TZ);
+  if (!kibanaTimeZone || kibanaTimeZone === 'Browser') {
+    return 'local';
+  }
+
+  return kibanaTimeZone;
+}
+
 const END_ZONE_LABEL = i18n.translate('xpack.streams.esqlChart.endzone', {
   defaultMessage:
     'The selected time range does not include this entire bucket. It might contain partial data.',

x-pack/platform/plugins/shared/streams_app/public/components/header_menu/header_menu_portal.test.tsx

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { render } from '@testing-library/react';
+import React from 'react';
+import HeaderMenuPortal from './header_menu_portal';
+import { themeServiceMock } from '@kbn/core/public/mocks';
+
+describe('HeaderMenuPortal', () => {
+  describe('when unmounted', () => {
+    it('calls setHeaderActionMenu with undefined', () => {
+      const setHeaderActionMenu = jest.fn();
+      const theme$ = themeServiceMock.createTheme$();
+
+      const { unmount } = render(
+        <HeaderMenuPortal setHeaderActionMenu={setHeaderActionMenu} theme$={theme$}>
+          test
+        </HeaderMenuPortal>
+      );
+
+      unmount();
+
+      expect(setHeaderActionMenu).toHaveBeenCalledWith(undefined);
+    });
+  });
+});