[Data Usage] New plugin (#193466)

The initial serverless only plugin for viewing data usage and retention
in Management. The purpose of this PR is to provide a place for other
engineers to work on it, hidden from public use.

- Plugin is hidden by default and can be enabled through kibana.yml
`xpack.dataUsage.enabled: true`
- Currently it will show up in both stateful and serverless (if enabled
using config above). When we are ready to make the plugin available we
will enable it in config/serverless.yml
- Renders a card in Management (serverless) when enabled:
<img width="1269" alt="Screenshot 2024-09-19 at 4 14 15 PM"
src="https://github.com/user-attachments/assets/705e3866-bc88-436a-8532-2af53167f7b1">


https://github.com/elastic/kibana/issues/192965
https://github.com/elastic/kibana/issues/192966

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -354,6 +354,7 @@ x-pack/plugins/data_quality @elastic/obs-ux-logs-team
 test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery
 packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
 packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore
+x-pack/plugins/data_usage @elastic/obs-ai-assistant @elastic/security-solution
 src/plugins/data_view_editor @elastic/kibana-data-discovery
 examples/data_view_field_editor_example @elastic/kibana-data-discovery
 src/plugins/data_view_field_editor @elastic/kibana-data-discovery

docs/developer/plugin-list.asciidoc

@@ -536,6 +536,10 @@ Plugin server-side only. Plugin has three main functions:
 |In order to make ongoing maintenance of log collection easy we want to introduce the concept of data set quality, where users can easily get an overview on the data sets they have with information such as integration, size, last activity, among others.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/data_usage/README.md[dataUsage]
+|Serverless only plugin for users to view data usage
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/data_visualizer/README.md[dataVisualizer]
 |The data_visualizer plugin enables you to explore the fields in your data.
 

package.json

@@ -421,6 +421,7 @@
     "@kbn/data-search-plugin": "link:test/plugin_functional/plugins/data_search",
     "@kbn/data-service": "link:packages/kbn-data-service",
     "@kbn/data-stream-adapter": "link:packages/kbn-data-stream-adapter",
+    "@kbn/data-usage-plugin": "link:x-pack/plugins/data_usage",
     "@kbn/data-view-editor-plugin": "link:src/plugins/data_view_editor",
     "@kbn/data-view-field-editor-example-plugin": "link:examples/data_view_field_editor_example",
     "@kbn/data-view-field-editor-plugin": "link:src/plugins/data_view_field_editor",

packages/kbn-management/cards_navigation/src/consts.tsx

@@ -72,6 +72,14 @@ export const appDefinitions: Record<AppId, AppDefinition> = {
     icon: 'documents',
   },
 
+  [AppIds.DATA_USAGE]: {
+    category: appCategories.DATA,
+    description: i18n.translate('management.landing.withCardNavigation.dataUsageDescription', {
+      defaultMessage: 'View data usage and retention.',
+    }),
+    icon: 'documents',
+  },
+
   [AppIds.RULES]: {
     category: appCategories.ALERTS,
     description: i18n.translate('management.landing.withCardNavigation.rulesDescription', {

packages/kbn-management/cards_navigation/src/types.ts

@@ -31,6 +31,7 @@ export enum AppIds {
   API_KEYS = 'api_keys',
   DATA_QUALITY = 'data_quality',
   SPACES = 'spaces',
+  DATA_USAGE = 'data_usage',
 }
 
 // Create new type that is a union of all the appId values

packages/kbn-optimizer/limits.yml

@@ -29,6 +29,7 @@ pageLoadAssetSize:
   data: 454087
   dataQuality: 19384
   datasetQuality: 52000
+  dataUsage: 30000
   dataViewEditor: 28082
   dataViewFieldEditor: 42021
   dataViewManagement: 5300

tsconfig.base.json

@@ -702,6 +702,8 @@
       "@kbn/data-service/*": ["packages/kbn-data-service/*"],
       "@kbn/data-stream-adapter": ["packages/kbn-data-stream-adapter"],
       "@kbn/data-stream-adapter/*": ["packages/kbn-data-stream-adapter/*"],
+      "@kbn/data-usage-plugin": ["x-pack/plugins/data_usage"],
+      "@kbn/data-usage-plugin/*": ["x-pack/plugins/data_usage/*"],
       "@kbn/data-view-editor-plugin": ["src/plugins/data_view_editor"],
       "@kbn/data-view-editor-plugin/*": ["src/plugins/data_view_editor/*"],
       "@kbn/data-view-field-editor-example-plugin": ["examples/data_view_field_editor_example"],

x-pack/.i18nrc.json

@@ -31,6 +31,7 @@
     "xpack.dashboard": "plugins/dashboard_enhanced",
     "xpack.dataQuality": "plugins/data_quality",
     "xpack.datasetQuality": "plugins/observability_solution/dataset_quality",
+    "xpack.dataUsage": "plugins/data_usage",
     "xpack.discover": "plugins/discover_enhanced",
     "xpack.crossClusterReplication": "plugins/cross_cluster_replication",
     "xpack.elasticAssistant": "packages/kbn-elastic-assistant",

x-pack/plugins/data_usage/README.md

@@ -0,0 +1,3 @@
+# dataUsage
+Serverless only plugin for users to view data usage
+

x-pack/plugins/data_usage/common/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export const PLUGIN_ID = 'data_usage';
+export const PLUGIN_NAME = i18n.translate('xpack.dataUsage.name', {
+  defaultMessage: 'Data Usage',
+});

x-pack/plugins/data_usage/jest.config.js

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/plugins/data_usage'],
+  coverageDirectory: '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/data_usage',
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: ['<rootDir>/x-pack/plugins/datas_usage/{common,public}/**/*.{ts,tsx}'],
+};

x-pack/plugins/data_usage/kibana.jsonc

@@ -0,0 +1,16 @@
+{
+  "type": "plugin",
+  "id": "@kbn/data-usage-plugin",
+  "owner": ["@elastic/obs-ai-assistant", "@elastic/security-solution"],
+  "plugin": {
+    "id": "dataUsage",
+    "server": true,
+    "browser": true,
+    "configPath": ["xpack", "dataUsage"],
+    "requiredPlugins": ["home", "management", "features", "share"],
+    "optionalPlugins": [],
+    "requiredBundles": [
+      "kibanaReact",
+    ],
+  }
+}

x-pack/plugins/data_usage/public/application.tsx

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreStart } from '@kbn/core/public';
+import { ManagementAppMountParams } from '@kbn/management-plugin/public';
+import React from 'react';
+import ReactDOM from 'react-dom';
+import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
+import { Route, Router, Routes } from '@kbn/shared-ux-router';
+import { useExecutionContext } from '@kbn/kibana-react-plugin/public';
+import { PerformanceContextProvider } from '@kbn/ebt-tools';
+import { useKibanaContextForPluginProvider } from './utils/use_kibana';
+import { DataUsageStartDependencies, DataUsagePublicStart } from './types';
+import { PLUGIN_ID } from '../common';
+
+export const renderApp = (
+  core: CoreStart,
+  plugins: DataUsageStartDependencies,
+  pluginStart: DataUsagePublicStart,
+  params: ManagementAppMountParams
+) => {
+  ReactDOM.render(
+    <App params={params} core={core} plugins={plugins} pluginStart={pluginStart} />,
+    params.element
+  );
+
+  return () => {
+    ReactDOM.unmountComponentAtNode(params.element);
+  };
+};
+
+const AppWithExecutionContext = ({
+  core,
+  params,
+}: {
+  core: CoreStart;
+  params: ManagementAppMountParams;
+}) => {
+  const { executionContext } = core;
+
+  useExecutionContext(executionContext, {
+    type: 'application',
+    page: PLUGIN_ID,
+  });
+
+  return (
+    <Router history={params.history}>
+      <PerformanceContextProvider>
+        <Routes>
+          <Route path="/" exact={true} render={() => <div>Data Usage</div>} />
+        </Routes>
+      </PerformanceContextProvider>
+    </Router>
+  );
+};
+
+interface AppProps {
+  core: CoreStart;
+  plugins: DataUsageStartDependencies;
+  pluginStart: DataUsagePublicStart;
+  params: ManagementAppMountParams;
+}
+
+const App = ({ core, plugins, pluginStart, params }: AppProps) => {
+  const KibanaContextProviderForPlugin = useKibanaContextForPluginProvider(
+    core,
+    plugins,
+    pluginStart,
+    params
+  );
+
+  return (
+    <KibanaRenderContextProvider {...core} {...params}>
+      <KibanaContextProviderForPlugin>
+        <AppWithExecutionContext core={core} params={params} />
+      </KibanaContextProviderForPlugin>
+    </KibanaRenderContextProvider>
+  );
+};

x-pack/plugins/data_usage/public/index.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public';
+import type {
+  DataUsagePublicSetup,
+  DataUsagePublicStart,
+  DataUsageSetupDependencies,
+  DataUsageStartDependencies,
+  ConfigSchema,
+} from './types';
+import { DataUsagePlugin } from './plugin';
+
+export type { DataUsagePublicSetup, DataUsagePublicStart } from './types';
+
+export const plugin: PluginInitializer<
+  DataUsagePublicSetup,
+  DataUsagePublicStart,
+  DataUsageSetupDependencies,
+  DataUsageStartDependencies
+> = (pluginInitializerContext: PluginInitializerContext<ConfigSchema>) => new DataUsagePlugin();

x-pack/plugins/data_usage/public/plugin.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreSetup, CoreStart, Plugin } from '@kbn/core/public';
+import { ManagementAppMountParams } from '@kbn/management-plugin/public';
+import {
+  DataUsagePublicSetup,
+  DataUsagePublicStart,
+  DataUsageStartDependencies,
+  DataUsageSetupDependencies,
+} from './types';
+import { PLUGIN_ID, PLUGIN_NAME } from '../common';
+
+export class DataUsagePlugin
+  implements
+    Plugin<
+      DataUsagePublicSetup,
+      DataUsagePublicStart,
+      DataUsageSetupDependencies,
+      DataUsageStartDependencies
+    >
+{
+  public setup(
+    core: CoreSetup<DataUsageStartDependencies, DataUsagePublicStart>,
+    plugins: DataUsageSetupDependencies
+  ): DataUsagePublicSetup {
+    const { management } = plugins;
+    management.sections.section.data.registerApp({
+      id: PLUGIN_ID,
+      title: PLUGIN_NAME,
+      order: 6,
+      keywords: ['data usage', 'usage'],
+      async mount(params: ManagementAppMountParams) {
+        const [{ renderApp }, [coreStart, pluginsStartDeps, pluginStart]] = await Promise.all([
+          import('./application'),
+          core.getStartServices(),
+        ]);
+
+        return renderApp(coreStart, pluginsStartDeps, pluginStart, params);
+      },
+    });
+
+    return {};
+  }
+
+  public start(_core: CoreStart): DataUsagePublicStart {
+    return {};
+  }
+
+  public stop() {}
+}

x-pack/plugins/data_usage/public/types.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ManagementSetup, ManagementStart } from '@kbn/management-plugin/public';
+import { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public';
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface DataUsagePublicSetup {}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface DataUsagePublicStart {}
+
+export interface DataUsageSetupDependencies {
+  management: ManagementSetup;
+  share: SharePluginSetup;
+}
+
+export interface DataUsageStartDependencies {
+  management: ManagementStart;
+  share: SharePluginStart;
+}
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface ConfigSchema {}

x-pack/plugins/data_usage/public/utils/use_kibana.tsx

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreStart } from '@kbn/core/public';
+import {
+  createKibanaReactContext,
+  KibanaReactContextValue,
+  useKibana,
+} from '@kbn/kibana-react-plugin/public';
+import { ManagementAppMountParams } from '@kbn/management-plugin/public';
+import { useMemo } from 'react';
+import { DataUsageStartDependencies, DataUsagePublicStart } from '../types';
+
+export type PluginKibanaContextValue = CoreStart &
+  DataUsageStartDependencies &
+  DataUsagePublicStart & {
+    appParams: ManagementAppMountParams;
+  };
+
+export const createKibanaContextForPlugin = (
+  core: CoreStart,
+  plugins: DataUsageStartDependencies,
+  pluginStart: DataUsagePublicStart,
+  appParams: ManagementAppMountParams
+) => {
+  return createKibanaReactContext<PluginKibanaContextValue>({
+    ...core,
+    ...plugins,
+    ...pluginStart,
+    appParams,
+  });
+};
+
+export const useKibanaContextForPlugin =
+  useKibana as () => KibanaReactContextValue<PluginKibanaContextValue>;
+
+export const useKibanaContextForPluginProvider = (
+  core: CoreStart,
+  plugins: DataUsageStartDependencies,
+  pluginStart: DataUsagePublicStart,
+  appParams: ManagementAppMountParams
+) => {
+  const { Provider } = useMemo(
+    () => createKibanaContextForPlugin(core, plugins, pluginStart, appParams),
+    [appParams, core, pluginStart, plugins]
+  );
+
+  return Provider;
+};

x-pack/plugins/data_usage/server/config.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema, type TypeOf } from '@kbn/config-schema';
+
+export const config = schema.object({
+  enabled: schema.boolean({ defaultValue: false }),
+});
+
+export type DataUsageConfig = TypeOf<typeof config>;

x-pack/plugins/data_usage/server/index.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type {
+  PluginInitializer,
+  PluginInitializerContext,
+  PluginConfigDescriptor,
+} from '@kbn/core/server';
+import { DataUsageConfig } from './config';
+
+import { DataUsagePlugin } from './plugin';
+import type {
+  DataUsageServerSetup,
+  DataUsageServerStart,
+  DataUsageSetupDependencies,
+  DataUsageStartDependencies,
+} from './types';
+
+import { config as configSchema } from './config';
+
+export type { DataUsageServerSetup, DataUsageServerStart };
+
+export const config: PluginConfigDescriptor<DataUsageConfig> = {
+  schema: configSchema,
+};
+
+export const plugin: PluginInitializer<
+  DataUsageServerSetup,
+  DataUsageServerStart,
+  DataUsageSetupDependencies,
+  DataUsageStartDependencies
+> = async (pluginInitializerContext: PluginInitializerContext<DataUsageConfig>) =>
+  await new DataUsagePlugin(pluginInitializerContext);

x-pack/plugins/data_usage/server/plugin.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server';
+import type { Logger } from '@kbn/logging';
+import { DataUsageConfig } from './config';
+import type {
+  DataUsageServerSetup,
+  DataUsageServerStart,
+  DataUsageSetupDependencies,
+  DataUsageStartDependencies,
+} from './types';
+
+export class DataUsagePlugin
+  implements
+    Plugin<
+      DataUsageServerSetup,
+      DataUsageServerStart,
+      DataUsageSetupDependencies,
+      DataUsageStartDependencies
+    >
+{
+  logger: Logger;
+  constructor(context: PluginInitializerContext<DataUsageConfig>) {
+    this.logger = context.logger.get();
+  }
+  setup(coreSetup: CoreSetup, pluginsSetup: DataUsageSetupDependencies): DataUsageServerSetup {
+    return {};
+  }
+
+  start(coreStart: CoreStart, pluginsStart: DataUsageStartDependencies): DataUsageServerStart {
+    return {};
+  }
+
+  public stop() {}
+}

x-pack/plugins/data_usage/server/types.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable @typescript-eslint/no-empty-interface*/
+
+export interface DataUsageSetupDependencies {}
+
+export interface DataUsageStartDependencies {}
+
+export interface DataUsageServerSetup {}
+
+export interface DataUsageServerStart {}

x-pack/plugins/data_usage/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types"
+  },
+  "include": [
+    "index.ts",
+    "common/**/*",
+    "public/**/*",
+    "server/**/*",
+    "../../../typings/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/core",
+    "@kbn/i18n",
+    "@kbn/kibana-react-plugin",
+    "@kbn/management-plugin",
+    "@kbn/react-kibana-context-render",
+    "@kbn/shared-ux-router",
+    "@kbn/ebt-tools",
+    "@kbn/share-plugin",
+    "@kbn/config-schema",
+    "@kbn/logging",
+  ],
+  "exclude": ["target/**/*"]
+}

yarn.lock

@@ -4675,6 +4675,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/data-usage-plugin@link:x-pack/plugins/data_usage":
+  version "0.0.0"
+  uid ""
+
 "@kbn/data-view-editor-plugin@link:src/plugins/data_view_editor":
   version "0.0.0"
   uid ""