[8.x] [Data Usage] Enabled plugin for Serverless and added feature flag to manage availability (#201465) (#201498)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Data Usage] Enabled plugin for Serverless and added feature flag to
manage availability
(#201465)](https://github.com/elastic/kibana/pull/201465)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Yuliia
Naumenko","email":"jo.naumenko@gmail.com"},"sourceCommit":{"committedDate":"2024-11-23T12:06:23Z","message":"[Data
Usage] Enabled plugin for Serverless and added feature flag to manage
availability (#201465)\n\nThis PR enables data_usage plugin for
Serverless environment for all 3\r\nsolutions.\r\nTo manage feature
availability added feature flag, which is turning Data\r\nUsage off by
default.","sha":"5342f327ee62219c5cf12d50e424da9b91330d5e","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:all-open","ci:build-serverless-image","v8.18.0"],"title":"[Data
Usage] Enabled plugin for Serverless and added feature flag to manage
availability","number":201465,"url":"https://github.com/elastic/kibana/pull/201465","mergeCommit":{"message":"[Data
Usage] Enabled plugin for Serverless and added feature flag to manage
availability (#201465)\n\nThis PR enables data_usage plugin for
Serverless environment for all 3\r\nsolutions.\r\nTo manage feature
availability added feature flag, which is turning Data\r\nUsage off by
default.","sha":"5342f327ee62219c5cf12d50e424da9b91330d5e"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/201465","number":201465,"mergeCommit":{"message":"[Data
Usage] Enabled plugin for Serverless and added feature flag to manage
availability (#201465)\n\nThis PR enables data_usage plugin for
Serverless environment for all 3\r\nsolutions.\r\nTo manage feature
availability added feature flag, which is turning Data\r\nUsage off by
default.","sha":"5342f327ee62219c5cf12d50e424da9b91330d5e"}},{"branch":"8.x","label":"v8.18.0","branchLabelMappingKey":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Yuliia Naumenko <jo.naumenko@gmail.com>

config/serverless.yml

@@ -229,3 +229,8 @@ monitoring.ui.enabled: false
 xpack.securitySolution.enableUiSettingsValidations: true
 data.enableUiSettingsValidations: true
 discover.enableUiSettingsValidations: true
+
+## Data Usage in stack management
+xpack.dataUsage.enabled: true
+# This feature is disabled in Serverless until fully tested within a Serverless environment
+xpack.dataUsage.enableExperimental: ['dataUsageDisabled']

x-pack/plugins/data_usage/common/experimental_features.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export type ServerlessExperimentalFeatures = Record<
+  keyof typeof allowedExperimentalValues,
+  boolean
+>;
+
+/**
+ * A list of allowed values that can be used in `xpack.dataUsage.enableExperimental`.
+ * This object is then used to validate and parse the value entered.
+ */
+export const allowedExperimentalValues = Object.freeze({
+  /**
+   * <Add a description of the feature here>
+   *
+   * [This is a fake feature key to showcase how to add a new serverless-specific experimental flag.
+   * It also prevents `allowedExperimentalValues` from being empty. It should be removed once a real feature is added.]
+   */
+  dataUsageDisabled: false,
+});
+
+type ServerlessExperimentalConfigKeys = Array<keyof ServerlessExperimentalFeatures>;
+type Mutable<T> = { -readonly [P in keyof T]: T[P] };
+
+const allowedKeys = Object.keys(
+  allowedExperimentalValues
+) as Readonly<ServerlessExperimentalConfigKeys>;
+
+export type ExperimentalFeatures = ServerlessExperimentalFeatures;
+/**
+ * Parses the string value used in `xpack.dataUsage.enableExperimental` kibana configuration,
+ * which should be a string of values delimited by a comma (`,`)
+ * The generic experimental features are merged with the serverless values to ensure they are available
+ *
+ * @param configValue
+ * @throws DataUsagenvalidExperimentalValue
+ */
+export const parseExperimentalConfigValue = (
+  configValue: string[]
+): { features: ExperimentalFeatures; invalid: string[] } => {
+  const enabledFeatures: Mutable<Partial<ExperimentalFeatures>> = {};
+  const invalidKeys: string[] = [];
+
+  for (const value of configValue) {
+    if (!allowedKeys.includes(value as keyof ServerlessExperimentalFeatures)) {
+      invalidKeys.push(value);
+    } else {
+      enabledFeatures[value as keyof ServerlessExperimentalFeatures] = true;
+    }
+  }
+
+  return {
+    features: {
+      ...allowedExperimentalValues,
+      ...enabledFeatures,
+    },
+    invalid: invalidKeys,
+  };
+};
+
+export const getExperimentalAllowedValues = (): string[] => [...allowedKeys];

x-pack/plugins/data_usage/public/index.ts

@@ -11,7 +11,6 @@ import type {
   DataUsagePublicStart,
   DataUsageSetupDependencies,
   DataUsageStartDependencies,
-  ConfigSchema,
 } from './types';
 import { DataUsagePlugin } from './plugin';
 
@@ -22,4 +21,5 @@ export const plugin: PluginInitializer<
   DataUsagePublicStart,
   DataUsageSetupDependencies,
   DataUsageStartDependencies
-> = (pluginInitializerContext: PluginInitializerContext<ConfigSchema>) => new DataUsagePlugin();
+> = (pluginInitializerContext: PluginInitializerContext) =>
+  new DataUsagePlugin(pluginInitializerContext);

x-pack/plugins/data_usage/public/plugin.ts

@@ -5,17 +5,21 @@
  * 2.0.
  */
 
-import { CoreSetup, CoreStart, Plugin } from '@kbn/core/public';
+import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public';
 import { ManagementAppMountParams } from '@kbn/management-plugin/public';
 import {
   DataUsagePublicSetup,
   DataUsagePublicStart,
   DataUsageStartDependencies,
   DataUsageSetupDependencies,
+  DataUsagePublicConfigType,
 } from './types';
 import { PLUGIN_ID } from '../common';
 import { PLUGIN_NAME } from './translations';
-
+import {
+  ExperimentalFeatures,
+  parseExperimentalConfigValue,
+} from '../common/experimental_features';
 export class DataUsagePlugin
   implements
     Plugin<
@@ -25,30 +29,46 @@ export class DataUsagePlugin
       DataUsageStartDependencies
     >
 {
+  private config: DataUsagePublicConfigType;
+  private experimentalFeatures: ExperimentalFeatures;
+
+  constructor(private readonly initializerContext: PluginInitializerContext) {
+    this.config = this.initializerContext.config.get<DataUsagePublicConfigType>();
+    this.experimentalFeatures = {} as ExperimentalFeatures;
+  }
+
   public setup(
     core: CoreSetup<DataUsageStartDependencies, DataUsagePublicStart>,
     plugins: DataUsageSetupDependencies
   ): DataUsagePublicSetup {
     const { management } = plugins;
-    management.sections.section.data.registerApp({
-      id: PLUGIN_ID,
-      title: PLUGIN_NAME,
-      order: 6,
-      keywords: ['data usage', 'usage'],
-      async mount(params: ManagementAppMountParams) {
-        const [{ renderApp }, [coreStart, pluginsStartDeps, pluginStart]] = await Promise.all([
-          import('./application'),
-          core.getStartServices(),
-        ]);
 
-        return renderApp(coreStart, pluginsStartDeps, pluginStart, params);
-      },
-    });
+    this.experimentalFeatures = parseExperimentalConfigValue(
+      this.config.enableExperimental
+    ).features;
 
+    const experimentalFeatures = this.experimentalFeatures;
+
+    if (!experimentalFeatures.dataUsageDisabled) {
+      management.sections.section.data.registerApp({
+        id: PLUGIN_ID,
+        title: PLUGIN_NAME,
+        order: 6,
+        keywords: ['data usage', 'usage'],
+        async mount(params: ManagementAppMountParams) {
+          const [{ renderApp }, [coreStart, pluginsStartDeps, pluginStart]] = await Promise.all([
+            import('./application'),
+            core.getStartServices(),
+          ]);
+
+          return renderApp(coreStart, pluginsStartDeps, pluginStart, params);
+        },
+      });
+    }
     return {};
   }
 
-  public start(_core: CoreStart): DataUsagePublicStart {
+  public start(_core: CoreStart, plugins: DataUsageStartDependencies): DataUsagePublicStart {
     return {};
   }
 

x-pack/plugins/data_usage/public/types.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { schema, TypeOf } from '@kbn/config-schema';
 import { ManagementSetup, ManagementStart } from '@kbn/management-plugin/public';
 import { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public';
 
@@ -23,5 +24,21 @@ export interface DataUsageStartDependencies {
   management: ManagementStart;
   share: SharePluginStart;
 }
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface ConfigSchema {}
+
+const schemaObject = schema.object({
+  /**
+   * For internal use. A list of string values (comma delimited) that will enable experimental
+   * type of functionality that is not yet released. Valid values for this settings need to
+   * be defined in:
+   * `x-pack/plugins/dataUsage/common/experimental_features.ts`
+   * under the `allowedExperimentalValues` object
+   *
+   * @example
+   * xpack.dataUsage.enableExperimental: ['someFeature']
+   */
+  enableExperimental: schema.arrayOf(schema.string(), {
+    defaultValue: () => [],
+  }),
+});
+
+export type DataUsagePublicConfigType = TypeOf<typeof schemaObject>;

x-pack/plugins/data_usage/server/config.ts

@@ -26,6 +26,19 @@ export const configSchema = schema.object({
       ),
     })
   ),
+  /**
+   * For internal use. A list of string values (comma delimited) that will enable experimental
+   * type of functionality that is not yet released. Valid values for this settings need to
+   * be defined in:
+   * `x-pack/plugins/dataUsage/common/experimental_features.ts`
+   * under the `allowedExperimentalValues` object
+   *
+   * @example
+   * xpack.dataUsage.enableExperimental: ['someFeature']
+   */
+  enableExperimental: schema.arrayOf(schema.string(), {
+    defaultValue: () => [],
+  }),
 });
 
 export type DataUsageConfigType = TypeOf<typeof configSchema>;

x-pack/plugins/data_usage/server/index.ts

@@ -25,6 +25,9 @@ export type { DataUsageServerSetup, DataUsageServerStart };
 
 export const config: PluginConfigDescriptor<DataUsageConfigType> = {
   schema: configSchema,
+  exposeToBrowser: {
+    enableExperimental: true,
+  },
 };
 
 export const plugin: PluginInitializer<

x-pack/test_serverless/api_integration/test_suites/observability/config.ts

@@ -27,6 +27,7 @@ export default createTestConfig({
     // useful for testing (also enabled in MKI QA)
     '--coreApp.allowDynamicConfigOverrides=true',
     '--xpack.dataUsage.enabled=true',
+    '--xpack.dataUsage.enableExperimental=[]',
     // dataUsage.autoops* config is set in kibana controller
     '--xpack.dataUsage.autoops.enabled=true',
     '--xpack.dataUsage.autoops.api.url=http://localhost:9000',

x-pack/test_serverless/api_integration/test_suites/search/config.ts

@@ -23,6 +23,7 @@ export default createTestConfig({
     // useful for testing (also enabled in MKI QA)
     '--coreApp.allowDynamicConfigOverrides=true',
     '--xpack.dataUsage.enabled=true',
+    '--xpack.dataUsage.enableExperimental=[]',
     // dataUsage.autoops* config is set in kibana controller
     '--xpack.dataUsage.autoops.enabled=true',
     '--xpack.dataUsage.autoops.api.url=http://localhost:9000',

x-pack/test_serverless/api_integration/test_suites/security/config.ts

@@ -27,6 +27,7 @@ export default createTestConfig({
     `--xpack.securitySolutionServerless.cloudSecurityUsageReportingTaskInterval=5s`,
     `--xpack.securitySolutionServerless.usageReportingApiUrl=http://localhost:8081/api/v1/usage`,
     '--xpack.dataUsage.enabled=true',
+    '--xpack.dataUsage.enableExperimental=[]',
     // dataUsage.autoops* config is set in kibana controller
     '--xpack.dataUsage.autoops.enabled=true',
     '--xpack.dataUsage.autoops.api.url=http://localhost:9000',

x-pack/test_serverless/functional/test_suites/observability/config.ts

@@ -20,6 +20,7 @@ export default createTestConfig({
   esServerArgs: ['xpack.ml.dfa.enabled=false'],
   kbnServerArgs: [
     '--xpack.dataUsage.enabled=true',
+    '--xpack.dataUsage.enableExperimental=[]',
     // dataUsage.autoops* config is set in kibana controller
     '--xpack.dataUsage.autoops.enabled=true',
     '--xpack.dataUsage.autoops.api.url=http://localhost:9000',

x-pack/test_serverless/functional/test_suites/search/config.ts

@@ -24,6 +24,7 @@ export default createTestConfig({
     `--xpack.cloud.base_url=https://fake-cloud.elastic.co`,
     `--xpack.cloud.projects_url=/projects/`,
     '--xpack.dataUsage.enabled=true',
+    '--xpack.dataUsage.enableExperimental=[]',
     // dataUsage.autoops* config is set in kibana controller
     '--xpack.dataUsage.autoops.enabled=true',
     '--xpack.dataUsage.autoops.api.url=http://localhost:9000',

x-pack/test_serverless/functional/test_suites/security/config.ts

@@ -20,6 +20,7 @@ export default createTestConfig({
   esServerArgs: ['xpack.ml.nlp.enabled=true'],
   kbnServerArgs: [
     '--xpack.dataUsage.enabled=true',
+    '--xpack.dataUsage.enableExperimental=[]',
     // dataUsage.autoops* config is set in kibana controller
     '--xpack.dataUsage.autoops.enabled=true',
     '--xpack.dataUsage.autoops.api.url=http://localhost:9000',