[Dataset Quality] Introduce Kibana Management Feature (#194825)

closes [#3874](https://github.com/elastic/observability-dev/issues/3874)


## 📝  Summary

This PR adds new kibana privilege feature to control access to `Data Set
Quality` page under Stack Management's `Data` section.

Had to fix a lot of tests since the `kibana_admin` role gets access by
default to all kibana features one of which now is the `Data Set
Quality` page. At the same time this made the `Data` section visible to
any user with `kibana_admin` role.

## 🎥 Demo



https://github.com/user-attachments/assets/ce8c8110-f6f4-44b8-a4e7-5f2dd3deda66

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-es/src/serverless_resources/project_roles/security/roles.yml

@@ -55,6 +55,7 @@ viewer:
         - feature_dashboard.all
         - feature_maps.all
         - feature_visualize.all
+        - feature_dataQuality.all
       resources: '*'
   run_as: []
 

x-pack/plugins/data_quality/common/index.ts

@@ -8,6 +8,7 @@
 import { i18n } from '@kbn/i18n';
 
 export const PLUGIN_ID = 'data_quality';
+export const PLUGIN_FEATURE_ID = 'dataQuality';
 export const PLUGIN_NAME = i18n.translate('xpack.dataQuality.name', {
   defaultMessage: 'Data Set Quality',
 });

x-pack/plugins/data_quality/public/plugin.ts

@@ -5,10 +5,11 @@
  * 2.0.
  */
 
-import { CoreSetup, CoreStart, Plugin } from '@kbn/core/public';
+import { Capabilities, CoreSetup, CoreStart, Plugin } from '@kbn/core/public';
 import { ManagementAppMountParams } from '@kbn/management-plugin/public';
 import { MANAGEMENT_APP_LOCATOR } from '@kbn/deeplinks-management/constants';
 import { ManagementAppLocatorParams } from '@kbn/management-plugin/common/locator';
+import { Subject } from 'rxjs';
 import {
   DataQualityPluginSetup,
   DataQualityPluginStart,
@@ -30,6 +31,8 @@ export class DataQualityPlugin
       AppPluginStartDependencies
     >
 {
+  private capabilities$ = new Subject<Capabilities>();
+
   public setup(
     core: CoreSetup<AppPluginStartDependencies, DataQualityPluginStart>,
     plugins: AppPluginSetupDependencies
@@ -37,51 +40,56 @@ export class DataQualityPlugin
     const { management, share } = plugins;
     const useHash = core.uiSettings.get('state:storeInSessionStorage');
 
-    management.sections.section.data.registerApp({
-      id: PLUGIN_ID,
-      title: PLUGIN_NAME,
-      order: 2,
-      keywords: [
-        'data',
-        'quality',
-        'data quality',
-        'datasets',
-        'datasets quality',
-        'data set quality',
-      ],
-      async mount(params: ManagementAppMountParams) {
-        const [{ renderApp }, [coreStart, pluginsStartDeps, pluginStart]] = await Promise.all([
-          import('./application'),
-          core.getStartServices(),
-        ]);
+    this.capabilities$.subscribe((capabilities) => {
+      if (!capabilities.dataQuality.show) return;
 
-        return renderApp(coreStart, pluginsStartDeps, pluginStart, params);
-      },
-      hideFromSidebar: false,
+      management.sections.section.data.registerApp({
+        id: PLUGIN_ID,
+        title: PLUGIN_NAME,
+        order: 2,
+        keywords: [
+          'data',
+          'quality',
+          'data quality',
+          'datasets',
+          'datasets quality',
+          'data set quality',
+        ],
+        async mount(params: ManagementAppMountParams) {
+          const [{ renderApp }, [coreStart, pluginsStartDeps, pluginStart]] = await Promise.all([
+            import('./application'),
+            core.getStartServices(),
+          ]);
+
+          return renderApp(coreStart, pluginsStartDeps, pluginStart, params);
+        },
+        hideFromSidebar: false,
+      });
+
+      const managementLocator =
+        share.url.locators.get<ManagementAppLocatorParams>(MANAGEMENT_APP_LOCATOR);
+
+      if (managementLocator) {
+        share.url.locators.create(
+          new DatasetQualityLocatorDefinition({
+            useHash,
+            managementLocator,
+          })
+        );
+        share.url.locators.create(
+          new DatasetQualityDetailsLocatorDefinition({
+            useHash,
+            managementLocator,
+          })
+        );
+      }
     });
 
-    const managementLocator =
-      share.url.locators.get<ManagementAppLocatorParams>(MANAGEMENT_APP_LOCATOR);
-
-    if (managementLocator) {
-      share.url.locators.create(
-        new DatasetQualityLocatorDefinition({
-          useHash,
-          managementLocator,
-        })
-      );
-      share.url.locators.create(
-        new DatasetQualityDetailsLocatorDefinition({
-          useHash,
-          managementLocator,
-        })
-      );
-    }
-
     return {};
   }
 
-  public start(_core: CoreStart): DataQualityPluginStart {
+  public start(core: CoreStart): DataQualityPluginStart {
+    this.capabilities$.next(core.application.capabilities);
     return {};
   }
 

x-pack/plugins/data_quality/server/features.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DEFAULT_APP_CATEGORIES } from '@kbn/core-application-common';
+import {
+  KibanaFeatureConfig,
+  KibanaFeatureScope,
+  ElasticsearchFeatureConfig,
+} from '@kbn/features-plugin/common';
+import { PLUGIN_FEATURE_ID, PLUGIN_ID, PLUGIN_NAME } from '../common';
+
+export const KIBANA_FEATURE: KibanaFeatureConfig = {
+  id: PLUGIN_FEATURE_ID,
+  name: PLUGIN_NAME,
+  category: DEFAULT_APP_CATEGORIES.management,
+  scope: [KibanaFeatureScope.Spaces, KibanaFeatureScope.Security],
+  app: [PLUGIN_ID],
+  privileges: {
+    all: {
+      app: [PLUGIN_ID],
+      savedObject: {
+        all: [],
+        read: [],
+      },
+      ui: ['show'],
+    },
+    read: {
+      disabled: true,
+      savedObject: {
+        all: [],
+        read: [],
+      },
+      ui: ['show'],
+    },
+  },
+};
+
+export const ELASTICSEARCH_FEATURE: ElasticsearchFeatureConfig = {
+  id: PLUGIN_ID,
+  management: {
+    data: [PLUGIN_ID],
+  },
+  privileges: [
+    {
+      ui: [],
+      requiredClusterPrivileges: [],
+      requiredIndexPrivileges: {
+        ['logs-*-*']: ['read'],
+      },
+    },
+    {
+      ui: [],
+      requiredClusterPrivileges: [],
+      requiredIndexPrivileges: {
+        ['traces-*-*']: ['read'],
+      },
+    },
+    {
+      ui: [],
+      requiredClusterPrivileges: [],
+      requiredIndexPrivileges: {
+        ['metrics-*-*']: ['read'],
+      },
+    },
+    {
+      ui: [],
+      requiredClusterPrivileges: [],
+      requiredIndexPrivileges: {
+        ['synthetics-*-*']: ['read'],
+      },
+    },
+  ],
+};

x-pack/plugins/data_quality/server/plugin.ts

@@ -6,48 +6,14 @@
  */
 
 import { CoreSetup, Plugin } from '@kbn/core/server';
-import { PLUGIN_ID } from '../common';
 
 import { Dependencies } from './types';
+import { ELASTICSEARCH_FEATURE, KIBANA_FEATURE } from './features';
 
 export class DataQualityPlugin implements Plugin<void, void, any, any> {
-  public setup(coreSetup: CoreSetup, { features }: Dependencies) {
-    features.registerElasticsearchFeature({
-      id: PLUGIN_ID,
-      management: {
-        data: [PLUGIN_ID],
-      },
-      privileges: [
-        {
-          ui: [],
-          requiredClusterPrivileges: [],
-          requiredIndexPrivileges: {
-            ['logs-*-*']: ['read'],
-          },
-        },
-        {
-          ui: [],
-          requiredClusterPrivileges: [],
-          requiredIndexPrivileges: {
-            ['traces-*-*']: ['read'],
-          },
-        },
-        {
-          ui: [],
-          requiredClusterPrivileges: [],
-          requiredIndexPrivileges: {
-            ['metrics-*-*']: ['read'],
-          },
-        },
-        {
-          ui: [],
-          requiredClusterPrivileges: [],
-          requiredIndexPrivileges: {
-            ['synthetics-*-*']: ['read'],
-          },
-        },
-      ],
-    });
+  public setup(_coreSetup: CoreSetup, { features }: Dependencies) {
+    features.registerKibanaFeature(KIBANA_FEATURE);
+    features.registerElasticsearchFeature(ELASTICSEARCH_FEATURE);
   }
 
   public start() {}

x-pack/plugins/data_quality/tsconfig.json

@@ -28,6 +28,7 @@
     "@kbn/deeplinks-management",
     "@kbn/deeplinks-observability",
     "@kbn/ebt-tools",
+    "@kbn/core-application-common",
   ],
   "exclude": ["target/**/*"]
 }

x-pack/test/api_integration/apis/features/features/features.ts

@@ -98,6 +98,7 @@ export default function ({ getService }: FtrProviderContext) {
             'discover',
             'visualize',
             'dashboard',
+            'dataQuality',
             'dev_tools',
             'actions',
             'enterpriseSearch',
@@ -147,6 +148,7 @@ export default function ({ getService }: FtrProviderContext) {
           'discover',
           'visualize',
           'dashboard',
+          'dataQuality',
           'dev_tools',
           'actions',
           'enterpriseSearch',

x-pack/test/api_integration/apis/security/privileges.ts

@@ -90,6 +90,7 @@ export default function ({ getService }: FtrProviderContext) {
       ],
       infrastructure: ['all', 'read', 'minimal_all', 'minimal_read'],
       logs: ['all', 'read', 'minimal_all', 'minimal_read'],
+      dataQuality: ['all', 'read', 'minimal_all', 'minimal_read'],
       apm: ['all', 'read', 'minimal_all', 'minimal_read'],
       discover: [
         'all',

x-pack/test/api_integration/apis/security/privileges_basic.ts

@@ -59,6 +59,7 @@ export default function ({ getService }: FtrProviderContext) {
             guidedOnboardingFeature: ['all', 'read', 'minimal_all', 'minimal_read'],
             aiAssistantManagementSelection: ['all', 'read', 'minimal_all', 'minimal_read'],
             inventory: ['all', 'read', 'minimal_all', 'minimal_read'],
+            dataQuality: ['all', 'read', 'minimal_all', 'minimal_read'],
           },
           global: ['all', 'read'],
           space: ['all', 'read'],
@@ -177,6 +178,7 @@ export default function ({ getService }: FtrProviderContext) {
             ],
             infrastructure: ['all', 'read', 'minimal_all', 'minimal_read'],
             logs: ['all', 'read', 'minimal_all', 'minimal_read'],
+            dataQuality: ['all', 'read', 'minimal_all', 'minimal_read'],
             apm: ['all', 'read', 'minimal_all', 'minimal_read'],
             discover: [
               'all',

x-pack/test/functional/apps/api_keys/feature_controls/api_keys_security.ts

@@ -35,8 +35,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
 
       it('should not render the "Security" section', async () => {
         await PageObjects.common.navigateToApp('management');
-        const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-        expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+        const sections = await managementMenu.getSections();
+
+        const sectionIds = sections.map((section) => section.sectionId);
+        expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+        const dataSection = sections.find((section) => section.sectionId === 'data');
+        expect(dataSection?.sectionLinks).to.eql(['data_quality']);
       });
     });
 

x-pack/test/functional/apps/cross_cluster_replication/feature_controls/ccr_security.ts

@@ -40,10 +40,15 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
 
       describe('"Data" section', function () {
         this.tags('skipFIPS');
-        it('should not render', async () => {
+        it('should render only data_quality section', async () => {
           await PageObjects.common.navigateToApp('management');
-          const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-          expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+          const sections = await managementMenu.getSections();
+
+          const sectionIds = sections.map((section) => section.sectionId);
+          expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+          const dataSection = sections.find((section) => section.sectionId === 'data');
+          expect(dataSection?.sectionLinks).to.eql(['data_quality']);
         });
       });
     });

x-pack/test/functional/apps/dataset_quality/dataset_quality_privileges.ts

@@ -41,7 +41,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
         // Index logs for synth-* and apache.access datasets
         await synthtrace.index(getInitialTestLogs({ to, count: 4 }));
 
-        await createDatasetQualityUserWithRole(security, 'dataset_quality_no_read', []);
+        await createDatasetQualityUserWithRole(security, 'dataset_quality_no_read', [], false);
 
         // Logout in order to re-login with a different user
         await PageObjects.security.forceLogout();
@@ -197,7 +197,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
 async function createDatasetQualityUserWithRole(
   security: ReturnType<DatasetQualityFtrProviderContext['getService']>,
   username: string,
-  indices: Array<{ names: string[]; privileges: string[] }>
+  indices: Array<{ names: string[]; privileges: string[] }>,
+  hasDataQualityPrivileges = true
 ) {
   const role = `${username}-role`;
   const password = `${username}-password`;
@@ -211,6 +212,7 @@ async function createDatasetQualityUserWithRole(
     kibana: [
       {
         feature: {
+          dataQuality: [hasDataQualityPrivileges ? 'all' : 'none'],
           discover: ['all'],
           fleet: ['none'],
         },

x-pack/test/functional/apps/index_lifecycle_management/feature_controls/ilm_security.ts

@@ -40,10 +40,15 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
 
       describe('"Data" section', function () {
         this.tags('skipFIPS');
-        it('should not render', async () => {
+        it('should render only data_quality section', async () => {
           await PageObjects.common.navigateToApp('management');
-          const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-          expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+          const sections = await managementMenu.getSections();
+
+          const sectionIds = sections.map((section) => section.sectionId);
+          expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+          const dataSection = sections.find((section) => section.sectionId === 'data');
+          expect(dataSection?.sectionLinks).to.eql(['data_quality']);
         });
       });
     });

x-pack/test/functional/apps/index_management/feature_controls/index_management_security.ts

@@ -40,10 +40,15 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
 
       describe('"Data" section', function () {
         this.tags('skipFIPS');
-        it('should not render', async () => {
+        it('should render only data_quality section', async () => {
           await PageObjects.common.navigateToApp('management');
-          const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-          expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+          const sections = await managementMenu.getSections();
+
+          const sectionIds = sections.map((section) => section.sectionId);
+          expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+          const dataSection = sections.find((section) => section.sectionId === 'data');
+          expect(dataSection?.sectionLinks).to.eql(['data_quality']);
         });
       });
     });
@@ -71,7 +76,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
           expect(sections).to.have.length(2);
           expect(sections[0]).to.eql({
             sectionId: 'data',
-            sectionLinks: ['index_management', 'data_quality', 'transform'],
+            sectionLinks: ['index_management', 'transform'],
           });
         });
       });

x-pack/test/functional/apps/ingest_pipelines/feature_controls/ingest_pipelines_security.ts

@@ -43,8 +43,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         this.tags('skipFIPS');
         it('should not render', async () => {
           await PageObjects.common.navigateToApp('management');
-          const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-          expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+          const sections = await managementMenu.getSections();
+
+          const sectionIds = sections.map((section) => section.sectionId);
+          expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+          const dataSection = sections.find((section) => section.sectionId === 'data');
+          expect(dataSection?.sectionLinks).to.eql(['data_quality']);
         });
       });
     });

x-pack/test/functional/apps/license_management/feature_controls/license_management_security.ts

@@ -42,8 +42,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         this.tags('skipFIPS');
         it('should not render', async () => {
           await PageObjects.common.navigateToApp('management');
-          const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-          expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+          const sections = await managementMenu.getSections();
+
+          const sectionIds = sections.map((section) => section.sectionId);
+          expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+          const dataSection = sections.find((section) => section.sectionId === 'data');
+          expect(dataSection?.sectionLinks).to.eql(['data_quality']);
         });
       });
     });

x-pack/test/functional/apps/logstash/feature_controls/logstash_security.ts

@@ -42,8 +42,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         this.tags('skipFIPS');
         it('should not render', async function () {
           await PageObjects.common.navigateToApp('management');
-          const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-          expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+          const sections = await managementMenu.getSections();
+
+          const sectionIds = sections.map((section) => section.sectionId);
+          expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+          const dataSection = sections.find((section) => section.sectionId === 'data');
+          expect(dataSection?.sectionLinks).to.eql(['data_quality']);
         });
       });
     });

x-pack/test/functional/apps/management/feature_controls/management_security.ts

@@ -63,8 +63,9 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       it('should only render management entries controllable via Kibana privileges', async () => {
         await PageObjects.common.navigateToApp('management');
         const sections = await managementMenu.getSections();
-        expect(sections).to.have.length(2);
-        expect(sections[0]).to.eql({
+        expect(sections).to.have.length(3);
+        expect(sections[0]).to.eql({ sectionId: 'data', sectionLinks: ['data_quality'] });
+        expect(sections[1]).to.eql({
           sectionId: 'insightsAndAlerting',
           sectionLinks: [
             'triggersActionsAlerts',
@@ -75,7 +76,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
             'maintenanceWindows',
           ],
         });
-        expect(sections[1]).to.eql({
+        expect(sections[2]).to.eql({
           sectionId: 'kibana',
           sectionLinks: [
             'dataViews',

x-pack/test/functional/apps/remote_clusters/feature_controls/remote_clusters_security.ts

@@ -42,8 +42,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         this.tags('skipFIPS');
         it('should not render', async () => {
           await PageObjects.common.navigateToApp('management');
-          const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-          expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+          const sections = await managementMenu.getSections();
+
+          const sectionIds = sections.map((section) => section.sectionId);
+          expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+          const dataSection = sections.find((section) => section.sectionId === 'data');
+          expect(dataSection?.sectionLinks).to.eql(['data_quality']);
         });
       });
     });

x-pack/test/functional/apps/transform/feature_controls/transform_security.ts

@@ -44,8 +44,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
 
         it('should not render', async () => {
           await pageObjects.common.navigateToApp('management');
-          const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-          expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+          const sections = await managementMenu.getSections();
+
+          const sectionIds = sections.map((section) => section.sectionId);
+          expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+          const dataSection = sections.find((section) => section.sectionId === 'data');
+          expect(dataSection?.sectionLinks).to.eql(['data_quality']);
         });
       });
     });

x-pack/test/functional/apps/upgrade_assistant/feature_controls/upgrade_assistant_security.ts

@@ -36,8 +36,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
 
       it('should not render the "Stack" section', async () => {
         await PageObjects.common.navigateToApp('management');
-        const sections = (await managementMenu.getSections()).map((section) => section.sectionId);
-        expect(sections).to.eql(['insightsAndAlerting', 'kibana']);
+        const sections = await managementMenu.getSections();
+
+        const sectionIds = sections.map((section) => section.sectionId);
+        expect(sectionIds).to.eql(['data', 'insightsAndAlerting', 'kibana']);
+
+        const dataSection = sections.find((section) => section.sectionId === 'data');
+        expect(dataSection?.sectionLinks).to.eql(['data_quality']);
       });
     });
 

x-pack/test/spaces_api_integration/spaces_only/telemetry/telemetry.ts

@@ -96,6 +96,7 @@ export default function ({ getService }: FtrProviderContext) {
         filesSharedImage: 0,
         savedObjectsManagement: 1,
         savedQueryManagement: 0,
+        dataQuality: 0,
       });
     });
 

x-pack/test/ui_capabilities/security_and_spaces/tests/nav_links.ts

@@ -67,7 +67,8 @@ export default function navLinksTests({ getService }: FtrProviderContext) {
                 'searchInferenceEndpoints',
                 'guidedOnboardingFeature',
                 'securitySolutionAssistant',
-                'securitySolutionAttackDiscovery'
+                'securitySolutionAttackDiscovery',
+                'dataQuality'
               )
             );
             break;