[8.x] [Data Usage] functional tests (#203166) (#204019)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Data Usage] functional tests
(#203166)](https://github.com/elastic/kibana/pull/203166)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Sandra
G","email":"neptunian@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-12-12T13:08:08Z","message":"[Data
Usage] functional tests (#203166)\n\n## Summary\r\n\r\nFunctional tests
for data usage UI.\r\n\r\n- `data_streams` route is intercepted, due to
filtering out zero size\r\ndata streams which will happen because
metering api needs time to\r\naggregate data\r\n- `autoops_api` is using
the mock server as there will be no data for it\r\nto return\r\n- tests
will only run in local serverless and not MKI due to using
the\r\nautoops mock server that won't return data for created data
streams\r\n- adds `interceptRequest` functionality to FTR `browser`
service\r\n\r\n## Tests\r\n- data stream filter dropdown renders with
created data streams of\r\n`data_streams` response and are checked\r\n-
data stream filter dropdown renders badge with correct number
of\r\nselected data streams\r\n- charts render from `data_streams` route
response\r\n- chart legends render with correct items\r\n- popover
renders for legend items\r\n- links in popovers correctly navigate and
update navigation between\r\ndifferent data stream
items","sha":"ba945c98510dc6de460b297beaa55e980456fc30","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor"],"title":"[Data
Usage] functional
tests","number":203166,"url":"https://github.com/elastic/kibana/pull/203166","mergeCommit":{"message":"[Data
Usage] functional tests (#203166)\n\n## Summary\r\n\r\nFunctional tests
for data usage UI.\r\n\r\n- `data_streams` route is intercepted, due to
filtering out zero size\r\ndata streams which will happen because
metering api needs time to\r\naggregate data\r\n- `autoops_api` is using
the mock server as there will be no data for it\r\nto return\r\n- tests
will only run in local serverless and not MKI due to using
the\r\nautoops mock server that won't return data for created data
streams\r\n- adds `interceptRequest` functionality to FTR `browser`
service\r\n\r\n## Tests\r\n- data stream filter dropdown renders with
created data streams of\r\n`data_streams` response and are checked\r\n-
data stream filter dropdown renders badge with correct number
of\r\nselected data streams\r\n- charts render from `data_streams` route
response\r\n- chart legends render with correct items\r\n- popover
renders for legend items\r\n- links in popovers correctly navigate and
update navigation between\r\ndifferent data stream
items","sha":"ba945c98510dc6de460b297beaa55e980456fc30"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/203166","number":203166,"mergeCommit":{"message":"[Data
Usage] functional tests (#203166)\n\n## Summary\r\n\r\nFunctional tests
for data usage UI.\r\n\r\n- `data_streams` route is intercepted, due to
filtering out zero size\r\ndata streams which will happen because
metering api needs time to\r\naggregate data\r\n- `autoops_api` is using
the mock server as there will be no data for it\r\nto return\r\n- tests
will only run in local serverless and not MKI due to using
the\r\nautoops mock server that won't return data for created data
streams\r\n- adds `interceptRequest` functionality to FTR `browser`
service\r\n\r\n## Tests\r\n- data stream filter dropdown renders with
created data streams of\r\n`data_streams` response and are checked\r\n-
data stream filter dropdown renders badge with correct number
of\r\nselected data streams\r\n- charts render from `data_streams` route
response\r\n- chart legends render with correct items\r\n- popover
renders for legend items\r\n- links in popovers correctly navigate and
update navigation between\r\ndifferent data stream
items","sha":"ba945c98510dc6de460b297beaa55e980456fc30"}}]}] BACKPORT-->

Co-authored-by: Sandra G <neptunian@users.noreply.github.com>

.github/CODEOWNERS

@@ -1560,6 +1560,7 @@ x-pack/test/api_integration/apis/management/index_management/inference_endpoints
 /x-pack/test/functional/es_archives/auditbeat/default @elastic/security-solution
 /x-pack/test/functional/es_archives/auditbeat/hosts @elastic/security-solution
 /x-pack/test_serverless/functional/page_objects/svl_management_page.ts @elastic/security-solution
+/x-pack/test_serverless/functional/page_objects/svl_data_usage.ts @elastic/security-solution @elastic/obs-ai-assistant
 /x-pack/test_serverless/api_integration/test_suites/security @elastic/security-solution
 
 /x-pack/test_serverless/functional/test_suites/security/cypress @elastic/security-solution

packages/kbn-ftr-common-functional-ui-services/index.ts

@@ -15,7 +15,7 @@ export type {
   CustomCheerioStatic,
 } from './services/web_element_wrapper/custom_cheerio_api';
 export { Browsers } from './services/remote/browsers';
-export { type Browser } from './services/browser';
+export { type Browser, type InterceptResponseFactory } from './services/browser';
 export {
   NETWORK_PROFILES,
   type NetworkOptions,

packages/kbn-ftr-common-functional-ui-services/services/browser.ts

@@ -13,6 +13,7 @@ import { Key, Origin, type WebDriver } from 'selenium-webdriver';
 import { Driver as ChromiumWebDriver } from 'selenium-webdriver/chrome';
 import { setTimeout as setTimeoutAsync } from 'timers/promises';
 import Url from 'url';
+import { Protocol } from 'devtools-protocol';
 
 import { NoSuchSessionError } from 'selenium-webdriver/lib/error';
 import sharp from 'sharp';
@@ -26,7 +27,12 @@ import {
 import { FtrService, type FtrProviderContext } from './ftr_provider_context';
 
 export type Browser = BrowserService;
-
+export interface InterceptResponseFactory {
+  fail: () => ['Fetch.failRequest', Protocol.Fetch.FailRequestRequest];
+  fulfill: (
+    responseOptions: Omit<Protocol.Fetch.FulfillRequestRequest, 'requestId'>
+  ) => ['Fetch.fulfillRequest', Protocol.Fetch.FulfillRequestRequest];
+}
 class BrowserService extends FtrService {
   /**
    * Keyboard events
@@ -837,6 +843,59 @@ class BrowserService extends FtrService {
       throw new Error(message);
     }
   }
+
+  /**
+   * Intercept network requests using the Chrome DevTools Protocol (CDP).
+   * @param pattern - URL pattern to match intercepted requests.
+   * @param onIntercept - Callback defining how to handle intercepted requests.
+   * @param cb - Callback to trigger actions that make requests.
+   */
+
+  public async interceptRequest(
+    pattern: string,
+    onIntercept: (responseFactory: InterceptResponseFactory) => [string, Record<string, any>],
+    cb: () => Promise<void>
+  ): Promise<void> {
+    const connection = await this.driver.createCDPConnection('page');
+
+    return new Promise<void>((resolve, reject) => {
+      connection._wsConnection.on('message', async (data: Buffer) => {
+        try {
+          const parsed = JSON.parse(data.toString());
+          this.log.debug(`CDP Event: ${parsed.method} ${parsed.params?.request?.url}`);
+
+          if (parsed.method === 'Fetch.requestPaused') {
+            const requestId = parsed.params.requestId;
+
+            const [method, params] = onIntercept({
+              fail: () => ['Fetch.failRequest', { requestId, errorReason: 'Failed' }],
+              fulfill: (responseOptions) => [
+                'Fetch.fulfillRequest',
+                { requestId, ...responseOptions },
+              ],
+            });
+
+            connection.execute(method, params, () => {
+              this.log.debug(`Executed command: ${method}`);
+            });
+          }
+        } catch (error) {
+          this.log.error(`Error in Fetch.requestPaused handler: ${error.message}`);
+        }
+      });
+
+      connection.execute('Fetch.enable', { patterns: [{ urlPattern: pattern }] }, (result: any) => {
+        this.log.debug('Fetch.enable result:', result);
+
+        cb()
+          .then(resolve)
+          .catch((error) => {
+            this.log.error(`Error in callback: ${error.message}`);
+            reject(error);
+          });
+      });
+    });
+  }
 }
 
 export async function BrowserProvider(ctx: FtrProviderContext) {

x-pack/plugins/data_usage/public/app/components/chart_panel.tsx

@@ -91,7 +91,7 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
 
   return (
     <EuiFlexItem grow={false} key={metricType}>
-      <EuiPanel hasShadow={false} hasBorder={true}>
+      <EuiPanel hasShadow={false} hasBorder={true} data-test-subj={`${metricType}-chart`}>
         <EuiTitle size="xs">
           <h5>{chartKeyToTitleMap[metricType as ChartKey] || metricType}</h5>
         </EuiTitle>

x-pack/plugins/data_usage/public/app/components/dataset_quality_link.tsx

@@ -41,7 +41,11 @@ export const DatasetQualityLink: React.FC<DatasetQualityLinkProps> = React.memo(
       }
     };
     return (
-      <LegendActionItem label={UX_LABELS.dataQualityPopup.view} onClick={onClickDataQuality} />
+      <LegendActionItem
+        label={UX_LABELS.dataQualityPopup.view}
+        onClick={onClickDataQuality}
+        dataTestSubj="DatasetQualityAction"
+      />
     );
   }
 );

x-pack/plugins/data_usage/public/app/components/legend_action.tsx

@@ -52,6 +52,7 @@ export const LegendAction: React.FC<LegendActionProps> = React.memo(
     return (
       <EuiFlexGroup gutterSize="s" alignItems="center">
         <EuiPopover
+          data-test-subj="legendActionPopover"
           button={
             <EuiFlexGroup gutterSize="s" alignItems="center">
               <EuiFlexItem grow={false}>
@@ -59,6 +60,7 @@ export const LegendAction: React.FC<LegendActionProps> = React.memo(
                   iconType="boxesHorizontal"
                   aria-label={UX_LABELS.dataQualityPopup.open}
                   onClick={() => togglePopover(uniqueStreamName)}
+                  data-test-subj="legendActionButton"
                 />
               </EuiFlexItem>
             </EuiFlexGroup>
@@ -71,11 +73,13 @@ export const LegendAction: React.FC<LegendActionProps> = React.memo(
             <LegendActionItem
               label={UX_LABELS.dataQualityPopup.copy}
               onClick={onCopyDataStreamName}
+              dataTestSubj="copyDataStreamNameAction"
             />
             {hasIndexManagementFeature && (
               <LegendActionItem
                 label={UX_LABELS.dataQualityPopup.manage}
                 onClick={onClickIndexManagement}
+                dataTestSubj="manageDataStreamAction"
               />
             )}
             {hasDataSetQualityFeature && <DatasetQualityLink dataStreamName={label} />}

x-pack/plugins/data_usage/public/app/components/legend_action_item.tsx

@@ -9,9 +9,15 @@ import React, { memo } from 'react';
 import { EuiListGroupItem } from '@elastic/eui';
 
 export const LegendActionItem = memo(
-  ({ label, onClick }: { label: string; onClick: () => Promise<void> | void }) => (
-    <EuiListGroupItem label={label} onClick={onClick} size="s" />
-  )
+  ({
+    label,
+    onClick,
+    dataTestSubj,
+  }: {
+    label: string;
+    onClick: () => Promise<void> | void;
+    dataTestSubj: string;
+  }) => <EuiListGroupItem label={label} onClick={onClick} data-test-subj={dataTestSubj} size="s" />
 );
 
 LegendActionItem.displayName = 'LegendActionItem';

x-pack/test_serverless/api_integration/test_suites/common/data_usage/mock_data.ts

@@ -15,6 +15,14 @@ export const mockAutoOpsResponse = {
         [1726862130000, 14657904],
       ],
     },
+    {
+      name: 'metrics-system.core.total.pct-default',
+      error: null,
+      data: [
+        [1726858530000, 13756849],
+        [1726862130000, 14657904],
+      ],
+    },
     {
       name: 'logs-nginx.access-default',
       error: null,
@@ -33,6 +41,14 @@ export const mockAutoOpsResponse = {
         [1726862130000, 13956423],
       ],
     },
+    {
+      name: 'metrics-system.core.total.pct-default',
+      error: null,
+      data: [
+        [1726858530000, 13756849],
+        [1726862130000, 14657904],
+      ],
+    },
     {
       name: 'logs-nginx.access-default',
       error: null,

x-pack/test_serverless/api_integration/test_suites/common/data_usage/tests/metrics.ts

@@ -29,7 +29,7 @@ export default function ({ getService }: FtrProviderContext) {
   const mockAutoopsApiService = setupMockServer();
   describe('Metrics', function () {
     let mockApiServer: http.Server;
-    // due to the plugin depending on yml config (xpack.dataUsage.enabled), we cannot test in MKI until it is on by default
+    // MKI has a different config in the QA environment and will ignore the mock service
     this.tags(['skipMKI']);
 
     before(async () => {

x-pack/test_serverless/functional/page_objects/index.ts

@@ -26,6 +26,7 @@ import { SvlSearchElasticsearchStartPageProvider } from './svl_search_elasticsea
 import { SvlApiKeysProvider } from './svl_api_keys';
 import { SvlSearchCreateIndexPageProvider } from './svl_search_create_index_page';
 import { SvlSearchInferenceManagementPageProvider } from './svl_search_inference_management_page';
+import { SvlDataUsagePageProvider } from './svl_data_usage';
 
 export const pageObjects = {
   ...xpackFunctionalPageObjects,
@@ -49,4 +50,5 @@ export const pageObjects = {
   svlApiKeys: SvlApiKeysProvider,
   svlSearchCreateIndexPage: SvlSearchCreateIndexPageProvider,
   svlSearchInferenceManagementPage: SvlSearchInferenceManagementPageProvider,
+  svlDataUsagePage: SvlDataUsagePageProvider,
 };

x-pack/test_serverless/functional/page_objects/svl_data_usage.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { WebElementWrapper } from '@kbn/ftr-common-functional-ui-services';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export function SvlDataUsagePageProvider({ getService }: FtrProviderContext) {
+  const testSubjects = getService('testSubjects');
+
+  return {
+    async assertDataUsagePageExists(): Promise<boolean> {
+      return await testSubjects.exists('DataUsagePage');
+    },
+    async clickDatastreamsDropdown() {
+      await testSubjects.click('data-usage-metrics-filter-dataStreams-popoverButton');
+    },
+    async findDatastreamsDropdownOptions() {
+      return await testSubjects.findAll('dataStreams-filter-option');
+    },
+    async findDatastreamsDropdownFilterButton() {
+      return await testSubjects.find('data-usage-metrics-filter-dataStreams-popoverButton');
+    },
+    async findIngestRateChart() {
+      return await testSubjects.find('ingest_rate-chart');
+    },
+    async storageRetainedChart() {
+      return await testSubjects.find('storage_retained-chart');
+    },
+    async findLegendItemsInChart(chartElement: WebElementWrapper) {
+      return await chartElement.findAllByCssSelector('li.echLegendItem');
+    },
+    async findLegendActionButton(legendItemElement: WebElementWrapper) {
+      return legendItemElement.findByTestSubject('legendActionButton');
+    },
+    async clickLegendActionButtonAtIndex(chartElement: WebElementWrapper, index: number) {
+      const legendItems = await this.findLegendItemsInChart(chartElement);
+      if (index < 0 || index >= legendItems.length) {
+        throw new Error(
+          `Invalid legend item index: ${index}. There are only ${legendItems.length} legend items.`
+        );
+      }
+      const legendItem = legendItems[index];
+      const actionButton = await this.findLegendActionButton(legendItem);
+      await actionButton.click();
+    },
+
+    async assertLegendActionPopoverExists() {
+      await testSubjects.existOrFail('legendActionPopover');
+    },
+  };
+}

x-pack/test_serverless/functional/test_suites/common/data_usage/main.ts

@@ -4,30 +4,200 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import expect from '@kbn/expect';
+import http from 'http';
+import { InterceptResponseFactory } from '@kbn/ftr-common-functional-ui-services';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import { setupMockServer } from '../../../../api_integration/test_suites/common/data_usage/mock_api';
 
 export default ({ getPageObjects, getService }: FtrProviderContext) => {
-  const pageObjects = getPageObjects(['svlCommonPage', 'svlManagementPage', 'common']);
+  const pageObjects = getPageObjects([
+    'svlDataUsagePage',
+    'svlCommonPage',
+    'svlManagementPage',
+    'common',
+  ]);
   const testSubjects = getService('testSubjects');
   const retry = getService('retry');
+  const mockAutoopsApiService = setupMockServer();
+  const es = getService('es');
+  const browser = getService('browser');
+  let mockApiServer: http.Server;
 
+  const dataStreamsMockResponse = [
+    {
+      name: 'metrics-system.cpu-default',
+      storageSizeBytes: 6197,
+    },
+    {
+      name: 'metrics-system.core.total.pct-default',
+      storageSizeBytes: 5197,
+    },
+    {
+      name: 'logs-nginx.access-default',
+      storageSizeBytes: 1938,
+    },
+  ];
   describe('Main page', function () {
     this.tags(['skipMKI']);
     before(async () => {
+      // create test data streams from the mock data streams response
+      // so the metrics api can verify they exist
+      for (const { name } of dataStreamsMockResponse) {
+        await es.indices.putIndexTemplate({
+          name,
+          body: {
+            index_patterns: [name],
+            data_stream: {},
+            priority: 200,
+          },
+        });
+        await es.indices.createDataStream({ name });
+      }
       await pageObjects.svlCommonPage.loginAsAdmin();
       await pageObjects.common.navigateToApp('management');
       await retry.waitFor('page to be visible', async () => {
         return await testSubjects.exists('cards-navigation-page');
       });
       await pageObjects.svlManagementPage.assertDataUsageManagementCardExists();
-      await pageObjects.svlManagementPage.clickDataUsageManagementCard();
+
+      // mock external API request to autoops
+      mockApiServer = mockAutoopsApiService.listen(9000);
+
+      // intercept the data_streams request to bypass waiting for the metering api to aggregate a response
+      // otherwise storage sizes get filtered out if they are 0
+      await browser.interceptRequest(
+        '*data_streams*',
+        (responseFactory: InterceptResponseFactory) => {
+          return responseFactory.fulfill({
+            responseCode: 200,
+            responseHeaders: [{ name: 'Content-Type', value: 'application/json' }],
+            body: Buffer.from(JSON.stringify(dataStreamsMockResponse)).toString('base64'),
+          });
+        },
+        async () => {
+          await pageObjects.svlManagementPage.clickDataUsageManagementCard();
+        }
+      );
+    });
+    after(async () => {
+      mockApiServer.close();
+      for (const { name } of dataStreamsMockResponse) {
+        await es.indices.deleteDataStream({ name });
+      }
     });
 
     it('renders data usage page', async () => {
       await retry.waitFor('page to be visible', async () => {
-        return await testSubjects.exists('DataUsagePage');
+        return await pageObjects.svlDataUsagePage.assertDataUsagePageExists();
       });
     });
+    it('shows 3 data streams in the filter dropdown', async () => {
+      // Click the dropdown button to show the options
+      await pageObjects.svlDataUsagePage.clickDatastreamsDropdown();
+
+      const options = await pageObjects.svlDataUsagePage.findDatastreamsDropdownOptions();
+
+      // Assert that exactly 3 elements are present
+      expect(options.length).to.eql(3);
+
+      // Assert that each option is checked
+      for (const option of options) {
+        const ariaChecked = await option.getAttribute('aria-checked');
+        expect(ariaChecked).to.be('true');
+      }
+
+      // Locate the filter button using its data-test-subj
+      const datastreamsDropdownFilterButton =
+        await pageObjects.svlDataUsagePage.findDatastreamsDropdownFilterButton();
+
+      // Find the badge element within the button (using its CSS class)
+      const notificationBadge = await datastreamsDropdownFilterButton.findByCssSelector(
+        '.euiNotificationBadge'
+      );
+
+      // Retrieve the text content of the badge
+      const activeFiltersCount = await notificationBadge.getVisibleText();
+
+      // Assert the badge displays the expected count
+      expect(activeFiltersCount).to.be('3');
+    });
+    it('renders charts', async () => {
+      // Data is coming from the mocked autoops API
+      const chartContainer = await testSubjects.find('data-usage-metrics');
+      await testSubjects.existOrFail('data-usage-metrics');
+
+      // Check 2 charts rendered
+      await retry.waitFor('chart to render', async () => {
+        const chartStatus = await chartContainer.findAllByCssSelector(
+          '.echChartStatus[data-ech-render-complete="true"]'
+        );
+        return chartStatus.length === 2;
+      });
+    });
+    it('renders legend', async () => {
+      const ingestRateChart = await pageObjects.svlDataUsagePage.findIngestRateChart();
+      const storageRetainedChart = await pageObjects.svlDataUsagePage.storageRetainedChart();
+
+      const ingestLegendItems = await pageObjects.svlDataUsagePage.findLegendItemsInChart(
+        ingestRateChart
+      );
+
+      expect(ingestLegendItems.length).to.eql(4); // 3 data streams + 1 Total line series
+
+      const storageLegendItems = await pageObjects.svlDataUsagePage.findLegendItemsInChart(
+        storageRetainedChart
+      );
+      expect(storageLegendItems.length).to.eql(4); // same number of data streams + total line series
+    });
+    it('renders actions popover with correct links', async () => {
+      // Open the first legend item actions popover
+      const ingestRateChart = await pageObjects.svlDataUsagePage.findIngestRateChart();
+      await pageObjects.svlDataUsagePage.clickLegendActionButtonAtIndex(ingestRateChart, 0);
+      await pageObjects.svlDataUsagePage.assertLegendActionPopoverExists();
+      // Check for links
+      await testSubjects.existOrFail('copyDataStreamNameAction');
+      await testSubjects.existOrFail('manageDataStreamAction');
+      await testSubjects.existOrFail('DatasetQualityAction');
+
+      const manageLink = await testSubjects.find('manageDataStreamAction');
+      await manageLink.click();
+
+      // Wait for navigation to the data stream details page
+      await retry.waitFor('URL to update (index management)', async () => {
+        const currentUrl = await browser.getCurrentUrl();
+        return currentUrl.includes(
+          `/app/management/data/index_management/data_streams/${dataStreamsMockResponse[0].name}`
+        );
+      });
+      await browser.goBack();
+      // test second link to ensure state changed
+      await pageObjects.svlDataUsagePage.clickLegendActionButtonAtIndex(ingestRateChart, 1);
+      await pageObjects.svlDataUsagePage.assertLegendActionPopoverExists();
+
+      await manageLink.click();
+
+      // Wait for navigation to the data stream details page
+      await retry.waitFor('URL to update (index management)', async () => {
+        const currentUrl = await browser.getCurrentUrl();
+        return currentUrl.includes(
+          `/app/management/data/index_management/data_streams/${dataStreamsMockResponse[1].name}`
+        );
+      });
+      await browser.goBack();
+
+      // Test navigation for the data quality link
+      await pageObjects.svlDataUsagePage.clickLegendActionButtonAtIndex(ingestRateChart, 0);
+      await pageObjects.svlDataUsagePage.assertLegendActionPopoverExists();
+      const dataQualityLink = await testSubjects.find('DatasetQualityAction');
+      await dataQualityLink.click();
+
+      // Wait for navigation to the data quality details page
+      await retry.waitFor('URL to update (data quality)', async () => {
+        const currentUrl = await browser.getCurrentUrl();
+        return currentUrl.includes('/app/management/data/data_quality/details');
+      });
+      await browser.goBack();
+    });
   });
 };