[Dataset Quality]Add logic to display spark plot on degraded fields (#184514)

## Summary

Closes: https://github.com/elastic/kibana/issues/183604

The PR adds a Spark Plot to the Degraded Fields Table in the Dataset
Quality Flyout

## Screenshot

<img width="818" alt="image"
src="45636e33-e6e8-4096-af2f-8cc42b1bd2e6">

x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts

@@ -82,6 +82,12 @@ export const degradedFieldRt = rt.type({
   name: rt.string,
   count: rt.number,
   lastOccurrence: rt.union([rt.null, rt.number]),
+  timeSeries: rt.array(
+    rt.type({
+      x: rt.number,
+      y: rt.number,
+    })
+  ),
 });
 
 export type DegradedField = rt.TypeOf<typeof degradedFieldRt>;

x-pack/plugins/observability_solution/dataset_quality/common/translations.ts

@@ -147,7 +147,7 @@ export const flyoutShowAllText = i18n.translate('xpack.datasetQuality.flyoutShow
 export const flyoutImprovementText = i18n.translate(
   'xpack.datasetQuality.flyoutDegradedFieldsSectionTitle',
   {
-    defaultMessage: 'Degraded Fields',
+    defaultMessage: 'Degraded fields',
   }
 );
 

x-pack/plugins/observability_solution/dataset_quality/common/types/common.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export type SortDirection = 'asc' | 'desc';
+
+export type Maybe<T> = T | null | undefined;
+
+export interface Coordinate {
+  x: number;
+  y: Maybe<number>;
+}

x-pack/plugins/observability_solution/dataset_quality/common/types/index.ts

@@ -7,5 +7,4 @@
 
 export * from './dataset_types';
 export * from './quality_types';
-
-export type SortDirection = 'asc' | 'desc';
+export * from './common';

x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx

@@ -5,31 +5,37 @@
  * 2.0.
  */
 
+import React from 'react';
+import { i18n } from '@kbn/i18n';
 import { EuiBasicTableColumn } from '@elastic/eui';
 import { FieldFormat } from '@kbn/field-formats-plugin/common';
-import { i18n } from '@kbn/i18n';
+import { formatNumber } from '@elastic/eui';
 
 import { DegradedField } from '../../../../common/api_types';
+import { SparkPlot } from './spark_plot';
+import { NUMBER_FORMAT } from '../../../../common/constants';
 
 const fieldColumnName = i18n.translate('xpack.datasetQuality.flyout.degradedField.field', {
   defaultMessage: 'Field',
 });
 
 const countColumnName = i18n.translate('xpack.datasetQuality.flyout.degradedField.count', {
-  defaultMessage: 'Count',
+  defaultMessage: 'Docs count',
 });
 
 const lastOccurrenceColumnName = i18n.translate(
   'xpack.datasetQuality.flyout.degradedField.lastOccurrence',
   {
-    defaultMessage: 'Last Occurrence',
+    defaultMessage: 'Last occurrence',
   }
 );
 
 export const getDegradedFieldsColumns = ({
   dateFormatter,
+  isLoading,
 }: {
   dateFormatter: FieldFormat;
+  isLoading: boolean;
 }): Array<EuiBasicTableColumn<DegradedField>> => [
   {
     name: fieldColumnName,
@@ -39,7 +45,10 @@ export const getDegradedFieldsColumns = ({
     name: countColumnName,
     sortable: true,
     field: 'count',
-    truncateText: true,
+    render: (_, { count, timeSeries }) => {
+      const countValue = formatNumber(count, NUMBER_FORMAT);
+      return <SparkPlot series={timeSeries} valueLabel={countValue} isLoading={isLoading} />;
+    },
   },
   {
     name: lastOccurrenceColumnName,

x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/spark_plot.tsx

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiIcon,
+  EuiLoadingChart,
+  euiPaletteColorBlind,
+  useEuiTheme,
+} from '@elastic/eui';
+import React from 'react';
+import { ScaleType, Settings, Tooltip, Chart, BarSeries } from '@elastic/charts';
+import { i18n } from '@kbn/i18n';
+import { Coordinate } from '../../../../common/types';
+
+export function SparkPlot({
+  valueLabel,
+  isLoading,
+  series,
+}: {
+  valueLabel: React.ReactNode;
+  isLoading: boolean;
+  series?: Coordinate[] | null;
+}) {
+  return (
+    <EuiFlexGroup justifyContent="flexStart" gutterSize="s" responsive={false} alignItems="center">
+      <EuiFlexItem grow={false}>
+        <SparkPlotItem isLoading={isLoading} series={series} />
+      </EuiFlexItem>
+      <EuiFlexItem grow={false}>{valueLabel}</EuiFlexItem>
+    </EuiFlexGroup>
+  );
+}
+
+function SparkPlotItem({
+  isLoading,
+  series,
+}: {
+  isLoading: boolean;
+  series?: Coordinate[] | null;
+}) {
+  const { euiTheme } = useEuiTheme();
+  const chartSize = {
+    height: euiTheme.size.l,
+    width: '80px',
+  };
+
+  const commonStyle = {
+    ...chartSize,
+    display: 'flex',
+    alignItems: 'center',
+    justifyContent: 'center',
+  };
+  const palette = euiPaletteColorBlind({ rotations: 2 });
+
+  if (isLoading) {
+    return (
+      <div style={commonStyle}>
+        <EuiLoadingChart mono />
+      </div>
+    );
+  }
+
+  if (hasValidTimeSeries(series)) {
+    return (
+      <div
+        style={{ backgroundColor: `${palette[0]}`, padding: 1, height: '100%' }}
+        data-test-subj="datasetQualitySparkPlot"
+      >
+        <Chart size={chartSize}>
+          <Settings showLegend={false} locale={i18n.getLocale()} />
+          <Tooltip type="none" />
+          <BarSeries
+            id="barseries"
+            xScaleType={ScaleType.Linear}
+            yScaleType={ScaleType.Linear}
+            xAccessor="x"
+            yAccessors={['y']}
+            data={series}
+            color={palette[1]}
+          />
+        </Chart>
+      </div>
+    );
+  }
+
+  return (
+    <div style={commonStyle}>
+      <EuiIcon type="visLine" color={euiTheme.colors.mediumShade} />
+    </div>
+  );
+}
+
+function hasValidTimeSeries(series?: Coordinate[] | null): series is Coordinate[] {
+  return !!series?.some((point) => point.y !== 0);
+}

x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx

@@ -21,7 +21,7 @@ export const DegradedFieldTable = () => {
   const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [
     ES_FIELD_TYPES.DATE,
   ]);
-  const columns = getDegradedFieldsColumns({ dateFormatter });
+  const columns = getDegradedFieldsColumns({ dateFormatter, isLoading });
 
   return (
     <EuiBasicTable

x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts

@@ -43,10 +43,6 @@ import {
   DefaultDatasetQualityControllerState,
   FlyoutDataset,
 } from './types';
-import {
-  DEFAULT_DEGRADED_FIELD_SORT_DIRECTION,
-  DEFAULT_DEGRADED_FIELD_SORT_FIELD,
-} from '../../../../common/constants';
 
 export const createPureDatasetQualityControllerStateMachine = (
   initialContext: DatasetQualityControllerContext
@@ -413,12 +409,9 @@ export const createPureDatasetQualityControllerStateMachine = (
             degradedFields: {
               ...context.flyout.degradedFields,
               table: {
+                ...context.flyout.degradedFields.table,
                 page: 0,
                 rowsPerPage: 10,
-                sort: {
-                  field: DEFAULT_DEGRADED_FIELD_SORT_FIELD,
-                  direction: DEFAULT_DEGRADED_FIELD_SORT_DIRECTION,
-                },
               },
             },
           },

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/get_interval.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment/moment';
+import { calculateAuto } from '@kbn/calculate-auto';
+
+export const getFieldIntervalInSeconds = ({
+  start,
+  end,
+  buckets = 10,
+  minIntervalSeconds = 60,
+}: {
+  start: number;
+  end: number;
+  buckets?: number;
+  minIntervalSeconds?: number;
+}) => {
+  const duration = moment.duration(end - start, 'ms');
+
+  return Math.max(calculateAuto.near(buckets, duration)?.asSeconds() ?? 0, minIntervalSeconds);
+};

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts

@@ -11,6 +11,7 @@ import { DegradedFieldResponse } from '../../../../common/api_types';
 import { MAX_DEGRADED_FIELDS } from '../../../../common/constants';
 import { createDatasetQualityESClient } from '../../../utils';
 import { _IGNORED, TIMESTAMP } from '../../../../common/es_fields';
+import { getFieldIntervalInSeconds } from './get_interval';
 
 export async function getDegradedFields({
   esClient,
@@ -23,6 +24,7 @@ export async function getDegradedFields({
   end: number;
   dataStream: string;
 }): Promise<DegradedFieldResponse> {
+  const fieldInterval = getFieldIntervalInSeconds({ start, end });
   const datasetQualityESClient = createDatasetQualityESClient(esClient);
 
   const filterQuery = [...rangeQuery(start, end)];
@@ -41,6 +43,17 @@ export async function getDegradedFields({
             field: TIMESTAMP,
           },
         },
+        timeSeries: {
+          date_histogram: {
+            field: TIMESTAMP,
+            fixed_interval: `${fieldInterval}s`,
+            min_doc_count: 0,
+            extended_bounds: {
+              min: start,
+              max: end,
+            },
+          },
+        },
       },
     },
   };
@@ -63,6 +76,10 @@ export async function getDegradedFields({
         name: bucket.key as string,
         count: bucket.doc_count,
         lastOccurrence: bucket.lastOccurrence.value,
+        timeSeries: bucket.timeSeries.buckets.map((timeSeriesBucket) => ({
+          x: timeSeriesBucket.key,
+          y: timeSeriesBucket.doc_count,
+        })),
       })) ?? [],
   };
 }

x-pack/plugins/observability_solution/dataset_quality/tsconfig.json

@@ -3,7 +3,12 @@
   "compilerOptions": {
     "outDir": "target/types"
   },
-  "include": ["common/**/*", "public/**/*", "server/**/*", "../../../../typings/**/*"],
+  "include": [
+    "common/**/*",
+    "public/**/*",
+    "server/**/*",
+    "../../../../typings/**/*"
+  ],
   "kbn_references": [
     "@kbn/core",
     "@kbn/core-plugins-server",
@@ -42,8 +47,11 @@
     "@kbn/core-elasticsearch-server",
     "@kbn/ui-actions-plugin",
     "@kbn/metrics-data-access-plugin",
+    "@kbn/calculate-auto",
     "@kbn/discover-plugin",
     "@kbn/shared-ux-prompt-no-data-views-types"
   ],
-  "exclude": ["target/**/*"]
+  "exclude": [
+    "target/**/*"
+  ]
 }

x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts

@@ -102,6 +102,31 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         expect(resp.body.degradedFields.length).to.be(2);
         expect(degradedFields).to.eql(expectedDegradedFields);
       });
+
+      it('returns proper timeSeries data for degraded fields', async () => {
+        const logsTimeSeriesData = [
+          { x: 1716357600000, y: 60 },
+          { x: 1716368400000, y: 180 },
+          { x: 1716379200000, y: 180 },
+          { x: 1716390000000, y: 180 },
+          { x: 1716400800000, y: 180 },
+          { x: 1716411600000, y: 180 },
+          { x: 1716422400000, y: 180 },
+          { x: 1716433200000, y: 180 },
+          { x: 1716444000000, y: 122 },
+        ];
+
+        const resp = await callApiAs(
+          'datasetQualityLogsUser',
+          `${type}-${degradedFieldDataset}-${namespace}`
+        );
+
+        const logLevelTimeSeries = resp.body.degradedFields.find(
+          (dFields) => dFields.name === 'log.level'
+        )?.timeSeries;
+
+        expect(logLevelTimeSeries).to.eql(logsTimeSeriesData);
+      });
     });
   });
 }

x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts

@@ -458,94 +458,135 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
       const goodDatasetName = 'good';
       const degradedDatasetName = 'degraded';
       const today = new Date().toISOString();
-      before(async () => {
-        await synthtrace.index([
-          getLogsForDataset({
-            to: today,
-            count: 2,
-            dataset: goodDatasetName,
-            isMalformed: false,
-          }),
-          createDegradedFieldsRecord({
-            to: today,
-            count: 2,
-            dataset: degradedDatasetName,
-          }),
-        ]);
-        await PageObjects.datasetQuality.navigateTo();
+
+      describe('Degraded Fields Table with common data', () => {
+        before(async () => {
+          await synthtrace.index([
+            getLogsForDataset({
+              to: today,
+              count: 2,
+              dataset: goodDatasetName,
+              isMalformed: false,
+            }),
+            createDegradedFieldsRecord({
+              to: today,
+              count: 2,
+              dataset: degradedDatasetName,
+            }),
+          ]);
+          await PageObjects.datasetQuality.navigateTo();
+        });
+
+        after(async () => {
+          await synthtrace.clean();
+        });
+
+        it('shows the degraded fields table with no data when no degraded fields are present', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(goodDatasetName);
+
+          await testSubjects.existOrFail(
+            PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedTableNoData
+          );
+
+          await PageObjects.datasetQuality.closeFlyout();
+        });
+
+        it('should load the degraded fields table with data', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+
+          await testSubjects.existOrFail(
+            PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedFieldTable
+          );
+
+          const rows =
+            await PageObjects.datasetQuality.getDatasetQualityFlyoutDegradedFieldTableRows();
+
+          expect(rows.length).to.eql(2);
+
+          await PageObjects.datasetQuality.closeFlyout();
+        });
+
+        it('should display Spark Plot for every row of degraded fields', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+
+          const rows =
+            await PageObjects.datasetQuality.getDatasetQualityFlyoutDegradedFieldTableRows();
+
+          const sparkPlots = await testSubjects.findAll(
+            PageObjects.datasetQuality.testSubjectSelectors.datasetQualitySparkPlot
+          );
+
+          expect(rows.length).to.be(sparkPlots.length);
+
+          await PageObjects.datasetQuality.closeFlyout();
+        });
+
+        it('should sort the table when the count table header is clicked', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+
+          const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
+
+          const countColumn = table['Docs count'];
+          const cellTexts = await countColumn.getCellTexts();
+
+          await countColumn.sort('ascending');
+          const sortedCellTexts = await countColumn.getCellTexts();
+
+          expect(cellTexts.reverse()).to.eql(sortedCellTexts);
+
+          await PageObjects.datasetQuality.closeFlyout();
+        });
       });
 
-      after(async () => {
-        await synthtrace.clean();
-      });
+      describe('Degraded Fields Table with data ingestion', () => {
+        before(async () => {
+          await synthtrace.index([
+            getLogsForDataset({
+              to: today,
+              count: 2,
+              dataset: goodDatasetName,
+              isMalformed: false,
+            }),
+            createDegradedFieldsRecord({
+              to: today,
+              count: 2,
+              dataset: degradedDatasetName,
+            }),
+          ]);
+          await PageObjects.datasetQuality.navigateTo();
+        });
 
-      it('shows the degraded fields table with no data when no degraded fields are present', async () => {
-        await PageObjects.datasetQuality.openDatasetFlyout(goodDatasetName);
+        after(async () => {
+          await synthtrace.clean();
+        });
 
-        await testSubjects.existOrFail(
-          PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedTableNoData
-        );
+        it('should update the table when new data is ingested and the flyout is refreshed using the time selector', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
 
-        await PageObjects.datasetQuality.closeFlyout();
-      });
+          const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
 
-      it('should load the degraded fields table with data', async () => {
-        await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+          const countColumn = table['Docs count'];
+          const cellTexts = await countColumn.getCellTexts();
 
-        await testSubjects.existOrFail(
-          PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedFieldTable
-        );
+          await synthtrace.index([
+            createDegradedFieldsRecord({
+              to: today,
+              count: 2,
+              dataset: degradedDatasetName,
+            }),
+          ]);
 
-        const rows =
-          await PageObjects.datasetQuality.getDatasetQualityFlyoutDegradedFieldTableRows();
+          await PageObjects.datasetQuality.refreshFlyout();
 
-        expect(rows.length).to.eql(2);
+          const updatedCellTexts = await countColumn.getCellTexts();
 
-        await PageObjects.datasetQuality.closeFlyout();
-      });
+          const singleValuePreviously = parseInt(cellTexts[0], 10);
+          const singleValueNow = parseInt(updatedCellTexts[0], 10);
 
-      it('should sort the table when the count table header is clicked', async () => {
-        await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+          expect(singleValueNow).to.be(singleValuePreviously * 2);
 
-        const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
-
-        const countColumn = table.Count;
-        const cellTexts = await countColumn.getCellTexts();
-
-        await countColumn.sort('ascending');
-        const sortedCellTexts = await countColumn.getCellTexts();
-
-        expect(cellTexts.reverse()).to.eql(sortedCellTexts);
-
-        await PageObjects.datasetQuality.closeFlyout();
-      });
-
-      it('should update the table when new data is ingested and the flyout is refreshed using the time selector', async () => {
-        await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
-
-        const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
-
-        const countColumn = table.Count;
-        const cellTexts = await countColumn.getCellTexts();
-
-        await synthtrace.index([
-          createDegradedFieldsRecord({
-            to: today,
-            count: 2,
-            dataset: degradedDatasetName,
-          }),
-        ]);
-
-        await PageObjects.datasetQuality.refreshFlyout();
-
-        const updatedCellTexts = await countColumn.getCellTexts();
-
-        const singleValuePreviously = parseInt(cellTexts[0], 10);
-        const singleValueNow = parseInt(updatedCellTexts[0], 10);
-
-        expect(singleValueNow).to.be(singleValuePreviously * 2);
-
-        await PageObjects.datasetQuality.closeFlyout();
+          await PageObjects.datasetQuality.closeFlyout();
+        });
       });
     });
   });

x-pack/test/functional/page_objects/dataset_quality.ts

@@ -77,6 +77,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
     datasetQualityFlyoutTitle: 'datasetQualityFlyoutTitle',
     datasetQualityFlyoutDegradedFieldTable: 'datasetQualityFlyoutDegradedFieldTable',
     datasetQualityFlyoutDegradedTableNoData: 'datasetQualityFlyoutDegradedTableNoData',
+    datasetQualitySparkPlot: 'datasetQualitySparkPlot',
     datasetQualityHeaderButton: 'datasetQualityHeaderButton',
     datasetQualityFlyoutFieldValue: 'datasetQualityFlyoutFieldValue',
     datasetQualityFlyoutIntegrationActionsButton: 'datasetQualityFlyoutIntegrationActionsButton',
@@ -226,7 +227,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
 
     async parseDegradedFieldTable() {
       const table = await this.getDatasetQualityFlyoutDegradedFieldTable();
-      return parseDatasetTable(table, ['Field', 'Count', 'Last Occurrence']);
+      return parseDatasetTable(table, ['Field', 'Docs count', 'Last Occurrence']);
     },
 
     async filterForIntegrations(integrations: string[]) {

x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts

@@ -479,95 +479,134 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       const goodDatasetName = 'good';
       const degradedDatasetName = 'degraded';
       const today = new Date().toISOString();
-      before(async () => {
-        await PageObjects.svlCommonPage.loginWithRole('admin');
-        await synthtrace.index([
-          getLogsForDataset({
-            to: today,
-            count: 2,
-            dataset: goodDatasetName,
-            isMalformed: false,
-          }),
-          createDegradedFieldsRecord({
-            to: today,
-            count: 2,
-            dataset: degradedDatasetName,
-          }),
-        ]);
-        await PageObjects.datasetQuality.navigateTo();
+      describe('Degraded Fields Table with common data', () => {
+        before(async () => {
+          await PageObjects.svlCommonPage.loginWithRole('admin');
+          await synthtrace.index([
+            getLogsForDataset({
+              to: today,
+              count: 2,
+              dataset: goodDatasetName,
+              isMalformed: false,
+            }),
+            createDegradedFieldsRecord({
+              to: today,
+              count: 2,
+              dataset: degradedDatasetName,
+            }),
+          ]);
+          await PageObjects.datasetQuality.navigateTo();
+        });
+
+        after(async () => {
+          await synthtrace.clean();
+        });
+        it('shows the degraded fields table with no data when no degraded fields are present', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(goodDatasetName);
+
+          await testSubjects.existOrFail(
+            PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedTableNoData
+          );
+
+          await PageObjects.datasetQuality.closeFlyout();
+        });
+
+        it('should load the degraded fields table with data', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+
+          await testSubjects.existOrFail(
+            PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedFieldTable
+          );
+
+          const rows =
+            await PageObjects.datasetQuality.getDatasetQualityFlyoutDegradedFieldTableRows();
+
+          expect(rows.length).to.eql(2);
+
+          await PageObjects.datasetQuality.closeFlyout();
+        });
+
+        it('should display Spark Plot for every row of degraded fields', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+
+          const rows =
+            await PageObjects.datasetQuality.getDatasetQualityFlyoutDegradedFieldTableRows();
+
+          const sparkPlots = await testSubjects.findAll(
+            PageObjects.datasetQuality.testSubjectSelectors.datasetQualitySparkPlot
+          );
+
+          expect(rows.length).to.be(sparkPlots.length);
+
+          await PageObjects.datasetQuality.closeFlyout();
+        });
+
+        it('should sort the table when the count table header is clicked', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+
+          const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
+
+          const countColumn = table['Docs count'];
+          const cellTexts = await countColumn.getCellTexts();
+
+          await countColumn.sort('ascending');
+          const sortedCellTexts = await countColumn.getCellTexts();
+
+          expect(cellTexts.reverse()).to.eql(sortedCellTexts);
+
+          await PageObjects.datasetQuality.closeFlyout();
+        });
       });
 
-      after(async () => {
-        await synthtrace.clean();
-      });
+      describe('Degraded Fields Table with data ingestion', () => {
+        before(async () => {
+          await PageObjects.svlCommonPage.loginWithRole('admin');
+          await synthtrace.index([
+            getLogsForDataset({
+              to: today,
+              count: 2,
+              dataset: goodDatasetName,
+              isMalformed: false,
+            }),
+            createDegradedFieldsRecord({
+              to: today,
+              count: 2,
+              dataset: degradedDatasetName,
+            }),
+          ]);
+          await PageObjects.datasetQuality.navigateTo();
+        });
 
-      it('shows the degraded fields table with no data when no degraded fields are present', async () => {
-        await PageObjects.datasetQuality.openDatasetFlyout(goodDatasetName);
+        after(async () => {
+          await synthtrace.clean();
+        });
+        it('should update the table when new data is ingested and the flyout is refreshed using the time selector', async () => {
+          await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
 
-        await testSubjects.existOrFail(
-          PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedTableNoData
-        );
+          const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
 
-        await PageObjects.datasetQuality.closeFlyout();
-      });
+          const countColumn = table['Docs count'];
+          const cellTexts = await countColumn.getCellTexts();
+          const singleValuePreviously = parseInt(cellTexts[0], 10);
 
-      it('should load the degraded fields table with data', async () => {
-        await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
+          await synthtrace.index([
+            createDegradedFieldsRecord({
+              to: today,
+              count: 2,
+              dataset: degradedDatasetName,
+            }),
+          ]);
 
-        await testSubjects.existOrFail(
-          PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedFieldTable
-        );
+          await PageObjects.datasetQuality.refreshFlyout();
 
-        const rows =
-          await PageObjects.datasetQuality.getDatasetQualityFlyoutDegradedFieldTableRows();
+          const updatedCellTexts = await countColumn.getCellTexts();
 
-        expect(rows.length).to.eql(2);
+          const singleValueNow = parseInt(updatedCellTexts[0], 10);
 
-        await PageObjects.datasetQuality.closeFlyout();
-      });
+          expect(singleValueNow).to.be(singleValuePreviously * 2);
 
-      it('should sort the table when the count table header is clicked', async () => {
-        await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
-
-        const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
-
-        const countColumn = table.Count;
-        const cellTexts = await countColumn.getCellTexts();
-
-        await countColumn.sort('ascending');
-        const sortedCellTexts = await countColumn.getCellTexts();
-
-        expect(cellTexts.reverse()).to.eql(sortedCellTexts);
-
-        await PageObjects.datasetQuality.closeFlyout();
-      });
-
-      it('should update the table when new data is ingested and the flyout is refreshed using the time selector', async () => {
-        await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName);
-
-        const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
-
-        const countColumn = table.Count;
-        const cellTexts = await countColumn.getCellTexts();
-
-        await synthtrace.index([
-          createDegradedFieldsRecord({
-            to: today,
-            count: 2,
-            dataset: degradedDatasetName,
-          }),
-        ]);
-
-        await PageObjects.datasetQuality.refreshFlyout();
-
-        const updatedCellTexts = await countColumn.getCellTexts();
-
-        const singleValuePreviously = parseInt(cellTexts[0], 10);
-        const singleValueNow = parseInt(updatedCellTexts[0], 10);
-
-        expect(singleValueNow).to.be(singleValuePreviously * 2);
-
-        await PageObjects.datasetQuality.closeFlyout();
+          await PageObjects.datasetQuality.closeFlyout();
+        });
       });
     });
   });