[8.9] [Lens] Relax counter field checks for saved visualizations with unsupported operations (#163515) - manual backport (#164191)

# Backport This will backport the following commits from `main` to `8.9`: - [[Lens] Relax counter field checks for saved visualizations with unsupported operations (#163515)](https://github.com/elastic/kibana/pull/163515) This is a manual backport which adds a missing mock function, not available in the automated backport cherry-pick of the PR. The extra code is here: `x-pack/plugins/lens/public/datasources/form_based/mocks.ts`  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  --------- Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
2025-04-24 17:59:23 -04:00 · 2023-08-21 15:24:25 +02:00 · 2023-08-21 15:24:25 +02:00 · 01f486e0d8
commit 01f486e0d8
parent 2774922290
5 changed files with 338 additions and 19 deletions
--- a/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx
+++ b/x-pack/plugins/lens/public/datasources/form_based/form_based.tsx
@ -71,6 +71,7 @@ import {
  isColumnInvalid,
  cloneLayer,
  getNotifiableFeatures,
+  getUnsupportedOperationsWarningMessage,
 } from './utils';
 import { getUniqueLabelGenerator, isDraggedDataViewField } from '../../utils';
 import { hasField, normalizeOperationDataType } from './pure_utils';
@ -891,6 +892,7 @@ export function getFormBasedDatasource({
          core.docLinks,
          setState
        ),
+        ...getUnsupportedOperationsWarningMessage(state, frameDatasourceAPI, core.docLinks),
      ];

      const infoMessages = getNotifiableFeatures(state, frameDatasourceAPI, visualizationInfo);
--- a/x-pack/plugins/lens/public/datasources/form_based/mocks.ts
+++ b/x-pack/plugins/lens/public/datasources/form_based/mocks.ts
@ -232,3 +232,15 @@ export function createMockedDragDropContext(): jest.Mocked<DragContextState> {
    registerDropTarget: jest.fn(),
  };
 }
+
+export const createMockedIndexPatternWithAdditionalFields = (
+  newFields: IndexPatternField[]
+): IndexPattern => {
+  const { fields, ...otherIndexPatternProps } = createMockedIndexPattern();
+  const completeFields = fields.concat(newFields);
+  return {
+    ...otherIndexPatternProps,
+    fields: completeFields,
+    getFieldByName: getFieldByNameFactory(completeFields),
+  };
+};
--- a/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx
+++ b/x-pack/plugins/lens/public/datasources/form_based/operations/definitions/metrics.tsx
@ -126,7 +126,6 @@ function buildMetricOperation<T extends MetricColumn<string>>({
        newField &&
          supportedTypes.includes(newField.type) &&
          newField.aggregatable &&
-          isTimeSeriesCompatible(type, newField.timeSeriesMetric) &&
          (!newField.aggregationRestrictions || newField.aggregationRestrictions![type])
      );
    },
--- a/x-pack/plugins/lens/public/datasources/form_based/utils.test.tsx
+++ b/x-pack/plugins/lens/public/datasources/form_based/utils.test.tsx
@ -8,15 +8,20 @@
 import React from 'react';
 import { shallow } from 'enzyme';
 import { createDatatableUtilitiesMock } from '@kbn/data-plugin/common/mocks';
-import { getPrecisionErrorWarningMessages, cloneLayer } from './utils';
+import {
+  getPrecisionErrorWarningMessages,
+  cloneLayer,
+  getUnsupportedOperationsWarningMessage,
+} from './utils';
 import type { FormBasedPrivateState, GenericIndexPatternColumn } from './types';
-import type { FramePublicAPI } from '../../types';
+import type { FramePublicAPI, IndexPattern } from '../../types';
 import type { DocLinksStart } from '@kbn/core/public';
 import { EuiLink } from '@elastic/eui';
 import { TermsIndexPatternColumn } from './operations';
 import { mountWithIntl } from '@kbn/test-jest-helpers';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { FormBasedLayer } from './types';
+import { createMockedIndexPatternWithAdditionalFields } from './mocks';

 describe('indexpattern_datasource utils', () => {
  describe('getPrecisionErrorWarningMessages', () => {
@ -240,4 +245,195 @@ describe('indexpattern_datasource utils', () => {
      ).toMatchSnapshot();
    });
  });
+
+  describe('getUnsupportedOperationsWarningMessage', () => {
+    let docLinks: DocLinksStart;
+    const affectedOperations = [
+      'sum',
+      'average',
+      'percentile',
+      'percentile_rank',
+      'count',
+      'unique_count',
+      'standard_deviation',
+    ];
+
+    function createColumnsForField(field: string, colOffset: number = 0) {
+      return Object.fromEntries(
+        affectedOperations.map((operationType, i) => [
+          `col_${i + colOffset}`,
+          { operationType, sourceField: field, label: `${operationType} of ${field}` },
+        ])
+      );
+    }
+
+    function createState(fields: string[]) {
+      return {
+        layers: {
+          id: {
+            indexPatternId: '0',
+            columns: Object.assign(
+              {},
+              ...fields.map((field, i) =>
+                createColumnsForField(field, i * affectedOperations.length)
+              )
+            ),
+          },
+        },
+      } as unknown as FormBasedPrivateState;
+    }
+
+    function createFramePublic(indexPattern: IndexPattern): FramePublicAPI {
+      return {
+        dataViews: {
+          indexPatterns: Object.fromEntries([indexPattern].map((dataView, i) => [i, dataView])),
+        },
+      } as unknown as FramePublicAPI;
+    }
+
+    function createFormulaColumns(formulaParts: string[], field: string, colOffset: number = 0) {
+      const fullFormula = formulaParts.map((part) => `${part}(${field})`).join(' + ');
+      // just assume it's a sum of all the parts for testing
+      const rootId = `col-formula${colOffset}`;
+      return Object.fromEntries([
+        [
+          rootId,
+          {
+            operationType: 'formula',
+            label: `Formula: ${fullFormula}`,
+            params: { formula: fullFormula },
+          },
+        ],
+        ...formulaParts.map((part, i) => [
+          `${rootId}X${i}`,
+          { operationType: part, sourceField: field, label: 'Part of formula' },
+        ]),
+        [
+          `${rootId}X${formulaParts.length}`,
+          { operationType: 'math', references: formulaParts.map((_, i) => `${rootId}X${i}`) },
+        ],
+      ]);
+    }
+
+    beforeEach(() => {
+      docLinks = {
+        links: {
+          fleet: {
+            datastreamsTSDSMetrics: 'http://tsdb_metric_doc',
+          },
+        },
+      } as DocLinksStart;
+    });
+
+    it.each([['bytes'], ['bytes_gauge']])(
+      'should return no warning for non-counter fields: %s',
+      (fieldName: string) => {
+        const warnings = getUnsupportedOperationsWarningMessage(
+          createState([fieldName]),
+          createFramePublic(
+            createMockedIndexPatternWithAdditionalFields([
+              {
+                name: 'bytes_gauge',
+                displayName: 'bytes_gauge',
+                type: 'number',
+                aggregatable: true,
+                searchable: true,
+                timeSeriesMetric: 'gauge',
+              },
+            ])
+          ),
+          docLinks
+        );
+        expect(warnings).toHaveLength(0);
+      }
+    );
+
+    it('should return a warning for a counter field grouped by field', () => {
+      const warnings = getUnsupportedOperationsWarningMessage(
+        createState(['bytes_counter']),
+        createFramePublic(
+          createMockedIndexPatternWithAdditionalFields([
+            {
+              name: 'bytes_counter',
+              displayName: 'bytes_counter',
+              type: 'number',
+              aggregatable: true,
+              searchable: true,
+              timeSeriesMetric: 'counter',
+            },
+          ])
+        ),
+        docLinks
+      );
+      expect(warnings).toHaveLength(1);
+    });
+
+    it('should group multiple warnings by field', () => {
+      const warnings = getUnsupportedOperationsWarningMessage(
+        createState(['bytes_counter', 'bytes_counter2']),
+        createFramePublic(
+          createMockedIndexPatternWithAdditionalFields([
+            {
+              name: 'bytes_counter',
+              displayName: 'bytes_counter',
+              type: 'number',
+              aggregatable: true,
+              searchable: true,
+              timeSeriesMetric: 'counter',
+            },
+            {
+              name: 'bytes_counter2',
+              displayName: 'bytes_counter2',
+              type: 'number',
+              aggregatable: true,
+              searchable: true,
+              timeSeriesMetric: 'counter',
+            },
+          ])
+        ),
+        docLinks
+      );
+      expect(warnings).toHaveLength(2);
+    });
+
+    it('should handle formula reporting only the top visible dimension', () => {
+      const warnings = getUnsupportedOperationsWarningMessage(
+        {
+          layers: {
+            id: {
+              indexPatternId: '0',
+              columns: Object.assign(
+                {},
+                ...['bytes_counter', 'bytes_counter2'].map((field, i) =>
+                  createFormulaColumns(affectedOperations, field, i * affectedOperations.length)
+                )
+              ),
+            },
+          },
+        } as unknown as FormBasedPrivateState,
+        createFramePublic(
+          createMockedIndexPatternWithAdditionalFields([
+            {
+              name: 'bytes_counter',
+              displayName: 'bytes_counter',
+              type: 'number',
+              aggregatable: true,
+              searchable: true,
+              timeSeriesMetric: 'counter',
+            },
+            {
+              name: 'bytes_counter2',
+              displayName: 'bytes_counter2',
+              type: 'number',
+              aggregatable: true,
+              searchable: true,
+              timeSeriesMetric: 'counter',
+            },
+          ])
+        ),
+        docLinks
+      );
+      expect(warnings).toHaveLength(2);
+    });
+  });
 });
--- a/x-pack/plugins/lens/public/datasources/form_based/utils.tsx
+++ b/x-pack/plugins/lens/public/datasources/form_based/utils.tsx
@ -14,7 +14,7 @@ import { TimeRange } from '@kbn/es-query';
 import { EuiLink, EuiSpacer, EuiText } from '@elastic/eui';

 import type { DatatableColumn } from '@kbn/expressions-plugin/common';
-import { groupBy, escape, uniq } from 'lodash';
+import { groupBy, escape, uniq, uniqBy } from 'lodash';
 import type { Query } from '@kbn/data-plugin/common';
 import { SearchRequest } from '@kbn/data-plugin/common';

@ -40,16 +40,19 @@ import type { ReferenceBasedIndexPatternColumn } from './operations/definitions/

 import {
  operationDefinitionMap,
-  GenericIndexPatternColumn,
-  TermsIndexPatternColumn,
-  CountIndexPatternColumn,
+  getReferenceRoot,
  updateColumnParam,
  updateDefaultLabels,
-  RangeIndexPatternColumn,
-  FormulaIndexPatternColumn,
-  DateHistogramIndexPatternColumn,
-  MaxIndexPatternColumn,
-  MinIndexPatternColumn,
+  type GenericIndexPatternColumn,
+  type TermsIndexPatternColumn,
+  type CountIndexPatternColumn,
+  type RangeIndexPatternColumn,
+  type FormulaIndexPatternColumn,
+  type DateHistogramIndexPatternColumn,
+  type MaxIndexPatternColumn,
+  type MinIndexPatternColumn,
+  type GenericOperationDefinition,
+  type FieldBasedIndexPatternColumn,
 } from './operations';

 import { getInvalidFieldMessage, isColumnOfType } from './operations/definitions/helpers';
@ -338,6 +341,113 @@ export function getShardFailuresWarningMessages(
  return [];
 }

+export function getUnsupportedOperationsWarningMessage(
+  state: FormBasedPrivateState,
+  { dataViews }: FramePublicAPI,
+  docLinks: DocLinksStart
+) {
+  const warningMessages: UserMessage[] = [];
+  const columnsWithUnsupportedOperations: Array<
+    [FieldBasedIndexPatternColumn, ReferenceBasedIndexPatternColumn | undefined]
+  > = Object.values(state.layers)
+    // filter layers without dataView loaded yet
+    .filter(({ indexPatternId }) => dataViews.indexPatterns[indexPatternId])
+    .flatMap((layer) => {
+      const dataView = dataViews.indexPatterns[layer.indexPatternId];
+      const columnsEntries = Object.entries(layer.columns);
+      return columnsEntries
+        .filter(([_, column]) => {
+          if (!hasField(column)) {
+            return false;
+          }
+          const field = dataView.getFieldByName(column.sourceField);
+          if (!field) {
+            return false;
+          }
+          return (
+            !(
+              operationDefinitionMap[column.operationType] as Extract<
+                GenericOperationDefinition,
+                { input: 'field' }
+              >
+            ).getPossibleOperationForField(field) && field?.timeSeriesMetric === 'counter'
+          );
+        })
+        .map(
+          ([id, fieldColumn]) =>
+            [fieldColumn, layer.columns[getReferenceRoot(layer, id)]] as [
+              FieldBasedIndexPatternColumn,
+              ReferenceBasedIndexPatternColumn | undefined
+            ]
+        );
+    });
+  if (columnsWithUnsupportedOperations.length) {
+    // group the columns by field
+    // then group together columns of a formula/referenced operation who use the same field
+    const columnsGroupedByField = Object.values(
+      groupBy(columnsWithUnsupportedOperations, ([column]) => column.sourceField)
+    ).map((columnsList) => uniqBy(columnsList, ([column, rootColumn]) => rootColumn ?? column));
+
+    for (const columnsGrouped of columnsGroupedByField) {
+      const sourceField = columnsGrouped[0][0].sourceField;
+      warningMessages.push({
+        severity: 'warning',
+        fixableInEditor: false,
+        displayLocations: [{ id: 'toolbar' }, { id: 'embeddableBadge' }],
+        shortMessage: i18n.translate(
+          'xpack.lens.indexPattern.tsdbErrorWarning.unsupportedCounterOperationErrorWarning.shortMessage',
+          {
+            defaultMessage:
+              'The result of {count} {count, plural, one {operation} other {operations}} might be meaningless for {field}: {operations}',
+            values: {
+              count: columnsGrouped.length,
+              operations: columnsGrouped
+                .map(([affectedColumn, rootColumn]) => (rootColumn ?? affectedColumn).label)
+                .join(', '),
+              field: sourceField,
+            },
+          }
+        ),
+        longMessage: (
+          <>
+            <FormattedMessage
+              id="xpack.lens.indexPattern.unsupportedCounterOperationErrorWarning"
+              defaultMessage="While {count} {count, plural, one {operation} other {operations}} for {field} {count, plural, one {is} other {are}} allowed the result might be meaningless: {operations}. To learn more about this, {link}."
+              values={{
+                count: columnsGrouped.length,
+                operations: (
+                  <>
+                    {columnsGrouped.map(([affectedColumn, rootColumn], i) => (
+                      <React.Fragment key={(rootColumn ?? affectedColumn).label}>
+                        <strong>{(rootColumn ?? affectedColumn).label}</strong>
+                        {i < columnsGrouped.length - 1 ? ', ' : ''}
+                      </React.Fragment>
+                    ))}
+                  </>
+                ),
+                field: sourceField,
+                link: (
+                  <EuiLink
+                    href={docLinks.links.fleet.datastreamsTSDSMetrics}
+                    target="_blank"
+                    external={true}
+                  >
+                    <FormattedMessage
+                      defaultMessage="visit the Time series documentation"
+                      id="xpack.lens.indexPattern.unsupportedCounterOperationErrorWarning.link"
+                    />
+                  </EuiLink>
+                ),
+              }}
+            />
+          </>
+        ),
+      });
+    }
+  }
+  return warningMessages;
+}
+
 export function getPrecisionErrorWarningMessages(
  datatableUtilities: DatatableUtilitiesService,
  state: FormBasedPrivateState,
@ -349,18 +459,18 @@ export function getPrecisionErrorWarningMessages(

  if (state && activeData) {
    Object.entries(activeData)
-      .reduce(
-        (acc, [layerId, { columns }]) => [
-          ...acc,
-          ...columns.map((column) => ({ layerId, column })),
-        ],
-        [] as Array<{ layerId: string; column: DatatableColumn }>
-      )
+      .reduce((acc, [layerId, { columns }]) => {
+        acc.push(...columns.map((column) => ({ layerId, column })));
+        return acc;
+      }, [] as Array<{ layerId: string; column: DatatableColumn }>)
      .forEach(({ layerId, column }) => {
        const currentLayer = state.layers[layerId];
        const currentColumn = currentLayer?.columns[column.id];
        if (currentLayer && currentColumn && datatableUtilities.hasPrecisionError(column)) {
          const indexPattern = dataViews.indexPatterns[currentLayer.indexPatternId];
+          if (!indexPattern) {
+            return;
+          }
          // currentColumnIsTerms is mostly a type guard. If there's a precision error,
          // we already know that we're dealing with a terms-based operation (at least for now).
          const currentColumnIsTerms = isColumnOfType<TermsIndexPatternColumn>(