[Lens] Add support for decimals in percentiles (#165703)

## Summary

Fixes #98853

This PR adds support for decimals (2 digits) in percentile operation.


![percentile_decimals_support](cd0d2901-ba6f-452e-955c-f9d774a4e27f)

Features:
* ✨ Add decimals support in percentile
  * 🐛 Fixed aggs optimization to work with decimals
* 💄 Show Toast for ranking reset when using decimals in both
percentile and percentile rank
* ✅ Extended `isValidNumber` to support digits check and added unit
tests for it
* ♻️ Added support also to `convert to Lens` feature

Added both unit and functional tests.


![percentile_rank_toast](a9be1f9f-a1b1-4f9f-90dc-55e2af8933e1)

When trying to add more digits than what is supported then it will show
the input as invalid:
<img width="347" alt="Screenshot 2023-09-05 at 12 24 03"
src="3c38474f-b78f-4144-bca7-3dc192313c09">

Also it works now as custom ranking column:
<img width="264" alt="Screenshot 2023-09-05 at 16 14 25"
src="cb7be312-7f7b-4dc1-95a3-d893de344585">
<img width="264" alt="Screenshot 2023-09-05 at 16 14 20"
src="1c13f66e-da78-4df6-bb4f-e811d47b66d5">

**Notes**: need to specify exact digits in percentile (2) because the
`any` step is not supported and need to specify a number. I guess
alternatives here are to either extend it to 4 digits or make it a
configurable thing.

### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>

src/plugins/vis_default_editor/public/components/controls/percentiles.tsx

@@ -42,6 +42,7 @@ function PercentilesEditor({
       id={`visEditorPercentileLabel${agg.id}`}
       isInvalid={showValidation ? !isValid : false}
       display="rowCompressed"
+      data-test-subj="visEditorPercentile"
     >
       <NumberList
         labelledbyId={`visEditorPercentileLabel${agg.id}-legend`}

src/plugins/visualizations/common/convert_to_lens/lib/convert/percentile.test.ts

@@ -17,11 +17,15 @@ const mockGetFieldByName = jest.fn();
 const mockGetLabel = jest.fn();
 const mockGetLabelForPercentile = jest.fn();
 
-jest.mock('../utils', () => ({
-  getFieldNameFromField: jest.fn(() => mockGetFieldNameFromField()),
-  getLabel: jest.fn(() => mockGetLabel()),
-  getLabelForPercentile: jest.fn(() => mockGetLabelForPercentile()),
-}));
+jest.mock('../utils', () => {
+  const utils = jest.requireActual('../utils');
+  return {
+    ...utils,
+    getFieldNameFromField: jest.fn(() => mockGetFieldNameFromField()),
+    getLabel: jest.fn(() => mockGetLabel()),
+    getLabelForPercentile: jest.fn(() => mockGetLabelForPercentile()),
+  };
+});
 
 describe('convertToPercentileColumn', () => {
   const visType = 'heatmap';

src/plugins/visualizations/common/convert_to_lens/lib/convert/percentile.ts

@@ -9,7 +9,7 @@
 import { METRIC_TYPES } from '@kbn/data-plugin/common';
 import { SchemaConfig } from '../../..';
 import { isFieldValid, PercentileParams } from '../..';
-import { getFieldNameFromField, getLabelForPercentile } from '../utils';
+import { getAggIdAndValue, getFieldNameFromField, getLabelForPercentile } from '../utils';
 import { createColumn, getFormat } from './column';
 import { PercentileColumn, CommonColumnConverterArgs } from './types';
 import { SUPPORTED_METRICS } from './supported_metrics';
@@ -40,7 +40,7 @@ const getPercent = (
 
   const { percents } = aggParams;
 
-  const [, percentStr] = aggId.split('.');
+  const [, percentStr] = getAggIdAndValue(aggId);
 
   const percent = Number(percentStr);
   if (!percents || !percents.length || percentStr === '' || isNaN(percent)) {

src/plugins/visualizations/common/convert_to_lens/lib/metrics/formula.test.ts

@@ -19,14 +19,18 @@ const mockIsStdDevAgg = jest.fn();
 const mockGetFieldByName = jest.fn();
 const originalGetFieldByName = stubLogstashDataView.getFieldByName;
 
-jest.mock('../utils', () => ({
-  getFieldNameFromField: jest.fn((field) => field),
-  getMetricFromParentPipelineAgg: jest.fn(() => mockGetMetricFromParentPipelineAgg()),
-  isPercentileAgg: jest.fn(() => mockIsPercentileAgg()),
-  isPercentileRankAgg: jest.fn(() => mockIsPercentileRankAgg()),
-  isPipeline: jest.fn(() => mockIsPipeline()),
-  isStdDevAgg: jest.fn(() => mockIsStdDevAgg()),
-}));
+jest.mock('../utils', () => {
+  const utils = jest.requireActual('../utils');
+  return {
+    ...utils,
+    getFieldNameFromField: jest.fn((field) => field),
+    getMetricFromParentPipelineAgg: jest.fn(() => mockGetMetricFromParentPipelineAgg()),
+    isPercentileAgg: jest.fn(() => mockIsPercentileAgg()),
+    isPercentileRankAgg: jest.fn(() => mockIsPercentileRankAgg()),
+    isPipeline: jest.fn(() => mockIsPipeline()),
+    isStdDevAgg: jest.fn(() => mockIsStdDevAgg()),
+  };
+});
 
 const dataView = stubLogstashDataView;
 const visType = 'heatmap';

src/plugins/visualizations/common/convert_to_lens/lib/metrics/formula.ts

@@ -12,6 +12,7 @@ import { Operations } from '../../constants';
 import { isMetricWithField, getStdDeviationFormula, ExtendedColumnConverterArgs } from '../convert';
 import { getFormulaFromMetric, SUPPORTED_METRICS } from '../convert/supported_metrics';
 import {
+  getAggIdAndValue,
   getFieldNameFromField,
   getMetricFromParentPipelineAgg,
   isPercentileAgg,
@@ -125,7 +126,7 @@ const getFormulaForPercentile = (
   selector: string,
   reducedTimeRange?: string
 ) => {
-  const percentile = Number(agg.aggId?.split('.')[1]);
+  const percentile = Number(getAggIdAndValue(agg.aggId)[1]);
   const op = SUPPORTED_METRICS[agg.aggType];
   if (!isValidAgg(visType, agg, dataView) || !op) {
     return null;

src/plugins/visualizations/common/convert_to_lens/lib/utils.ts

@@ -199,3 +199,17 @@ export const getMetricFromParentPipelineAgg = (
 
   return metric as SchemaConfig<METRIC_TYPES>;
 };
+
+const aggIdWithDecimalsRegexp = /^(\w)+\['([0-9]+\.[0-9]+)'\]$/;
+
+export const getAggIdAndValue = (aggId?: string) => {
+  if (!aggId) {
+    return [];
+  }
+  // agg value contains decimals
+  if (/\['/.test(aggId)) {
+    const [_, id, value] = aggId.match(aggIdWithDecimalsRegexp) || [];
+    return [id, value];
+  }
+  return aggId.split('.');
+};

src/plugins/visualizations/public/convert_to_lens/schemas.test.ts

@@ -40,9 +40,13 @@ jest.mock('../../common/convert_to_lens/lib/buckets', () => ({
   convertBucketToColumns: jest.fn(() => mockConvertBucketToColumns()),
 }));
 
-jest.mock('../../common/convert_to_lens/lib/utils', () => ({
-  getCustomBucketsFromSiblingAggs: jest.fn(() => mockGetCutomBucketsFromSiblingAggs()),
-}));
+jest.mock('../../common/convert_to_lens/lib/utils', () => {
+  const utils = jest.requireActual('../../common/convert_to_lens/lib/utils');
+  return {
+    ...utils,
+    getCustomBucketsFromSiblingAggs: jest.fn(() => mockGetCutomBucketsFromSiblingAggs()),
+  };
+});
 
 jest.mock('../vis_schemas', () => ({
   getVisSchemas: jest.fn(() => mockGetVisSchemas()),

src/plugins/visualizations/public/convert_to_lens/schemas.ts

@@ -10,7 +10,10 @@ import type { DataView } from '@kbn/data-views-plugin/common';
 import { IAggConfig, METRIC_TYPES, TimefilterContract } from '@kbn/data-plugin/public';
 import { AggBasedColumn, PercentageModeConfig, SchemaConfig } from '../../common';
 import { convertMetricToColumns } from '../../common/convert_to_lens/lib/metrics';
-import { getCustomBucketsFromSiblingAggs } from '../../common/convert_to_lens/lib/utils';
+import {
+  getAggIdAndValue,
+  getCustomBucketsFromSiblingAggs,
+} from '../../common/convert_to_lens/lib/utils';
 import { BucketColumn } from '../../common/convert_to_lens/lib';
 import type { Vis } from '../types';
 import { getVisSchemas, Schemas } from '../vis_schemas';
@@ -178,11 +181,12 @@ export const getColumnsFromVis = <T>(
 
   if (series && series.length) {
     for (const { metrics: metricAggIds } of series) {
+      const metricAggIdsLookup = new Set(metricAggIds);
       const metrics = aggs.filter(
-        (agg) => agg.aggId && metricAggIds.includes(agg.aggId.split('.')[0])
+        (agg) => agg.aggId && metricAggIdsLookup.has(getAggIdAndValue(agg.aggId)[0])
       );
       const customBucketsForLayer = customBucketsWithMetricIds.filter((c) =>
-        c.metricIds.some((m) => metricAggIds.includes(m))
+        c.metricIds.some((m) => metricAggIdsLookup.has(m))
       );
       const layer = createLayer(
         vis.type.name,

test/functional/page_objects/visualize_editor_page.ts

@@ -211,10 +211,21 @@ export class VisualizeEditorPageObject extends FtrService {
     const input = await this.find.byCssSelector(
       '[data-test-subj="visEditorPercentileRanks"] input'
     );
+    this.log.debug(`Setting percentile rank value of ${newValue}`);
     await input.clearValue();
     await input.type(newValue);
   }
 
+  public async setPercentileValue(newValue: string, index: number = 0) {
+    const correctIndex = index * 2 + 1;
+    const input = await this.find.byCssSelector(
+      `[data-test-subj="visEditorPercentile"]>div:nth-child(2)>div:nth-child(${correctIndex}) input`
+    );
+    this.log.debug(`Setting percentile value at ${index}th input of ${newValue}`);
+    await input.clearValueWithKeyboard();
+    await input.type(newValue, { charByChar: true });
+  }
+
   public async clickEditorSidebarCollapse() {
     await this.testSubjects.click('collapseSideBarButton');
   }

x-pack/plugins/lens/public/datasources/form_based/dimension_panel/dimension_editor.tsx

@@ -66,6 +66,7 @@ import {
   DimensionEditorButtonGroups,
   CalloutWarning,
   DimensionEditorGroupsOptions,
+  isLayerChangingDueToDecimalsPercentile,
 } from './dimensions_editor_helpers';
 import type { TemporaryState } from './dimensions_editor_helpers';
 import { FieldInput } from './field_input';
@@ -124,11 +125,14 @@ export function DimensionEditor(props: DimensionEditorProps) {
 
   const [temporaryState, setTemporaryState] = useState<TemporaryState>('none');
   const [isHelpOpen, setIsHelpOpen] = useState(false);
+
   // If a layer has sampling disabled, assume the toast has already fired in the past
   const [hasRandomSamplingToastFired, setSamplingToastAsFired] = useState(
     !isSamplingValueEnabled(state.layers[layerId])
   );
 
+  const [hasRankingToastFired, setRankingToastAsFired] = useState(false);
+
   const onHelpClick = () => setIsHelpOpen((prevIsHelpOpen) => !prevIsHelpOpen);
   const closeHelp = () => setIsHelpOpen(false);
 
@@ -163,6 +167,32 @@ export function DimensionEditor(props: DimensionEditorProps) {
     [hasRandomSamplingToastFired, layerId, props.notifications.toasts, state.layers]
   );
 
+  const fireOrResetRankingToast = useCallback(
+    (newLayer: FormBasedLayer) => {
+      if (isLayerChangingDueToDecimalsPercentile(state.layers[layerId], newLayer)) {
+        props.notifications.toasts.add({
+          title: i18n.translate('xpack.lens.uiInfo.rankingResetTitle', {
+            defaultMessage: 'Ranking changed to alphabetical',
+          }),
+          text: i18n.translate('xpack.lens.uiInfo.rankingResetToAlphabetical', {
+            defaultMessage: 'To rank by percentile, use whole numbers only.',
+          }),
+        });
+      }
+      // reset the flag if the user switches to another supported operation
+      setRankingToastAsFired(!hasRankingToastFired);
+    },
+    [hasRankingToastFired, layerId, props.notifications.toasts, state.layers]
+  );
+
+  const fireOrResetToastChecks = useCallback(
+    (newLayer: FormBasedLayer) => {
+      fireOrResetRandomSamplingToast(newLayer);
+      fireOrResetRankingToast(newLayer);
+    },
+    [fireOrResetRandomSamplingToast, fireOrResetRankingToast]
+  );
+
   const setStateWrapper = useCallback(
     (
       setter:
@@ -203,7 +233,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
           }
           const newLayer = adjustColumnReferencesForChangedColumn(outputLayer, columnId);
           // Fire an info toast (eventually) on layer update
-          fireOrResetRandomSamplingToast(newLayer);
+          fireOrResetToastChecks(newLayer);
 
           return mergeLayer({
             state: prevState,
@@ -217,7 +247,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
         }
       );
     },
-    [columnId, fireOrResetRandomSamplingToast, layerId, setState, state.layers]
+    [columnId, fireOrResetToastChecks, layerId, setState, state.layers]
   );
 
   const incompleteInfo = (state.layers[layerId].incompleteColumns ?? {})[columnId];
@@ -811,7 +841,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
                     field,
                     visualizationGroups: dimensionGroups,
                   });
-                  fireOrResetRandomSamplingToast(newLayer);
+                  fireOrResetToastChecks(newLayer);
                   updateLayer(newLayer);
                 }}
                 onChooseField={(choice: FieldChoiceWithOperationType) => {
@@ -846,7 +876,7 @@ export function DimensionEditor(props: DimensionEditorProps) {
                   } else {
                     newLayer = setter;
                   }
-                  fireOrResetRandomSamplingToast(newLayer);
+                  fireOrResetToastChecks(newLayer);
                   return updateLayer(adjustColumnReferencesForChangedColumn(newLayer, referenceId));
                 }}
                 validation={validation}

x-pack/plugins/lens/public/datasources/form_based/dimension_panel/dimensions_editor_helpers.tsx

@@ -16,15 +16,71 @@ import './dimension_editor.scss';
 import React from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiCallOut, EuiButtonGroup, EuiFormRow } from '@elastic/eui';
-import { operationDefinitionMap } from '../operations';
+import { nonNullable } from '../../../utils';
+import {
+  operationDefinitionMap,
+  type PercentileIndexPatternColumn,
+  type PercentileRanksIndexPatternColumn,
+  type TermsIndexPatternColumn,
+} from '../operations';
+import { isColumnOfType } from '../operations/definitions/helpers';
+import { FormBasedLayer } from '../types';
 
 export const formulaOperationName = 'formula';
 export const staticValueOperationName = 'static_value';
 export const quickFunctionsName = 'quickFunctions';
+export const termsOperationName = 'terms';
+export const optionallySortableOperationNames = ['percentile', 'percentile_ranks'];
 export const nonQuickFunctions = new Set([formulaOperationName, staticValueOperationName]);
 
 export type TemporaryState = typeof quickFunctionsName | typeof staticValueOperationName | 'none';
 
+export function isLayerChangingDueToDecimalsPercentile(
+  prevLayer: FormBasedLayer,
+  newLayer: FormBasedLayer
+) {
+  // step 1: find the ranking column in prevState and return its value
+  const termsRiskyColumns = Object.entries(prevLayer.columns)
+    .map(([id, column]) => {
+      if (
+        isColumnOfType<TermsIndexPatternColumn>('terms', column) &&
+        column.params?.orderBy.type === 'column' &&
+        column.params.orderBy.columnId != null
+      ) {
+        const rankingColumn = prevLayer.columns[column.params.orderBy.columnId];
+        if (isColumnOfType<PercentileIndexPatternColumn>('percentile', rankingColumn)) {
+          if (Number.isInteger(rankingColumn.params.percentile)) {
+            return { id, rankId: column.params.orderBy.columnId };
+          }
+        }
+        if (isColumnOfType<PercentileRanksIndexPatternColumn>('percentile_rank', rankingColumn)) {
+          if (Number.isInteger(rankingColumn.params.value)) {
+            return { id, rankId: column.params.orderBy.columnId };
+          }
+        }
+      }
+    })
+    .filter(nonNullable);
+  // now check again the terms risky column in the new layer and verify that at
+  // least one changed due to decimals
+  const hasChangedDueToDecimals = termsRiskyColumns.some(({ id, rankId }) => {
+    const termsColumn = newLayer.columns[id];
+    if (!isColumnOfType<TermsIndexPatternColumn>('terms', termsColumn)) {
+      return false;
+    }
+    if (termsColumn.params.orderBy.type === 'alphabetical') {
+      const rankingColumn = newLayer.columns[rankId];
+      if (isColumnOfType<PercentileIndexPatternColumn>('percentile', rankingColumn)) {
+        return !Number.isInteger(rankingColumn.params.percentile);
+      }
+      if (isColumnOfType<PercentileRanksIndexPatternColumn>('percentile_rank', rankingColumn)) {
+        return !Number.isInteger(rankingColumn.params.value);
+      }
+    }
+  });
+  return hasChangedDueToDecimals;
+}
+
 export function isQuickFunction(operationType: string) {
   return !nonQuickFunctions.has(operationType);
 }

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.test.ts

@@ -8,7 +8,7 @@
 import { createMockedIndexPattern } from '../../mocks';
 import type { FormBasedLayer } from '../../types';
 import type { GenericIndexPatternColumn } from './column_types';
-import { getInvalidFieldMessage } from './helpers';
+import { getInvalidFieldMessage, isValidNumber } from './helpers';
 import type { TermsIndexPatternColumn } from './terms';
 
 describe('helpers', () => {
@@ -248,4 +248,77 @@ describe('helpers', () => {
       expect(messages).toBeUndefined();
     });
   });
+
+  describe('isValidNumber', () => {
+    it('should work for integers', () => {
+      const number = 99;
+      for (const value of [number, `${number}`]) {
+        expect(isValidNumber(value)).toBeTruthy();
+        expect(isValidNumber(value, true)).toBeTruthy();
+        expect(isValidNumber(value, false)).toBeTruthy();
+        expect(isValidNumber(value, true, number, 1)).toBeTruthy();
+        expect(isValidNumber(value, true, number + 1, number)).toBeTruthy();
+        expect(isValidNumber(value, false, number, 1)).toBeTruthy();
+        expect(isValidNumber(value, false, number + 1, number)).toBeTruthy();
+        expect(isValidNumber(value, false, number + 1, number, 2)).toBeTruthy();
+        expect(isValidNumber(value, false, number - 1, number - 2)).toBeFalsy();
+      }
+    });
+
+    it('should work correctly for numeric falsy values', () => {
+      expect(isValidNumber(0)).toBeTruthy();
+      expect(isValidNumber(0, true)).toBeTruthy();
+      expect(isValidNumber(0, false)).toBeTruthy();
+      expect(isValidNumber(0, true, 1, 0)).toBeTruthy();
+    });
+
+    it('should work for decimals', () => {
+      const number = 99.9;
+      for (const value of [number, `${number}`]) {
+        expect(isValidNumber(value)).toBeTruthy();
+        expect(isValidNumber(value, true)).toBeFalsy();
+        expect(isValidNumber(value, false)).toBeTruthy();
+        expect(isValidNumber(value, true, number, 1)).toBeFalsy();
+        expect(isValidNumber(value, true, number + 1, number)).toBeFalsy();
+        expect(isValidNumber(value, false, number, 1)).toBeTruthy();
+        expect(isValidNumber(value, false, number + 1, number)).toBeTruthy();
+        expect(isValidNumber(value, false, number + 1, number, 0)).toBeFalsy();
+        expect(isValidNumber(value, false, number + 1, number, 1)).toBeTruthy();
+        expect(isValidNumber(value, false, number + 1, number, 2)).toBeTruthy();
+        expect(isValidNumber(value, false, number - 1, number - 2)).toBeFalsy();
+      }
+    });
+
+    it('should work for negative values', () => {
+      const number = -10.1;
+      for (const value of [number, `${number}`]) {
+        expect(isValidNumber(value)).toBeTruthy();
+        expect(isValidNumber(value, true)).toBeFalsy();
+        expect(isValidNumber(value, false)).toBeTruthy();
+        expect(isValidNumber(value, true, number, -20)).toBeFalsy();
+        expect(isValidNumber(value, true, number + 1, number)).toBeFalsy();
+        expect(isValidNumber(value, false, number, -20)).toBeTruthy();
+        expect(isValidNumber(value, false, number + 1, number)).toBeTruthy();
+        expect(isValidNumber(value, false, number + 1, number, 0)).toBeFalsy();
+        expect(isValidNumber(value, false, number + 1, number, 1)).toBeTruthy();
+        expect(isValidNumber(value, false, number + 1, number, 2)).toBeTruthy();
+        expect(isValidNumber(value, false, number - 1, number - 2)).toBeFalsy();
+      }
+    });
+
+    it('should spot invalid values', () => {
+      for (const value of [NaN, ``, undefined, null, Infinity, -Infinity]) {
+        expect(isValidNumber(value)).toBeFalsy();
+        expect(isValidNumber(value, true)).toBeFalsy();
+        expect(isValidNumber(value, false)).toBeFalsy();
+        expect(isValidNumber(value, true, 99, 1)).toBeFalsy();
+        expect(isValidNumber(value, true, 99, 1)).toBeFalsy();
+        expect(isValidNumber(value, false, 99, 1)).toBeFalsy();
+        expect(isValidNumber(value, false, 99, 1)).toBeFalsy();
+        expect(isValidNumber(value, false, 99, 1, 0)).toBeFalsy();
+        expect(isValidNumber(value, false, 99, 1, 1)).toBeFalsy();
+        expect(isValidNumber(value, false, 99, 1, 2)).toBeFalsy();
+      }
+    });
+  });
 });

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/helpers.tsx

@@ -138,11 +138,17 @@ export function getSafeName(name: string, indexPattern: IndexPattern | undefined
       });
 }
 
+function areDecimalsValid(inputValue: string | number, digits: number) {
+  const [, decimals = ''] = `${inputValue}`.split('.');
+  return decimals.length <= digits;
+}
+
 export function isValidNumber(
   inputValue: string | number | null | undefined,
   integer?: boolean,
   upperBound?: number,
-  lowerBound?: number
+  lowerBound?: number,
+  digits: number = 2
 ) {
   const inputValueAsNumber = Number(inputValue);
   return (
@@ -152,7 +158,8 @@ export function isValidNumber(
     Number.isFinite(inputValueAsNumber) &&
     (!integer || Number.isInteger(inputValueAsNumber)) &&
     (upperBound === undefined || inputValueAsNumber <= upperBound) &&
-    (lowerBound === undefined || inputValueAsNumber >= lowerBound)
+    (lowerBound === undefined || inputValueAsNumber >= lowerBound) &&
+    areDecimalsValid(inputValue, integer ? 0 : digits)
   );
 }
 

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.test.tsx

@@ -654,7 +654,7 @@ describe('percentile', () => {
       });
     });
 
-    it('should not update on invalid input, but show invalid value locally', () => {
+    it('should update on decimals input up to 2 digits', () => {
       const updateLayerSpy = jest.fn();
       const instance = mount(
         <InlineOptions
@@ -679,6 +679,41 @@ describe('percentile', () => {
 
       instance.update();
 
+      expect(updateLayerSpy).toHaveBeenCalled();
+
+      expect(
+        instance
+          .find('[data-test-subj="lns-indexPattern-percentile-input"]')
+          .find(EuiRange)
+          .prop('value')
+      ).toEqual('12.12');
+    });
+
+    it('should not update on invalid input, but show invalid value locally', () => {
+      const updateLayerSpy = jest.fn();
+      const instance = mount(
+        <InlineOptions
+          {...defaultProps}
+          layer={layer}
+          paramEditorUpdater={updateLayerSpy}
+          columnId="col2"
+          currentColumn={layer.columns.col2 as PercentileIndexPatternColumn}
+        />
+      );
+
+      const input = instance
+        .find('[data-test-subj="lns-indexPattern-percentile-input"]')
+        .find(EuiRange);
+
+      act(() => {
+        input.prop('onChange')!(
+          { currentTarget: { value: '12.1212312312312312' } } as ChangeEvent<HTMLInputElement>,
+          true
+        );
+      });
+
+      instance.update();
+
       expect(updateLayerSpy).not.toHaveBeenCalled();
 
       expect(
@@ -692,7 +727,7 @@ describe('percentile', () => {
           .find('[data-test-subj="lns-indexPattern-percentile-input"]')
           .find(EuiRange)
           .prop('value')
-      ).toEqual('12.12');
+      ).toEqual('12.1212312312312312');
     });
   });
 });

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/percentile.tsx

@@ -67,6 +67,40 @@ function ofName(
 }
 
 const DEFAULT_PERCENTILE_VALUE = 95;
+const ALLOWED_DECIMAL_DIGITS = 4;
+
+function getInvalidErrorMessage(
+  value: string | undefined,
+  isInline: boolean | undefined,
+  max: number,
+  min: number
+) {
+  if (
+    !isInline &&
+    isValidNumber(
+      value,
+      false,
+      max,
+      min,
+      15 // max supported digits in JS
+    )
+  ) {
+    return i18n.translate('xpack.lens.indexPattern.percentile.errorMessageTooManyDigits', {
+      defaultMessage: 'Only {digits} numbers allowed after the decimal point.',
+      values: {
+        digits: ALLOWED_DECIMAL_DIGITS,
+      },
+    });
+  }
+
+  return i18n.translate('xpack.lens.indexPattern.percentile.errorMessage', {
+    defaultMessage: 'Percentile has to be an integer between {min} and {max}',
+    values: {
+      min,
+      max,
+    },
+  });
+}
 
 const supportedFieldTypes = ['number', 'histogram'];
 
@@ -309,10 +343,13 @@ export const percentileOperation: OperationDefinition<
       i18n.translate('xpack.lens.indexPattern.percentile.percentileValue', {
         defaultMessage: 'Percentile',
       });
+
+    const step = isInline ? 1 : 0.0001;
+    const upperBound = isInline ? 99 : 99.9999;
     const onChange = useCallback(
       (value) => {
         if (
-          !isValidNumber(value, true, 99, 1) ||
+          !isValidNumber(value, isInline, upperBound, step, ALLOWED_DECIMAL_DIGITS) ||
           Number(value) === currentColumn.params.percentile
         ) {
           return;
@@ -334,7 +371,7 @@ export const percentileOperation: OperationDefinition<
           },
         } as PercentileIndexPatternColumn);
       },
-      [paramEditorUpdater, currentColumn, indexPattern]
+      [isInline, upperBound, step, currentColumn, paramEditorUpdater, indexPattern]
     );
     const { inputValue, handleInputChange: handleInputChangeWithoutValidation } = useDebouncedValue<
       string | undefined
@@ -342,7 +379,13 @@ export const percentileOperation: OperationDefinition<
       onChange,
       value: String(currentColumn.params.percentile),
     });
-    const inputValueIsValid = isValidNumber(inputValue, true, 99, 1);
+    const inputValueIsValid = isValidNumber(
+      inputValue,
+      isInline,
+      upperBound,
+      step,
+      ALLOWED_DECIMAL_DIGITS
+    );
 
     const handleInputChange = useCallback(
       (e) => handleInputChangeWithoutValidation(String(e.currentTarget.value)),
@@ -357,12 +400,7 @@ export const percentileOperation: OperationDefinition<
         display="rowCompressed"
         fullWidth
         isInvalid={!inputValueIsValid}
-        error={
-          !inputValueIsValid &&
-          i18n.translate('xpack.lens.indexPattern.percentile.errorMessage', {
-            defaultMessage: 'Percentile has to be an integer between 1 and 99',
-          })
-        }
+        error={!inputValueIsValid && getInvalidErrorMessage(inputValue, isInline, upperBound, step)}
       >
         {isInline ? (
           <EuiFieldNumber
@@ -370,9 +408,9 @@ export const percentileOperation: OperationDefinition<
             data-test-subj="lns-indexPattern-percentile-input"
             compressed
             value={inputValue ?? ''}
-            min={1}
-            max={99}
-            step={1}
+            min={step}
+            max={upperBound}
+            step={step}
             onChange={handleInputChange}
             aria-label={percentileLabel}
           />
@@ -382,9 +420,9 @@ export const percentileOperation: OperationDefinition<
             data-test-subj="lns-indexPattern-percentile-input"
             compressed
             value={inputValue ?? ''}
-            min={1}
-            max={99}
-            step={1}
+            min={step}
+            max={upperBound}
+            step={step}
             onChange={handleInputChange}
             showInput
             aria-label={percentileLabel}

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/terms/helpers.ts

@@ -22,6 +22,7 @@ import type { FiltersIndexPatternColumn } from '..';
 import type { TermsIndexPatternColumn } from './types';
 import type { LastValueIndexPatternColumn } from '../last_value';
 import type { PercentileRanksIndexPatternColumn } from '../percentile_ranks';
+import type { PercentileIndexPatternColumn } from '../percentile';
 
 import type { FormBasedLayer } from '../../../types';
 import { MULTI_KEY_VISUAL_SEPARATOR, supportedTypes } from './constants';
@@ -231,13 +232,21 @@ function checkLastValue(column: GenericIndexPatternColumn) {
   );
 }
 
+// allow the rank by metric only if the percentile rank value is integer
+// https://github.com/elastic/elasticsearch/issues/66677
+
+export function isPercentileSortable(column: GenericIndexPatternColumn) {
+  const isPercentileColumn = isColumnOfType<PercentileIndexPatternColumn>('percentile', column);
+  return !isPercentileColumn || (isPercentileColumn && Number.isInteger(column.params.percentile));
+}
+
 export function isPercentileRankSortable(column: GenericIndexPatternColumn) {
-  // allow the rank by metric only if the percentile rank value is integer
-  // https://github.com/elastic/elasticsearch/issues/66677
+  const isPercentileRankColumn = isColumnOfType<PercentileRanksIndexPatternColumn>(
+    'percentile_rank',
+    column
+  );
   return (
-    column.operationType !== 'percentile_rank' ||
-    (column.operationType === 'percentile_rank' &&
-      Number.isInteger((column as PercentileRanksIndexPatternColumn).params.value))
+    !isPercentileRankColumn || (isPercentileRankColumn && Number.isInteger(column.params.value))
   );
 }
 
@@ -248,6 +257,7 @@ export function isSortableByColumn(layer: FormBasedLayer, columnId: string) {
     !column.isBucketed &&
     checkLastValue(column) &&
     isPercentileRankSortable(column) &&
+    isPercentileSortable(column) &&
     !('references' in column) &&
     !isReferenced(layer, columnId)
   );

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/terms/index.tsx

@@ -45,6 +45,7 @@ import {
   getFieldsByValidationState,
   isSortableByColumn,
   isPercentileRankSortable,
+  isPercentileSortable,
 } from './helpers';
 import {
   DEFAULT_MAX_DOC_COUNT,
@@ -310,7 +311,11 @@ export const termsOperation: OperationDefinition<
       const orderColumn = layer.columns[column.params.orderBy.columnId];
       orderBy = String(orderedColumnIds.indexOf(column.params.orderBy.columnId));
       // percentile rank with non integer value should default to alphabetical order
-      if (!orderColumn || !isPercentileRankSortable(orderColumn)) {
+      if (
+        !orderColumn ||
+        !isPercentileRankSortable(orderColumn) ||
+        !isPercentileSortable(orderColumn)
+      ) {
         orderBy = '_key';
       }
     }

x-pack/plugins/lens/public/datasources/form_based/to_expression.ts

@@ -47,11 +47,21 @@ declare global {
 
 // esAggs column ID manipulation functions
 export const extractAggId = (id: string) => id.split('.')[0].split('-')[2];
+// Need a more complex logic for decimals percentiles
+function getAggIdPostFixForPercentile(percentile: string, decimals?: string) {
+  if (!percentile && !decimals) {
+    return '';
+  }
+  if (!decimals) {
+    return `.${percentile}`;
+  }
+  return `['${percentile}.${decimals}']`;
+}
 const updatePositionIndex = (currentId: string, newIndex: number) => {
-  const [fullId, percentile] = currentId.split('.');
+  const [fullId, percentile, percentileDecimals] = currentId.split('.');
   const idParts = fullId.split('-');
   idParts[1] = String(newIndex);
-  return idParts.join('-') + (percentile ? `.${percentile}` : '');
+  return idParts.join('-') + getAggIdPostFixForPercentile(percentile, percentileDecimals);
 };
 
 function getExpressionForLayer(

x-pack/plugins/translations/translations/fr-FR.json

@@ -21050,7 +21050,6 @@
     "xpack.lens.indexPattern.percentFormatLabel": "Pourcent",
     "xpack.lens.indexPattern.percentile": "Centile",
     "xpack.lens.indexPattern.percentile.documentation.quick": "\n      La plus grande valeur qui est inférieure à n pour cent des valeurs présentes dans tous les documents.\n      ",
-    "xpack.lens.indexPattern.percentile.errorMessage": "Le centile doit être un entier compris entre 1 et 99",
     "xpack.lens.indexPattern.percentile.percentileRanksValue": "Valeur des rangs centiles",
     "xpack.lens.indexPattern.percentile.percentileValue": "Centile",
     "xpack.lens.indexPattern.percentile.signature": "champ : chaîne, [percentile] : nombre",

x-pack/plugins/translations/translations/ja-JP.json

@@ -21065,7 +21065,6 @@
     "xpack.lens.indexPattern.percentFormatLabel": "割合（%）",
     "xpack.lens.indexPattern.percentile": "パーセンタイル",
     "xpack.lens.indexPattern.percentile.documentation.quick": "\n      すべてのドキュメントで発生する値のnパーセントよりも小さい最大値。\n      ",
-    "xpack.lens.indexPattern.percentile.errorMessage": "パーセンタイルは1～99の範囲の整数でなければなりません。",
     "xpack.lens.indexPattern.percentile.percentileRanksValue": "パーセンタイル順位値",
     "xpack.lens.indexPattern.percentile.percentileValue": "パーセンタイル",
     "xpack.lens.indexPattern.percentile.signature": "フィールド：文字列、[percentile]：数値",

x-pack/plugins/translations/translations/zh-CN.json

@@ -21065,7 +21065,6 @@
     "xpack.lens.indexPattern.percentFormatLabel": "百分比",
     "xpack.lens.indexPattern.percentile": "百分位数",
     "xpack.lens.indexPattern.percentile.documentation.quick": "\n      小于所有文档中出现值的 n% 的最大值。\n      ",
-    "xpack.lens.indexPattern.percentile.errorMessage": "百分位数必须是介于 1 到 99 之间的整数",
     "xpack.lens.indexPattern.percentile.percentileRanksValue": "百分位等级值",
     "xpack.lens.indexPattern.percentile.percentileValue": "百分位数",
     "xpack.lens.indexPattern.percentile.signature": "field: string, [percentile]: number",

x-pack/test/functional/apps/lens/group1/smokescreen.ts

@@ -761,5 +761,36 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       const hasVisualOptionsButton = await PageObjects.lens.hasVisualOptionsButton();
       expect(hasVisualOptionsButton).to.be(false);
     });
+
+    it('should correctly optimize multiple percentile metrics', async () => {
+      await PageObjects.visualize.navigateToNewVisualization();
+      await PageObjects.visualize.clickVisType('lens');
+      for (const percentileValue of [90, 95.5, 99.9]) {
+        await PageObjects.lens.configureDimension({
+          dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension',
+          operation: 'percentile',
+          field: 'bytes',
+          keepOpen: true,
+        });
+
+        await retry.try(async () => {
+          const value = `${percentileValue}`;
+          // Can not use testSubjects because data-test-subj is placed range input and number input
+          const percentileInput = await PageObjects.lens.getNumericFieldReady(
+            'lns-indexPattern-percentile-input'
+          );
+          await percentileInput.type(value);
+
+          const attrValue = await percentileInput.getAttribute('value');
+          if (attrValue !== value) {
+            throw new Error(`layerPanelTopHitsSize not set to ${value}`);
+          }
+        });
+
+        await PageObjects.lens.closeDimensionEditor();
+      }
+      await PageObjects.lens.waitForVisualization('xyVisChart');
+      expect(await PageObjects.lens.getWorkspaceErrorCount()).to.eql(0);
+    });
   });
 }

x-pack/test/functional/apps/lens/group3/terms.ts

@@ -96,62 +96,128 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         await PageObjects.lens.closeDimensionEditor();
       });
     });
-    describe('sorting by custom metric', () => {
-      it('should allow sort by custom metric', async () => {
-        await PageObjects.visualize.navigateToNewVisualization();
-        await PageObjects.visualize.clickVisType('lens');
-        await elasticChart.setNewChartUiDebugFlag(true);
-        await PageObjects.lens.goToTimeRange();
+    describe('rank by', () => {
+      describe('reset rank on metric change', () => {
+        it('should reset the ranking when using decimals on percentile', async () => {
+          await PageObjects.visualize.navigateToNewVisualization();
+          await PageObjects.visualize.clickVisType('lens');
 
-        await PageObjects.lens.configureDimension({
-          dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension',
-          operation: 'average',
-          field: 'bytes',
-        });
+          await PageObjects.lens.configureDimension({
+            dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension',
+            operation: 'terms',
+            field: 'geo.src',
+          });
 
-        await PageObjects.lens.configureDimension({
-          dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension',
-          operation: 'terms',
-          field: 'geo.src',
-          keepOpen: true,
-        });
-        await find.clickByCssSelector(
-          'select[data-test-subj="indexPattern-terms-orderBy"] > option[value="custom"]'
-        );
+          await PageObjects.lens.configureDimension({
+            dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension',
+            operation: 'percentile',
+            field: 'bytes',
+            keepOpen: true,
+          });
 
-        const fnTarget = await testSubjects.find('indexPattern-reference-function');
-        await comboBox.openOptionsList(fnTarget);
-        await comboBox.setElement(fnTarget, 'percentile');
+          await retry.try(async () => {
+            const value = '60.5';
+            // Can not use testSubjects because data-test-subj is placed range input and number input
+            const percentileInput = await PageObjects.lens.getNumericFieldReady(
+              'lns-indexPattern-percentile-input'
+            );
+            await percentileInput.clearValueWithKeyboard();
+            await percentileInput.type(value);
 
-        const fieldTarget = await testSubjects.find(
-          'indexPattern-reference-field-selection-row>indexPattern-dimension-field'
-        );
-        await comboBox.openOptionsList(fieldTarget);
-        await comboBox.setElement(fieldTarget, 'bytes');
+            const percentileValue = await percentileInput.getAttribute('value');
+            if (percentileValue !== value) {
+              throw new Error(
+                `[date-test-subj="lns-indexPattern-percentile-input"] not set to ${value}`
+              );
+            }
+          });
 
-        await retry.try(async () => {
-          // Can not use testSubjects because data-test-subj is placed range input and number input
-          const percentileInput = await PageObjects.lens.getNumericFieldReady(
-            'lns-indexPattern-percentile-input'
+          // close the toast about reset ranking
+          // note: this has also the side effect to close the dimension editor
+          await testSubjects.click('toastCloseButton');
+
+          await PageObjects.lens.openDimensionEditor(
+            'lnsXY_yDimensionPanel > lns-dimensionTrigger'
           );
-          await percentileInput.type('60');
 
-          const percentileValue = await percentileInput.getAttribute('value');
-          if (percentileValue !== '60') {
-            throw new Error('layerPanelTopHitsSize not set to 60');
-          }
+          await PageObjects.lens.selectOperation('percentile_rank');
+
+          await retry.try(async () => {
+            const value = '600.5';
+            const percentileRankInput = await testSubjects.find(
+              'lns-indexPattern-percentile_ranks-input'
+            );
+            await percentileRankInput.clearValueWithKeyboard();
+            await percentileRankInput.type(value);
+
+            const percentileRankValue = await percentileRankInput.getAttribute('value');
+            if (percentileRankValue !== value) {
+              throw new Error(
+                `[date-test-subj="lns-indexPattern-percentile_ranks-input"] not set to ${value}`
+              );
+            }
+          });
+          // note: this has also the side effect to close the dimension editor
+          await testSubjects.click('toastCloseButton');
         });
+      });
+      describe('sorting by custom metric', () => {
+        it('should allow sort by custom metric', async () => {
+          await PageObjects.visualize.navigateToNewVisualization();
+          await PageObjects.visualize.clickVisType('lens');
+          await elasticChart.setNewChartUiDebugFlag(true);
+          await PageObjects.lens.goToTimeRange();
 
-        await PageObjects.lens.waitForVisualization('xyVisChart');
-        await PageObjects.lens.closeDimensionEditor();
+          await PageObjects.lens.configureDimension({
+            dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension',
+            operation: 'average',
+            field: 'bytes',
+          });
 
-        expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_xDimensionPanel', 0)).to.eql(
-          'Top 5 values of geo.src'
-        );
+          await PageObjects.lens.configureDimension({
+            dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension',
+            operation: 'terms',
+            field: 'geo.src',
+            keepOpen: true,
+          });
+          await find.clickByCssSelector(
+            'select[data-test-subj="indexPattern-terms-orderBy"] > option[value="custom"]'
+          );
 
-        const data = await PageObjects.lens.getCurrentChartDebugState('xyVisChart');
-        expect(data!.bars![0].bars[0].x).to.eql('BN');
-        expect(data!.bars![0].bars[0].y).to.eql(19265);
+          const fnTarget = await testSubjects.find('indexPattern-reference-function');
+          await comboBox.openOptionsList(fnTarget);
+          await comboBox.setElement(fnTarget, 'percentile');
+
+          const fieldTarget = await testSubjects.find(
+            'indexPattern-reference-field-selection-row>indexPattern-dimension-field'
+          );
+          await comboBox.openOptionsList(fieldTarget);
+          await comboBox.setElement(fieldTarget, 'bytes');
+
+          await retry.try(async () => {
+            // Can not use testSubjects because data-test-subj is placed range input and number input
+            const percentileInput = await PageObjects.lens.getNumericFieldReady(
+              'lns-indexPattern-percentile-input'
+            );
+            await percentileInput.type('60');
+
+            const percentileValue = await percentileInput.getAttribute('value');
+            if (percentileValue !== '60') {
+              throw new Error('layerPanelTopHitsSize not set to 60');
+            }
+          });
+
+          await PageObjects.lens.waitForVisualization('xyVisChart');
+          await PageObjects.lens.closeDimensionEditor();
+
+          expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_xDimensionPanel', 0)).to.eql(
+            'Top 5 values of geo.src'
+          );
+
+          const data = await PageObjects.lens.getCurrentChartDebugState('xyVisChart');
+          expect(data!.bars![0].bars[0].x).to.eql('BN');
+          expect(data!.bars![0].bars[0].y).to.eql(19265);
+        });
       });
     });
 

x-pack/test/functional/apps/lens/open_in_lens/agg_based/xy.ts

@@ -357,5 +357,17 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       });
       expect(data?.legend?.items.map((item) => item.name)).to.eql(expectedData);
     });
+
+    it('should convert correctly percentiles with decimals', async () => {
+      await visEditor.clickBucket('Y-axis', 'metrics');
+      await visEditor.selectAggregation('Percentiles', 'metrics');
+      await visEditor.selectField('memory', 'metrics');
+      await visEditor.setPercentileValue('99.99', 6);
+      await visEditor.clickGo();
+      await header.waitUntilLoadingHasFinished();
+      await visualize.navigateToLensFromAnotherVisulization();
+      await lens.waitForVisualization('xyVisChart');
+      expect(await lens.getWorkspaceErrorCount()).to.eql(0);
+    });
   });
 }

x-pack/test/functional/page_objects/lens_page.ts

@@ -691,7 +691,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont
     },
     async getNumericFieldReady(testSubj: string) {
       const numericInput = await find.byCssSelector(
-        `input[data-test-subj=${testSubj}][type='number']`
+        `input[data-test-subj="${testSubj}"][type='number']`
       );
       await numericInput.click();
       await numericInput.clearValue();