[ML] Handle Empty Partition Field Values in Single Metric Viewer (#61649) (#61983)

* [ML] WIP support empty partition fields values * [ML] support empty field in anomaly table * [ML] remove comments * [ML] fix context chart * [ML] rename empty field label, render as italic * [ML] rename empty field label * [ML] fix focus chart * [ML] add time range capping for fields_service.ts * [ML] empty string labels in anomaly explorer
2025-04-24 09:48:58 -04:00 · 2020-03-31 23:11:53 +02:00 · 2020-03-31 23:11:53 +02:00 · 874fe9d7e4
commit 874fe9d7e4
parent e08cf61bcc
7 changed files with 96 additions and 73 deletions
--- a/x-pack/plugins/ml/public/application/components/entity_cell/entity_cell.js
+++ b/x-pack/plugins/ml/public/application/components/entity_cell/entity_cell.js
@ -10,6 +10,8 @@ import React from 'react';
 import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem, EuiText, EuiToolTip } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
+import { EMPTY_FIELD_VALUE_LABEL } from '../../timeseriesexplorer/components/entity_control/entity_control';
+import { MLCATEGORY } from '../../../../common/constants/field_types';

 function getAddFilter({ entityName, entityValue, filter }) {
  return (
@ -68,7 +70,11 @@ export const EntityCell = function EntityCell({
  filter,
  wrapText = false,
 }) {
-  const valueText = entityName !== 'mlcategory' ? entityValue : `mlcategory ${entityValue}`;
+  let valueText = entityValue === '' ? <i>{EMPTY_FIELD_VALUE_LABEL}</i> : entityValue;
+  if (entityName === MLCATEGORY) {
+    valueText = `${MLCATEGORY} ${valueText}`;
+  }
+
  const textStyle = { maxWidth: '100%' };
  const textWrapperClass = wrapText ? 'field-value-long' : 'field-value-short';
  const shouldDisplayIcons =
--- a/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.js
+++ b/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.js
@ -25,6 +25,7 @@ import { mlChartTooltipService } from '../components/chart_tooltip/chart_tooltip
 import { ALLOW_CELL_RANGE_SELECTION, dragSelect$ } from './explorer_dashboard_service';
 import { DRAG_SELECT_ACTION } from './explorer_constants';
 import { i18n } from '@kbn/i18n';
+import { EMPTY_FIELD_VALUE_LABEL } from '../timeseriesexplorer/components/entity_control/entity_control';

 const SCSS = {
  mlDragselectDragging: 'mlDragselectDragging',
@ -309,6 +310,7 @@ export class ExplorerSwimlane extends React.Component {
      return function(lane) {
        const bucketScore = getBucketScore(lane, time);
        if (bucketScore !== 0) {
+          lane = lane === '' ? EMPTY_FIELD_VALUE_LABEL : lane;
          cellMouseover(this, lane, bucketScore, i, time);
        }
      };
@ -376,7 +378,7 @@ export class ExplorerSwimlane extends React.Component {
            values: { label: mlEscape(label) },
          });
        } else {
-          return mlEscape(label);
+          return label === '' ? `<i>${EMPTY_FIELD_VALUE_LABEL}</i>` : mlEscape(label);
        }
      })
      .on('click', () => {
@ -393,7 +395,7 @@ export class ExplorerSwimlane extends React.Component {
                  { skipHeader: true },
                  {
                    label: swimlaneData.fieldName,
-                    value,
+                    value: value === '' ? EMPTY_FIELD_VALUE_LABEL : value,
                    seriesIdentifier: { key: value },
                    valueAccessor: 'fieldName',
                  },
--- a/x-pack/plugins/ml/public/application/services/results_service/results_service.js
+++ b/x-pack/plugins/ml/public/application/services/results_service/results_service.js
@ -1259,39 +1259,13 @@ export function getRecordMaxScoreByTime(jobId, criteriaFields, earliestMs, lates
      },
      { term: { job_id: jobId } },
    ];
-    const shouldCriteria = [];

    _.each(criteriaFields, criteria => {
-      if (criteria.fieldValue.length !== 0) {
-        mustCriteria.push({
-          term: {
-            [criteria.fieldName]: criteria.fieldValue,
-          },
-        });
-      } else {
-        // Add special handling for blank entity field values, checking for either
-        // an empty string or the field not existing.
-        const emptyFieldCondition = {
-          bool: {
-            must: [
-              {
-                term: {},
-              },
-            ],
-          },
-        };
-        emptyFieldCondition.bool.must[0].term[criteria.fieldName] = '';
-        shouldCriteria.push(emptyFieldCondition);
-        shouldCriteria.push({
-          bool: {
-            must_not: [
-              {
-                exists: { field: criteria.fieldName },
-              },
-            ],
-          },
-        });
-      }
+      mustCriteria.push({
+        term: {
+          [criteria.fieldName]: criteria.fieldValue,
+        },
+      });
    });

    ml.esSearch({
--- a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx
+++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx
@ -16,6 +16,7 @@ import {
  EuiFormRow,
  EuiToolTip,
 } from '@elastic/eui';
+import { EuiSelectableOption } from '@elastic/eui/src/components/selectable/selectable_option';

 export interface Entity {
  fieldName: string;
@ -29,15 +30,22 @@ interface EntityControlProps {
  isLoading: boolean;
  onSearchChange: (entity: Entity, queryTerm: string) => void;
  forceSelection: boolean;
-  options: EuiComboBoxOptionOption[];
+  options: Array<EuiComboBoxOptionOption<string>>;
 }

 interface EntityControlState {
-  selectedOptions: EuiComboBoxOptionOption[] | undefined;
+  selectedOptions: Array<EuiComboBoxOptionOption<string>> | undefined;
  isLoading: boolean;
-  options: EuiComboBoxOptionOption[] | undefined;
+  options: Array<EuiComboBoxOptionOption<string>> | undefined;
 }

+export const EMPTY_FIELD_VALUE_LABEL = i18n.translate(
+  'xpack.ml.timeSeriesExplorer.emptyPartitionFieldLabel.',
+  {
+    defaultMessage: '"" (empty string)',
+  }
+);
+
 export class EntityControl extends Component<EntityControlProps, EntityControlState> {
  inputRef: any;

@ -53,16 +61,18 @@ export class EntityControl extends Component<EntityControlProps, EntityControlSt

    const { fieldValue } = entity;

-    let selectedOptionsUpdate: EuiComboBoxOptionOption[] | undefined = selectedOptions;
+    let selectedOptionsUpdate: Array<EuiComboBoxOptionOption<string>> | undefined = selectedOptions;
    if (
-      (selectedOptions === undefined && fieldValue.length > 0) ||
+      (selectedOptions === undefined && fieldValue !== null) ||
      (Array.isArray(selectedOptions) &&
        // @ts-ignore
-        selectedOptions[0].label !== fieldValue &&
-        fieldValue.length > 0)
+        selectedOptions[0].value !== fieldValue &&
+        fieldValue !== null)
    ) {
-      selectedOptionsUpdate = [{ label: fieldValue }];
-    } else if (Array.isArray(selectedOptions) && fieldValue.length === 0) {
+      selectedOptionsUpdate = [
+        { label: fieldValue === '' ? EMPTY_FIELD_VALUE_LABEL : fieldValue, value: fieldValue },
+      ];
+    } else if (Array.isArray(selectedOptions) && fieldValue === null) {
      selectedOptionsUpdate = undefined;
    }

@ -84,14 +94,14 @@ export class EntityControl extends Component<EntityControlProps, EntityControlSt
    }
  }

-  onChange = (selectedOptions: EuiComboBoxOptionOption[]) => {
+  onChange = (selectedOptions: Array<EuiComboBoxOptionOption<string>>) => {
    const options = selectedOptions.length > 0 ? selectedOptions : undefined;
    this.setState({
      selectedOptions: options,
    });

    const fieldValue =
-      Array.isArray(options) && options[0].label.length > 0 ? options[0].label : '';
+      Array.isArray(options) && options[0].value !== null ? options[0].value : null;
    this.props.entityFieldValueChanged(this.props.entity, fieldValue);
  };

@ -103,6 +113,11 @@ export class EntityControl extends Component<EntityControlProps, EntityControlSt
    this.props.onSearchChange(this.props.entity, searchValue);
  };

+  renderOption = (option: EuiSelectableOption) => {
+    const { label } = option;
+    return label === EMPTY_FIELD_VALUE_LABEL ? <i>{label}</i> : label;
+  };
+
  render() {
    const { entity, forceSelection } = this.props;
    const { isLoading, options, selectedOptions } = this.state;
@ -126,6 +141,7 @@ export class EntityControl extends Component<EntityControlProps, EntityControlSt
        onChange={this.onChange}
        onSearchChange={this.onSearchChange}
        isClearable={false}
+        renderOption={this.renderOption}
      />
    );

--- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts
+++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseries_search_service.ts
@ -128,7 +128,7 @@ function getChartDetails(
    obj.results.functionLabel = functionLabel;

    const blankEntityFields = _.filter(entityFields, entity => {
-      return entity.fieldValue.length === 0;
+      return entity.fieldValue === null;
    });

    // Look to see if any of the entity fields have defined values
--- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js
+++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js
@ -78,6 +78,7 @@ import {
  processRecordScoreResults,
  getFocusData,
 } from './timeseriesexplorer_utils';
+import { EMPTY_FIELD_VALUE_LABEL } from './components/entity_control/entity_control';

 // Used to indicate the chart is being plotted across
 // all partition field values, where the cardinality of the field cannot be
@ -94,7 +95,7 @@ function getEntityControlOptions(fieldValues) {
  fieldValues.sort();

  return fieldValues.map(value => {
-    return { label: value };
+    return { label: value === '' ? EMPTY_FIELD_VALUE_LABEL : value, value };
  });
 }

@ -192,7 +193,7 @@ export class TimeSeriesExplorer extends React.Component {
  getFieldNamesWithEmptyValues = () => {
    const latestEntityControls = this.getControlsForDetector();
    return latestEntityControls
-      .filter(({ fieldValue }) => !fieldValue)
+      .filter(({ fieldValue }) => fieldValue === null)
      .map(({ fieldName }) => fieldName);
  };

@ -249,7 +250,7 @@ export class TimeSeriesExplorer extends React.Component {
    if (operator === '+' && entity.fieldValue !== value) {
      resultValue = value;
    } else if (operator === '-' && entity.fieldValue === value) {
-      resultValue = '';
+      resultValue = null;
    } else {
      return;
    }
@ -302,7 +303,7 @@ export class TimeSeriesExplorer extends React.Component {
      focusAggregationInterval,
      selectedForecastId,
      modelPlotEnabled,
-      entityControls.filter(entity => entity.fieldValue.length > 0),
+      entityControls.filter(entity => entity.fieldValue !== null),
      searchBounds,
      selectedJob,
      TIME_FIELD_NAME
@ -576,7 +577,7 @@ export class TimeSeriesExplorer extends React.Component {
        };

        const nonBlankEntities = entityControls.filter(entity => {
-          return entity.fieldValue.length > 0;
+          return entity.fieldValue !== null;
        });

        if (
@ -739,7 +740,7 @@ export class TimeSeriesExplorer extends React.Component {
    const overFieldName = get(detector, 'over_field_name');
    const byFieldName = get(detector, 'by_field_name');
    if (partitionFieldName !== undefined) {
-      const partitionFieldValue = get(entitiesState, partitionFieldName, '');
+      const partitionFieldValue = get(entitiesState, partitionFieldName, null);
      entities.push({
        fieldType: 'partition_field',
        fieldName: partitionFieldName,
@ -747,7 +748,7 @@ export class TimeSeriesExplorer extends React.Component {
      });
    }
    if (overFieldName !== undefined) {
-      const overFieldValue = get(entitiesState, overFieldName, '');
+      const overFieldValue = get(entitiesState, overFieldName, null);
      entities.push({
        fieldType: 'over_field',
        fieldName: overFieldName,
@ -761,7 +762,7 @@ export class TimeSeriesExplorer extends React.Component {
    // TODO - metric data can be filtered by this field, so should only exclude
    // from filter for the anomaly records.
    if (byFieldName !== undefined && overFieldName === undefined) {
-      const byFieldValue = get(entitiesState, byFieldName, '');
+      const byFieldValue = get(entitiesState, byFieldName, null);
      entities.push({ fieldType: 'by_field', fieldName: byFieldName, fieldValue: byFieldValue });
    }

@ -775,7 +776,7 @@ export class TimeSeriesExplorer extends React.Component {
   */
  getCriteriaFields(detectorIndex, entities) {
    // Only filter on the entity if the field has a value.
-    const nonBlankEntities = entities.filter(entity => entity.fieldValue.length > 0);
+    const nonBlankEntities = entities.filter(entity => entity.fieldValue !== null);
    return [
      {
        fieldName: 'detector_index',
@ -1150,7 +1151,7 @@ export class TimeSeriesExplorer extends React.Component {
            </EuiFlexItem>
            {entityControls.map(entity => {
              const entityKey = `${entity.fieldName}`;
-              const forceSelection = !hasEmptyFieldValues && !entity.fieldValue;
+              const forceSelection = !hasEmptyFieldValues && entity.fieldValue === null;
              hasEmptyFieldValues = !hasEmptyFieldValues && forceSelection;
              return (
                <EntityControl
--- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts
+++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts
@ -6,6 +6,7 @@

 import Boom from 'boom';
 import { APICaller } from 'kibana/server';
+import { duration } from 'moment';
 import { parseInterval } from '../../../common/util/parse_interval';
 import { initCardinalityFieldsCache } from './fields_aggs_cache';

@ -16,6 +17,19 @@ import { initCardinalityFieldsCache } from './fields_aggs_cache';
 export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
  const fieldsAggsCache = initCardinalityFieldsCache();

+  /**
+   * Caps the time range to the last 90 days if necessary
+   */
+  function getSafeTimeRange(earliestMs: number, latestMs: number): { start: number; end: number } {
+    const capOffsetMs = duration(3, 'months').asMilliseconds();
+    const capRangeStart = latestMs - capOffsetMs;
+
+    return {
+      start: Math.max(earliestMs, capRangeStart),
+      end: latestMs,
+    };
+  }
+
  /**
   * Gets aggregatable fields.
   */
@ -61,12 +75,14 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
      return {};
    }

+    const { start, end } = getSafeTimeRange(earliestMs, latestMs);
+
    const cachedValues =
      fieldsAggsCache.getValues(
        index,
        timeFieldName,
-        earliestMs,
-        latestMs,
+        start,
+        end,
        'overallCardinality',
        fieldNames
      ) ?? {};
@ -84,8 +100,8 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
      {
        range: {
          [timeFieldName]: {
-            gte: earliestMs,
-            lte: latestMs,
+            gte: start,
+            lte: end,
            format: 'epoch_millis',
          },
        },
@ -130,7 +146,7 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
      return obj;
    }, {} as { [field: string]: number });

-    fieldsAggsCache.updateValues(index, timeFieldName, earliestMs, latestMs, {
+    fieldsAggsCache.updateValues(index, timeFieldName, start, end, {
      overallCardinality: aggResult,
    });

@ -185,15 +201,16 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
  }

  /**
-   * Caps provided time boundaries based on the interval.
-   * @param earliestMs
-   * @param latestMs
-   * @param interval
+   * Caps provided time boundaries based on the interval
   */
-  function getSafeTimeRange(
+  function getSafeTimeRangeForInterval(
+    interval: string,
+    ...timeRange: number[]
+  ): { start: number; end: number };
+  function getSafeTimeRangeForInterval(
+    interval: string,
    earliestMs: number,
-    latestMs: number,
-    interval: string
+    latestMs: number
  ): { start: number; end: number } {
    const maxNumberOfBuckets = 1000;
    const end = latestMs;
@ -234,7 +251,7 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
    interval: string | undefined
  ): Promise<{ [key: string]: number }> {
    if (!interval) {
-      throw new Error('Interval is required to retrieve max bucket cardinalities.');
+      throw Boom.badRequest('Interval is required to retrieve max bucket cardinalities.');
    }

    const aggregatableFields = await getAggregatableFields(index, fieldNames);
@ -243,12 +260,17 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
      return {};
    }

+    const { start, end } = getSafeTimeRangeForInterval(
+      interval,
+      ...Object.values(getSafeTimeRange(earliestMs, latestMs))
+    );
+
    const cachedValues =
      fieldsAggsCache.getValues(
        index,
        timeFieldName,
-        earliestMs,
-        latestMs,
+        start,
+        end,
        'maxBucketCardinality',
        fieldNames
      ) ?? {};
@ -260,8 +282,6 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
      return cachedValues;
    }

-    const { start, end } = getSafeTimeRange(earliestMs, latestMs, interval);
-
    const mustCriteria = [
      {
        range: {
@ -334,6 +354,10 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) {
      return obj;
    }, {} as { [field: string]: number });

+    fieldsAggsCache.updateValues(index, timeFieldName, start, end, {
+      maxBucketCardinality: aggResult,
+    });
+
    return {
      ...cachedValues,
      ...aggResult,