[Response Ops][Alerting] Allow runtime fields to be selected for Elasticsearch query rule type group by or aggregate over options (#160319)

Resolves https://github.com/elastic/kibana/issues/157258

## Summary

Gets the runtime_mappings from the es query, and includes them in the
query to retrieve the fields.
Also gets runtime mappings from data views.


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios


### To verify

- Create a new Es Query rule using DSL
- Include runtime mappings in your query and verify that the runtime
fields are listed in the group by or aggregate over options below the
query
- Create a dataview and include runtime fields
- Create a new Es Query rule using KQL
- Verify that the runtime fields are listed in the group by or aggregate
over options below the query

x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/es_query_expression.tsx

@@ -6,6 +6,7 @@
  */
 
 import React, { useState, Fragment, useEffect, useCallback } from 'react';
+import { get, sortBy } from 'lodash';
 import { lastValueFrom } from 'rxjs';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
@@ -32,7 +33,7 @@ import { EsQueryRuleParams, EsQueryRuleMetaData, SearchType } from '../types';
 import { IndexSelectPopover } from '../../components/index_select_popover';
 import { DEFAULT_VALUES } from '../constants';
 import { RuleCommonExpressions } from '../rule_common_expressions';
-import { useTriggerUiActionServices } from '../util';
+import { convertRawRuntimeFieldtoFieldOption, useTriggerUiActionServices } from '../util';
 
 const { useXJsonMode } = XJson;
 
@@ -89,6 +90,8 @@ export const EsQueryExpression: React.FC<
   const { http, docLinks } = services;
 
   const [esFields, setEsFields] = useState<FieldOption[]>([]);
+  const [runtimeFields, setRuntimeFields] = useState<FieldOption[]>([]);
+  const [combinedFields, setCombinedFields] = useState<FieldOption[]>([]);
   const { convertToJson, setXJson, xJson } = useXJsonMode(DEFAULT_VALUES.QUERY);
 
   const setDefaultExpressionValues = async () => {
@@ -108,6 +111,21 @@ export const EsQueryExpression: React.FC<
   const refreshEsFields = async (indices: string[]) => {
     const currentEsFields = await getFields(http, indices);
     setEsFields(currentEsFields);
+    setCombinedFields(sortBy(currentEsFields.concat(runtimeFields), 'name'));
+  };
+
+  const getRuntimeFields = () => {
+    let runtimeMappings;
+    try {
+      runtimeMappings = get(JSON.parse(xJson), 'runtime_mappings');
+    } catch (e) {
+      // ignore error
+    }
+    if (runtimeMappings) {
+      const currentRuntimeFields = convertRawRuntimeFieldtoFieldOption(runtimeMappings);
+      setRuntimeFields(currentRuntimeFields);
+      setCombinedFields(sortBy(esFields.concat(currentRuntimeFields), 'name'));
+    }
   };
 
   const onTestQuery = useCallback(async () => {
@@ -252,6 +270,7 @@ export const EsQueryExpression: React.FC<
           onChange={(xjson: string) => {
             setXJson(xjson);
             setParam('esQuery', convertToJson(xjson));
+            getRuntimeFields();
           }}
           options={{
             ariaLabel: i18n.translate('xpack.stackAlerts.esQuery.ui.queryEditor', {
@@ -276,7 +295,7 @@ export const EsQueryExpression: React.FC<
         timeWindowSize={timeWindowSize}
         timeWindowUnit={timeWindowUnit}
         size={size}
-        esFields={esFields}
+        esFields={combinedFields}
         aggType={aggType}
         aggField={aggField}
         groupBy={groupBy}

x-pack/plugins/stack_alerts/public/rule_types/es_query/util.test.ts

@@ -5,12 +5,30 @@
  * 2.0.
  */
 
-import { convertFieldSpecToFieldOption } from './util';
+import { convertFieldSpecToFieldOption, convertRawRuntimeFieldtoFieldOption } from './util';
 
-describe('convertFieldSpecToFieldOption', () => {
+describe('Es Query utils', () => {
   test('should correctly convert FieldSpec to FieldOption', () => {
     expect(
       convertFieldSpecToFieldOption([
+        {
+          count: 0,
+          name: 'day_of_week',
+          type: 'string',
+          esTypes: ['keyword'],
+          scripted: false,
+          searchable: true,
+          aggregatable: true,
+          readFromDocValues: false,
+          shortDotsEnable: false,
+          runtimeField: {
+            type: 'keyword',
+            script: {
+              source:
+                "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
+            },
+          },
+        },
         {
           count: 0,
           name: '@timestamp',
@@ -85,6 +103,13 @@ describe('convertFieldSpecToFieldOption', () => {
         },
       ])
     ).toEqual([
+      {
+        name: 'day_of_week',
+        type: 'keyword',
+        normalizedType: 'keyword',
+        aggregatable: true,
+        searchable: true,
+      },
       {
         name: '@timestamp',
         type: 'date',
@@ -122,4 +147,60 @@ describe('convertFieldSpecToFieldOption', () => {
       },
     ]);
   });
+
+  test('should correctly convert raw runtime field to FieldOption', () => {
+    expect(
+      convertRawRuntimeFieldtoFieldOption({
+        day_of_week: {
+          type: 'keyword',
+          script: {
+            source:
+              "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
+          },
+        },
+        location: {
+          type: 'lookup',
+          target_index: 'ip_location',
+          input_field: 'host',
+          target_field: 'ip',
+          fetch_fields: ['country', 'city'],
+        },
+      })
+    ).toEqual([
+      {
+        name: 'day_of_week',
+        type: 'keyword',
+        normalizedType: 'keyword',
+        aggregatable: true,
+        searchable: true,
+      },
+      {
+        name: 'location',
+        type: 'lookup',
+        normalizedType: 'lookup',
+        aggregatable: false,
+        searchable: false,
+      },
+    ]);
+  });
+
+  test('should return an empty array if raw runtime fields are malformed JSON', () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const rawFields: any = null;
+    expect(convertRawRuntimeFieldtoFieldOption(rawFields)).toEqual([]);
+  });
+
+  test('should not return FieldOption if raw runtime fields do not include the type', () => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const rawFields: any = {
+      day_of_week: {
+        test: 'keyword',
+        script: {
+          source:
+            "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
+        },
+      },
+    };
+    expect(convertRawRuntimeFieldtoFieldOption(rawFields)).toEqual([]);
+  });
 });

x-pack/plugins/stack_alerts/public/rule_types/es_query/util.ts

@@ -7,7 +7,8 @@
 
 import { FieldSpec } from '@kbn/data-views-plugin/common';
 import { useKibana } from '@kbn/triggers-actions-ui-plugin/public';
-import { FieldOption } from '@kbn/triggers-actions-ui-plugin/public/common';
+import { FieldOption, NORMALIZED_FIELD_TYPES } from '@kbn/triggers-actions-ui-plugin/public/common';
+import { estypes } from '@elastic/elasticsearch';
 import { EsQueryRuleParams, SearchType } from './types';
 
 export const isSearchSourceRule = (
@@ -18,7 +19,7 @@ export const isSearchSourceRule = (
 
 export const convertFieldSpecToFieldOption = (fieldSpec: FieldSpec[]): FieldOption[] => {
   return (fieldSpec ?? [])
-    .filter((spec: FieldSpec) => spec.isMapped)
+    .filter((spec: FieldSpec) => spec.isMapped || spec.runtimeField)
     .map((spec: FieldSpec) => {
       const converted = {
         name: spec.name,
@@ -41,4 +42,31 @@ export const convertFieldSpecToFieldOption = (fieldSpec: FieldSpec[]): FieldOpti
     });
 };
 
+export const convertRawRuntimeFieldtoFieldOption = (
+  rawFields: Record<string, estypes.MappingRuntimeField>
+): FieldOption[] => {
+  const result: FieldOption[] = [];
+
+  // verifying that the raw fields are an object
+  let keys;
+  try {
+    keys = Object.keys(rawFields);
+  } catch (e) {
+    return result;
+  }
+
+  for (const name of keys) {
+    const rawField = rawFields[name];
+    const type = rawField.type;
+
+    const normalizedType = NORMALIZED_FIELD_TYPES[type] || type;
+    const isAggregatableAndSearchable = type !== 'lookup';
+    const aggregatable = isAggregatableAndSearchable;
+    const searchable = isAggregatableAndSearchable;
+
+    if (type) result.push({ name, type, normalizedType, aggregatable, searchable });
+  }
+  return result;
+};
+
 export const useTriggerUiActionServices = () => useKibana().services;

x-pack/plugins/triggers_actions_ui/common/index.ts

@@ -12,3 +12,4 @@ export * from './data';
 export const BASE_TRIGGERS_ACTIONS_UI_API_PATH = '/internal/triggers_actions_ui';
 export * from './parse_interval';
 export * from './experimental_features';
+export * from './normalized_field_types';

x-pack/plugins/triggers_actions_ui/common/normalized_field_types.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const NORMALIZED_FIELD_TYPES: Record<string, string> = {
+  long: 'number',
+  integer: 'number',
+  short: 'number',
+  byte: 'number',
+  double: 'number',
+  float: 'number',
+  half_float: 'number',
+  scaled_float: 'number',
+  unsigned_long: 'number',
+};

x-pack/plugins/triggers_actions_ui/public/common/index.ts

@@ -39,5 +39,6 @@ export {
   isCountAggregation,
   isGroupAggregation,
   parseAggregationResults,
+  NORMALIZED_FIELD_TYPES,
 } from '../../common';
 export type { ParsedAggregationGroup } from '../../common';

x-pack/plugins/triggers_actions_ui/server/data/routes/fields.ts

@@ -15,6 +15,7 @@ import {
   ElasticsearchClient,
 } from '@kbn/core/server';
 import { Logger } from '@kbn/core/server';
+import { NORMALIZED_FIELD_TYPES } from '../../../common';
 
 const bodySchema = schema.object({
   indexPatterns: schema.arrayOf(schema.string()),
@@ -118,7 +119,7 @@ function getFieldsFromRawFields(rawFields: RawFields): Field[] {
     if (!type || type.startsWith('_')) continue;
     if (!values) continue;
 
-    const normalizedType = normalizedFieldTypes[type] || type;
+    const normalizedType = NORMALIZED_FIELD_TYPES[type] || type;
     const aggregatable = values.aggregatable;
     const searchable = values.searchable;
 
@@ -128,15 +129,3 @@ function getFieldsFromRawFields(rawFields: RawFields): Field[] {
   result.sort((a, b) => a.name.localeCompare(b.name));
   return result;
 }
-
-const normalizedFieldTypes: Record<string, string> = {
-  long: 'number',
-  integer: 'number',
-  short: 'number',
-  byte: 'number',
-  double: 'number',
-  float: 'number',
-  half_float: 'number',
-  scaled_float: 'number',
-  unsigned_long: 'number',
-};