[ML] AIOps: Fix missing field caps filters for log rate analysis. (#181109)

## Summary

Part of #172981.

Field caps requests can be heavy calls in larger clusters. For all other
queries for log rate analysis we were applying filters based on the time
range selection. This was missing from the field caps call. The
following parameters were added to improve the call:

- `index_filter`: Adds a range filter to only get field caps from
indices spanning the deviation time range.
- `filters`: `-metadata` was added to not return fields like `_id` and
esp. `_tier`. We previously had a manually check for `_tier` which is
now unnecessary using this option.
- `types`: Previously we fetched all field types and then filtered out
the ones we don't support. This option allows us to pass in the
supported fields right away and not return unsupported ones in the first
place.

----

Here are examples that show how `index_filter` get applied correctly:

Here the deviation selection spans only 1 month and that is reflected in
the response from the field caps call:

<img width="1026" alt="image"
src="50a00e5a-2b59-4ae5-9d50-e2ed766f68f5">

```
{
  indices: [ 'gallery-2021-11' ],
  fields: {
   ...
  }
}
```

Now the deviation selection covers more months:

<img width="1010" alt="image"
src="0034a19e-b136-4261-9761-6b48fdf45989">

```
{
  indices: [
    'gallery-2021-09',
    'gallery-2021-10',
    'gallery-2021-11',
    'gallery-2021-12',
    'gallery-2022-01'
  ],
  fields: {
   ...
  }
}
```

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

x-pack/packages/ml/aiops_log_rate_analysis/queries/__mocks__/field_caps_ecommerce.ts

@@ -6,228 +6,100 @@
  */
 
 export const fieldCapsEcommerceMock = {
-  indices: ['ft_ecommerce'],
+  indices: ['kibana_sample_data_ecommerce'],
   fields: {
     'products.manufacturer': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
-    'products.discount_amount': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    'products.base_unit_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    type: {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'products.discount_percentage': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    'products._id.keyword': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    day_of_week_i: {
-      integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    total_quantity: {
-      integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    total_unique_products: {
-      integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    taxless_total_price: {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    'geoip.continent_name': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    sku: {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    _version: {
-      _version: { type: '_version', metadata_field: true, searchable: false, aggregatable: true },
-    },
-    'customer_full_name.keyword': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'category.keyword': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'products.taxless_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    'products.quantity': {
-      integer: { type: 'integer', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'products.price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
-    customer_first_name: {
-      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
-    },
-    customer_phone: {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'geoip.region_name': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    _tier: {
-      keyword: { type: 'keyword', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    _seq_no: {
-      _seq_no: { type: '_seq_no', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    customer_full_name: {
-      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
-    },
-    'geoip.country_iso_code': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    _source: {
-      _source: { type: '_source', metadata_field: true, searchable: false, aggregatable: false },
-    },
-    _id: { _id: { type: '_id', metadata_field: true, searchable: true, aggregatable: false } },
-    order_id: {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
     'products._id': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
     'products.product_name.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    _index: {
-      _index: { type: '_index', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    'products.product_id': {
-      long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true },
-    },
     'products.category': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
     'products.manufacturer.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    type: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     manufacturer: {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
     products: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
-    'products.unit_discount_amount': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
     customer_last_name: {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
-    'geoip.location': {
+    'products._id.keyword': {
-      geo_point: { type: 'geo_point', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    'products.tax_amount': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
     },
     'products.product_name': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
-    'products.min_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
     'manufacturer.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'products.taxful_price': {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
     currency: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'products.base_price': {
+    'geoip.continent_name': {
-      half_float: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-        type: 'half_float',
+    },
-        metadata_field: false,
+    event: {
-        searchable: true,
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
-        aggregatable: true,
+    },
-      },
+    sku: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     email: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'customer_full_name.keyword': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     day_of_week: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'customer_last_name.keyword': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     'products.sku': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'customer_last_name.keyword': {
+    'category.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     geoip: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
+    customer_first_name: {
+      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
+    },
+    customer_phone: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     'products.category.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'geoip.city_name': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    order_date: {
+    'geoip.region_name': {
-      date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'customer_first_name.keyword': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'products.created_on': {
+    customer_full_name: {
-      date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
+      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
+    },
+    'geoip.country_iso_code': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     category: {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
@@ -238,16 +110,14 @@ export const fieldCapsEcommerceMock = {
     user: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    order_id: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'event.dataset': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     customer_gender: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    taxful_total_price: {
-      half_float: {
-        type: 'half_float',
-        metadata_field: false,
-        searchable: true,
-        aggregatable: true,
-      },
-    },
   },
 };

x-pack/packages/ml/aiops_log_rate_analysis/queries/__mocks__/field_caps_large_arrays.ts

@@ -6,27 +6,8 @@
  */
 
 export const fieldCapsLargeArraysMock = {
-  indices: ['large_arrays'],
+  indices: ['large_array'],
   fields: {
-    _tier: {
-      keyword: { type: 'keyword', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    _seq_no: {
-      _seq_no: { type: '_seq_no', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    '@timestamp': {
-      date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
-    },
-    _index: {
-      _index: { type: '_index', metadata_field: true, searchable: true, aggregatable: true },
-    },
-    _source: {
-      _source: { type: '_source', metadata_field: true, searchable: false, aggregatable: false },
-    },
-    _id: { _id: { type: '_id', metadata_field: true, searchable: true, aggregatable: false } },
-    _version: {
-      _version: { type: '_version', metadata_field: true, searchable: false, aggregatable: true },
-    },
     items: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },

x-pack/packages/ml/aiops_log_rate_analysis/queries/__mocks__/field_caps_pgbench.ts

@@ -6,37 +6,113 @@
  */
 
 export const fieldCapsPgBenchMock = {
-  indices: ['my-index'],
+  indices: ['.ds-filebeat-8.2.0-2022.06.07-000082'],
   fields: {
-    stack: {
+    'kubernetes.node.uid': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    metadata: {
+    stack: {
-      flattened: { type: 'flattened', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'kubernetes.namespace_uid': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'host.os.name.text': {
+      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
+    },
+    'kubernetes.labels': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
     'host.hostname': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'host.mac': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     'kubernetes.node.labels.kubernetes_io/os': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'container.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'service.type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'transaction.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     hostname: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'host.os.version': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.node.labels.beta_kubernetes_io/os': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     _metadata: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
-    _version: {
+    'kubernetes.node.labels.topology_kubernetes_io/region': {
-      _version: { type: '_version', metadata_field: true, searchable: false, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'host.os.type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'fileset.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'cloud.account': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'span.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'agent.hostname': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'req.headers.x-real-ip': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    amount_f: {
+    'req.headers.connection': {
-      float: { type: 'float', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    labels: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'cloud.service': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    '_metadata.message_template': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    input: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'log.origin.function': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'host.containerized': {
+      boolean: { type: 'boolean', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.node.labels.beta_kubernetes_io/instance-type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.node.labels.failure-domain_beta_kubernetes_io/region': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.node.hostname': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'elasticapm_labels.trace.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'host.ip': { ip: { type: 'ip', metadata_field: false, searchable: true, aggregatable: true } },
+    'agent.type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'process.executable.text': {
+      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
     'kubernetes.node.labels.addon_gke_io/node-local-dns-ds-ready': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
@@ -50,10 +126,10 @@ export const fieldCapsPgBenchMock = {
     '_metadata.user_id': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'kubernetes.container.name': {
+    'postgresql.log.database': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'postgresql.log.database': {
+    'kubernetes.container.name': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'container.labels.annotation_io_kubernetes_container_restartCount': {
@@ -68,24 +144,15 @@ export const fieldCapsPgBenchMock = {
     'host.os.platform': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    _field_names: {
-      _field_names: {
-        type: '_field_names',
-        metadata_field: true,
-        searchable: true,
-        aggregatable: false,
-      },
-    },
     'cloud.account.id': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    v: { long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true } },
-    'error.message': {
-      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
-    },
     elasticapm_transaction_id: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'error.message': {
+      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
+    },
     'log.file.path': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
@@ -95,9 +162,6 @@ export const fieldCapsPgBenchMock = {
     'container.labels.io_kubernetes_container_name': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'user.name': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
     'user.name.text': {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
@@ -116,6 +180,9 @@ export const fieldCapsPgBenchMock = {
     'cloud.instance': {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
+    'process.name.text': {
+      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
+    },
     'container.labels.io_kubernetes_pod_namespace': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
@@ -137,9 +204,6 @@ export const fieldCapsPgBenchMock = {
     'host.os.name': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'host.os.name.text': {
-      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
-    },
     'log.level': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
@@ -164,21 +228,15 @@ export const fieldCapsPgBenchMock = {
     '_metadata.elastic_apm_trace_id': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'log.file': {
-      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
-    },
-    'log.offset': {
-      long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true },
-    },
     'client.ip': {
       ip: { type: 'ip', metadata_field: false, searchable: true, aggregatable: true },
     },
+    'log.file': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
     'process.name': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'process.name.text': {
-      text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
-    },
     name: {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
@@ -197,18 +255,18 @@ export const fieldCapsPgBenchMock = {
     'req.headers.tracestate': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
+    '_metadata.metadata_event_dataset': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     'postgresql.log.timestamp': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    '_metadata.metadata_event_dataset': {
+    'event.module': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     related: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
-    'event.module': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
     'req.headers': {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
@@ -218,21 +276,18 @@ export const fieldCapsPgBenchMock = {
     'kubernetes.node.labels.cloud_google_com/gke-container-runtime': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'kubernetes.pod.name': {
-      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
-    },
     client: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
+    'kubernetes.pod.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     'req.headers.cache-control': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     'event.timezone': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'log.origin.file.line': {
-      long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true },
-    },
     'service.name': {
       keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
@@ -242,45 +297,332 @@ export const fieldCapsPgBenchMock = {
     message: {
       text: { type: 'text', metadata_field: false, searchable: true, aggregatable: false },
     },
-    _source: {
+    'kubernetes.node.labels.kubernetes_io/hostname': {
-      _source: { type: '_source', metadata_field: true, searchable: false, aggregatable: false },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'req.headers.traceparent': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.namespace_labels': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    service: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'kubernetes.node.labels.node_type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    container: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'event.category': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'elasticapm_labels.trace': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'kubernetes.node.labels.topology_kubernetes_io/zone': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'client.geo.country_iso_code': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'client.geo': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    type: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'req.method': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'container.image.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.labels.app': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'agent.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'log.original': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'process.thread.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'container.labels.io_kubernetes_pod_uid': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.node': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'kubernetes.node.labels.failure-domain_beta_kubernetes_io/zone': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'input.type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'log.flags': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'related.user': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'host.architecture': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    elasticapm_labels: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'req.url': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'cloud.provider': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'cloud.machine.type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'agent.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'cloud.machine': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'container.labels.io_kubernetes_sandbox_id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'req.headers.pragma': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'container.labels.io_kubernetes_docker_type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    '_metadata.elastic_apm_transaction_id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.node.labels.cloud_google_com/gke-os-distribution': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
     log: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
+    'kubernetes.pod': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'container.labels.annotation_io_kubernetes_container_hash': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'req.remoteAddress': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'user.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'log.logger': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'postgresql.log.query_step': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'cloud.instance.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'client.geo.region_name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    stream: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'log.origin.file': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'kubernetes.node.labels.cloud_google_com/gke-nodepool': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     event: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
-    'event.duration': {
+    'req.headers.host': {
-      long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    'event.ingested': {
+    'req.headers.content-type': {
-      date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    '@timestamp': {
+    'kubernetes.replicaset.name': {
-      date: { type: 'date', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'host.os.codename': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'req.headers.referer': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'req.headers.cookie': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'elasticapm_labels.span': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'log.origin.file.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    data_stream: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'data_stream.dataset': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'agent.ephemeral_id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'cloud.project': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'container.image': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
     transaction: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
+    'cloud.project.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
     span: {
       object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
-    '_metadata.sum': {
+    'container.labels.annotation_io_kubernetes_container_terminationMessagePolicy': {
-      long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    _tier: {
+    'elasticapm_labels.transaction': {
-      keyword: { type: 'keyword', metadata_field: true, searchable: true, aggregatable: true },
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
     },
-    _seq_no: {
+    'cloud.availability_zone': {
-      _seq_no: { type: '_seq_no', metadata_field: true, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
-    code: { long: { type: 'long', metadata_field: false, searchable: true, aggregatable: true } },
+    cloud: {
-    _index: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
-      _index: { type: '_index', metadata_field: true, searchable: true, aggregatable: true },
     },
-    'client.geo.location': {
+    'container.name': {
-      geo_point: { type: 'geo_point', metadata_field: false, searchable: true, aggregatable: true },
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    ecs: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'kubernetes.namespace': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    host: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'host.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'req.headers.accept': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'client.geo.country_name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'event.kind': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.replicaset': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'elasticapm_labels.transaction.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'data_stream.type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'container.runtime': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'cloud.service.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'ecs.version': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'container.labels.io_kubernetes_pod_name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'labels.userId': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'container.labels.annotation_io_kubernetes_container_terminationMessagePath': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.node.name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'client.geo.continent_name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'postgresql.log': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'req.headers.user-agent': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.pod.uid': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    error: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'kubernetes.node.labels': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    trace: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'trace.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    postgresql: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'elasticapm_labels.span.id': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'kubernetes.container': {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    elasticapm_trace_id: {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'process.executable': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    process: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'client.geo.city_name': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'client.geo.region_iso_code': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'data_stream.namespace': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'req.headers.content-length': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'event.type': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    user: {
+      object: { type: 'object', metadata_field: false, searchable: false, aggregatable: false },
+    },
+    'event.dataset': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
+    },
+    'container.labels.io_kubernetes_container_logpath': {
+      keyword: { type: 'keyword', metadata_field: false, searchable: true, aggregatable: true },
     },
   },
 };

x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.test.ts

@@ -80,55 +80,154 @@ describe('fetch_index_info', () => {
 
       expect(fieldCandidates).toEqual([
         '_metadata.elastic_apm_trace_id',
+        '_metadata.elastic_apm_transaction_id',
+        '_metadata.message_template',
         '_metadata.metadata_event_dataset',
         '_metadata.user_id',
+        'agent.ephemeral_id',
+        'agent.hostname',
+        'agent.id',
+        'agent.name',
+        'agent.type',
         'agent.version',
+        'client.geo.city_name',
+        'client.geo.continent_name',
+        'client.geo.country_iso_code',
+        'client.geo.country_name',
+        'client.geo.region_iso_code',
+        'client.geo.region_name',
         'client.ip',
         'cloud.account.id',
+        'cloud.availability_zone',
+        'cloud.instance.id',
         'cloud.instance.name',
+        'cloud.machine.type',
+        'cloud.project.id',
+        'cloud.provider',
+        'cloud.service.name',
+        'container.id',
+        'container.image.name',
+        'container.labels.annotation_io_kubernetes_container_hash',
         'container.labels.annotation_io_kubernetes_container_restartCount',
+        'container.labels.annotation_io_kubernetes_container_terminationMessagePath',
+        'container.labels.annotation_io_kubernetes_container_terminationMessagePolicy',
         'container.labels.annotation_io_kubernetes_pod_terminationGracePeriod',
+        'container.labels.io_kubernetes_container_logpath',
         'container.labels.io_kubernetes_container_name',
+        'container.labels.io_kubernetes_docker_type',
+        'container.labels.io_kubernetes_pod_name',
         'container.labels.io_kubernetes_pod_namespace',
+        'container.labels.io_kubernetes_pod_uid',
+        'container.labels.io_kubernetes_sandbox_id',
+        'container.name',
+        'container.runtime',
+        'data_stream.dataset',
+        'data_stream.namespace',
+        'data_stream.type',
         'details',
+        'ecs.version',
+        'elasticapm_labels.span.id',
+        'elasticapm_labels.trace.id',
+        'elasticapm_labels.transaction.id',
         'elasticapm_span_id',
+        'elasticapm_trace_id',
         'elasticapm_transaction_id',
+        'event.category',
+        'event.dataset',
+        'event.kind',
         'event.module',
         'event.timezone',
+        'event.type',
+        'fileset.name',
+        'host.architecture',
+        'host.containerized',
         'host.hostname',
+        'host.ip',
+        'host.mac',
+        'host.name',
+        'host.os.codename',
         'host.os.family',
         'host.os.kernel',
         'host.os.name',
         'host.os.platform',
+        'host.os.type',
+        'host.os.version',
         'hostname',
+        'input.type',
         'kubernetes.container.name',
+        'kubernetes.labels.app',
         'kubernetes.labels.pod-template-hash',
+        'kubernetes.namespace',
         'kubernetes.namespace_labels.kubernetes_io/metadata_name',
         'kubernetes.namespace_uid',
+        'kubernetes.node.hostname',
         'kubernetes.node.labels.addon_gke_io/node-local-dns-ds-ready',
         'kubernetes.node.labels.beta_kubernetes_io/arch',
+        'kubernetes.node.labels.beta_kubernetes_io/instance-type',
+        'kubernetes.node.labels.beta_kubernetes_io/os',
         'kubernetes.node.labels.cloud_google_com/gke-boot-disk',
         'kubernetes.node.labels.cloud_google_com/gke-container-runtime',
+        'kubernetes.node.labels.cloud_google_com/gke-nodepool',
+        'kubernetes.node.labels.cloud_google_com/gke-os-distribution',
         'kubernetes.node.labels.cloud_google_com/machine-family',
+        'kubernetes.node.labels.failure-domain_beta_kubernetes_io/region',
+        'kubernetes.node.labels.failure-domain_beta_kubernetes_io/zone',
         'kubernetes.node.labels.kubernetes_io/arch',
+        'kubernetes.node.labels.kubernetes_io/hostname',
         'kubernetes.node.labels.kubernetes_io/os',
         'kubernetes.node.labels.node_kubernetes_io/instance-type',
+        'kubernetes.node.labels.node_type',
+        'kubernetes.node.labels.topology_kubernetes_io/region',
+        'kubernetes.node.labels.topology_kubernetes_io/zone',
+        'kubernetes.node.name',
+        'kubernetes.node.uid',
         'kubernetes.pod.ip',
         'kubernetes.pod.name',
+        'kubernetes.pod.uid',
+        'kubernetes.replicaset.name',
+        'labels.userId',
         'log.file.path',
+        'log.flags',
         'log.level',
+        'log.logger',
+        'log.origin.file.name',
+        'log.origin.function',
+        'log.original',
         'name',
         'postgresql.log.database',
         'postgresql.log.query',
+        'postgresql.log.query_step',
         'postgresql.log.timestamp',
+        'process.executable',
         'process.name',
+        'process.thread.name',
+        'related.user',
+        'req.headers.accept',
         'req.headers.accept-encoding',
         'req.headers.cache-control',
+        'req.headers.connection',
+        'req.headers.content-length',
+        'req.headers.content-type',
+        'req.headers.cookie',
+        'req.headers.host',
         'req.headers.origin',
+        'req.headers.pragma',
+        'req.headers.referer',
+        'req.headers.traceparent',
         'req.headers.tracestate',
+        'req.headers.user-agent',
         'req.headers.x-real-ip',
+        'req.method',
+        'req.remoteAddress',
+        'req.url',
         'service.name',
+        'service.type',
+        'span.id',
         'stack',
+        'stream',
+        'trace.id',
+        'transaction.id',
+        'type',
         'user.name',
       ]);
       expect(textFieldCandidates).toEqual(['error.message', 'message']);
@@ -172,6 +271,7 @@ describe('fetch_index_info', () => {
         'customer_phone',
         'day_of_week',
         'email',
+        'event.dataset',
         'geoip.city_name',
         'geoip.continent_name',
         'geoip.country_iso_code',

x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts

@@ -25,8 +25,6 @@ const SUPPORTED_ES_FIELD_TYPES = [
 
 const SUPPORTED_ES_FIELD_TYPES_TEXT = [ES_FIELD_TYPES.TEXT, ES_FIELD_TYPES.MATCH_ONLY_TEXT];
 
-const IGNORE_FIELD_NAMES = ['_tier'];
-
 interface IndexInfo {
   fieldCandidates: string[];
   textFieldCandidates: string[];
@@ -45,9 +43,19 @@ export const fetchIndexInfo = async (
   // Get all supported fields
   const respMapping = await esClient.fieldCaps(
     {
-      index,
       fields: '*',
+      filters: '-metadata',
       include_empty_fields: false,
+      index,
+      index_filter: {
+        range: {
+          [params.timeFieldName]: {
+            gte: params.deviationMin,
+            lte: params.deviationMax,
+          },
+        },
+      },
+      types: [...SUPPORTED_ES_FIELD_TYPES, ...SUPPORTED_ES_FIELD_TYPES_TEXT],
     },
     { signal: abortSignal, maxRetries: 0 }
   );
@@ -64,11 +72,11 @@ export const fetchIndexInfo = async (
     const isTextField = fieldTypes.some((type) => SUPPORTED_ES_FIELD_TYPES_TEXT.includes(type));
 
     // Check if fieldName is something we can aggregate on
-    if (isSupportedType && isAggregatable && !IGNORE_FIELD_NAMES.includes(key)) {
+    if (isSupportedType && isAggregatable) {
       acceptableFields.add(key);
     }
 
-    if (isTextField && !IGNORE_FIELD_NAMES.includes(key)) {
+    if (isTextField) {
       acceptableTextFields.add(key);
     }