[ML] Removing use of ignore_throttled (#199107)

`ignore_throttled` is being removed, we can remove it from the various
searches we run.

I'm leaving it in the schema for the datafeed config for now, just in
case we have a situation where a very old job is used in the UI with an
endpoint which validates the datafeed config. Even though the setting
would be ignored, we don't want the route to reject the datafeed because
it doesn't know what `ignore_throttled` is.

When cloning very old jobs jobs, I'm removing `ignore_throttled` from
the `indices_options`

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/packages/ml/aiops_log_pattern_analysis/schema.ts

@@ -38,7 +38,6 @@ export const indicesOptionsSchema = schema.object({
   ),
   ignore_unavailable: schema.maybe(schema.boolean()),
   allow_no_indices: schema.maybe(schema.boolean()),
-  ignore_throttled: schema.maybe(schema.boolean()),
 });
 
 /**

x-pack/packages/ml/aiops_log_rate_analysis/api/schema_v2.ts

@@ -45,6 +45,7 @@ export const aiopsLogRateAnalysisBase = schema.object({
   end: schema.number(),
   searchQuery: schema.string(),
   timeFieldName: schema.string(),
+  // when v2 is removed, includeFrozen should not carry over to v3+
   includeFrozen: schema.maybe(schema.boolean()),
   grouping: schema.maybe(schema.boolean()),
   /** Analysis selection time ranges */

x-pack/packages/ml/aiops_log_rate_analysis/queries/__mocks__/params_match_all.ts

@@ -14,6 +14,5 @@ export const paramsMock = {
   baselineMax: 20,
   deviationMin: 30,
   deviationMax: 40,
-  includeFrozen: false,
   searchQuery: '{ "match_all": {} }',
 };

x-pack/packages/ml/aiops_log_rate_analysis/queries/get_request_base.test.ts

@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { paramsMock } from './__mocks__/params_match_all';
-
-import { getRequestBase } from './get_request_base';
-
-describe('getRequestBase', () => {
-  it('defaults to not setting `ignore_throttled`', () => {
-    const requestBase = getRequestBase(paramsMock);
-    expect(requestBase.ignore_throttled).toEqual(undefined);
-  });
-
-  it('adds `ignore_throttled=false` when `includeFrozen=true`', () => {
-    const requestBase = getRequestBase({
-      ...paramsMock,
-      includeFrozen: true,
-    });
-    expect(requestBase.ignore_throttled).toEqual(false);
-  });
-});

x-pack/packages/ml/aiops_log_rate_analysis/queries/get_request_base.ts

@@ -7,8 +7,7 @@
 
 import type { AiopsLogRateAnalysisSchema } from '../api/schema';
 
-export const getRequestBase = ({ index, includeFrozen }: AiopsLogRateAnalysisSchema) => ({
+export const getRequestBase = ({ index }: AiopsLogRateAnalysisSchema) => ({
   index,
-  ...(includeFrozen ? { ignore_throttled: false } : {}),
   ignore_unavailable: true,
 });

x-pack/packages/ml/json_schemas/src/put___ml_anomaly_detectors__job_id__schema.json

@@ -10765,10 +10765,6 @@
           "ignore_unavailable": {
             "description": "If true, missing or closed indices are not included in the response.",
             "type": "boolean"
-          },
-          "ignore_throttled": {
-            "description": "If true, concrete, expanded or aliased indices are ignored when frozen.",
-            "type": "boolean"
           }
         }
       },
@@ -10884,4 +10880,4 @@
       }
     }
   }
-}
+}

x-pack/packages/ml/json_schemas/src/put___ml_datafeeds__datafeed_id__schema.json

@@ -7932,10 +7932,6 @@
           "ignore_unavailable": {
             "description": "If true, missing or closed indices are not included in the response.",
             "type": "boolean"
-          },
-          "ignore_throttled": {
-            "description": "If true, concrete, expanded or aliased indices are ignored when frozen.",
-            "type": "boolean"
           }
         }
       },
@@ -8051,4 +8047,4 @@
       }
     }
   }
-}
+}

x-pack/plugins/ml/server/models/job_service/jobs.ts

@@ -354,6 +354,10 @@ export function jobsProvider(
     const result: { datafeed?: Datafeed; job?: Job } = { job: undefined, datafeed: undefined };
     if (datafeedResult && datafeedResult.job_id === jobId) {
       result.datafeed = datafeedResult;
+      if (result.datafeed.indices_options?.ignore_throttled !== undefined) {
+        // ignore_throttled is a deprecated setting, remove it from the response
+        delete result.datafeed.indices_options.ignore_throttled;
+      }
     }
 
     if (jobResults?.jobs?.length > 0) {

x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts

@@ -27,7 +27,10 @@ export const indicesOptionsSchema = schema.object({
   ),
   ignore_unavailable: schema.maybe(schema.boolean()),
   allow_no_indices: schema.maybe(schema.boolean()),
+  // retaining the deprecated ignore_throttled in case an older jobs are used
+  // we don't want to fail the schema validation if this is present
   ignore_throttled: schema.maybe(schema.boolean()),
+  failure_store: schema.maybe(schema.string()),
 });
 
 export const datafeedConfigSchema = schema.object({

x-pack/test/functional/apps/ml/anomaly_detection_result_views/aggregated_scripted_job.ts

@@ -59,7 +59,6 @@ export default function ({ getService }: FtrProviderContext) {
           expand_wildcards: ['open'],
           ignore_unavailable: false,
           allow_no_indices: true,
-          ignore_throttled: true,
         },
         query: {
           match_all: {},
@@ -140,7 +139,6 @@ export default function ({ getService }: FtrProviderContext) {
           expand_wildcards: ['open'],
           ignore_unavailable: false,
           allow_no_indices: true,
-          ignore_throttled: true,
         },
         query: {
           bool: {
@@ -217,7 +215,6 @@ export default function ({ getService }: FtrProviderContext) {
           expand_wildcards: ['open'],
           ignore_unavailable: false,
           allow_no_indices: true,
-          ignore_throttled: true,
         },
         query: {
           match_all: {},
@@ -317,7 +314,6 @@ export default function ({ getService }: FtrProviderContext) {
           expand_wildcards: ['open'],
           ignore_unavailable: false,
           allow_no_indices: true,
-          ignore_throttled: true,
         },
         query: {
           bool: {