[8.17] Exclude unrecognized tasks from the task manager aggregate API (#202163) (#202682)

# Backport

This will backport the following commits from `main` to `8.17`:
- [Exclude unrecognized tasks from the task manager aggregate API
(#202163)](https://github.com/elastic/kibana/pull/202163)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Mike
Côté","email":"mikecote@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-12-03T11:21:49Z","message":"Exclude
unrecognized tasks from the task manager aggregate API (#202163)\n\nIn
this PR, I'm removing tasks with a status `unrecognized`
from\r\nreturning on any `taskStore.aggregate` calls. Without this, we
had\r\nunrecognized recurring tasks that were still part of the task
manager\r\ncapacity calculation
under\r\n`assumedAverageRecurringRequiredThroughputPerMinutePerKibana`.\r\n\r\n##
To Verify\r\n1. Create a few ES Query alerting rules running every
1s\r\n2. Capture the task manager health API report
via\r\n`/api/task_manager/_health`\r\n3. Apply the following diff to
mark es query alerting tasks as\r\nunrecognized\r\n```\r\ndiff --git
a/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts
b/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\nindex
1988eebc21a..8d649f4c6a5 100644\r\n---
a/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\n+++
b/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\n@@
-10,5 +10,5 @@ import { getRuleType } from './rule_type';\r\n\r\n export
function register(params: RegisterRuleTypesParams, isServerless:
boolean) {\r\n const { alerting, core } = params;\r\n-
alerting.registerType(getRuleType(core, isServerless));\r\n+ //
alerting.registerType(getRuleType(core, isServerless));\r\n }\r\ndiff
--git
a/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts
b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\nindex
e28d5221e72..dbfc1bbd135 100644\r\n---
a/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\n+++
b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\n@@
-33,6 +33,11 @@ export async function
scheduleMarkRemovedTasksAsUnrecognizedDefinition(\r\n state: {},\r\n
params: {},\r\n });\r\n+ try {\r\n+ await
taskScheduling.runSoon(TASK_ID);\r\n+ } catch (e) {\r\n+ // Ignore\r\n+
}\r\n } catch (e) {\r\n logger.error(`Error scheduling ${TASK_ID} task,
received ${e.message}`);\r\n }\r\ndiff --git
a/x-pack/plugins/task_manager/server/task_type_dictionary.ts
b/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\nindex
e0b28eccea3..142c07bb507 100644\r\n---
a/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\n+++
b/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\n@@ -32,6
+32,8 @@ export const REMOVED_TYPES: string[] = [\r\n\r\n
'cleanup_failed_action_executions',\r\n 'reports:monitor',\r\n+\r\n+
'alerting:.es-query',\r\n ];\r\n\r\n /**\r\n```\r\n5. Capture the task
manager health API report again via\r\n`/api/task_manager/_health`\r\n6.
Notice the number dropped
for\r\n`capacity_estimation.value.observed.avg_recurring_required_throughput_per_minute`","sha":"3b670980431da414535940aeeb0088d2ae7ff89c","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Feature:Task
Manager","Team:ResponseOps","v9.0.0","backport:version","v8.17.0","v8.18.0","v8.16.2"],"title":"Exclude
unrecognized tasks from the task manager aggregate
API","number":202163,"url":"https://github.com/elastic/kibana/pull/202163","mergeCommit":{"message":"Exclude
unrecognized tasks from the task manager aggregate API (#202163)\n\nIn
this PR, I'm removing tasks with a status `unrecognized`
from\r\nreturning on any `taskStore.aggregate` calls. Without this, we
had\r\nunrecognized recurring tasks that were still part of the task
manager\r\ncapacity calculation
under\r\n`assumedAverageRecurringRequiredThroughputPerMinutePerKibana`.\r\n\r\n##
To Verify\r\n1. Create a few ES Query alerting rules running every
1s\r\n2. Capture the task manager health API report
via\r\n`/api/task_manager/_health`\r\n3. Apply the following diff to
mark es query alerting tasks as\r\nunrecognized\r\n```\r\ndiff --git
a/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts
b/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\nindex
1988eebc21a..8d649f4c6a5 100644\r\n---
a/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\n+++
b/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\n@@
-10,5 +10,5 @@ import { getRuleType } from './rule_type';\r\n\r\n export
function register(params: RegisterRuleTypesParams, isServerless:
boolean) {\r\n const { alerting, core } = params;\r\n-
alerting.registerType(getRuleType(core, isServerless));\r\n+ //
alerting.registerType(getRuleType(core, isServerless));\r\n }\r\ndiff
--git
a/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts
b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\nindex
e28d5221e72..dbfc1bbd135 100644\r\n---
a/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\n+++
b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\n@@
-33,6 +33,11 @@ export async function
scheduleMarkRemovedTasksAsUnrecognizedDefinition(\r\n state: {},\r\n
params: {},\r\n });\r\n+ try {\r\n+ await
taskScheduling.runSoon(TASK_ID);\r\n+ } catch (e) {\r\n+ // Ignore\r\n+
}\r\n } catch (e) {\r\n logger.error(`Error scheduling ${TASK_ID} task,
received ${e.message}`);\r\n }\r\ndiff --git
a/x-pack/plugins/task_manager/server/task_type_dictionary.ts
b/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\nindex
e0b28eccea3..142c07bb507 100644\r\n---
a/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\n+++
b/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\n@@ -32,6
+32,8 @@ export const REMOVED_TYPES: string[] = [\r\n\r\n
'cleanup_failed_action_executions',\r\n 'reports:monitor',\r\n+\r\n+
'alerting:.es-query',\r\n ];\r\n\r\n /**\r\n```\r\n5. Capture the task
manager health API report again via\r\n`/api/task_manager/_health`\r\n6.
Notice the number dropped
for\r\n`capacity_estimation.value.observed.avg_recurring_required_throughput_per_minute`","sha":"3b670980431da414535940aeeb0088d2ae7ff89c"}},"sourceBranch":"main","suggestedTargetBranches":["8.17","8.x","8.16"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/202163","number":202163,"mergeCommit":{"message":"Exclude
unrecognized tasks from the task manager aggregate API (#202163)\n\nIn
this PR, I'm removing tasks with a status `unrecognized`
from\r\nreturning on any `taskStore.aggregate` calls. Without this, we
had\r\nunrecognized recurring tasks that were still part of the task
manager\r\ncapacity calculation
under\r\n`assumedAverageRecurringRequiredThroughputPerMinutePerKibana`.\r\n\r\n##
To Verify\r\n1. Create a few ES Query alerting rules running every
1s\r\n2. Capture the task manager health API report
via\r\n`/api/task_manager/_health`\r\n3. Apply the following diff to
mark es query alerting tasks as\r\nunrecognized\r\n```\r\ndiff --git
a/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts
b/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\nindex
1988eebc21a..8d649f4c6a5 100644\r\n---
a/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\n+++
b/x-pack/plugins/stack_alerts/server/rule_types/es_query/index.ts\r\n@@
-10,5 +10,5 @@ import { getRuleType } from './rule_type';\r\n\r\n export
function register(params: RegisterRuleTypesParams, isServerless:
boolean) {\r\n const { alerting, core } = params;\r\n-
alerting.registerType(getRuleType(core, isServerless));\r\n+ //
alerting.registerType(getRuleType(core, isServerless));\r\n }\r\ndiff
--git
a/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts
b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\nindex
e28d5221e72..dbfc1bbd135 100644\r\n---
a/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\n+++
b/x-pack/plugins/task_manager/server/removed_tasks/mark_removed_tasks_as_unrecognized.ts\r\n@@
-33,6 +33,11 @@ export async function
scheduleMarkRemovedTasksAsUnrecognizedDefinition(\r\n state: {},\r\n
params: {},\r\n });\r\n+ try {\r\n+ await
taskScheduling.runSoon(TASK_ID);\r\n+ } catch (e) {\r\n+ // Ignore\r\n+
}\r\n } catch (e) {\r\n logger.error(`Error scheduling ${TASK_ID} task,
received ${e.message}`);\r\n }\r\ndiff --git
a/x-pack/plugins/task_manager/server/task_type_dictionary.ts
b/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\nindex
e0b28eccea3..142c07bb507 100644\r\n---
a/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\n+++
b/x-pack/plugins/task_manager/server/task_type_dictionary.ts\r\n@@ -32,6
+32,8 @@ export const REMOVED_TYPES: string[] = [\r\n\r\n
'cleanup_failed_action_executions',\r\n 'reports:monitor',\r\n+\r\n+
'alerting:.es-query',\r\n ];\r\n\r\n /**\r\n```\r\n5. Capture the task
manager health API report again via\r\n`/api/task_manager/_health`\r\n6.
Notice the number dropped
for\r\n`capacity_estimation.value.observed.avg_recurring_required_throughput_per_minute`","sha":"3b670980431da414535940aeeb0088d2ae7ff89c"}},{"branch":"8.17","label":"v8.17.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.x","label":"v8.18.0","branchLabelMappingKey":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.16","label":"v8.16.2","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Mike Côté <mikecote@users.noreply.github.com>

x-pack/plugins/task_manager/server/task_store.test.ts

@@ -555,7 +555,14 @@ describe('TaskStore', () => {
         body: {
           size: 0,
           query: {
-            bool: { filter: [{ term: { type: 'task' } }, { term: { 'task.enabled': true } }] },
+            bool: {
+              filter: {
+                bool: {
+                  must: [{ term: { type: 'task' } }, { term: { 'task.enabled': true } }],
+                  must_not: [{ term: { 'task.status': 'unrecognized' } }],
+                },
+              },
+            },
           },
           aggs: { testAgg: { terms: { field: 'task.taskType' } } },
         },
@@ -578,7 +585,12 @@ describe('TaskStore', () => {
               must: [
                 {
                   bool: {
-                    filter: [{ term: { type: 'task' } }, { term: { 'task.enabled': true } }],
+                    filter: {
+                      bool: {
+                        must: [{ term: { type: 'task' } }, { term: { 'task.enabled': true } }],
+                        must_not: [{ term: { 'task.status': 'unrecognized' } }],
+                      },
+                    },
                   },
                 },
                 { term: { 'task.taskType': 'bar' } },
@@ -600,7 +612,14 @@ describe('TaskStore', () => {
         body: {
           size: 0,
           query: {
-            bool: { filter: [{ term: { type: 'task' } }, { term: { 'task.enabled': true } }] },
+            bool: {
+              filter: {
+                bool: {
+                  must: [{ term: { type: 'task' } }, { term: { 'task.enabled': true } }],
+                  must_not: [{ term: { 'task.status': 'unrecognized' } }],
+                },
+              },
+            },
           },
           aggs: { testAgg: { terms: { field: 'task.taskType' } } },
           runtime_mappings: { testMapping: { type: 'long', script: { source: `` } } },

x-pack/plugins/task_manager/server/task_store.ts

@@ -34,6 +34,7 @@ import {
   ConcreteTaskInstance,
   ConcreteTaskInstanceVersion,
   TaskInstance,
+  TaskStatus,
   TaskLifecycle,
   TaskLifecycleResult,
   SerializedConcreteTaskInstance,
@@ -842,7 +843,12 @@ function ensureAggregationOnlyReturnsEnabledTaskObjects(opts: AggregationOpts):
   const originalQuery = opts.query;
   const filterToOnlyTasks = {
     bool: {
-      filter: [{ term: { type: 'task' } }, { term: { 'task.enabled': true } }],
+      filter: {
+        bool: {
+          must: [{ term: { type: 'task' } }, { term: { 'task.enabled': true } }],
+          must_not: [{ term: { 'task.status': TaskStatus.Unrecognized } }],
+        },
+      },
     },
   };
   const query = originalQuery