[Security solution] Fix alert grouping pagination bug (#151941)

.github/CODEOWNERS

@@ -1003,6 +1003,8 @@ packages/kbn-yarn-lock-validator @elastic/kibana-operations
 
 /x-pack/plugins/security_solution/public/common/components/guided_onboarding_tour @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/public/common/components/charts @elastic/security-threat-hunting-explore
+/x-pack/plugins/security_solution/public/common/components/grouping @elastic/security-threat-hunting-explore
+/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/public/common/components/header_page @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/public/common/components/header_section @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/public/common/components/inspect @elastic/security-threat-hunting-explore

x-pack/plugins/security_solution/public/common/components/grouping/index.tsx

@@ -9,6 +9,7 @@ import { NONE_GROUP_KEY } from './types';
 
 export * from './container';
 export * from './query';
+export * from './query/types';
 export * from './groups_selector';
 export * from './types';
 

x-pack/plugins/security_solution/public/common/components/grouping/query/index.test.ts

@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { GroupingQueryArgs } from '..';
+import { getGroupingQuery, MAX_QUERY_SIZE } from '..';
+
+const testProps: GroupingQueryArgs = {
+  additionalFilters: [],
+  additionalAggregationsRoot: [],
+  additionalStatsAggregationsFields0: [
+    {
+      alertsCount: {
+        cardinality: {
+          field: 'kibana.alert.uuid',
+        },
+      },
+    },
+    {
+      rulesCountAggregation: {
+        cardinality: {
+          field: 'kibana.alert.rule.rule_id',
+        },
+      },
+    },
+    {
+      countSeveritySubAggregation: {
+        cardinality: {
+          field: 'kibana.alert.severity',
+        },
+      },
+    },
+    {
+      severitiesSubAggregation: {
+        terms: {
+          field: 'kibana.alert.severity',
+        },
+      },
+    },
+    {
+      usersCountAggregation: {
+        cardinality: {
+          field: 'user.name',
+        },
+      },
+    },
+  ],
+  additionalStatsAggregationsFields1: [],
+  from: '2022-12-28T15:35:32.871Z',
+  runtimeMappings: {},
+  stackByMultipleFields0: ['host.name'],
+  stackByMultipleFields0Size: 25,
+  stackByMultipleFields0From: 0,
+  stackByMultipleFields1: [],
+  stackByMultipleFields1Size: 10,
+  to: '2023-02-23T06:59:59.999Z',
+};
+describe('group selector', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+  it('Sets terms query when single stackBy field requested', () => {
+    const result = getGroupingQuery(testProps);
+    expect(result.aggs.stackByMultipleFields0.multi_terms).toBeUndefined();
+    expect(result.aggs.stackByMultipleFields0.terms).toEqual({
+      field: 'host.name',
+      size: MAX_QUERY_SIZE,
+    });
+    expect(result.aggs.stackByMultipleFields0.aggs).toEqual({
+      bucket_truncate: { bucket_sort: { from: 0, size: 25 } },
+      alertsCount: { cardinality: { field: 'kibana.alert.uuid' } },
+      rulesCountAggregation: { cardinality: { field: 'kibana.alert.rule.rule_id' } },
+      countSeveritySubAggregation: { cardinality: { field: 'kibana.alert.severity' } },
+      severitiesSubAggregation: { terms: { field: 'kibana.alert.severity' } },
+      usersCountAggregation: { cardinality: { field: 'user.name' } },
+    });
+    expect(result.aggs.alertsCount).toBeUndefined();
+    expect(result.aggs.groupsNumber).toBeUndefined();
+    expect(result.query.bool.filter.length).toEqual(1);
+  });
+  it('Sets terms query when single stackBy field requested additionalAggregationsRoot', () => {
+    const result = getGroupingQuery({
+      ...testProps,
+      additionalAggregationsRoot: [
+        {
+          alertsCount: {
+            terms: {
+              field: 'kibana.alert.rule.producer',
+              exclude: ['alerts'],
+            },
+          },
+        },
+        {
+          groupsNumber: {
+            cardinality: {
+              field: 'host.name',
+            },
+          },
+        },
+      ],
+    });
+    expect(result.aggs.alertsCount).toEqual({
+      terms: { field: 'kibana.alert.rule.producer', exclude: ['alerts'] },
+    });
+    expect(result.aggs.groupsNumber).toEqual({ cardinality: { field: 'host.name' } });
+  });
+  it('Sets terms query when multiple stackBy fields requested', () => {
+    const result = getGroupingQuery({
+      ...testProps,
+      stackByMultipleFields0: ['kibana.alert.rule.name', 'kibana.alert.rule.description'],
+    });
+    expect(result.aggs.stackByMultipleFields0.terms).toBeUndefined();
+    expect(result.aggs.stackByMultipleFields0.multi_terms).toEqual({
+      terms: [{ field: 'kibana.alert.rule.name' }, { field: 'kibana.alert.rule.description' }],
+      size: MAX_QUERY_SIZE,
+    });
+  });
+  it('Additional filters get added to the query', () => {
+    const result = getGroupingQuery({
+      ...testProps,
+      additionalFilters: [
+        {
+          bool: {
+            must: [],
+            filter: [
+              {
+                term: {
+                  'kibana.alert.workflow_status': 'open',
+                },
+              },
+            ],
+            should: [],
+            must_not: [
+              {
+                exists: {
+                  field: 'kibana.alert.building_block_type',
+                },
+              },
+            ],
+          },
+        },
+      ],
+    });
+    expect(result.query.bool.filter.length).toEqual(2);
+  });
+});

x-pack/plugins/security_solution/public/common/components/grouping/query/index.ts

@@ -6,40 +6,17 @@
  */
 
 import { isEmpty } from 'lodash/fp';
-import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-
+import type {
+  GroupingQueryArgs,
+  GroupingQuery,
+  SubAggregation,
+  TermsOrCardinalityAggregation,
+} from './types';
 /** The maximum number of items to render */
 export const DEFAULT_STACK_BY_FIELD0_SIZE = 10;
 export const DEFAULT_STACK_BY_FIELD1_SIZE = 10;
 
-interface OptionalSubAggregation {
-  stackByMultipleFields1: {
-    multi_terms: {
-      terms: Array<{
-        field: string;
-      }>;
-    };
-  };
-}
-
-export interface CardinalitySubAggregation {
-  [category: string]: {
-    cardinality: {
-      field: string;
-    };
-  };
-}
-
-export interface TermsSubAggregation {
-  [category: string]: {
-    terms: {
-      field: string;
-      exclude?: string[];
-    };
-  };
-}
-
-export const getOptionalSubAggregation = ({
+const getOptionalSubAggregation = ({
   stackByMultipleFields1,
   stackByMultipleFields1Size,
   stackByMultipleFields1From = 0,
@@ -50,8 +27,8 @@ export const getOptionalSubAggregation = ({
   stackByMultipleFields1Size: number;
   stackByMultipleFields1From?: number;
   stackByMultipleFields1Sort?: Array<{ [category: string]: { order: 'asc' | 'desc' } }>;
-  additionalStatsAggregationsFields1: Array<CardinalitySubAggregation | TermsSubAggregation>;
-}): OptionalSubAggregation | {} =>
+  additionalStatsAggregationsFields1: TermsOrCardinalityAggregation[];
+}): SubAggregation | {} =>
   stackByMultipleFields1 != null && !isEmpty(stackByMultipleFields1)
     ? {
         stackByMultipleFields1: {
@@ -77,6 +54,9 @@ export const getOptionalSubAggregation = ({
       }
     : {};
 
+// our pagination will be broken if the stackBy field cardinality exceeds 10,000
+// https://github.com/elastic/kibana/issues/151913
+export const MAX_QUERY_SIZE = 10000;
 export const getGroupingQuery = ({
   additionalFilters = [],
   additionalAggregationsRoot,
@@ -93,25 +73,7 @@ export const getGroupingQuery = ({
   stackByMultipleFields1From,
   stackByMultipleFields1Sort,
   to,
-}: {
-  additionalFilters: Array<{
-    bool: { filter: unknown[]; should: unknown[]; must_not: unknown[]; must: unknown[] };
-  }>;
-  from: string;
-  runtimeMappings?: MappingRuntimeFields;
-  additionalAggregationsRoot?: Array<CardinalitySubAggregation | TermsSubAggregation>;
-  stackByMultipleFields0: string[];
-  stackByMultipleFields0Size?: number;
-  stackByMultipleFields0From?: number;
-  stackByMultipleFields0Sort?: Array<{ [category: string]: { order: 'asc' | 'desc' } }>;
-  additionalStatsAggregationsFields0: Array<CardinalitySubAggregation | TermsSubAggregation>;
-  stackByMultipleFields1: string[] | undefined;
-  stackByMultipleFields1Size?: number;
-  stackByMultipleFields1From?: number;
-  stackByMultipleFields1Sort?: Array<{ [category: string]: { order: 'asc' | 'desc' } }>;
-  additionalStatsAggregationsFields1: Array<CardinalitySubAggregation | TermsSubAggregation>;
-  to: string;
-}) => ({
+}: GroupingQueryArgs): GroupingQuery => ({
   size: 0,
   aggs: {
     stackByMultipleFields0: {
@@ -121,12 +83,13 @@ export const getGroupingQuery = ({
               terms: stackByMultipleFields0.map((stackByMultipleField0) => ({
                 field: stackByMultipleField0,
               })),
+              size: MAX_QUERY_SIZE,
             },
           }
         : {
             terms: {
               field: stackByMultipleFields0[0],
-              size: 10000,
+              size: MAX_QUERY_SIZE,
             },
           }),
       aggs: {

x-pack/plugins/security_solution/public/common/components/grouping/query/types.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { BoolQuery } from '@kbn/es-query';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+interface BoolAgg {
+  bool: BoolQuery;
+}
+
+interface RangeAgg {
+  range: { '@timestamp': { gte: string; lte: string } };
+}
+
+export interface TermsOrCardinalityAggregation {
+  [category: string]:
+    | {
+        cardinality: estypes.AggregationsAggregationContainer['cardinality'];
+      }
+    | {
+        terms: estypes.AggregationsAggregationContainer['terms'];
+      };
+}
+
+export interface GroupingQueryArgs {
+  additionalFilters: BoolAgg[];
+  from: string;
+  runtimeMappings?: MappingRuntimeFields;
+  additionalAggregationsRoot?: TermsOrCardinalityAggregation[];
+  stackByMultipleFields0: string[];
+  stackByMultipleFields0Size?: number;
+  stackByMultipleFields0From?: number;
+  stackByMultipleFields0Sort?: Array<{ [category: string]: { order: 'asc' | 'desc' } }>;
+  additionalStatsAggregationsFields0: TermsOrCardinalityAggregation[];
+  stackByMultipleFields1: string[] | undefined;
+  stackByMultipleFields1Size?: number;
+  stackByMultipleFields1From?: number;
+  stackByMultipleFields1Sort?: Array<{ [category: string]: { order: estypes.SortOrder } }>;
+  additionalStatsAggregationsFields1: TermsOrCardinalityAggregation[];
+  to: string;
+}
+
+export interface SubAggregation extends Record<string, estypes.AggregationsAggregationContainer> {
+  bucket_truncate: { bucket_sort: estypes.AggregationsAggregationContainer['bucket_sort'] };
+}
+
+export interface MainAggregation extends Record<string, estypes.AggregationsAggregationContainer> {
+  stackByMultipleFields0: {
+    terms?: estypes.AggregationsAggregationContainer['terms'];
+    multi_terms?: estypes.AggregationsAggregationContainer['multi_terms'];
+    aggs: SubAggregation;
+  };
+}
+
+export interface GroupingQuery extends estypes.QueryDslQueryContainer {
+  size: number;
+  runtime_mappings: MappingRuntimeFields | undefined;
+  query: {
+    bool: {
+      filter: Array<BoolAgg | RangeAgg>;
+    };
+  };
+  _source: boolean;
+  aggs: MainAggregation;
+}

x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/query_builder.test.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MAX_QUERY_SIZE } from '../../../../common/components/grouping';
 import { getAlertsGroupingQuery } from '.';
 
 describe('getAlertsGroupingQuery', () => {
@@ -95,6 +96,7 @@ describe('getAlertsGroupingQuery', () => {
             },
           },
           multi_terms: {
+            size: MAX_QUERY_SIZE,
             terms: [
               {
                 field: 'kibana.alert.rule.name',

x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings/query_builder.ts

@@ -7,10 +7,7 @@
 
 import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 import type { BoolQuery } from '@kbn/es-query';
-import type {
-  CardinalitySubAggregation,
-  TermsSubAggregation,
-} from '../../../../common/components/grouping';
+import type { TermsOrCardinalityAggregation } from '../../../../common/components/grouping';
 import { getGroupingQuery } from '../../../../common/components/grouping';
 
 const getGroupFields = (groupValue: string) => {
@@ -77,10 +74,8 @@ export const getAlertsGroupingQuery = ({
     stackByMultipleFields1: [],
   });
 
-const getAggregationsByGroupField = (
-  field: string
-): Array<CardinalitySubAggregation | TermsSubAggregation> => {
-  const aggMetrics: Array<CardinalitySubAggregation | TermsSubAggregation> = [
+const getAggregationsByGroupField = (field: string): TermsOrCardinalityAggregation[] => {
+  const aggMetrics: TermsOrCardinalityAggregation[] = [
     {
       alertsCount: {
         cardinality: {