[Security Solution] Value list exceptions (#133254)

packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx

@@ -16,7 +16,7 @@ import { getField } from '../fields/index.mock';
 import { AutocompleteFieldListsComponent } from '.';
 import {
   getListResponseMock,
-  getFoundListSchemaMock,
+  getFoundListsBySizeSchemaMock,
   DATE_NOW,
   IMMUTABLE,
   VERSION,
@@ -34,14 +34,15 @@ const mockKeywordList: ListSchema = {
   name: 'keyword list',
   type: 'keyword',
 };
-const mockResult = { ...getFoundListSchemaMock() };
-mockResult.data = [...mockResult.data, mockKeywordList];
+const mockResult = { ...getFoundListsBySizeSchemaMock() };
+mockResult.smallLists = [...mockResult.smallLists, mockKeywordList];
+mockResult.largeLists = [];
 jest.mock('@kbn/securitysolution-list-hooks', () => {
   const originalModule = jest.requireActual('@kbn/securitysolution-list-hooks');
 
   return {
     ...originalModule,
-    useFindLists: () => ({
+    useFindListsBySize: () => ({
       error: undefined,
       loading: false,
       result: mockResult,
@@ -116,7 +117,7 @@ describe('AutocompleteFieldListsComponent', () => {
       wrapper
         .find('EuiComboBox[data-test-subj="valuesAutocompleteComboBox listsComboxBox"]')
         .prop('options')
-    ).toEqual([{ label: 'some name' }]);
+    ).toEqual([{ label: 'some name', disabled: false }]);
   });
 
   test('it correctly displays lists that match the selected "keyword" field esType', () => {
@@ -139,7 +140,7 @@ describe('AutocompleteFieldListsComponent', () => {
       wrapper
         .find('EuiComboBox[data-test-subj="valuesAutocompleteComboBox listsComboxBox"]')
         .prop('options')
-    ).toEqual([{ label: 'keyword list' }]);
+    ).toEqual([{ label: 'keyword list', disabled: false }]);
   });
 
   test('it correctly displays lists that match the selected "ip" field esType', () => {
@@ -162,7 +163,7 @@ describe('AutocompleteFieldListsComponent', () => {
       wrapper
         .find('EuiComboBox[data-test-subj="valuesAutocompleteComboBox listsComboxBox"]')
         .prop('options')
-    ).toEqual([{ label: 'some name' }]);
+    ).toEqual([{ label: 'some name', disabled: false }]);
   });
 
   test('it correctly displays selected list', async () => {
@@ -206,7 +207,7 @@ describe('AutocompleteFieldListsComponent', () => {
       wrapper.find(EuiComboBox).props() as unknown as {
         onChange: (a: EuiComboBoxOptionOption[]) => void;
       }
-    ).onChange([{ label: 'some name' }]);
+    ).onChange([{ label: 'some name', disabled: false }]);
 
     await waitFor(() => {
       expect(mockOnChange).toHaveBeenCalledWith({

packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.tsx

@@ -7,9 +7,9 @@
  */
 
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
-import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow } from '@elastic/eui';
+import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow, EuiLink, EuiText } from '@elastic/eui';
 import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types';
-import { useFindLists } from '@kbn/securitysolution-list-hooks';
+import { useFindListsBySize } from '@kbn/securitysolution-list-hooks';
 import { DataViewFieldBase } from '@kbn/es-query';
 
 import { filterFieldToList } from '../filter_field_to_list';
@@ -33,6 +33,12 @@ interface AutocompleteFieldListsProps {
   rowLabel?: string;
   selectedField: DataViewFieldBase | undefined;
   selectedValue: string | undefined;
+  allowLargeValueLists?: boolean;
+}
+
+export interface AutocompleteListsData {
+  smallLists: ListSchema[];
+  largeLists: ListSchema[];
 }
 
 export const AutocompleteFieldListsComponent: React.FC<AutocompleteFieldListsProps> = ({
@@ -45,37 +51,44 @@ export const AutocompleteFieldListsComponent: React.FC<AutocompleteFieldListsPro
   rowLabel,
   selectedField,
   selectedValue,
+  allowLargeValueLists = false,
 }): JSX.Element => {
   const [error, setError] = useState<string | undefined>(undefined);
-  const [lists, setLists] = useState<ListSchema[]>([]);
-  const { loading, result, start } = useFindLists();
+  const [listData, setListData] = useState<AutocompleteListsData>({
+    smallLists: [],
+    largeLists: [],
+  });
+  const { loading, result, start } = useFindListsBySize();
   const getLabel = useCallback(({ name }) => name, []);
 
   const optionsMemo = useMemo(
-    () => filterFieldToList(lists, selectedField),
-    [lists, selectedField]
+    () => filterFieldToList(listData, selectedField),
+    [listData, selectedField]
   );
   const selectedOptionsMemo = useMemo(() => {
     if (selectedValue != null) {
-      const list = lists.filter(({ id }) => id === selectedValue);
+      const combinedLists = [...listData.smallLists, ...listData.largeLists];
+      const list = combinedLists.filter(({ id }) => id === selectedValue);
       return list ?? [];
     } else {
       return [];
     }
-  }, [selectedValue, lists]);
+  }, [selectedValue, listData]);
   const { comboOptions, labels, selectedComboOptions } = useMemo(
     () =>
       getGenericComboBoxProps<ListSchema>({
         getLabel,
-        options: optionsMemo,
+        options: [...optionsMemo.smallLists, ...optionsMemo.largeLists],
         selectedOptions: selectedOptionsMemo,
+        disabledOptions: allowLargeValueLists ? undefined : optionsMemo.largeLists, // Disable large lists if the rule type doesn't allow it
       }),
-    [optionsMemo, selectedOptionsMemo, getLabel]
+    [optionsMemo, selectedOptionsMemo, getLabel, allowLargeValueLists]
   );
 
   const handleValuesChange = useCallback(
     (newOptions: EuiComboBoxOptionOption[]) => {
-      const [newValue] = newOptions.map(({ label }) => optionsMemo[labels.indexOf(label)]);
+      const combinedLists = [...optionsMemo.smallLists, ...optionsMemo.largeLists];
+      const [newValue] = newOptions.map(({ label }) => combinedLists[labels.indexOf(label)]);
       onChange(newValue ?? '');
     },
     [labels, optionsMemo, onChange]
@@ -87,7 +100,7 @@ export const AutocompleteFieldListsComponent: React.FC<AutocompleteFieldListsPro
 
   useEffect(() => {
     if (result != null) {
-      setLists(result.data);
+      setListData(result);
     }
   }, [result]);
 
@@ -103,8 +116,27 @@ export const AutocompleteFieldListsComponent: React.FC<AutocompleteFieldListsPro
 
   const isLoadingState = useMemo((): boolean => isLoading || loading, [isLoading, loading]);
 
+  const helpText = useMemo(() => {
+    return (
+      !allowLargeValueLists && (
+        <EuiText size="xs">
+          {i18n.LISTS_TOOLTIP_INFO}{' '}
+          <EuiLink external target="_blank" href="https://www.elastic.co/">
+            {i18n.SEE_DOCUMENTATION}
+          </EuiLink>
+        </EuiText>
+      )
+    );
+  }, [allowLargeValueLists]);
+
   return (
-    <EuiFormRow label={rowLabel} error={error} isInvalid={error != null} fullWidth>
+    <EuiFormRow
+      label={rowLabel}
+      error={error}
+      isInvalid={error != null}
+      helpText={helpText}
+      fullWidth
+    >
       <EuiComboBox
         async
         data-test-subj="valuesAutocompleteComboBox listsComboxBox"

packages/kbn-securitysolution-autocomplete/src/filter_field_to_list/index.test.ts

@@ -8,83 +8,99 @@
 
 import { filterFieldToList } from '.';
 
-import type { ListSchema } from '@kbn/securitysolution-io-ts-list-types';
 import { getListResponseMock } from '../list_schema/index.mock';
 import { DataViewFieldBase } from '@kbn/es-query';
+import { AutocompleteListsData } from '../field_value_lists';
+
+const emptyListData: AutocompleteListsData = { smallLists: [], largeLists: [] };
 
 describe('#filterFieldToList', () => {
-  test('it returns empty array if given a undefined for field', () => {
-    const filter = filterFieldToList([], undefined);
-    expect(filter).toEqual([]);
+  test('it returns empty list data object if given a undefined for field', () => {
+    const filter = filterFieldToList(emptyListData, undefined);
+    expect(filter).toEqual(emptyListData);
   });
 
-  test('it returns empty array if filed does not contain esTypes', () => {
+  test('it returns empty list data object if filed does not contain esTypes', () => {
     const field: DataViewFieldBase = {
       name: 'some-name',
       type: 'some-type',
     };
-    const filter = filterFieldToList([], field);
-    expect(filter).toEqual([]);
+    const filter = filterFieldToList(emptyListData, field);
+    expect(filter).toEqual(emptyListData);
   });
 
-  test('it returns single filtered list of ip_range -> ip', () => {
+  test('it returns filtered lists of ip_range -> ip', () => {
     const field: DataViewFieldBase & { esTypes: string[] } = {
       esTypes: ['ip'],
       name: 'some-name',
       type: 'ip',
     };
-    const listItem: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
-    const filter = filterFieldToList([listItem], field);
-    const expected: ListSchema[] = [listItem];
+    const listData: AutocompleteListsData = {
+      smallLists: [{ ...getListResponseMock(), type: 'ip_range' }],
+      largeLists: [],
+    };
+    const filter = filterFieldToList(listData, field);
+    const expected = listData;
     expect(filter).toEqual(expected);
   });
 
-  test('it returns single filtered list of ip -> ip', () => {
+  test('it returns filtered lists of ip -> ip', () => {
     const field: DataViewFieldBase & { esTypes: string[] } = {
       esTypes: ['ip'],
       name: 'some-name',
       type: 'ip',
     };
-    const listItem: ListSchema = { ...getListResponseMock(), type: 'ip' };
-    const filter = filterFieldToList([listItem], field);
-    const expected: ListSchema[] = [listItem];
+    const listData: AutocompleteListsData = {
+      smallLists: [{ ...getListResponseMock(), type: 'ip' }],
+      largeLists: [],
+    };
+    const filter = filterFieldToList(listData, field);
+    const expected = listData;
     expect(filter).toEqual(expected);
   });
 
-  test('it returns single filtered list of keyword -> keyword', () => {
+  test('it returns filtered lists of keyword -> keyword', () => {
     const field: DataViewFieldBase & { esTypes: string[] } = {
       esTypes: ['keyword'],
       name: 'some-name',
       type: 'keyword',
     };
-    const listItem: ListSchema = { ...getListResponseMock(), type: 'keyword' };
-    const filter = filterFieldToList([listItem], field);
-    const expected: ListSchema[] = [listItem];
+    const listData: AutocompleteListsData = {
+      smallLists: [{ ...getListResponseMock(), type: 'keyword' }],
+      largeLists: [],
+    };
+    const filter = filterFieldToList(listData, field);
+    const expected = listData;
     expect(filter).toEqual(expected);
   });
 
-  test('it returns single filtered list of text -> text', () => {
+  test('it returns filtered lists of text -> text', () => {
     const field: DataViewFieldBase & { esTypes: string[] } = {
       esTypes: ['text'],
       name: 'some-name',
       type: 'text',
     };
-    const listItem: ListSchema = { ...getListResponseMock(), type: 'text' };
-    const filter = filterFieldToList([listItem], field);
-    const expected: ListSchema[] = [listItem];
+    const listData: AutocompleteListsData = {
+      smallLists: [{ ...getListResponseMock(), type: 'text' }],
+      largeLists: [],
+    };
+    const filter = filterFieldToList(listData, field);
+    const expected = listData;
     expect(filter).toEqual(expected);
   });
 
-  test('it returns 2 filtered lists of ip_range -> ip', () => {
+  test('it returns small and large filtered lists of ip_range -> ip', () => {
     const field: DataViewFieldBase & { esTypes: string[] } = {
       esTypes: ['ip'],
       name: 'some-name',
       type: 'ip',
     };
-    const listItem1: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
-    const listItem2: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
-    const filter = filterFieldToList([listItem1, listItem2], field);
-    const expected: ListSchema[] = [listItem1, listItem2];
+    const listData: AutocompleteListsData = {
+      smallLists: [{ ...getListResponseMock(), type: 'ip_range' }],
+      largeLists: [{ ...getListResponseMock(), type: 'ip_range' }],
+    };
+    const filter = filterFieldToList(listData, field);
+    const expected = listData;
     expect(filter).toEqual(expected);
   });
 
@@ -94,10 +110,15 @@ describe('#filterFieldToList', () => {
       name: 'some-name',
       type: 'ip',
     };
-    const listItem1: ListSchema = { ...getListResponseMock(), type: 'ip_range' };
-    const listItem2: ListSchema = { ...getListResponseMock(), type: 'text' };
-    const filter = filterFieldToList([listItem1, listItem2], field);
-    const expected: ListSchema[] = [listItem1];
+    const listData: AutocompleteListsData = {
+      smallLists: [{ ...getListResponseMock(), type: 'ip_range' }],
+      largeLists: [{ ...getListResponseMock(), type: 'text' }],
+    };
+    const filter = filterFieldToList(listData, field);
+    const expected: AutocompleteListsData = {
+      smallLists: [{ ...getListResponseMock(), type: 'ip_range' }],
+      largeLists: [],
+    };
     expect(filter).toEqual(expected);
   });
 });

packages/kbn-securitysolution-autocomplete/src/filter_field_to_list/index.ts

@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 
-import { ListSchema } from '@kbn/securitysolution-io-ts-list-types';
 import { DataViewFieldBase } from '@kbn/es-query';
 import { typeMatch } from '../type_match';
+import { AutocompleteListsData } from '../field_value_lists';
 
 /**
  * Given an array of lists and optionally a field this will return all
@@ -21,13 +21,20 @@ import { typeMatch } from '../type_match';
  * @param field The field to check against the list to see if they are compatible
  */
 export const filterFieldToList = (
-  lists: ListSchema[],
+  lists: AutocompleteListsData,
   field?: DataViewFieldBase & { esTypes?: string[] }
-): ListSchema[] => {
+): AutocompleteListsData => {
   if (field != null) {
     const { esTypes = [] } = field;
-    return lists.filter(({ type }) => esTypes.some((esType: string) => typeMatch(type, esType)));
+    return {
+      smallLists: lists.smallLists.filter(({ type }) =>
+        esTypes.some((esType: string) => typeMatch(type, esType))
+      ),
+      largeLists: lists.largeLists.filter(({ type }) =>
+        esTypes.some((esType: string) => typeMatch(type, esType))
+      ),
+    };
   } else {
-    return [];
+    return { smallLists: [], largeLists: [] };
   }
 };

packages/kbn-securitysolution-autocomplete/src/get_generic_combo_box_props/index.test.tsx

@@ -67,7 +67,7 @@ describe('get_generic_combo_box_props', () => {
     });
   });
 
-  test('it return "selectedOptions" items that do appear in "options"', () => {
+  test('it returns "selectedOptions" items that do appear in "options"', () => {
     const result = getGenericComboBoxProps<string>({
       options: ['option1', 'option2', 'option3'],
       selectedOptions: ['option2'],
@@ -94,4 +94,32 @@ describe('get_generic_combo_box_props', () => {
       ],
     });
   });
+
+  test('it returns "disabledOptions" items that do appear in "options" as disabled', () => {
+    const result = getGenericComboBoxProps<string>({
+      options: ['option1', 'option2', 'option3'],
+      selectedOptions: [],
+      disabledOptions: ['option2'],
+      getLabel: (t: string) => t,
+    });
+
+    expect(result).toEqual({
+      comboOptions: [
+        {
+          label: 'option1',
+          disabled: false,
+        },
+        {
+          label: 'option2',
+          disabled: true,
+        },
+        {
+          label: 'option3',
+          disabled: false,
+        },
+      ],
+      labels: ['option1', 'option2', 'option3'],
+      selectedComboOptions: [],
+    });
+  });
 });

packages/kbn-securitysolution-autocomplete/src/get_generic_combo_box_props/index.ts

@@ -24,13 +24,19 @@ export const getGenericComboBoxProps = <T>({
   getLabel,
   options,
   selectedOptions,
+  disabledOptions,
 }: {
   getLabel: (value: T) => string;
   options: T[];
   selectedOptions: T[];
+  disabledOptions?: T[];
 }): GetGenericComboBoxPropsReturn => {
   const newLabels = options.map(getLabel);
-  const newComboOptions: EuiComboBoxOptionOption[] = newLabels.map((label) => ({ label }));
+  const disabledLabels = disabledOptions?.map(getLabel);
+  const newComboOptions: EuiComboBoxOptionOption[] = newLabels.map((label) => ({
+    label,
+    disabled: disabledLabels && disabledLabels.length !== 0 && disabledLabels.includes(label),
+  }));
   const newSelectedComboOptions = selectedOptions
     .map(getLabel)
     .filter((option) => {

packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts

@@ -6,7 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { FoundListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types';
+import {
+  FoundListSchema,
+  ListSchema,
+  FoundListsBySizeSchema,
+} from '@kbn/securitysolution-io-ts-list-types';
 
 // TODO: Once this mock is available within packages, use it instead, https://github.com/elastic/kibana/issues/100715
 // import { getFoundListSchemaMock } from '../../../../../lists/common/schemas/response/found_list_schema.mock';
@@ -18,6 +22,11 @@ export const getFoundListSchemaMock = (): FoundListSchema => ({
   total: 1,
 });
 
+export const getFoundListsBySizeSchemaMock = (): FoundListsBySizeSchema => ({
+  smallLists: [getListResponseMock()],
+  largeLists: [getListResponseMock()],
+});
+
 // TODO: Once these mocks are available from packages use it instead, https://github.com/elastic/kibana/issues/100715
 export const DATE_NOW = '2020-04-20T15:25:31.830Z';
 export const USER = 'some user';

packages/kbn-securitysolution-autocomplete/src/translations/index.ts

@@ -35,6 +35,14 @@ export const FIELD_SPACE_WARNING = i18n.translate('autocomplete.fieldSpaceWarnin
   defaultMessage: "Warning: Spaces at the start or end of this value aren't being displayed.",
 });
 
+export const LISTS_TOOLTIP_INFO = i18n.translate('autocomplete.listsTooltipWarning', {
+  defaultMessage: "Lists that aren't able to be processed by this rule type will be disabled.",
+});
+
+export const SEE_DOCUMENTATION = i18n.translate('autocomplete.seeDocumentation', {
+  defaultMessage: 'See Documentation',
+});
+
 // eslint-disable-next-line import/no-default-export
 export default {
   LOADING,

packages/kbn-securitysolution-io-ts-list-types/BUILD.bazel

@@ -48,6 +48,7 @@ TYPES_DEPS = [
   "//packages/kbn-securitysolution-io-ts-types:npm_module_types",
   "//packages/kbn-securitysolution-io-ts-utils:npm_module_types",
   "//packages/kbn-securitysolution-list-constants:npm_module_types",
+  "//packages/kbn-es-query:npm_module_types",
   "@npm//fp-ts",
   "@npm//io-ts",
   "@npm//tslib",

packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts

@@ -41,6 +41,7 @@ export * from './lists_default_array';
 export * from './max_size';
 export * from './meta';
 export * from './name';
+export * from './namespace_type';
 export * from './non_empty_entries_array';
 export * from './non_empty_nested_entries_array';
 export * from './os_type';

packages/kbn-securitysolution-io-ts-list-types/src/request/get_exception_filter_schema/index.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as t from 'io-ts';
+import { namespaceType } from '../../common/default_namespace';
+import { exceptionListItemSchema } from '../../response';
+import { createExceptionListItemSchema } from '../create_exception_list_item_schema';
+
+const exceptionListId = t.type({
+  exception_list_id: t.string,
+  namespace_type: namespaceType,
+});
+
+export const exceptionListIds = t.type({
+  exception_list_ids: t.array(exceptionListId),
+  type: t.literal('exception_list_ids'),
+});
+
+export const exceptions = t.type({
+  exceptions: t.array(t.union([exceptionListItemSchema, createExceptionListItemSchema])),
+  type: t.literal('exception_items'),
+});
+
+const optionalExceptionParams = t.exact(
+  t.partial({ alias: t.string, chunk_size: t.number, exclude_exceptions: t.boolean })
+);
+
+export const getExceptionFilterSchema = t.intersection([
+  t.union([exceptions, exceptionListIds]),
+  optionalExceptionParams,
+]);
+
+export type GetExceptionFilterSchema = t.TypeOf<typeof getExceptionFilterSchema>;
+export type ExceptionListId = t.TypeOf<typeof exceptionListId>;

packages/kbn-securitysolution-io-ts-list-types/src/request/index.ts

@@ -23,6 +23,7 @@ export * from './find_exception_list_schema';
 export * from './find_exception_list_item_schema';
 export * from './find_list_item_schema';
 export * from './find_list_schema';
+export * from './get_exception_filter_schema';
 export * from './import_list_item_query_schema';
 export * from './import_exception_list_schema';
 export * from './import_exception_item_schema';

packages/kbn-securitysolution-io-ts-list-types/src/response/found_all_list_items_schema/index.mock.ts

@@ -6,9 +6,10 @@
  * Side Public License, v 1.
  */
 
-describe('build_exceptions_filter', () => {
-  test('Tests should be ported', () => {
-    // TODO: Port all the tests from: x-pack/plugins/lists/common/exceptions/build_exceptions_filter.test.ts here once mocks are figured out and kbn package mocks are figured out
-    expect(true).toBe(true);
-  });
+import { FoundAllListItemsSchema } from '.';
+import { getListItemResponseMock } from '../list_item_schema/index.mock';
+
+export const getFoundAllListItemsSchemaMock = (): FoundAllListItemsSchema => ({
+  data: [getListItemResponseMock()],
+  total: 1,
 });

packages/kbn-securitysolution-io-ts-list-types/src/response/found_all_list_items_schema/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as t from 'io-ts';
+
+import { listItemSchema } from '../list_item_schema';
+import { total } from '../../common/total';
+
+export const foundAllListItemsSchema = t.exact(
+  t.type({
+    data: t.array(listItemSchema),
+    total,
+  })
+);
+
+export type FoundAllListItemsSchema = t.TypeOf<typeof foundAllListItemsSchema>;

packages/kbn-securitysolution-io-ts-list-types/src/response/found_lists_by_size_schema/index.mock.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { FoundListsBySizeSchema } from '.';
+import { getListResponseMock } from '../list_schema/index.mock';
+
+export const getFoundListsBySizeSchemaMock = (): FoundListsBySizeSchema => ({
+  smallLists: [getListResponseMock()],
+  largeLists: [getListResponseMock()],
+});

packages/kbn-securitysolution-io-ts-list-types/src/response/found_lists_by_size_schema/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as t from 'io-ts';
+
+import { listSchema } from '../list_schema';
+
+export const foundListsBySizeSchema = t.exact(
+  t.type({
+    largeLists: t.array(listSchema),
+    smallLists: t.array(listSchema),
+  })
+);
+
+export type FoundListsBySizeSchema = t.TypeOf<typeof foundListsBySizeSchema>;

packages/kbn-securitysolution-io-ts-list-types/src/response/index.ts

@@ -12,6 +12,8 @@ export * from './exception_list_schema';
 export * from './exception_list_item_schema';
 export * from './found_exception_list_item_schema';
 export * from './found_exception_list_schema';
+export * from './found_all_list_items_schema';
+export * from './found_lists_by_size_schema';
 export * from './found_list_item_schema';
 export * from './found_list_schema';
 export * from './import_exceptions_schema';

packages/kbn-securitysolution-io-ts-list-types/src/typescript_types/index.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import type { Filter } from '@kbn/es-query';
 import { NamespaceType } from '../common/default_namespace';
 import { ExceptionListType } from '../common/exception_list';
 import { Page } from '../common/page';
@@ -13,6 +14,7 @@ import { PerPage } from '../common/per_page';
 import { TotalOrUndefined } from '../common/total';
 import { CreateExceptionListItemSchema } from '../request/create_exception_list_item_schema';
 import { CreateExceptionListSchema } from '../request/create_exception_list_schema';
+import { ExceptionListId } from '../request/get_exception_filter_schema';
 import { UpdateExceptionListItemSchema } from '../request/update_exception_list_item_schema';
 import { UpdateExceptionListSchema } from '../request/update_exception_list_schema';
 import { ExceptionListItemSchema } from '../response/exception_list_item_schema';
@@ -107,6 +109,19 @@ export interface ApiCallFindListsItemsMemoProps {
   onSuccess: (arg: UseExceptionListItemsSuccess) => void;
 }
 
+export interface ApiCallGetExceptionFilterFromIdsMemoProps extends GetExceptionFilterOptionalProps {
+  exceptionListIds: ExceptionListId[];
+  onError: (arg: string[]) => void;
+  onSuccess: (arg: Filter) => void;
+}
+
+export interface ApiCallGetExceptionFilterFromExceptionsMemoProps
+  extends GetExceptionFilterOptionalProps {
+  exceptions: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
+  onError: (arg: string[]) => void;
+  onSuccess: (arg: Filter) => void;
+}
+
 export interface ExportExceptionListProps {
   http: HttpStart;
   id: string;
@@ -184,3 +199,25 @@ export interface PersistHookProps {
 export interface ExceptionList extends ExceptionListSchema {
   totalItems: number;
 }
+
+export interface GetExceptionFilterOptionalProps {
+  signal?: AbortSignal;
+  chunkSize?: number;
+  alias?: string;
+  excludeExceptions?: boolean;
+}
+
+export interface GetExceptionFilterFromExceptionListIdsProps
+  extends GetExceptionFilterOptionalProps {
+  http: HttpStart;
+  exceptionListIds: ExceptionListId[];
+}
+
+export interface GetExceptionFilterFromExceptionsProps extends GetExceptionFilterOptionalProps {
+  http: HttpStart;
+  exceptions: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
+}
+
+export interface ExceptionFilterResponse {
+  filter: Filter;
+}

packages/kbn-securitysolution-list-api/src/api/index.ts

@@ -29,10 +29,14 @@ import {
   ExportExceptionListProps,
   UpdateExceptionListItemProps,
   UpdateExceptionListProps,
+  GetExceptionFilterFromExceptionListIdsProps,
+  GetExceptionFilterFromExceptionsProps,
+  ExceptionFilterResponse,
 } from '@kbn/securitysolution-io-ts-list-types';
 
 import {
   ENDPOINT_LIST_URL,
+  EXCEPTION_FILTER,
   EXCEPTION_LIST_ITEM_URL,
   EXCEPTION_LIST_URL,
 } from '@kbn/securitysolution-list-constants';
@@ -547,3 +551,59 @@ export const exportExceptionList = async ({
     query: { id, list_id: listId, namespace_type: namespaceType },
     signal,
   });
+
+/**
+ * Create a Filter query from an exception list id
+ *
+ * @param exceptionListId The id of the exception list from which create a Filter query
+ * @param signal AbortSignal for cancelling request
+ *
+ * @throws An error if response is not OK
+ */
+export const getExceptionFilterFromExceptionListIds = async ({
+  alias,
+  chunkSize,
+  exceptionListIds,
+  excludeExceptions,
+  http,
+  signal,
+}: GetExceptionFilterFromExceptionListIdsProps): Promise<ExceptionFilterResponse> =>
+  http.fetch(EXCEPTION_FILTER, {
+    method: 'POST',
+    body: JSON.stringify({
+      exception_list_ids: exceptionListIds,
+      type: 'exception_list_ids',
+      alias,
+      exclude_exceptions: excludeExceptions,
+      chunk_size: chunkSize,
+    }),
+    signal,
+  });
+
+/**
+ * Create a Filter query from a list of exceptions
+ *
+ * @param exceptions Exception items to be made into a `Filter` query
+ * @param signal AbortSignal for cancelling request
+ *
+ * @throws An error if response is not OK
+ */
+export const getExceptionFilterFromExceptions = async ({
+  exceptions,
+  alias,
+  excludeExceptions,
+  http,
+  chunkSize,
+  signal,
+}: GetExceptionFilterFromExceptionsProps): Promise<ExceptionFilterResponse> =>
+  http.fetch(EXCEPTION_FILTER, {
+    method: 'POST',
+    body: JSON.stringify({
+      exceptions,
+      type: 'exception_items',
+      alias,
+      exclude_exceptions: excludeExceptions,
+      chunk_size: chunkSize,
+    }),
+    signal,
+  });

packages/kbn-securitysolution-list-api/src/list_api/index.ts

@@ -29,12 +29,15 @@ import {
   importListItemSchema,
   listItemIndexExistSchema,
   listSchema,
+  foundListsBySizeSchema,
+  FoundListsBySizeSchema,
 } from '@kbn/securitysolution-io-ts-list-types';
 import {
   LIST_INDEX,
   LIST_ITEM_URL,
   LIST_PRIVILEGES_URL,
   LIST_URL,
+  FIND_LISTS_BY_SIZE,
 } from '@kbn/securitysolution-list-constants';
 import { toError, toPromise } from '../fp_utils';
 
@@ -104,6 +107,46 @@ const findListsWithValidation = async ({
 
 export { findListsWithValidation as findLists };
 
+const findListsBySize = async ({
+  http,
+  cursor,
+  page,
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  per_page,
+  signal,
+}: ApiParams & FindListSchemaEncoded): Promise<FoundListsBySizeSchema> => {
+  return http.fetch(`${FIND_LISTS_BY_SIZE}`, {
+    method: 'GET',
+    query: {
+      cursor,
+      page,
+      per_page,
+    },
+    signal,
+  });
+};
+
+const findListsBySizeWithValidation = async ({
+  cursor,
+  http,
+  pageIndex,
+  pageSize,
+  signal,
+}: FindListsParams): Promise<FoundListsBySizeSchema> =>
+  pipe(
+    {
+      cursor: cursor != null ? cursor.toString() : undefined,
+      page: pageIndex != null ? pageIndex.toString() : undefined,
+      per_page: pageSize != null ? pageSize.toString() : undefined,
+    },
+    (payload) => fromEither(validateEither(findListSchema, payload)),
+    chain((payload) => tryCatch(() => findListsBySize({ http, signal, ...payload }), toError)),
+    chain((response) => fromEither(validateEither(foundListsBySizeSchema, response))),
+    flow(toPromise)
+  );
+
+export { findListsBySizeWithValidation as findListsBySize };
+
 const importList = async ({
   file,
   http,

packages/kbn-securitysolution-list-constants/index.ts

@@ -14,6 +14,13 @@ export const LIST_INDEX = `${LIST_URL}/index`;
 export const LIST_ITEM_URL = `${LIST_URL}/items`;
 export const LIST_PRIVILEGES_URL = `${LIST_URL}/privileges`;
 
+/**
+ * Internal value list routes
+ */
+export const INTERNAL_LIST_URL = '/internal/lists';
+export const FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const;
+export const EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const;
+
 /**
  * Exception list routes
  */
@@ -59,6 +66,10 @@ export const ENDPOINT_LIST_DESCRIPTION = 'Endpoint Security Exception List';
 
 export const MAX_EXCEPTION_LIST_SIZE = 10000;
 
+export const MAXIMUM_SMALL_VALUE_LIST_SIZE = 65536;
+
+export const MAXIMUM_SMALL_IP_RANGE_VALUE_LIST_DASH_SIZE = 200;
+
 /** ID of trusted apps agnostic list */
 export const ENDPOINT_TRUSTED_APPS_LIST_ID = 'endpoint_trusted_apps';
 

packages/kbn-securitysolution-list-hooks/index.ts

@@ -13,6 +13,7 @@ export * from './src/use_delete_list';
 export * from './src/use_exception_lists';
 export * from './src/use_export_list';
 export * from './src/use_find_lists';
+export * from './src/use_find_lists_by_size';
 export * from './src/use_import_list';
 export * from './src/use_persist_exception_item';
 export * from './src/use_persist_exception_list';

packages/kbn-securitysolution-list-hooks/src/use_api/index.ts

@@ -15,6 +15,8 @@ import type {
   ApiCallFindListsItemsMemoProps,
   ApiCallMemoProps,
   ApiListExportProps,
+  ApiCallGetExceptionFilterFromIdsMemoProps,
+  ApiCallGetExceptionFilterFromExceptionsMemoProps,
 } from '@kbn/securitysolution-io-ts-list-types';
 import * as Api from '@kbn/securitysolution-list-api';
 
@@ -44,6 +46,10 @@ export interface ExceptionsApi {
     arg: ApiCallMemoProps & { onSuccess: (arg: ExceptionListSchema) => void }
   ) => Promise<void>;
   getExceptionListsItems: (arg: ApiCallFindListsItemsMemoProps) => Promise<void>;
+  getExceptionFilterFromIds: (arg: ApiCallGetExceptionFilterFromIdsMemoProps) => Promise<void>;
+  getExceptionFilterFromExceptions: (
+    arg: ApiCallGetExceptionFilterFromExceptionsMemoProps
+  ) => Promise<void>;
   exportExceptionList: (arg: ApiListExportProps) => Promise<void>;
 }
 
@@ -224,6 +230,52 @@ export const useApi = (http: HttpStart): ExceptionsApi => {
           onError(error);
         }
       },
+      async getExceptionFilterFromIds({
+        exceptionListIds,
+        chunkSize,
+        alias,
+        excludeExceptions,
+        onSuccess,
+        onError,
+      }: ApiCallGetExceptionFilterFromIdsMemoProps): Promise<void> {
+        const abortCtrl = new AbortController();
+        try {
+          const { filter } = await Api.getExceptionFilterFromExceptionListIds({
+            http,
+            exceptionListIds,
+            signal: abortCtrl.signal,
+            chunkSize,
+            alias,
+            excludeExceptions,
+          });
+          onSuccess(filter);
+        } catch (error) {
+          onError(error);
+        }
+      },
+      async getExceptionFilterFromExceptions({
+        exceptions,
+        chunkSize,
+        alias,
+        excludeExceptions,
+        onSuccess,
+        onError,
+      }: ApiCallGetExceptionFilterFromExceptionsMemoProps): Promise<void> {
+        const abortCtrl = new AbortController();
+        try {
+          const { filter } = await Api.getExceptionFilterFromExceptions({
+            http,
+            exceptions,
+            signal: abortCtrl.signal,
+            chunkSize,
+            alias,
+            excludeExceptions,
+          });
+          onSuccess(filter);
+        } catch (error) {
+          onError(error);
+        }
+      },
       async updateExceptionListItem({
         listItem,
       }: {

packages/kbn-securitysolution-list-hooks/src/use_find_lists_by_size/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { findListsBySize } from '@kbn/securitysolution-list-api';
+import { useAsync, withOptionalSignal } from '@kbn/securitysolution-hook-utils';
+
+const findListsBySizeWithOptionalSignal = withOptionalSignal(findListsBySize);
+
+export const useFindListsBySize = () => useAsync(findListsBySizeWithOptionalSignal);

packages/kbn-securitysolution-list-utils/index.ts

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 export * from './src/autocomplete_operators';
-export * from './src/build_exception_filter';
 export * from './src/get_exception_list_type';
 export * from './src/get_filters';
 export * from './src/get_general_filters';

packages/kbn-securitysolution-list-utils/src/build_exception_filter/index.ts

@@ -1,390 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { chunk } from 'lodash/fp';
-
-import {
-  CreateExceptionListItemSchema,
-  EntryExists,
-  EntryMatch,
-  EntryMatchAny,
-  EntryNested,
-  ExceptionListItemSchema,
-  entriesExists,
-  entriesMatch,
-  entriesMatchAny,
-  entriesNested,
-  OsTypeArray,
-  entriesMatchWildcard,
-  EntryMatchWildcard,
-} from '@kbn/securitysolution-io-ts-list-types';
-import { Filter } from '@kbn/es-query';
-
-import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { hasLargeValueList } from '../has_large_value_list';
-
-type NonListEntry = EntryMatch | EntryMatchAny | EntryNested | EntryExists | EntryMatchWildcard;
-interface ExceptionListItemNonLargeList extends ExceptionListItemSchema {
-  entries: NonListEntry[];
-}
-
-interface CreateExceptionListItemNonLargeList extends CreateExceptionListItemSchema {
-  entries: NonListEntry[];
-}
-
-export type ExceptionItemSansLargeValueLists =
-  | ExceptionListItemNonLargeList
-  | CreateExceptionListItemNonLargeList;
-
-export interface BooleanFilter {
-  bool: estypes.QueryDslBoolQuery;
-}
-
-export interface NestedFilter {
-  nested: estypes.QueryDslNestedQuery;
-}
-
-export const chunkExceptions = (
-  exceptions: ExceptionItemSansLargeValueLists[],
-  chunkSize: number
-): ExceptionItemSansLargeValueLists[][] => {
-  return chunk(chunkSize, exceptions);
-};
-
-/**
- * Transforms the os_type into a regular filter as if the user had created it
- * from the fields for the next state of transforms which will create the elastic filters
- * from it.
- *
- * Note: We use two types of fields, the "host.os.type" and "host.os.name.caseless"
- * The endpoint/endgame agent has been using "host.os.name.caseless" as the same value as the ECS
- * value of "host.os.type" where the auditbeat, winlogbeat, etc... (other agents) are all using
- * "host.os.type". In order to be compatible with both, I create an "OR" between these two data types
- * where if either has a match then we will exclude it as part of the match. This should also be
- * forwards compatible for endpoints/endgame agents when/if they upgrade to using "host.os.type"
- * rather than using "host.os.name.caseless" values.
- *
- * Also we create another "OR" from the osType names so that if there are multiples such as ['windows', 'linux']
- * this will exclude anything with either 'windows' or with 'linux'
- * @param osTypes The os_type array from the REST interface that is an array such as ['windows', 'linux']
- * @param entries The entries to join the OR's with before the elastic filter change out
- */
-export const transformOsType = (
-  osTypes: OsTypeArray,
-  entries: NonListEntry[]
-): NonListEntry[][] => {
-  const hostTypeTransformed = osTypes.map<NonListEntry[]>((osType) => {
-    return [
-      { field: 'host.os.type', operator: 'included', type: 'match', value: osType },
-      ...entries,
-    ];
-  });
-  const caseLessTransformed = osTypes.map<NonListEntry[]>((osType) => {
-    return [
-      { field: 'host.os.name.caseless', operator: 'included', type: 'match', value: osType },
-      ...entries,
-    ];
-  });
-  return [...hostTypeTransformed, ...caseLessTransformed];
-};
-
-/**
- * This builds an exception item filter with the os type
- * @param osTypes The os_type array from the REST interface that is an array such as ['windows', 'linux']
- * @param entries The entries to join the OR's with before the elastic filter change out
- */
-export const buildExceptionItemFilterWithOsType = (
-  osTypes: OsTypeArray,
-  entries: NonListEntry[]
-): BooleanFilter[] => {
-  const entriesWithOsTypes = transformOsType(osTypes, entries);
-  return entriesWithOsTypes.map((entryWithOsType) => {
-    return {
-      bool: {
-        filter: entryWithOsType.map((entry) => createInnerAndClauses(entry)),
-      },
-    };
-  });
-};
-
-export const buildExceptionItemFilter = (
-  exceptionItem: ExceptionItemSansLargeValueLists
-): Array<BooleanFilter | NestedFilter> => {
-  const { entries, os_types: osTypes } = exceptionItem;
-  if (osTypes != null && osTypes.length > 0) {
-    return buildExceptionItemFilterWithOsType(osTypes, entries);
-  } else {
-    if (entries.length === 1) {
-      return [createInnerAndClauses(entries[0])];
-    } else {
-      return [
-        {
-          bool: {
-            filter: entries.map((entry) => createInnerAndClauses(entry)),
-          },
-        },
-      ];
-    }
-  }
-};
-
-export const createOrClauses = (
-  exceptionItems: ExceptionItemSansLargeValueLists[]
-): Array<BooleanFilter | NestedFilter> => {
-  return exceptionItems.flatMap((exceptionItem) => buildExceptionItemFilter(exceptionItem));
-};
-
-export const buildExceptionFilter = ({
-  lists,
-  excludeExceptions,
-  chunkSize,
-  alias = null,
-}: {
-  lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
-  excludeExceptions: boolean;
-  chunkSize: number;
-  alias: string | null;
-}): Filter | undefined => {
-  // Remove exception items with large value lists. These are evaluated
-  // elsewhere for the moment being.
-  const exceptionsWithoutLargeValueLists = lists.filter(
-    (item): item is ExceptionItemSansLargeValueLists => !hasLargeValueList(item.entries)
-  );
-
-  const exceptionFilter: Filter = {
-    meta: {
-      alias,
-      disabled: false,
-      negate: excludeExceptions,
-    },
-    query: {
-      bool: {
-        should: undefined,
-      },
-    },
-  };
-
-  if (exceptionsWithoutLargeValueLists.length === 0) {
-    return undefined;
-  } else if (exceptionsWithoutLargeValueLists.length <= chunkSize) {
-    const clause = createOrClauses(exceptionsWithoutLargeValueLists);
-    exceptionFilter.query!.bool!.should = clause;
-    return exceptionFilter;
-  } else {
-    const chunks = chunkExceptions(exceptionsWithoutLargeValueLists, chunkSize);
-
-    const filters = chunks.map((exceptionsChunk) => {
-      const orClauses = createOrClauses(exceptionsChunk);
-
-      return {
-        meta: {
-          alias: null,
-          disabled: false,
-          negate: false,
-        },
-        query: {
-          bool: {
-            should: orClauses,
-          },
-        },
-      };
-    });
-
-    const clauses = filters.map<BooleanFilter>(({ query }) => query);
-
-    return {
-      meta: {
-        alias,
-        disabled: false,
-        negate: excludeExceptions,
-      },
-      query: {
-        bool: {
-          should: clauses,
-        },
-      },
-    };
-  }
-};
-
-export const buildExclusionClause = (booleanFilter: BooleanFilter): BooleanFilter => {
-  return {
-    bool: {
-      must_not: booleanFilter,
-    },
-  };
-};
-
-export const buildMatchClause = (entry: EntryMatch): BooleanFilter => {
-  const { field, operator, value } = entry;
-  const matchClause = {
-    bool: {
-      minimum_should_match: 1,
-      should: [
-        {
-          match_phrase: {
-            [field]: value,
-          },
-        },
-      ],
-    },
-  };
-
-  if (operator === 'excluded') {
-    return buildExclusionClause(matchClause);
-  } else {
-    return matchClause;
-  }
-};
-
-export const getBaseMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
-  const { field, value } = entry;
-
-  if (value.length === 1) {
-    return {
-      bool: {
-        minimum_should_match: 1,
-        should: [
-          {
-            match_phrase: {
-              [field]: value[0],
-            },
-          },
-        ],
-      },
-    };
-  }
-
-  return {
-    bool: {
-      minimum_should_match: 1,
-      should: value.map((val) => {
-        return {
-          bool: {
-            minimum_should_match: 1,
-            should: [
-              {
-                match_phrase: {
-                  [field]: val,
-                },
-              },
-            ],
-          },
-        };
-      }),
-    },
-  };
-};
-
-export const buildMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
-  const { operator } = entry;
-  const matchAnyClause = getBaseMatchAnyClause(entry);
-
-  if (operator === 'excluded') {
-    return buildExclusionClause(matchAnyClause);
-  } else {
-    return matchAnyClause;
-  }
-};
-
-export const buildMatchWildcardClause = (entry: EntryMatchWildcard): BooleanFilter => {
-  const { field, operator, value } = entry;
-  const wildcardClause = {
-    bool: {
-      filter: {
-        wildcard: {
-          [field]: value,
-        },
-      },
-    },
-  };
-
-  if (operator === 'excluded') {
-    return buildExclusionClause(wildcardClause);
-  } else {
-    return wildcardClause;
-  }
-};
-
-export const buildExistsClause = (entry: EntryExists): BooleanFilter => {
-  const { field, operator } = entry;
-  const existsClause = {
-    bool: {
-      minimum_should_match: 1,
-      should: [
-        {
-          exists: {
-            field,
-          },
-        },
-      ],
-    },
-  };
-
-  if (operator === 'excluded') {
-    return buildExclusionClause(existsClause);
-  } else {
-    return existsClause;
-  }
-};
-
-const isBooleanFilter = (clause: object): clause is BooleanFilter => {
-  const keys = Object.keys(clause);
-  return keys.includes('bool') != null;
-};
-
-export const getBaseNestedClause = (
-  entries: NonListEntry[],
-  parentField: string
-): BooleanFilter => {
-  if (entries.length === 1) {
-    const [singleNestedEntry] = entries;
-    const innerClause = createInnerAndClauses(singleNestedEntry, parentField);
-    return isBooleanFilter(innerClause) ? innerClause : { bool: {} };
-  }
-
-  return {
-    bool: {
-      filter: entries.map((nestedEntry) => createInnerAndClauses(nestedEntry, parentField)),
-    },
-  };
-};
-
-export const buildNestedClause = (entry: EntryNested): NestedFilter => {
-  const { field, entries } = entry;
-
-  const baseNestedClause = getBaseNestedClause(entries, field);
-
-  return {
-    nested: {
-      path: field,
-      query: baseNestedClause,
-      score_mode: 'none',
-    },
-  };
-};
-
-export const createInnerAndClauses = (
-  entry: NonListEntry,
-  parent?: string
-): BooleanFilter | NestedFilter => {
-  const field = parent != null ? `${parent}.${entry.field}` : entry.field;
-  if (entriesExists.is(entry)) {
-    return buildExistsClause({ ...entry, field });
-  } else if (entriesMatch.is(entry)) {
-    return buildMatchClause({ ...entry, field });
-  } else if (entriesMatchAny.is(entry)) {
-    return buildMatchAnyClause({ ...entry, field });
-  } else if (entriesMatchWildcard.is(entry)) {
-    return buildMatchWildcardClause({ ...entry, field });
-  } else if (entriesNested.is(entry)) {
-    return buildNestedClause(entry);
-  } else {
-    throw new TypeError(`Unexpected exception entry: ${entry}`);
-  }
-};

x-pack/plugins/lists/common/schemas/request/get_exception_filter_schema.mock.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GetExceptionFilterSchema } from '@kbn/securitysolution-io-ts-list-types';
+
+import { LIST_ID, NAMESPACE_TYPE } from '../../constants.mock';
+import { getExceptionListItemSchemaMock } from '../response/exception_list_item_schema.mock';
+
+export const getExceptionFilterFromExceptionItemsSchemaMock = (): GetExceptionFilterSchema => ({
+  exceptions: [getExceptionListItemSchemaMock()],
+  type: 'exception_items',
+});
+
+export const getExceptionFilterFromExceptionIdsSchemaMock = (): GetExceptionFilterSchema => ({
+  exception_list_ids: [{ exception_list_id: LIST_ID, namespace_type: NAMESPACE_TYPE }],
+  type: 'exception_list_ids',
+});

x-pack/plugins/lists/common/schemas/response/found_all_list_items_schema.mock.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FoundAllListItemsSchema } from '@kbn/securitysolution-io-ts-list-types';
+
+import { getListItemResponseMock } from './list_item_schema.mock';
+
+export const getFoundAllListItemsSchemaMock = (): FoundAllListItemsSchema => ({
+  data: [getListItemResponseMock()],
+  total: 1,
+});

x-pack/plugins/lists/common/schemas/response/found_lists_by_size_schema.mock.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FoundListsBySizeSchema } from '@kbn/securitysolution-io-ts-list-types';
+
+import { getListResponseMock } from './list_schema.mock';
+
+export const getFoundListsBySizeSchemaMock = (): FoundListsBySizeSchema => ({
+  largeLists: [getListResponseMock()],
+  smallLists: [getListResponseMock()],
+});

x-pack/plugins/lists/common/schemas/types/entry_list.mock.ts

@@ -15,3 +15,10 @@ export const getEntryListMock = (): EntryList => ({
   operator: OPERATOR,
   type: LIST,
 });
+
+export const getEntryListExcludedMock = (): EntryList => ({
+  field: FIELD,
+  list: { id: LIST_ID, type: TYPE },
+  operator: 'excluded',
+  type: LIST,
+});

x-pack/plugins/lists/public/exceptions/components/builder/entry_renderer.test.tsx

@@ -21,14 +21,14 @@ import {
   matchesOperator,
 } from '@kbn/securitysolution-list-utils';
 import { validateFilePathInput } from '@kbn/securitysolution-utils';
-import { useFindLists } from '@kbn/securitysolution-list-hooks';
+import { useFindListsBySize } from '@kbn/securitysolution-list-hooks';
 import type { FieldSpec } from '@kbn/data-plugin/common';
 import { fields, getField } from '@kbn/data-plugin/common/mocks';
 import { unifiedSearchPluginMock } from '@kbn/unified-search-plugin/public/mocks';
 import { waitFor } from '@testing-library/dom';
 import { ReactWrapper, mount } from 'enzyme';
 
-import { getFoundListSchemaMock } from '../../../../common/schemas/response/found_list_schema.mock';
+import { getFoundListsBySizeSchemaMock } from '../../../../common/schemas/response/found_lists_by_size_schema.mock';
 
 import { BuilderEntryItem } from './entry_renderer';
 
@@ -37,15 +37,17 @@ jest.mock('@kbn/securitysolution-utils');
 
 const mockKibanaHttpService = coreMock.createStart().http;
 const { autocomplete: autocompleteStartMock } = unifiedSearchPluginMock.createStartContract();
+const mockResult = getFoundListsBySizeSchemaMock();
+mockResult.largeLists = [];
 
 describe('BuilderEntryItem', () => {
   let wrapper: ReactWrapper;
 
   beforeEach(() => {
-    (useFindLists as jest.Mock).mockReturnValue({
+    (useFindListsBySize as jest.Mock).mockReturnValue({
       error: undefined,
       loading: false,
-      result: getFoundListSchemaMock(),
+      result: mockResult,
       start: jest.fn(),
     });
   });

x-pack/plugins/lists/public/exceptions/components/builder/entry_renderer.tsx

@@ -241,7 +241,7 @@ export const BuilderEntryItem: React.FC<EntryItemProps> = ({
           entry,
           listType,
           entry.field != null && entry.field.type === 'boolean',
-          isFirst && allowLargeValueLists
+          isFirst
         );
 
     const comboBox = (
@@ -383,6 +383,7 @@ export const BuilderEntryItem: React.FC<EntryItemProps> = ({
             isClearable={false}
             onChange={handleFieldListValueChange}
             data-test-subj="exceptionBuilderEntryFieldList"
+            allowLargeValueLists={allowLargeValueLists}
           />
         );
       case OperatorTypeEnum.EXISTS:

x-pack/plugins/lists/public/exceptions/components/builder/exception_items_renderer.test.tsx

@@ -154,7 +154,7 @@ describe('ExceptionBuilderComponent', () => {
     ).toEqual(expect.arrayContaining([{ label: 'is in list' }, { label: 'is not in list' }]));
   });
 
-  test('it does not display "is in list" operators if "allowLargeValueLists" is false', async () => {
+  test('it still displays "is in list" operators if "allowLargeValueLists" is false', async () => {
     wrapper = mount(
       <EuiThemeProvider>
         <ExceptionBuilderComponent
@@ -188,7 +188,7 @@ describe('ExceptionBuilderComponent', () => {
 
     expect(
       wrapper.find('[data-test-subj="operatorAutocompleteComboBox"]').at(0).prop('options')
-    ).not.toEqual(expect.arrayContaining([{ label: 'is in list' }, { label: 'is not in list' }]));
+    ).toEqual(expect.arrayContaining([{ label: 'is in list' }, { label: 'is not in list' }]));
   });
 
   test('it displays "or", "and" and "add nested button" enabled', () => {

x-pack/plugins/lists/server/routes/find_list_item_route.ts

@@ -52,7 +52,7 @@ export const findListItemRoute = (router: ListsPluginRouter): void => {
         const {
           isValid,
           errorMessage,
-          cursor: [currentIndexPosition, searchAfter],
+          cursor: [currentIndexPosition, searchAfter = []],
         } = decodeCursor({
           cursor,
           page,
@@ -72,6 +72,7 @@ export const findListItemRoute = (router: ListsPluginRouter): void => {
             listId,
             page,
             perPage,
+            runtimeMappings: undefined,
             searchAfter,
             sortField,
             sortOrder,

x-pack/plugins/lists/server/routes/find_list_route.ts

@@ -63,6 +63,7 @@ export const findListRoute = (router: ListsPluginRouter): void => {
             filter,
             page,
             perPage,
+            runtimeMappings: undefined,
             searchAfter,
             sortField,
             sortOrder,

x-pack/plugins/lists/server/routes/find_lists_by_size_route.ts

@@ -0,0 +1,161 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { validate } from '@kbn/securitysolution-io-ts-utils';
+import { transformError } from '@kbn/securitysolution-es-utils';
+import { findListSchema, foundListsBySizeSchema } from '@kbn/securitysolution-io-ts-list-types';
+import {
+  FIND_LISTS_BY_SIZE,
+  MAXIMUM_SMALL_IP_RANGE_VALUE_LIST_DASH_SIZE,
+  MAXIMUM_SMALL_VALUE_LIST_SIZE,
+} from '@kbn/securitysolution-list-constants';
+import { chunk } from 'lodash';
+
+import type { ListsPluginRouter } from '../types';
+import { decodeCursor } from '../services/utils';
+
+import { buildRouteValidation, buildSiemResponse, getListClient } from './utils';
+
+export const findListsBySizeRoute = (router: ListsPluginRouter): void => {
+  router.get(
+    {
+      options: {
+        tags: ['access:lists-read'],
+      },
+      path: `${FIND_LISTS_BY_SIZE}`,
+      validate: {
+        query: buildRouteValidation(findListSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const listClient = await getListClient(context);
+        const {
+          cursor,
+          filter: filterOrUndefined,
+          page: pageOrUndefined,
+          per_page: perPageOrUndefined,
+          sort_field: sortField,
+          sort_order: sortOrder,
+        } = request.query;
+
+        const page = pageOrUndefined ?? 1;
+        const perPage = perPageOrUndefined ?? 20;
+        const filter = filterOrUndefined ?? '';
+        const {
+          isValid,
+          errorMessage,
+          cursor: [currentIndexPosition, searchAfter],
+        } = decodeCursor({
+          cursor,
+          page,
+          perPage,
+          sortField,
+        });
+        if (!isValid) {
+          return siemResponse.error({
+            body: errorMessage,
+            statusCode: 400,
+          });
+        } else {
+          const valueLists = await listClient.findList({
+            currentIndexPosition,
+            filter,
+            page,
+            perPage,
+            runtimeMappings: undefined,
+            searchAfter,
+            sortField,
+            sortOrder,
+          });
+
+          const listBooleans: boolean[] = [];
+
+          const chunks = chunk(valueLists.data, 10);
+          for (const listChunk of chunks) {
+            const booleans = await Promise.all(
+              listChunk.map(async (valueList) => {
+                // Currently the only list types we support for exceptions
+                if (
+                  valueList.type !== 'ip_range' &&
+                  valueList.type !== 'ip' &&
+                  valueList.type !== 'keyword'
+                ) {
+                  return false;
+                }
+
+                const list = await listClient.findListItem({
+                  currentIndexPosition: 0,
+                  filter: '',
+                  listId: valueList.id,
+                  page: 0,
+                  perPage: 0,
+                  runtimeMappings: undefined,
+                  searchAfter: [],
+                  sortField: undefined,
+                  sortOrder: undefined,
+                });
+
+                if (
+                  valueList.type === 'ip_range' &&
+                  list &&
+                  list.total < MAXIMUM_SMALL_VALUE_LIST_SIZE
+                ) {
+                  const rangeList = await listClient.findListItem({
+                    currentIndexPosition: 0,
+                    filter: 'is_cidr: false',
+                    listId: valueList.id,
+                    page: 0,
+                    perPage: 0,
+                    runtimeMappings: {
+                      is_cidr: {
+                        script: `
+                          if (params._source["ip_range"] instanceof String) {
+                            emit(true);
+                          } else {
+                            emit(false);
+                          }
+                          `,
+                        type: 'boolean',
+                      },
+                    },
+                    searchAfter: [],
+                    sortField: undefined,
+                    sortOrder: undefined,
+                  });
+
+                  return rangeList && rangeList.total < MAXIMUM_SMALL_IP_RANGE_VALUE_LIST_DASH_SIZE
+                    ? true
+                    : false;
+                }
+                return list && list.total < MAXIMUM_SMALL_VALUE_LIST_SIZE ? true : false;
+              })
+            );
+            listBooleans.push(...booleans);
+          }
+
+          const smallLists = valueLists.data.filter((valueList, index) => listBooleans[index]);
+          const largeLists = valueLists.data.filter((valueList, index) => !listBooleans[index]);
+
+          const [validated, errors] = validate({ largeLists, smallLists }, foundListsBySizeSchema);
+          if (errors != null) {
+            return siemResponse.error({ body: errors, statusCode: 500 });
+          } else {
+            return response.ok({ body: validated ?? {} });
+          }
+        }
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/get_exception_filter_route.ts

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { transformError } from '@kbn/securitysolution-es-utils';
+import {
+  CreateExceptionListItemSchema,
+  ExceptionListItemSchema,
+  FoundExceptionListItemSchema,
+  getExceptionFilterSchema,
+} from '@kbn/securitysolution-io-ts-list-types';
+import { EXCEPTION_FILTER } from '@kbn/securitysolution-list-constants';
+
+import { buildExceptionFilter } from '../services/exception_lists/build_exception_filter';
+import { ListsPluginRouter } from '../types';
+
+import { buildRouteValidation, buildSiemResponse } from './utils';
+
+export const getExceptionFilterRoute = (router: ListsPluginRouter): void => {
+  router.post(
+    {
+      options: {
+        tags: ['access:securitySolution'],
+      },
+      path: `${EXCEPTION_FILTER}`,
+      validate: {
+        body: buildRouteValidation(getExceptionFilterSchema),
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+      try {
+        const ctx = await context.resolve(['lists']);
+        const listClient = ctx.lists?.getListClient();
+        if (!listClient) {
+          return siemResponse.error({ body: 'Cannot retrieve list client', statusCode: 500 });
+        }
+        const exceptionListClient = ctx.lists?.getExceptionListClient();
+        const exceptionItems: Array<ExceptionListItemSchema | CreateExceptionListItemSchema> = [];
+        const {
+          type,
+          alias = null,
+          exclude_exceptions: excludeExceptions = true,
+          chunk_size: chunkSize = 10,
+        } = request.body;
+        if (type === 'exception_list_ids') {
+          const listIds = request.body.exception_list_ids.map(
+            ({ exception_list_id: listId }) => listId
+          );
+          const namespaceTypes = request.body.exception_list_ids.map(
+            ({ namespace_type: namespaceType }) => namespaceType
+          );
+
+          // Stream the results from the Point In Time (PIT) finder into this array
+          let items: ExceptionListItemSchema[] = [];
+          const executeFunctionOnStream = (responseBody: FoundExceptionListItemSchema): void => {
+            items = [...items, ...responseBody.data];
+          };
+
+          await exceptionListClient?.findExceptionListsItemPointInTimeFinder({
+            executeFunctionOnStream,
+            filter: [],
+            listId: listIds,
+            maxSize: undefined, // NOTE: This is unbounded when it is "undefined"
+            namespaceType: namespaceTypes,
+            perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k
+            sortField: undefined,
+            sortOrder: undefined,
+          });
+          exceptionItems.push(...items);
+        } else {
+          const { exceptions } = request.body;
+          exceptionItems.push(...exceptions);
+        }
+
+        const { filter } = await buildExceptionFilter({
+          alias,
+          chunkSize,
+          excludeExceptions,
+          listClient,
+          lists: exceptionItems,
+        });
+
+        return response.ok({ body: { filter } ?? {} });
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/lists/server/routes/index.ts

@@ -25,6 +25,8 @@ export * from './find_exception_list_item_route';
 export * from './find_exception_list_route';
 export * from './find_list_item_route';
 export * from './find_list_route';
+export * from './find_lists_by_size_route';
+export * from './get_exception_filter_route';
 export * from './import_exceptions_route';
 export * from './import_list_item_route';
 export * from './init_routes';

x-pack/plugins/lists/server/routes/init_routes.ts

@@ -29,6 +29,8 @@ import {
   findExceptionListRoute,
   findListItemRoute,
   findListRoute,
+  findListsBySizeRoute,
+  getExceptionFilterRoute,
   importExceptionsRoute,
   importListItemRoute,
   internalCreateExceptionListRoute,
@@ -58,6 +60,7 @@ export const initRoutes = (router: ListsPluginRouter, config: ConfigType): void
   patchListRoute(router);
   findListRoute(router);
   readPrivilegesRoute(router);
+  findListsBySizeRoute(router);
 
   // list items
   createListItemRoute(router);
@@ -92,6 +95,9 @@ export const initRoutes = (router: ListsPluginRouter, config: ConfigType): void
   deleteExceptionListItemRoute(router);
   findExceptionListItemRoute(router);
 
+  // exception filter
+  getExceptionFilterRoute(router);
+
   // endpoint list
   createEndpointListRoute(router);
 

x-pack/plugins/lists/server/services/exception_lists/build_exception_filter.ts

@@ -0,0 +1,617 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { chunk } from 'lodash/fp';
+import {
+  CreateExceptionListItemSchema,
+  Entry,
+  EntryExists,
+  EntryList,
+  EntryMatch,
+  EntryMatchAny,
+  EntryMatchWildcard,
+  EntryNested,
+  ExceptionListItemSchema,
+  OsTypeArray,
+  Type,
+  entriesExists,
+  entriesList,
+  entriesMatch,
+  entriesMatchAny,
+  entriesMatchWildcard,
+  entriesNested,
+} from '@kbn/securitysolution-io-ts-list-types';
+import type { Filter } from '@kbn/es-query';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { partition } from 'lodash';
+import { hasLargeValueList } from '@kbn/securitysolution-list-utils';
+import {
+  MAXIMUM_SMALL_IP_RANGE_VALUE_LIST_DASH_SIZE,
+  MAXIMUM_SMALL_VALUE_LIST_SIZE,
+} from '@kbn/securitysolution-list-constants';
+
+import type { ListClient } from '../..';
+
+type ExceptionEntry = Entry | EntryNested;
+export interface BooleanFilter {
+  bool: estypes.QueryDslBoolQuery;
+}
+
+export interface NestedFilter {
+  nested: estypes.QueryDslNestedQuery;
+}
+
+export const chunkExceptions = <T extends ExceptionListItemSchema | CreateExceptionListItemSchema>(
+  exceptions: T[],
+  chunkSize: number
+): T[][] => {
+  return chunk(chunkSize, exceptions);
+};
+
+/**
+ * Transforms the os_type into a regular filter as if the user had created it
+ * from the fields for the next state of transforms which will create the elastic filters
+ * from it.
+ *
+ * Note: We use two types of fields, the "host.os.type" and "host.os.name.caseless"
+ * The endpoint/endgame agent has been using "host.os.name.caseless" as the same value as the ECS
+ * value of "host.os.type" where the auditbeat, winlogbeat, etc... (other agents) are all using
+ * "host.os.type". In order to be compatible with both, I create an "OR" between these two data types
+ * where if either has a match then we will exclude it as part of the match. This should also be
+ * forwards compatible for endpoints/endgame agents when/if they upgrade to using "host.os.type"
+ * rather than using "host.os.name.caseless" values.
+ *
+ * Also we create another "OR" from the osType names so that if there are multiples such as ['windows', 'linux']
+ * this will exclude anything with either 'windows' or with 'linux'
+ * @param osTypes The os_type array from the REST interface that is an array such as ['windows', 'linux']
+ * @param entries The entries to join the OR's with before the elastic filter change out
+ */
+export const transformOsType = (
+  osTypes: OsTypeArray,
+  entries: ExceptionEntry[]
+): ExceptionEntry[][] => {
+  const hostTypeTransformed = osTypes.map<ExceptionEntry[]>((osType) => {
+    return [
+      { field: 'host.os.type', operator: 'included', type: 'match', value: osType },
+      ...entries,
+    ];
+  });
+  const caseLessTransformed = osTypes.map<ExceptionEntry[]>((osType) => {
+    return [
+      { field: 'host.os.name.caseless', operator: 'included', type: 'match', value: osType },
+      ...entries,
+    ];
+  });
+  return [...hostTypeTransformed, ...caseLessTransformed];
+};
+
+/**
+ * This builds an exception item filter with the os type
+ * @param osTypes The os_type array from the REST interface that is an array such as ['windows', 'linux']
+ * @param entries The entries to join the OR's with before the elastic filter change out
+ */
+export const buildExceptionItemFilterWithOsType = async (
+  osTypes: OsTypeArray,
+  entries: ExceptionEntry[],
+  listClient: ListClient
+): Promise<BooleanFilter[] | undefined> => {
+  let isUnprocessable = false;
+  const entriesWithOsTypes = transformOsType(osTypes, entries);
+  const exceptionItemFilter: BooleanFilter[] = [];
+  await Promise.all(
+    entriesWithOsTypes.map(async (entryWithOsType) => {
+      const esFilter: Array<BooleanFilter | NestedFilter> = [];
+      await Promise.all(
+        entryWithOsType.map(async (entry) => {
+          const filter = await createInnerAndClauses({
+            entry,
+            listClient,
+          });
+          if (!filter) {
+            isUnprocessable = true;
+            return;
+          }
+          esFilter.push(filter);
+        })
+      );
+      exceptionItemFilter.push({
+        bool: {
+          filter: esFilter,
+        },
+      });
+    })
+  );
+  return isUnprocessable ? undefined : exceptionItemFilter;
+};
+
+export const buildExceptionItemFilter = async <
+  T extends ExceptionListItemSchema | CreateExceptionListItemSchema
+>(
+  exceptionItem: T,
+  listClient: ListClient
+): Promise<Array<BooleanFilter | NestedFilter> | undefined> => {
+  const { entries, os_types: osTypes } = exceptionItem;
+  if (osTypes != null && osTypes.length > 0) {
+    return buildExceptionItemFilterWithOsType(osTypes, entries, listClient);
+  } else {
+    if (entries.length === 1) {
+      const filter = await createInnerAndClauses({
+        entry: entries[0],
+        listClient,
+      });
+      if (!filter) {
+        return undefined;
+      }
+      return [filter];
+    } else {
+      const esFilter: Array<BooleanFilter | NestedFilter> = [];
+
+      for (const entry of entries) {
+        const filter = await createInnerAndClauses({
+          entry,
+          listClient,
+        });
+        if (!filter) {
+          return undefined;
+        }
+        esFilter.push(filter);
+      }
+
+      return [
+        {
+          bool: {
+            filter: esFilter,
+          },
+        },
+      ];
+    }
+  }
+};
+
+export const createOrClauses = async <
+  T extends ExceptionListItemSchema | CreateExceptionListItemSchema
+>({
+  exceptionsWithoutValueLists,
+  exceptionsWithValueLists,
+  chunkSize,
+  listClient,
+}: {
+  exceptionsWithoutValueLists: T[];
+  exceptionsWithValueLists: T[];
+  chunkSize: number;
+  listClient: ListClient;
+}): Promise<{
+  orClauses: Array<BooleanFilter | NestedFilter>;
+  unprocessableExceptionItems: T[];
+}> => {
+  const unprocessableExceptionItems: T[] = [];
+  const orClauses: Array<Array<BooleanFilter | NestedFilter>> = [];
+
+  for (const exceptionItem of exceptionsWithoutValueLists) {
+    const filter = await buildExceptionItemFilter(exceptionItem, listClient);
+    if (!filter) {
+      unprocessableExceptionItems.push(exceptionItem);
+    } else {
+      orClauses.push(filter);
+    }
+  }
+
+  // Chunk the exceptions that will require list client requests
+  const chunks = chunkExceptions(exceptionsWithValueLists, chunkSize);
+  for (const exceptionsChunk of chunks) {
+    await Promise.all(
+      exceptionsChunk.map(async (exceptionItem) => {
+        const filter = await buildExceptionItemFilter(exceptionItem, listClient);
+        if (!filter) {
+          unprocessableExceptionItems.push(exceptionItem);
+          return;
+        }
+        orClauses.push(filter);
+      })
+    );
+  }
+
+  return { orClauses: orClauses.flat(), unprocessableExceptionItems };
+};
+
+const isListTypeProcessable = (type: Type): boolean =>
+  type === 'keyword' || type === 'ip' || type === 'ip_range';
+
+export const filterOutUnprocessableValueLists = async <
+  T extends ExceptionListItemSchema | CreateExceptionListItemSchema
+>(
+  exceptionItems: T[],
+  listClient: ListClient
+): Promise<{
+  filteredExceptions: T[];
+  unprocessableValueListExceptions: T[];
+}> => {
+  const exceptionBooleans = await Promise.all(
+    exceptionItems.map(async (exceptionItem) => {
+      const listEntries = exceptionItem.entries.filter((entry): entry is EntryList =>
+        entriesList.is(entry)
+      );
+      for await (const listEntry of listEntries) {
+        const {
+          list: { id, type },
+        } = listEntry;
+
+        if (!isListTypeProcessable(type)) {
+          return false;
+        }
+
+        // Don't want any items, just the total list size
+        const valueList = await listClient.findListItem({
+          currentIndexPosition: 0,
+          filter: '',
+          listId: id,
+          page: 0,
+          perPage: 0,
+          runtimeMappings: undefined,
+          searchAfter: [],
+          sortField: undefined,
+          sortOrder: undefined,
+        });
+
+        if (!valueList || (valueList && valueList.total > MAXIMUM_SMALL_VALUE_LIST_SIZE)) {
+          return false;
+        }
+      }
+      // If we're here, all the entries are processable
+      return true;
+    })
+  );
+  const filteredExceptions = exceptionItems.filter((item, index) => exceptionBooleans[index]);
+  const unprocessableValueListExceptions = exceptionItems.filter(
+    (item, index) => !exceptionBooleans[index]
+  );
+
+  return { filteredExceptions, unprocessableValueListExceptions };
+};
+
+export const buildExceptionFilter = async <
+  T extends ExceptionListItemSchema | CreateExceptionListItemSchema
+>({
+  lists,
+  excludeExceptions,
+  chunkSize,
+  alias = null,
+  listClient,
+}: {
+  lists: T[];
+  excludeExceptions: boolean;
+  chunkSize: number;
+  alias: string | null;
+  listClient: ListClient;
+}): Promise<{ filter: Filter | undefined; unprocessedExceptions: T[] }> => {
+  // Remove exception items with large value lists. These are evaluated
+  // elsewhere for the moment being.
+  const [exceptionsWithoutValueLists, valueListExceptions] = partition(
+    lists,
+    (item): item is T => !hasLargeValueList(item.entries)
+  );
+
+  // Exceptions that contain large value list exceptions and will be processed later on in rule execution
+  const unprocessedExceptions: T[] = [];
+
+  const { filteredExceptions: exceptionsWithValueLists, unprocessableValueListExceptions } =
+    await filterOutUnprocessableValueLists<T>(valueListExceptions, listClient);
+  unprocessedExceptions.push(...unprocessableValueListExceptions);
+
+  if (exceptionsWithoutValueLists.length === 0 && exceptionsWithValueLists.length === 0) {
+    return { filter: undefined, unprocessedExceptions };
+  }
+  const { orClauses, unprocessableExceptionItems } = await createOrClauses<T>({
+    chunkSize,
+    exceptionsWithValueLists,
+    exceptionsWithoutValueLists,
+    listClient,
+  });
+
+  const exceptionFilter: Filter = {
+    meta: {
+      alias,
+      disabled: false,
+      negate: excludeExceptions,
+    },
+    query: {
+      bool: {
+        should: orClauses,
+      },
+    },
+  };
+  unprocessedExceptions.concat(unprocessableExceptionItems);
+  return { filter: exceptionFilter, unprocessedExceptions };
+};
+
+export const buildExclusionClause = (booleanFilter: BooleanFilter): BooleanFilter => {
+  return {
+    bool: {
+      must_not: booleanFilter,
+    },
+  };
+};
+
+export const buildMatchClause = (entry: EntryMatch): BooleanFilter => {
+  const { field, operator, value } = entry;
+  const matchClause = {
+    bool: {
+      minimum_should_match: 1,
+      should: [
+        {
+          match_phrase: {
+            [field]: value,
+          },
+        },
+      ],
+    },
+  };
+
+  if (operator === 'excluded') {
+    return buildExclusionClause(matchClause);
+  } else {
+    return matchClause;
+  }
+};
+
+export const getBaseMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
+  const { field, value } = entry;
+
+  if (value.length === 1) {
+    return {
+      bool: {
+        minimum_should_match: 1,
+        should: [
+          {
+            match_phrase: {
+              [field]: value[0],
+            },
+          },
+        ],
+      },
+    };
+  }
+
+  return {
+    bool: {
+      minimum_should_match: 1,
+      should: value.map((val) => {
+        return {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match_phrase: {
+                  [field]: val,
+                },
+              },
+            ],
+          },
+        };
+      }),
+    },
+  };
+};
+
+export const buildMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
+  const { operator } = entry;
+  const matchAnyClause = getBaseMatchAnyClause(entry);
+
+  if (operator === 'excluded') {
+    return buildExclusionClause(matchAnyClause);
+  } else {
+    return matchAnyClause;
+  }
+};
+
+export const buildMatchWildcardClause = (entry: EntryMatchWildcard): BooleanFilter => {
+  const { field, operator, value } = entry;
+  const wildcardClause = {
+    bool: {
+      filter: {
+        wildcard: {
+          [field]: value,
+        },
+      },
+    },
+  };
+
+  if (operator === 'excluded') {
+    return buildExclusionClause(wildcardClause);
+  } else {
+    return wildcardClause;
+  }
+};
+
+export const buildExistsClause = (entry: EntryExists): BooleanFilter => {
+  const { field, operator } = entry;
+  const existsClause = {
+    bool: {
+      minimum_should_match: 1,
+      should: [
+        {
+          exists: {
+            field,
+          },
+        },
+      ],
+    },
+  };
+
+  if (operator === 'excluded') {
+    return buildExclusionClause(existsClause);
+  } else {
+    return existsClause;
+  }
+};
+
+const isBooleanFilter = (clause?: object): clause is BooleanFilter => {
+  if (!clause) {
+    return false;
+  }
+  const keys = Object.keys(clause);
+  return keys.includes('bool') != null;
+};
+
+export const buildIpRangeClauses = (
+  ranges: string[],
+  field: string
+): estypes.QueryDslQueryContainer[] =>
+  ranges.map((range) => {
+    const [gte, lte] = range.split('-');
+    return {
+      range: {
+        [field]: {
+          gte,
+          lte,
+        },
+      },
+    };
+  });
+
+export const buildListClause = async (
+  entry: EntryList,
+  listClient: ListClient
+): Promise<BooleanFilter | undefined> => {
+  const {
+    field,
+    operator,
+    list: { type },
+  } = entry;
+
+  const list = await listClient.findAllListItems({
+    filter: '',
+    listId: entry.list.id,
+  });
+  if (list == null) {
+    throw new TypeError(`Cannot find list: "${entry.list.id}"`);
+  }
+  const listValues = list.data.map((listItem) => listItem.value);
+
+  if (type === 'ip_range') {
+    const [dashNotationRange, slashNotationRange] = partition(listValues, (value) => {
+      return value.includes('-');
+    });
+    if (dashNotationRange.length > MAXIMUM_SMALL_IP_RANGE_VALUE_LIST_DASH_SIZE) {
+      return undefined;
+    }
+    const rangeClauses = buildIpRangeClauses(dashNotationRange, field);
+    if (slashNotationRange.length > 0) {
+      rangeClauses.push({
+        terms: {
+          [field]: slashNotationRange,
+        },
+      });
+    }
+    return {
+      bool: {
+        [operator === 'excluded' ? 'must_not' : 'should']: rangeClauses,
+        minimum_should_match: 1,
+      },
+    };
+  }
+
+  return {
+    bool: {
+      [operator === 'excluded' ? 'must_not' : 'filter']: {
+        terms: {
+          [field]: listValues,
+        },
+      },
+    },
+  };
+};
+
+export const getBaseNestedClause = async (
+  entries: ExceptionEntry[],
+  parentField: string,
+  listClient: ListClient
+): Promise<BooleanFilter | undefined> => {
+  if (entries.length === 1) {
+    const [singleNestedEntry] = entries;
+    const innerClause = await createInnerAndClauses({
+      entry: singleNestedEntry,
+      listClient,
+      parent: parentField,
+    });
+    return isBooleanFilter(innerClause) ? innerClause : { bool: {} };
+  }
+
+  const filter: Array<BooleanFilter | NestedFilter> = [];
+  let isUnprocessable = false;
+  await Promise.all(
+    entries.map(async (nestedEntry) => {
+      const clauses = await createInnerAndClauses({
+        entry: nestedEntry,
+        listClient,
+        parent: parentField,
+      });
+      if (!clauses) {
+        isUnprocessable = true;
+        return;
+      }
+      filter.push(clauses);
+    })
+  );
+
+  if (isUnprocessable) {
+    return undefined;
+  }
+  return {
+    bool: {
+      filter,
+    },
+  };
+};
+
+export const buildNestedClause = async (
+  entry: EntryNested,
+  listClient: ListClient
+): Promise<NestedFilter | undefined> => {
+  const { field, entries } = entry;
+
+  const baseNestedClause = await getBaseNestedClause(entries, field, listClient);
+
+  if (!baseNestedClause) {
+    return undefined;
+  }
+
+  return {
+    nested: {
+      path: field,
+      query: baseNestedClause,
+      score_mode: 'none',
+    },
+  };
+};
+
+export const createInnerAndClauses = async ({
+  entry,
+  parent,
+  listClient,
+}: {
+  entry: ExceptionEntry;
+  parent?: string;
+  listClient: ListClient;
+}): Promise<BooleanFilter | NestedFilter | undefined> => {
+  const field = parent != null ? `${parent}.${entry.field}` : entry.field;
+  if (entriesExists.is(entry)) {
+    return buildExistsClause({ ...entry, field });
+  } else if (entriesMatch.is(entry)) {
+    return buildMatchClause({ ...entry, field });
+  } else if (entriesMatchAny.is(entry)) {
+    return buildMatchAnyClause({ ...entry, field });
+  } else if (entriesMatchWildcard.is(entry)) {
+    return buildMatchWildcardClause({ ...entry, field });
+  } else if (entriesList.is(entry)) {
+    return buildListClause({ ...entry, field }, listClient);
+  } else if (entriesNested.is(entry)) {
+    return buildNestedClause(entry, listClient);
+  } else {
+    throw new TypeError(`Unexpected exception entry: ${entry}`);
+  }
+};

x-pack/plugins/lists/server/services/exception_lists/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+export * from './build_exception_filter';
 export * from './create_exception_list';
 export * from './create_exception_list_item';
 export * from './delete_exception_list';

x-pack/plugins/lists/server/services/items/find_all_list_items.mock.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
+
+import { LIST_ID, LIST_INDEX, LIST_ITEM_INDEX } from '../../../common/constants.mock';
+import { getShardMock } from '../../schemas/common/get_shard.mock';
+
+import { FindAllListItemsOptions } from './find_all_list_items';
+
+export const getFindCount = (): Promise<estypes.CountResponse> => {
+  return Promise.resolve({
+    _shards: getShardMock(),
+    count: 1,
+  });
+};
+
+export const getFindAllListItemsOptionsMock = (): FindAllListItemsOptions => {
+  return {
+    esClient: elasticsearchClientMock.createScopedClusterClient().asCurrentUser,
+    filter: '',
+    listId: LIST_ID,
+    listIndex: LIST_INDEX,
+    listItemIndex: LIST_ITEM_INDEX,
+    sortField: undefined,
+    sortOrder: undefined,
+  };
+};

x-pack/plugins/lists/server/services/items/find_all_list_items.test.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
+
+import { getEmptySearchListMock } from '../../schemas/elastic_response/search_es_list_schema.mock';
+
+import { getFindAllListItemsOptionsMock } from './find_all_list_items.mock';
+import { findAllListItems } from './find_all_list_items';
+
+describe('find_all_list_items', () => {
+  test('should return null if the list is null', async () => {
+    const options = getFindAllListItemsOptionsMock();
+    const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
+    esClient.search.mockResponse(getEmptySearchListMock());
+    const item = await findAllListItems({ ...options, esClient });
+    expect(item).toEqual(null);
+  });
+});

x-pack/plugins/lists/server/services/items/find_all_list_items.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from '@kbn/core/server';
+import type {
+  Filter,
+  FoundAllListItemsSchema,
+  ListId,
+  ListItemArraySchema,
+  SortFieldOrUndefined,
+  SortOrderOrUndefined,
+} from '@kbn/securitysolution-io-ts-list-types';
+
+import { SearchEsListItemSchema } from '../../schemas/elastic_response';
+import { getList } from '../lists';
+import {
+  getQueryFilterWithListId,
+  getSortWithTieBreaker,
+  transformElasticToListItem,
+} from '../utils';
+
+export interface FindAllListItemsOptions {
+  listId: ListId;
+  filter: Filter;
+  sortField: SortFieldOrUndefined;
+  sortOrder: SortOrderOrUndefined;
+  esClient: ElasticsearchClient;
+  listIndex: string;
+  listItemIndex: string;
+}
+
+export const findAllListItems = async ({
+  esClient,
+  filter,
+  listId,
+  sortField,
+  listIndex,
+  listItemIndex,
+  sortOrder,
+}: FindAllListItemsOptions): Promise<FoundAllListItemsSchema | null> => {
+  const list = await getList({ esClient, id: listId, listIndex });
+  if (list == null) {
+    return null;
+  } else {
+    const allListItems: ListItemArraySchema = [];
+    const query = getQueryFilterWithListId({ filter, listId });
+    const sort = getSortWithTieBreaker({ sortField, sortOrder });
+    const { count } = await esClient.count({
+      body: {
+        query,
+      },
+      ignore_unavailable: true,
+      index: listItemIndex,
+    });
+
+    let response = await esClient.search<SearchEsListItemSchema>({
+      body: {
+        query,
+        sort,
+      },
+      ignore_unavailable: true,
+      index: listItemIndex,
+      seq_no_primary_term: true,
+      size: 10000,
+    });
+
+    if (count > 100000) {
+      throw new TypeError('API route only supports up to 100,000 items');
+    }
+
+    while (response.hits.hits.length !== 0) {
+      allListItems.push(...transformElasticToListItem({ response, type: list.type }));
+
+      if (allListItems.length > 100000) {
+        throw new TypeError('API route only supports up to 100,000 items');
+      }
+
+      response = await esClient.search<SearchEsListItemSchema>({
+        body: {
+          query,
+          search_after: response.hits.hits[response.hits.hits.length - 1].sort,
+          sort,
+        },
+        ignore_unavailable: true,
+        index: listItemIndex,
+        seq_no_primary_term: true,
+        size: 10000,
+      });
+    }
+    return {
+      data: allListItems,
+      total: count,
+    };
+  }
+};

x-pack/plugins/lists/server/services/items/find_list_item.mock.ts

@@ -30,6 +30,7 @@ export const getFindListItemOptionsMock = (): FindListItemOptions => {
     listItemIndex: LIST_ITEM_INDEX,
     page: 1,
     perPage: 25,
+    runtimeMappings: undefined,
     searchAfter: undefined,
     sortField: undefined,
     sortOrder: undefined,

x-pack/plugins/lists/server/services/items/find_list_item.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 import { ElasticsearchClient } from '@kbn/core/server';
 import type {
   Filter,
@@ -27,6 +28,13 @@ import {
   transformElasticToListItem,
 } from '../utils';
 
+export const getTotalHitsValue = (totalHits: number | { value: number } | undefined): number =>
+  typeof totalHits === 'undefined'
+    ? -1
+    : typeof totalHits === 'number'
+    ? totalHits
+    : totalHits.value;
+
 export interface FindListItemOptions {
   listId: ListId;
   filter: Filter;
@@ -39,6 +47,7 @@ export interface FindListItemOptions {
   esClient: ElasticsearchClient;
   listIndex: string;
   listItemIndex: string;
+  runtimeMappings: MappingRuntimeFields | undefined;
 }
 
 export const findListItem = async ({
@@ -53,6 +62,7 @@ export const findListItem = async ({
   listIndex,
   listItemIndex,
   sortOrder,
+  runtimeMappings,
 }: FindListItemOptions): Promise<FoundListItemSchema | null> => {
   const list = await getList({ esClient, id: listId, listIndex });
   if (list == null) {
@@ -69,17 +79,21 @@ export const findListItem = async ({
       index: listItemIndex,
       page,
       perPage,
+      runtimeMappings,
       searchAfter,
       sortField,
       sortOrder,
     });
 
-    const respose = await esClient.count({
+    const respose = await esClient.search({
       body: {
         query,
+        runtime_mappings: runtimeMappings,
       },
       ignore_unavailable: true,
       index: listItemIndex,
+      size: 0,
+      track_total_hits: true,
     });
 
     if (scroll.validSearchAfterFound) {
@@ -106,7 +120,7 @@ export const findListItem = async ({
         data: transformElasticToListItem({ response, type: list.type }),
         page,
         per_page: perPage,
-        total: respose.count,
+        total: getTotalHitsValue(respose.hits.total),
       };
     } else {
       return {
@@ -114,7 +128,7 @@ export const findListItem = async ({
         data: [],
         page,
         per_page: perPage,
-        total: respose.count,
+        total: getTotalHitsValue(respose.hits.total),
       };
     }
   }

x-pack/plugins/lists/server/services/items/index.ts

@@ -10,6 +10,7 @@ export * from './create_list_item';
 export * from './create_list_items_bulk';
 export * from './delete_list_item_by_value';
 export * from './delete_list_item';
+export * from './find_all_list_items';
 export * from './find_list_item';
 export * from './get_list_item_by_value';
 export * from './get_list_item_by_values';

x-pack/plugins/lists/server/services/lists/find_list.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 import { ElasticsearchClient } from '@kbn/core/server';
 import type {
   Filter,
@@ -35,6 +36,7 @@ interface FindListOptions {
   sortOrder: SortOrderOrUndefined;
   esClient: ElasticsearchClient;
   listIndex: string;
+  runtimeMappings: MappingRuntimeFields | undefined;
 }
 
 export const findList = async ({
@@ -47,6 +49,7 @@ export const findList = async ({
   sortField,
   listIndex,
   sortOrder,
+  runtimeMappings,
 }: FindListOptions): Promise<FoundListSchema> => {
   const query = getQueryFilter({ filter });
 
@@ -58,6 +61,7 @@ export const findList = async ({
     index: listIndex,
     page,
     perPage,
+    runtimeMappings,
     searchAfter,
     sortField,
     sortOrder,

x-pack/plugins/lists/server/services/lists/list_client.mock.ts

@@ -7,6 +7,7 @@
 
 import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
 
+import { getFoundAllListItemsSchemaMock } from '../../../common/schemas/response/found_all_list_items_schema.mock';
 import { getFoundListItemSchemaMock } from '../../../common/schemas/response/found_list_item_schema.mock';
 import { getFoundListSchemaMock } from '../../../common/schemas/response/found_list_schema.mock';
 import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock';
@@ -61,6 +62,7 @@ export class ListClientMock extends ListClient {
   public getListItemByValues = jest.fn().mockResolvedValue([getListItemResponseMock()]);
   public findList = jest.fn().mockResolvedValue(getFoundListSchemaMock());
   public findListItem = jest.fn().mockResolvedValue(getFoundListItemSchemaMock());
+  public findAllListItems = jest.fn().mockResolvedValue(getFoundAllListItemsSchemaMock());
 }
 
 export const getListClientMock = (): ListClient => {

x-pack/plugins/lists/server/services/lists/list_client.ts

@@ -20,6 +20,7 @@ import {
   setPolicy,
 } from '@kbn/securitysolution-es-utils';
 import type {
+  FoundAllListItemsSchema,
   FoundListItemSchema,
   FoundListSchema,
   ListItemArraySchema,
@@ -34,6 +35,7 @@ import {
   deleteListItem,
   deleteListItemByValue,
   exportListItemsToStream,
+  findAllListItems,
   findListItem,
   getListItem,
   getListItemByValue,
@@ -56,6 +58,7 @@ import type {
   DeleteListItemOptions,
   DeleteListOptions,
   ExportListItemsToStreamOptions,
+  FindAllListItemsOptions,
   FindListItemOptions,
   FindListOptions,
   GetListItemByValueOptions,
@@ -807,6 +810,7 @@ export class ListClient {
     sortField,
     sortOrder,
     searchAfter,
+    runtimeMappings,
   }: FindListOptions): Promise<FoundListSchema> => {
     const { esClient } = this;
     const listIndex = this.getListIndex();
@@ -817,6 +821,7 @@ export class ListClient {
       listIndex,
       page,
       perPage,
+      runtimeMappings,
       searchAfter,
       sortField,
       sortOrder,
@@ -844,6 +849,7 @@ export class ListClient {
     currentIndexPosition,
     perPage,
     page,
+    runtimeMappings,
     sortField,
     sortOrder,
     searchAfter,
@@ -860,9 +866,30 @@ export class ListClient {
       listItemIndex,
       page,
       perPage,
+      runtimeMappings,
       searchAfter,
       sortField,
       sortOrder,
     });
   };
+
+  public findAllListItems = async ({
+    listId,
+    filter,
+    sortField,
+    sortOrder,
+  }: FindAllListItemsOptions): Promise<FoundAllListItemsSchema | null> => {
+    const { esClient } = this;
+    const listIndex = this.getListIndex();
+    const listItemIndex = this.getListItemIndex();
+    return findAllListItems({
+      esClient,
+      filter,
+      listId,
+      listIndex,
+      listItemIndex,
+      sortField,
+      sortOrder,
+    });
+  };
 }

x-pack/plugins/lists/server/services/lists/list_client_types.ts

@@ -30,6 +30,7 @@ import type {
   _VersionOrUndefined,
 } from '@kbn/securitysolution-io-ts-list-types';
 import type { Version, VersionOrUndefined } from '@kbn/securitysolution-io-ts-types';
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 
 import type { ConfigType } from '../../config';
 
@@ -272,6 +273,8 @@ export interface FindListOptions {
   perPage: PerPage;
   /** The current page number for the current find */
   page: Page;
+  /** Any runtime mappings to be included in the ES query */
+  runtimeMappings: MappingRuntimeFields | undefined;
   /** array of search_after terms, otherwise "undefined" if there is no search_after */
   searchAfter: string[] | undefined;
   /** Which field to sort on, "undefined" for no sort field */
@@ -295,14 +298,31 @@ export interface FindListItemOptions {
   perPage: PerPage;
   /** The current page number for the current find */
   page: Page;
+  /** Any runtime mappings to be included in the ES query */
+  runtimeMappings: MappingRuntimeFields | undefined;
   /** array of search_after terms, otherwise "undefined" if there is no search_after */
-  searchAfter: string[] | undefined;
+  searchAfter: string[];
   /** Which field to sort on, "undefined" for no sort field */
   sortField: SortFieldOrUndefined;
   /** "asc" or "desc" to sort, otherwise "undefined" if there is no sort order */
   sortOrder: SortOrderOrUndefined;
 }
 
+/**
+ * ListClient.findListItem
+ * {@link ListClient.findListItem}
+ */
+export interface FindAllListItemsOptions {
+  /** A KQL string filter to find list items. */
+  filter: Filter;
+  /** The list id to search for the list items */
+  listId: ListId;
+  /** Which field to sort on, "undefined" for no sort field */
+  sortField?: SortFieldOrUndefined;
+  /** "asc" or "desc" to sort, otherwise "undefined" if there is no sort order */
+  sortOrder?: SortOrderOrUndefined;
+}
+
 /**
  * ListClient.searchListItemByValues
  * {@link ListClient.findListItem}

x-pack/plugins/lists/server/services/utils/get_search_after_scroll.ts

@@ -11,6 +11,7 @@ import type {
   SortFieldOrUndefined,
   SortOrderOrUndefined,
 } from '@kbn/securitysolution-io-ts-list-types';
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 
 import { Scroll } from '../lists/types';
 
@@ -28,6 +29,7 @@ interface GetSearchAfterOptions {
   index: string;
   sortField: SortFieldOrUndefined;
   sortOrder: SortOrderOrUndefined;
+  runtimeMappings: MappingRuntimeFields | undefined;
 }
 
 export const getSearchAfterScroll = async <T>({
@@ -39,6 +41,7 @@ export const getSearchAfterScroll = async <T>({
   sortField,
   sortOrder,
   index,
+  runtimeMappings,
 }: GetSearchAfterOptions): Promise<Scroll> => {
   const query = getQueryFilter({ filter });
   let newSearchAfter = searchAfter;
@@ -47,6 +50,7 @@ export const getSearchAfterScroll = async <T>({
       body: {
         _source: getSourceWithTieBreaker({ sortField }),
         query,
+        runtime_mappings: runtimeMappings,
         search_after: newSearchAfter,
         sort: getSortWithTieBreaker({ sortField, sortOrder }),
       },

x-pack/plugins/lists/server/services/utils/scroll_to_start_page.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/types';
 import { ElasticsearchClient } from '@kbn/core/server';
 import type {
   Filter,
@@ -28,6 +29,7 @@ interface ScrollToStartPageOptions {
   index: string;
   currentIndexPosition: number;
   searchAfter: string[] | undefined;
+  runtimeMappings: MappingRuntimeFields | undefined;
 }
 
 export const scrollToStartPage = async ({
@@ -41,6 +43,7 @@ export const scrollToStartPage = async ({
   sortOrder,
   sortField,
   index,
+  runtimeMappings,
 }: ScrollToStartPageOptions): Promise<Scroll> => {
   const { hops, leftOverAfterHops } = calculateScrollMath({
     currentIndexPosition,
@@ -67,6 +70,7 @@ export const scrollToStartPage = async ({
       hopSize,
       hops,
       index,
+      runtimeMappings,
       searchAfter,
       sortField,
       sortOrder,
@@ -78,6 +82,7 @@ export const scrollToStartPage = async ({
         hopSize: leftOverAfterHops,
         hops: 1,
         index,
+        runtimeMappings,
         searchAfter: scroll.searchAfter,
         sortField,
         sortOrder,
@@ -92,6 +97,7 @@ export const scrollToStartPage = async ({
       hopSize: leftOverAfterHops,
       hops: 1,
       index,
+      runtimeMappings,
       searchAfter,
       sortField,
       sortOrder,

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -173,7 +173,7 @@ export const status = t.keyof({
   open: null,
   closed: null,
   acknowledged: null,
-  'in-progress': null, // TODO: Remove after `acknowledged` migrations
+  'in-progress': null,
 });
 export type Status = t.TypeOf<typeof status>;
 

x-pack/plugins/security_solution/public/detection_engine/rule_exceptions/components/add_exception_flyout/index.tsx

@@ -39,12 +39,7 @@ import type { ExceptionsBuilderExceptionItem } from '@kbn/securitysolution-list-
 import { getExceptionBuilderComponentLazy } from '@kbn/lists-plugin/public';
 import type { DataViewBase } from '@kbn/es-query';
 import { useRuleIndices } from '../../../../detections/containers/detection_engine/rules/use_rule_indices';
-import {
-  hasEqlSequenceQuery,
-  isEqlRule,
-  isNewTermsRule,
-  isThresholdRule,
-} from '../../../../../common/detection_engine/utils';
+import { hasEqlSequenceQuery, isEqlRule } from '../../../../../common/detection_engine/utils';
 import type { Status } from '../../../../../common/detection_engine/schemas/common/schemas';
 import * as i18nCommon from '../../../../common/translations';
 import * as i18n from './translations';
@@ -71,6 +66,7 @@ import type { ErrorInfo } from '../error_callout';
 import { ErrorCallout } from '../error_callout';
 import type { AlertData } from '../../utils/types';
 import { useFetchIndex } from '../../../../common/containers/source';
+import { ruleTypesThatAllowLargeValueLists } from '../../utils/constants';
 
 export interface AddExceptionFlyoutProps {
   ruleName: string;
@@ -444,6 +440,11 @@ export const AddExceptionFlyout = memo(function AddExceptionFlyout({
     return hasOsSelection && selectedOs === undefined;
   }, [hasOsSelection, selectedOs]);
 
+  const allowLargeValueLists = useMemo(
+    () => (maybeRule != null ? ruleTypesThatAllowLargeValueLists.includes(maybeRule.type) : false),
+    [maybeRule]
+  );
+
   return (
     <EuiFlyout
       ownFocus
@@ -524,10 +525,7 @@ export const AddExceptionFlyout = memo(function AddExceptionFlyout({
                 </>
               )}
               {getExceptionBuilderComponentLazy({
-                allowLargeValueLists:
-                  !isEqlRule(maybeRule?.type) &&
-                  !isThresholdRule(maybeRule?.type) &&
-                  !isNewTermsRule(maybeRule?.type),
+                allowLargeValueLists,
                 httpService: http,
                 autocompleteService: unifiedSearch.autocomplete,
                 exceptionListItems: initialExceptionItems,

x-pack/plugins/security_solution/public/detection_engine/rule_exceptions/components/edit_exception_flyout/index.tsx

@@ -39,12 +39,7 @@ import { getExceptionBuilderComponentLazy } from '@kbn/lists-plugin/public';
 import type { DataViewBase } from '@kbn/es-query';
 
 import { useRuleIndices } from '../../../../detections/containers/detection_engine/rules/use_rule_indices';
-import {
-  hasEqlSequenceQuery,
-  isEqlRule,
-  isNewTermsRule,
-  isThresholdRule,
-} from '../../../../../common/detection_engine/utils';
+import { hasEqlSequenceQuery, isEqlRule } from '../../../../../common/detection_engine/utils';
 import { useFetchIndex } from '../../../../common/containers/source';
 import { useSignalIndex } from '../../../../detections/containers/detection_engine/alerts/use_signal_index';
 import { useRuleAsync } from '../../../../detections/containers/detection_engine/rules/use_rule_async';
@@ -66,6 +61,7 @@ import {
 import { Loader } from '../../../../common/components/loader';
 import type { ErrorInfo } from '../error_callout';
 import { ErrorCallout } from '../error_callout';
+import { ruleTypesThatAllowLargeValueLists } from '../../utils/constants';
 
 interface EditExceptionFlyoutProps {
   ruleName: string;
@@ -338,6 +334,11 @@ export const EditExceptionFlyout = memo(function EditExceptionFlyout({
       .slice(0, -2);
   };
 
+  const allowLargeValueLists = useMemo(
+    () => (maybeRule != null ? ruleTypesThatAllowLargeValueLists.includes(maybeRule.type) : false),
+    [maybeRule]
+  );
+
   return (
     <EuiFlyout size="l" onClose={onCancel} data-test-subj="edit-exception-flyout">
       <FlyoutHeader>
@@ -386,10 +387,7 @@ export const EditExceptionFlyout = memo(function EditExceptionFlyout({
                 </>
               )}
               {getExceptionBuilderComponentLazy({
-                allowLargeValueLists:
-                  !isEqlRule(maybeRule?.type) &&
-                  !isThresholdRule(maybeRule?.type) &&
-                  !isNewTermsRule(maybeRule?.type),
+                allowLargeValueLists,
                 httpService: http,
                 autocompleteService: unifiedSearch.autocomplete,
                 exceptionListItems: [exceptionItem],

x-pack/plugins/security_solution/public/detection_engine/rule_exceptions/logic/use_add_exception.test.tsx

@@ -13,7 +13,7 @@ import { KibanaServices } from '../../../common/lib/kibana';
 
 import * as alertsApi from '../../../detections/containers/detection_engine/alerts/api';
 import * as listsApi from '@kbn/securitysolution-list-api';
-import * as getQueryFilterHelper from '../../../../common/detection_engine/get_query_filter';
+import * as getQueryFilterHelper from '../../../detections/containers/detection_engine/exceptions/get_es_query_filter';
 import * as buildFilterHelpers from '../../../detections/components/alerts_table/default_config';
 import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
 import { getCreateExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_item_schema.mock';
@@ -43,7 +43,7 @@ describe('useAddOrUpdateException', () => {
   let updateAlertStatus: jest.SpyInstance<Promise<estypes.UpdateByQueryResponse>>;
   let addExceptionListItem: jest.SpyInstance<Promise<ExceptionListItemSchema>>;
   let updateExceptionListItem: jest.SpyInstance<Promise<ExceptionListItemSchema>>;
-  let getQueryFilter: jest.SpyInstance<ReturnType<typeof getQueryFilterHelper.getQueryFilter>>;
+  let getQueryFilter: jest.SpyInstance<ReturnType<typeof getQueryFilterHelper.getEsQueryFilter>>;
   let buildAlertStatusesFilter: jest.SpyInstance<
     ReturnType<typeof buildFilterHelpers.buildAlertStatusesFilter>
   >;
@@ -125,7 +125,9 @@ describe('useAddOrUpdateException', () => {
       .spyOn(listsApi, 'updateExceptionListItem')
       .mockResolvedValue(getExceptionListItemSchemaMock());
 
-    getQueryFilter = jest.spyOn(getQueryFilterHelper, 'getQueryFilter');
+    getQueryFilter = jest
+      .spyOn(getQueryFilterHelper, 'getEsQueryFilter')
+      .mockResolvedValue({ bool: { must_not: [], must: [], filter: [], should: [] } });
 
     buildAlertStatusesFilter = jest.spyOn(buildFilterHelpers, 'buildAlertStatusesFilter');
 

x-pack/plugins/security_solution/public/detection_engine/rule_exceptions/logic/use_add_exception.tsx

@@ -11,7 +11,7 @@ import type {
   ExceptionListItemSchema,
   CreateExceptionListItemSchema,
 } from '@kbn/securitysolution-io-ts-list-types';
-import { useApi } from '@kbn/securitysolution-list-hooks';
+import { useApi, removeIdFromExceptionItemsEntries } from '@kbn/securitysolution-list-hooks';
 import type { HttpStart } from '@kbn/core/public';
 
 import { updateAlertStatus } from '../../../detections/containers/detection_engine/alerts/api';
@@ -20,7 +20,7 @@ import {
   buildAlertsFilter,
   buildAlertStatusesFilter,
 } from '../../../detections/components/alerts_table/default_config';
-import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter';
+import { getEsQueryFilter } from '../../../detections/containers/detection_engine/exceptions/get_es_query_filter';
 import type { Index } from '../../../../common/detection_engine/schemas/common/schemas';
 import { formatExceptionItemForUpdate, prepareExceptionItemsForBulkClose } from '../utils/helpers';
 import { useKibana } from '../../../common/lib/kibana';
@@ -133,12 +133,16 @@ export const useAddOrUpdateException = ({
             'in-progress',
           ]);
 
-          const filter = getQueryFilter(
+          const exceptionsToFilter = exceptionItemsToAddOrUpdate.map((exception) =>
+            removeIdFromExceptionItemsEntries(exception)
+          );
+
+          const filter = await getEsQueryFilter(
             '',
             'kuery',
             [...buildAlertsFilter(ruleStaticId), ...alertStatusFilter],
             bulkCloseIndex,
-            prepareExceptionItemsForBulkClose(exceptionItemsToAddOrUpdate),
+            prepareExceptionItemsForBulkClose(exceptionsToFilter),
             false
           );
 

x-pack/plugins/security_solution/public/detection_engine/rule_exceptions/utils/constants.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { Type } from '@kbn/securitysolution-io-ts-alerting-types';
+
+export const ruleTypesThatAllowLargeValueLists: Type[] = [
+  'query',
+  'machine_learning',
+  'saved_query',
+  'threat_match',
+];

x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx

@@ -89,7 +89,7 @@ describe('alert actions', () => {
   let searchStrategyClient: jest.Mocked<ISearchStart>;
   let clock: sinon.SinonFakeTimers;
   let mockKibanaServices: jest.Mock;
-  let mockGetExceptions: jest.Mock;
+  let mockGetExceptionFilter: jest.Mock;
   let fetchMock: jest.Mock;
   let toastMock: jest.Mock;
 
@@ -225,7 +225,7 @@ describe('alert actions', () => {
     jest.resetAllMocks();
     jest.restoreAllMocks();
     jest.clearAllMocks();
-    mockGetExceptions = jest.fn().mockResolvedValue([]);
+    mockGetExceptionFilter = jest.fn().mockResolvedValue(undefined);
 
     createTimeline = jest.fn() as jest.Mocked<CreateTimeline>;
     updateTimelineIsLoading = jest.fn() as jest.Mocked<UpdateTimelineLoading>;
@@ -260,10 +260,10 @@ describe('alert actions', () => {
           ecsData: mockEcsDataWithAlert,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
 
-        expect(mockGetExceptions).not.toHaveBeenCalled();
+        expect(mockGetExceptionFilter).not.toHaveBeenCalled();
         expect(updateTimelineIsLoading).toHaveBeenCalledTimes(1);
         expect(updateTimelineIsLoading).toHaveBeenCalledWith({
           id: TimelineId.active,
@@ -277,7 +277,7 @@ describe('alert actions', () => {
           ecsData: mockEcsDataWithAlert,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
         const expected = {
           from: '2018-11-05T18:58:25.937Z',
@@ -415,7 +415,7 @@ describe('alert actions', () => {
           ruleNote: '# this is some markdown documentation',
         };
 
-        expect(mockGetExceptions).not.toHaveBeenCalled();
+        expect(mockGetExceptionFilter).not.toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledWith(expected);
       });
 
@@ -437,11 +437,11 @@ describe('alert actions', () => {
           ecsData: mockEcsDataWithAlert,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
         const createTimelineArg = (createTimeline as jest.Mock).mock.calls[0][0];
 
-        expect(mockGetExceptions).not.toHaveBeenCalled();
+        expect(mockGetExceptionFilter).not.toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimelineArg.timeline.kqlQuery.filterQuery.kuery.kind).toEqual('kuery');
       });
@@ -456,7 +456,7 @@ describe('alert actions', () => {
           ecsData: mockEcsDataWithAlert,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
         const defaultTimelinePropsWithoutNote = { ...defaultTimelineProps };
 
@@ -470,7 +470,7 @@ describe('alert actions', () => {
           id: TimelineId.active,
           isLoading: false,
         });
-        expect(mockGetExceptions).not.toHaveBeenCalled();
+        expect(mockGetExceptionFilter).not.toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith({
           ...defaultTimelinePropsWithoutNote,
@@ -503,11 +503,11 @@ describe('alert actions', () => {
           ecsData: ecsDataMock,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
 
         expect(updateTimelineIsLoading).not.toHaveBeenCalled();
-        expect(mockGetExceptions).not.toHaveBeenCalled();
+        expect(mockGetExceptionFilter).not.toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith(defaultTimelineProps);
       });
@@ -530,11 +530,11 @@ describe('alert actions', () => {
           ecsData: ecsDataMock,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
 
         expect(updateTimelineIsLoading).not.toHaveBeenCalled();
-        expect(mockGetExceptions).not.toHaveBeenCalled();
+        expect(mockGetExceptionFilter).not.toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith(defaultTimelineProps);
       });
@@ -561,11 +561,11 @@ describe('alert actions', () => {
           ecsData: ecsDataMock,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
 
         expect(updateTimelineIsLoading).not.toHaveBeenCalled();
-        expect(mockGetExceptions).not.toHaveBeenCalled();
+        expect(mockGetExceptionFilter).not.toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith({
           ...defaultTimelineProps,
@@ -604,11 +604,11 @@ describe('alert actions', () => {
           ecsData: ecsDataMock,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
 
         expect(updateTimelineIsLoading).not.toHaveBeenCalled();
-        expect(mockGetExceptions).not.toHaveBeenCalled();
+        expect(mockGetExceptionFilter).not.toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith(defaultTimelineProps);
       });
@@ -627,20 +627,68 @@ describe('alert actions', () => {
             ],
           },
         });
-        mockGetExceptions.mockResolvedValue([getExceptionListItemSchemaMock()]);
+        mockGetExceptionFilter.mockResolvedValue({
+          meta: {
+            alias: 'Exceptions',
+            disabled: false,
+            negate: true,
+          },
+          query: {
+            bool: {
+              should: [
+                {
+                  bool: {
+                    filter: [
+                      {
+                        nested: {
+                          path: 'some.parentField',
+                          query: {
+                            bool: {
+                              minimum_should_match: 1,
+                              should: [
+                                {
+                                  match_phrase: {
+                                    'some.parentField.nested.field': 'some value',
+                                  },
+                                },
+                              ],
+                            },
+                          },
+                          score_mode: 'none',
+                        },
+                      },
+                      {
+                        bool: {
+                          minimum_should_match: 1,
+                          should: [
+                            {
+                              match_phrase: {
+                                'some.not.nested.field': 'some value',
+                              },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+            },
+          },
+        });
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: ecsDataMockWithNoTemplateTimeline,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
 
         const expectedFrom = '2021-01-10T21:11:45.839Z';
         const expectedTo = '2021-01-10T21:12:45.839Z';
 
         expect(updateTimelineIsLoading).not.toHaveBeenCalled();
-        expect(mockGetExceptions).toHaveBeenCalled();
+        expect(mockGetExceptionFilter).toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith({
           ...defaultTimelineProps,
@@ -756,13 +804,12 @@ describe('alert actions', () => {
             ],
           },
         });
-        mockGetExceptions.mockResolvedValue([getExceptionListItemSchemaMock()]);
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: ecsDataMockWithNoTemplateTimelineAndNoFilters,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
 
         expect(createTimeline).not.toThrow();
@@ -781,19 +828,20 @@ describe('alert actions', () => {
             ],
           },
         });
+
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: ecsDataMockWithTemplateTimeline,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
 
         const expectedFrom = '2021-01-10T21:11:45.839Z';
         const expectedTo = '2021-01-10T21:12:45.839Z';
 
         expect(updateTimelineIsLoading).toHaveBeenCalled();
-        expect(mockGetExceptions).toHaveBeenCalled();
+        expect(mockGetExceptionFilter).toHaveBeenCalled();
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith({
           ...defaultTimelineProps,
@@ -886,7 +934,7 @@ describe('alert actions', () => {
           ecsData: ecsDataMockWithNoTemplateTimeline,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith({
@@ -977,7 +1025,7 @@ describe('alert actions', () => {
           ecsData: ecsDataMockWithNoTemplateTimeline,
           updateTimelineIsLoading,
           searchStrategyClient,
-          getExceptions: mockGetExceptions,
+          getExceptionFilter: mockGetExceptionFilter,
         });
         expect(createTimeline).toHaveBeenCalledTimes(1);
         expect(createTimeline).toHaveBeenCalledWith(timelineProps);

x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx

@@ -24,9 +24,6 @@ import {
   ALERT_RULE_PARAMETERS,
 } from '@kbn/rule-data-utils';
 
-import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
-import { buildExceptionFilter } from '@kbn/securitysolution-list-utils';
-
 import type { TGridModel } from '@kbn/timelines-plugin/public';
 
 import { lastValueFrom } from 'rxjs';
@@ -46,6 +43,7 @@ import type {
   ThresholdAggregationData,
   UpdateAlertStatusActionProps,
   CreateTimelineProps,
+  GetExceptionFilter,
 } from './types';
 import type { Ecs } from '../../../../common/ecs';
 import type {
@@ -410,7 +408,7 @@ const createThresholdTimeline = async (
     dataProviders?: DataProvider[];
     columns?: TGridModel['columns'];
   },
-  getExceptions: (ecs: Ecs) => Promise<ExceptionListItemSchema[]>
+  getExceptionFilter: GetExceptionFilter
 ) => {
   try {
     const alertResponse = await KibanaServices.get().http.fetch<
@@ -448,15 +446,11 @@ const createThresholdTimeline = async (
     const indexNames = getField(alertDoc, ALERT_RULE_INDICES) ?? alertDoc.signal?.rule?.index ?? [];
 
     const { thresholdFrom, thresholdTo, dataProviders } = getThresholdAggregationData(alertDoc);
-    const exceptions = await getExceptions(ecsData);
-    const exceptionsFilter =
-      buildExceptionFilter({
-        lists: exceptions,
-        excludeExceptions: true,
-        chunkSize: 10000,
-        alias: 'Exceptions',
-      }) ?? [];
-    const allFilters = (templateValues.filters ?? augmentedFilters).concat(exceptionsFilter);
+    const exceptionsFilter = await getExceptionFilter(ecsData);
+
+    const allFilters = (templateValues.filters ?? augmentedFilters).concat(
+      !exceptionsFilter ? [] : [exceptionsFilter]
+    );
 
     return createTimeline({
       from: thresholdFrom,
@@ -560,7 +554,7 @@ const createNewTermsTimeline = async (
     dataProviders?: DataProvider[];
     columns?: TGridModel['columns'];
   },
-  getExceptions: (ecs: Ecs) => Promise<ExceptionListItemSchema[]>
+  getExceptionFilter: GetExceptionFilter
 ) => {
   try {
     const alertResponse = await KibanaServices.get().http.fetch<
@@ -598,15 +592,9 @@ const createNewTermsTimeline = async (
     const indexNames = getField(alertDoc, ALERT_RULE_INDICES) ?? alertDoc.signal?.rule?.index ?? [];
 
     const { from, to, dataProviders } = getNewTermsData(alertDoc);
-    const exceptions = await getExceptions(ecsData);
-    const exceptionsFilter =
-      buildExceptionFilter({
-        lists: exceptions,
-        excludeExceptions: true,
-        chunkSize: 10000,
-        alias: 'Exceptions',
-      }) ?? [];
-    const allFilters = (templateValues.filters ?? augmentedFilters).concat(exceptionsFilter);
+    const filter = await getExceptionFilter(ecsData);
+
+    const allFilters = (templateValues.filters ?? augmentedFilters).concat(!filter ? [] : [filter]);
     return createTimeline({
       from,
       notes: null,
@@ -678,7 +666,7 @@ export const sendAlertToTimelineAction = async ({
   ecsData: ecs,
   updateTimelineIsLoading,
   searchStrategyClient,
-  getExceptions,
+  getExceptionFilter,
 }: SendAlertToTimelineActionProps) => {
   /* FUTURE DEVELOPER
    * We are making an assumption here that if you have an array of ecs data they are all coming from the same rule
@@ -751,7 +739,7 @@ export const sendAlertToTimelineAction = async ({
               dataProviders,
               columns: timeline.columns,
             },
-            getExceptions
+            getExceptionFilter
           );
         } else if (isNewTermsAlert(ecsData)) {
           return createNewTermsTimeline(
@@ -764,7 +752,7 @@ export const sendAlertToTimelineAction = async ({
               dataProviders,
               columns: timeline.columns,
             },
-            getExceptions
+            getExceptionFilter
           );
         } else {
           return createTimeline({
@@ -819,9 +807,9 @@ export const sendAlertToTimelineAction = async ({
       });
     }
   } else if (isThresholdAlert(ecsData)) {
-    return createThresholdTimeline(ecsData, createTimeline, noteContent, {}, getExceptions);
+    return createThresholdTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter);
   } else if (isNewTermsAlert(ecsData)) {
-    return createNewTermsTimeline(ecsData, createTimeline, noteContent, {}, getExceptions);
+    return createNewTermsTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter);
   } else {
     let { dataProviders, filters } = buildTimelineDataProviderOrFilter(alertIds ?? [], ecsData._id);
     if (isEqlAlertWithGroupId(ecsData)) {

x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/use_investigate_in_timeline.tsx

@@ -11,13 +11,10 @@ import { EuiContextMenuItem } from '@elastic/eui';
 
 import { i18n } from '@kbn/i18n';
 import { ALERT_RULE_EXCEPTIONS_LIST } from '@kbn/rule-data-utils';
-import type {
-  ExceptionListIdentifiers,
-  ExceptionListItemSchema,
-} from '@kbn/securitysolution-io-ts-list-types';
-import { ExceptionListTypeEnum } from '@kbn/securitysolution-io-ts-list-types';
+import type { ExceptionListId } from '@kbn/securitysolution-io-ts-list-types';
 import { useApi } from '@kbn/securitysolution-list-hooks';
 
+import type { Filter } from '@kbn/es-query';
 import { useKibana } from '../../../../common/lib/kibana';
 import { TimelineId, TimelineType } from '../../../../../common/types/timeline';
 import type { Ecs } from '../../../../../common/ecs';
@@ -50,19 +47,17 @@ export const useInvestigateInTimeline = ({
   const { startTransaction } = useStartTransaction();
 
   const { services } = useKibana();
-  const { getExceptionListsItems } = useApi(services.http);
+  const { getExceptionFilterFromIds } = useApi(services.http);
 
-  const getExceptions = useCallback(
-    async (ecsData: Ecs): Promise<ExceptionListItemSchema[]> => {
+  const getExceptionFilter = useCallback(
+    async (ecsData: Ecs): Promise<Filter | undefined> => {
       const exceptionsLists = (getField(ecsData, ALERT_RULE_EXCEPTIONS_LIST) ?? []).reduce(
-        (acc: ExceptionListIdentifiers[], next: string) => {
+        (acc: ExceptionListId[], next: string) => {
           const parsedList = JSON.parse(next);
           if (parsedList.type === 'detection') {
             const formattedList = {
-              id: parsedList.id,
-              listId: parsedList.list_id,
-              type: ExceptionListTypeEnum.DETECTION,
-              namespaceType: parsedList.namespace_type,
+              exception_list_id: parsedList.list_id,
+              namespace_type: parsedList.namespace_type,
             };
             acc.push(formattedList);
           }
@@ -71,34 +66,28 @@ export const useInvestigateInTimeline = ({
         []
       );
 
-      const allExceptions: ExceptionListItemSchema[] = [];
-
       if (exceptionsLists.length > 0) {
-        await getExceptionListsItems({
-          lists: exceptionsLists,
-          pagination: {
-            page: 0,
-            perPage: 10000,
-            total: 10000,
-          },
-          showDetectionsListsOnly: true,
-          showEndpointListsOnly: false,
-          onSuccess: ({ exceptions }) => {
-            allExceptions.push(...exceptions);
+        await getExceptionFilterFromIds({
+          exceptionListIds: exceptionsLists,
+          excludeExceptions: true,
+          chunkSize: 20,
+          alias: 'Exceptions',
+          onSuccess: (filter) => {
+            return filter;
           },
           onError: (err: string[]) => {
             addError(err, {
               title: i18n.translate(
-                'xpack.securitySolution.detectionEngine.alerts.fetchExceptionsFailure',
-                { defaultMessage: 'Error fetching exceptions.' }
+                'xpack.securitySolution.detectionEngine.alerts.fetchExceptionFilterFailure',
+                { defaultMessage: 'Error fetching exception filter.' }
               ),
             });
           },
         });
       }
-      return allExceptions;
+      return undefined;
     },
-    [addError, getExceptionListsItems]
+    [addError, getExceptionFilterFromIds]
   );
 
   const filterManagerBackup = useMemo(() => query.filterManager, [query.filterManager]);
@@ -154,7 +143,7 @@ export const useInvestigateInTimeline = ({
         ecsData: ecsRowData,
         searchStrategyClient,
         updateTimelineIsLoading,
-        getExceptions,
+        getExceptionFilter,
       });
     }
   }, [
@@ -164,7 +153,7 @@ export const useInvestigateInTimeline = ({
     onInvestigateInTimelineAlertClick,
     searchStrategyClient,
     updateTimelineIsLoading,
-    getExceptions,
+    getExceptionFilter,
   ]);
 
   const investigateInTimelineActionItems = useMemo(

x-pack/plugins/security_solution/public/detections/components/alerts_table/types.ts

@@ -5,9 +5,8 @@
  * 2.0.
  */
 
-import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
-
 import type { ISearchStart } from '@kbn/data-plugin/public';
+import type { Filter } from '@kbn/es-query';
 import type { Status } from '../../../../common/detection_engine/schemas/common/schemas';
 import type { Ecs } from '../../../../common/ecs';
 import type { NoteResult } from '../../../../common/types/timeline/note';
@@ -57,7 +56,7 @@ export interface SendAlertToTimelineActionProps {
   ecsData: Ecs | Ecs[];
   updateTimelineIsLoading: UpdateTimelineLoading;
   searchStrategyClient: ISearchStart;
-  getExceptions: GetExceptions;
+  getExceptionFilter: GetExceptionFilter;
 }
 
 export type UpdateTimelineLoading = ({ id, isLoading }: { id: string; isLoading: boolean }) => void;
@@ -71,7 +70,7 @@ export interface CreateTimelineProps {
 }
 
 export type CreateTimeline = ({ from, timeline, to }: CreateTimelineProps) => void;
-export type GetExceptions = (ecsData: Ecs) => Promise<ExceptionListItemSchema[]>;
+export type GetExceptionFilter = (ecsData: Ecs) => Promise<Filter | undefined>;
 
 export interface ThresholdAggregationData {
   thresholdFrom: string;

x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/helpers.test.ts

@@ -7,12 +7,7 @@
 
 import moment from 'moment';
 import { DataSourceType } from '../../../pages/detection_engine/rules/types';
-import {
-  isNoisy,
-  getTimeframeOptions,
-  getInfoFromQueryBar,
-  getIsRulePreviewDisabled,
-} from './helpers';
+import { isNoisy, getTimeframeOptions, getIsRulePreviewDisabled } from './helpers';
 
 describe('query_preview/helpers', () => {
   const timeframeEnd = moment();
@@ -301,122 +296,4 @@ describe('query_preview/helpers', () => {
       expect(options).toEqual([{ value: 'h', text: 'Last hour' }]);
     });
   });
-
-  describe('getInfoFromQueryBar', () => {
-    test('returns queryFilter when ruleType is query', () => {
-      const { queryString, language, filters, queryFilter } = getInfoFromQueryBar(
-        {
-          query: { query: 'host.name:*', language: 'kuery' },
-          filters: [{ meta: { alias: '', disabled: false, negate: false } }],
-          saved_id: null,
-        },
-        ['foo-*'],
-        'query'
-      );
-
-      expect(queryString).toEqual('host.name:*');
-      expect(language).toEqual('kuery');
-      expect(filters).toEqual([{ meta: { alias: '', disabled: false, negate: false }, query: {} }]);
-      expect(queryFilter).toEqual({
-        bool: {
-          filter: [
-            { bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] } },
-            {},
-          ],
-          must: [],
-          must_not: [],
-          should: [],
-        },
-      });
-    });
-
-    test('returns queryFilter when ruleType is saved_query', () => {
-      const { queryString, language, filters, queryFilter } = getInfoFromQueryBar(
-        {
-          query: { query: 'host.name:*', language: 'kuery' },
-          filters: [{ meta: { alias: '', disabled: false, negate: false } }],
-          saved_id: null,
-        },
-        ['foo-*'],
-        'saved_query'
-      );
-
-      expect(queryString).toEqual('host.name:*');
-      expect(language).toEqual('kuery');
-      expect(filters).toEqual([{ meta: { alias: '', disabled: false, negate: false }, query: {} }]);
-      expect(queryFilter).toEqual({
-        bool: {
-          filter: [
-            { bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] } },
-            {},
-          ],
-          must: [],
-          must_not: [],
-          should: [],
-        },
-      });
-    });
-
-    test('returns queryFilter when ruleType is threshold', () => {
-      const { queryString, language, filters, queryFilter } = getInfoFromQueryBar(
-        {
-          query: { query: 'host.name:*', language: 'kuery' },
-          filters: [{ meta: { alias: '', disabled: false, negate: false } }],
-          saved_id: null,
-        },
-        ['foo-*'],
-        'threshold'
-      );
-
-      expect(queryString).toEqual('host.name:*');
-      expect(language).toEqual('kuery');
-      expect(filters).toEqual([{ meta: { alias: '', disabled: false, negate: false }, query: {} }]);
-      expect(queryFilter).toEqual({
-        bool: {
-          filter: [
-            { bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] } },
-            {},
-          ],
-          must: [],
-          must_not: [],
-          should: [],
-        },
-      });
-    });
-
-    test('returns undefined queryFilter when ruleType is eql', () => {
-      const { queryString, language, filters, queryFilter } = getInfoFromQueryBar(
-        {
-          query: { query: 'file where true', language: 'eql' },
-          filters: [{ meta: { alias: '', disabled: false, negate: false } }],
-          saved_id: null,
-        },
-        ['foo-*'],
-        'eql'
-      );
-
-      expect(queryString).toEqual('file where true');
-      expect(language).toEqual('eql');
-      expect(filters).toEqual([{ meta: { alias: '', disabled: false, negate: false } }]);
-      expect(queryFilter).toBeUndefined();
-    });
-
-    test('returns undefined queryFilter when getQueryFilter throws', () => {
-      // query is malformed, forcing error in getQueryFilter
-      const { queryString, language, filters, queryFilter } = getInfoFromQueryBar(
-        {
-          query: { query: 'host.name:', language: 'kuery' },
-          filters: [{ meta: { alias: '', disabled: false, negate: false } }],
-          saved_id: null,
-        },
-        ['foo-*'],
-        'threshold'
-      );
-
-      expect(queryString).toEqual('host.name:');
-      expect(language).toEqual('kuery');
-      expect(filters).toEqual([{ meta: { alias: '', disabled: false, negate: false } }]);
-      expect(queryFilter).toBeUndefined();
-    });
-  });
 });

x-pack/plugins/security_solution/public/detections/components/rules/rule_preview/helpers.ts

@@ -8,14 +8,11 @@
 import { isEmpty } from 'lodash';
 import { Position, ScaleType } from '@elastic/charts';
 import type { EuiSelectOption } from '@elastic/eui';
-import type { Type, Language, ThreatMapping } from '@kbn/securitysolution-io-ts-alerting-types';
-import type { Filter } from '@kbn/es-query';
+import type { Type, ThreatMapping } from '@kbn/securitysolution-io-ts-alerting-types';
 import * as i18n from './translations';
 import { histogramDateTimeFormatter } from '../../../../common/components/utils';
 import type { ChartSeriesConfigs } from '../../../../common/components/charts/common';
-import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter';
 import type { FieldValueQueryBar } from '../query_bar';
-import type { ESQuery } from '../../../../../common/typed_json';
 import type { TimeframePreviewOptions } from '../../../pages/detection_engine/rules/types';
 import { DataSourceType } from '../../../pages/detection_engine/rules/types';
 
@@ -63,52 +60,6 @@ export const getTimeframeOptions = (ruleType: Type): EuiSelectOption[] => {
   }
 };
 
-/**
- * Quick little helper to extract the query info from the
- * queryBar object.
- * @param queryBar Object containing all query info
- * @param index Indices searched
- * @param ruleType
- */
-export const getInfoFromQueryBar = (
-  queryBar: FieldValueQueryBar,
-  index: string[],
-  ruleType: Type
-): {
-  queryString: string;
-  language: Language;
-  filters: Filter[];
-  queryFilter: ESQuery | undefined;
-} => {
-  const queryString = typeof queryBar.query.query === 'string' ? queryBar.query.query : '';
-  const language = queryBar.query.language as Language;
-  const filters = queryBar.filters;
-
-  // hm?? Why a try catch here? Because if the
-  // query is invalid, it throws an error and
-  // entire UI shows gross KQLSyntax error screen
-  try {
-    const queryFilter =
-      ruleType !== 'eql'
-        ? getQueryFilter(queryString, language, filters, index, [], true)
-        : undefined;
-
-    return {
-      queryString,
-      language,
-      filters,
-      queryFilter,
-    };
-  } catch {
-    return {
-      queryString,
-      language,
-      filters,
-      queryFilter: undefined,
-    };
-  }
-};
-
 /**
  * Config passed into elastic-charts settings.
  * @param to

x-pack/plugins/security_solution/public/detections/containers/detection_engine/exceptions/get_es_query_filter.ts

@@ -10,21 +10,22 @@ import type {
   ExceptionListItemSchema,
   CreateExceptionListItemSchema,
 } from '@kbn/securitysolution-io-ts-list-types';
-import { buildExceptionFilter } from '@kbn/securitysolution-list-utils';
 import type { Filter, EsQueryConfig, DataViewBase } from '@kbn/es-query';
+import { getExceptionFilterFromExceptions } from '@kbn/securitysolution-list-api';
 import { buildEsQuery } from '@kbn/es-query';
+import { KibanaServices } from '../../../../common/lib/kibana';
 
-import type { ESBoolQuery } from '../typed_json';
-import type { Query, Index } from './schemas/common/schemas';
+import type { Query, Index } from '../../../../../common/detection_engine/schemas/common';
+import type { ESBoolQuery } from '../../../../../common/typed_json';
 
-export const getQueryFilter = (
+export const getEsQueryFilter = async (
   query: Query,
   language: Language,
   filters: unknown,
   index: Index,
   lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>,
   excludeExceptions: boolean = true
-): ESBoolQuery => {
+): Promise<ESBoolQuery> => {
   const indexPattern: DataViewBase = {
     fields: [],
     title: index.join(),
@@ -40,14 +41,14 @@ export const getQueryFilter = (
   // allowing us to make 1024-item chunks of exception list items.
   // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
   // very conservative value.
-  const exceptionFilter = buildExceptionFilter({
-    lists,
+  const { filter } = await getExceptionFilterFromExceptions({
+    http: KibanaServices.get().http,
+    exceptions: lists,
     excludeExceptions,
-    chunkSize: 1024,
-    alias: null,
+    chunkSize: 10,
   });
   const initialQuery = { query, language };
-  const allFilters = getAllFilters(filters as Filter[], exceptionFilter);
+  const allFilters = getAllFilters(filters as Filter[], filter);
 
   return buildEsQuery(indexPattern, initialQuery, allFilters, config);
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts

@@ -13,6 +13,7 @@ import { TIMESTAMP } from '@kbn/rule-data-utils';
 import { createPersistenceRuleTypeWrapper } from '@kbn/rule-registry-plugin/server';
 import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils';
 
+import { buildExceptionFilter } from '@kbn/lists-plugin/server/services/exception_lists';
 import {
   checkPrivilegesFromEsClient,
   getExceptions,
@@ -300,6 +301,14 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
               indicesToQuery: inputIndex,
             });
 
+            const { filter: exceptionFilter, unprocessedExceptions } = await buildExceptionFilter({
+              alias: null,
+              excludeExceptions: true,
+              chunkSize: 10,
+              lists: exceptionItems,
+              listClient,
+            });
+
             if (!skipExecution) {
               for (const tuple of tuples) {
                 const runResult = await type.executor({
@@ -309,7 +318,8 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
                   runOpts: {
                     completeRule,
                     inputIndex,
-                    exceptionItems,
+                    exceptionFilter,
+                    unprocessedExceptions,
                     runtimeMappings: {
                       ...runtimeMappings,
                       ...timestampRuntimeMappings,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts

@@ -67,24 +67,23 @@ export const createEqlAlertType = (
           tuple,
           inputIndex,
           runtimeMappings,
-          exceptionItems,
           ruleExecutionLogger,
           bulkCreate,
           wrapHits,
           wrapSequences,
           primaryTimestamp,
           secondaryTimestamp,
+          exceptionFilter,
+          unprocessedExceptions,
         },
         services,
         state,
       } = execOptions;
-
       const result = await eqlExecutor({
         completeRule,
         tuple,
         inputIndex,
         runtimeMappings,
-        exceptionItems,
         ruleExecutionLogger,
         services,
         version,
@@ -93,6 +92,8 @@ export const createEqlAlertType = (
         wrapSequences,
         primaryTimestamp,
         secondaryTimestamp,
+        exceptionFilter,
+        unprocessedExceptions,
       });
       return { ...result, state };
     },

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts

@@ -68,7 +68,6 @@ export const createIndicatorMatchAlertType = (
           runtimeMappings,
           completeRule,
           tuple,
-          exceptionItems,
           listClient,
           ruleExecutionLogger,
           searchAfterSize,
@@ -76,6 +75,8 @@ export const createIndicatorMatchAlertType = (
           wrapHits,
           primaryTimestamp,
           secondaryTimestamp,
+          exceptionFilter,
+          unprocessedExceptions,
         },
         services,
         state,
@@ -87,7 +88,6 @@ export const createIndicatorMatchAlertType = (
         completeRule,
         tuple,
         listClient,
-        exceptionItems,
         services,
         version,
         searchAfterSize,
@@ -97,6 +97,8 @@ export const createIndicatorMatchAlertType = (
         wrapHits,
         primaryTimestamp,
         secondaryTimestamp,
+        exceptionFilter,
+        unprocessedExceptions,
       });
       return { ...result, state };
     },

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts

@@ -53,11 +53,12 @@ export const createMlAlertType = (
         runOpts: {
           bulkCreate,
           completeRule,
-          exceptionItems,
           listClient,
           ruleExecutionLogger,
           tuple,
           wrapHits,
+          exceptionFilter,
+          unprocessedExceptions,
         },
         services,
         state,
@@ -68,11 +69,12 @@ export const createMlAlertType = (
         tuple,
         ml,
         listClient,
-        exceptionItems,
         services,
         ruleExecutionLogger,
         bulkCreate,
         wrapHits,
+        exceptionFilter,
+        unprocessedExceptions,
       });
       return { ...result, state };
     },

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts

@@ -29,7 +29,11 @@ import {
 import type { SignalSource } from '../../signals/types';
 import { validateIndexPatterns } from '../utils';
 import { parseDateString, validateHistoryWindowStart } from './utils';
-import { addToSearchAfterReturn, createSearchAfterReturnType } from '../../signals/utils';
+import {
+  addToSearchAfterReturn,
+  createSearchAfterReturnType,
+  logUnprocessedExceptionsWarnings,
+} from '../../signals/utils';
 import { createEnrichEventsFunction } from '../../signals/enrichments';
 
 export const createNewTermsAlertType = (
@@ -87,7 +91,6 @@ export const createNewTermsAlertType = (
           ruleExecutionLogger,
           bulkCreate,
           completeRule,
-          exceptionItems,
           tuple,
           mergeStrategy,
           inputIndex,
@@ -95,6 +98,8 @@ export const createNewTermsAlertType = (
           primaryTimestamp,
           secondaryTimestamp,
           aggregatableTimestampField,
+          exceptionFilter,
+          unprocessedExceptions,
         },
         services,
         params,
@@ -109,7 +114,9 @@ export const createNewTermsAlertType = (
         from: params.from,
       });
 
-      const filter = await getFilter({
+      logUnprocessedExceptionsWarnings(unprocessedExceptions, ruleExecutionLogger);
+
+      const esFilter = await getFilter({
         filters: params.filters,
         index: inputIndex,
         language: params.language,
@@ -117,7 +124,7 @@ export const createNewTermsAlertType = (
         services,
         type: params.type,
         query: params.query,
-        lists: exceptionItems,
+        exceptionFilter,
       });
 
       const parsedHistoryWindowSize = parseDateString({
@@ -152,7 +159,7 @@ export const createNewTermsAlertType = (
           to: tuple.to.toISOString(),
           services,
           ruleExecutionLogger,
-          filter,
+          filter: esFilter,
           pageSize: 0,
           primaryTimestamp,
           secondaryTimestamp,
@@ -202,7 +209,7 @@ export const createNewTermsAlertType = (
           to: tuple.to.toISOString(),
           services,
           ruleExecutionLogger,
-          filter,
+          filter: esFilter,
           pageSize: 0,
           primaryTimestamp,
           secondaryTimestamp,
@@ -244,7 +251,7 @@ export const createNewTermsAlertType = (
             to: tuple.to.toISOString(),
             services,
             ruleExecutionLogger,
-            filter,
+            filter: esFilter,
             pageSize: 0,
             primaryTimestamp,
             secondaryTimestamp,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts

@@ -66,7 +66,6 @@ export const createQueryAlertType = (
           runtimeMappings,
           completeRule,
           tuple,
-          exceptionItems,
           listClient,
           ruleExecutionLogger,
           searchAfterSize,
@@ -74,15 +73,15 @@ export const createQueryAlertType = (
           wrapHits,
           primaryTimestamp,
           secondaryTimestamp,
+          unprocessedExceptions,
+          exceptionFilter,
         },
         services,
         state,
       } = execOptions;
-
       const result = await queryExecutor({
         completeRule,
         tuple,
-        exceptionItems,
         listClient,
         experimentalFeatures,
         ruleExecutionLogger,
@@ -96,6 +95,8 @@ export const createQueryAlertType = (
         runtimeMappings,
         primaryTimestamp,
         secondaryTimestamp,
+        unprocessedExceptions,
+        exceptionFilter,
       });
       return { ...result, state };
     },

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/saved_query/create_saved_query_alert_type.ts

@@ -66,7 +66,6 @@ export const createSavedQueryAlertType = (
           runtimeMappings,
           completeRule,
           tuple,
-          exceptionItems,
           listClient,
           ruleExecutionLogger,
           searchAfterSize,
@@ -74,6 +73,8 @@ export const createSavedQueryAlertType = (
           wrapHits,
           primaryTimestamp,
           secondaryTimestamp,
+          exceptionFilter,
+          unprocessedExceptions,
         },
         services,
         state,
@@ -84,7 +85,6 @@ export const createSavedQueryAlertType = (
         runtimeMappings,
         completeRule: completeRule as CompleteRule<UnifiedQueryRuleParams>,
         tuple,
-        exceptionItems,
         experimentalFeatures,
         listClient,
         ruleExecutionLogger,
@@ -96,6 +96,8 @@ export const createSavedQueryAlertType = (
         wrapHits,
         primaryTimestamp,
         secondaryTimestamp,
+        exceptionFilter,
+        unprocessedExceptions,
       });
       return { ...result, state };
     },

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts

@@ -65,7 +65,6 @@ export const createThresholdAlertType = (
       const {
         runOpts: {
           bulkCreate,
-          exceptionItems,
           completeRule,
           tuple,
           wrapHits,
@@ -76,16 +75,16 @@ export const createThresholdAlertType = (
           secondaryTimestamp,
           ruleExecutionLogger,
           aggregatableTimestampField,
+          exceptionFilter,
+          unprocessedExceptions,
         },
         services,
         startedAt,
         state,
       } = execOptions;
-
       const result = await thresholdExecutor({
         completeRule,
         tuple,
-        exceptionItems,
         ruleExecutionLogger,
         services,
         version,
@@ -99,8 +98,9 @@ export const createThresholdAlertType = (
         primaryTimestamp,
         secondaryTimestamp,
         aggregatableTimestampField,
+        exceptionFilter,
+        unprocessedExceptions,
       });
-
       return result;
     },
   };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts

@@ -25,6 +25,7 @@ import type {
   IRuleDataReader,
 } from '@kbn/rule-registry-plugin/server';
 
+import type { Filter } from '@kbn/es-query';
 import type { ConfigType } from '../../../config';
 import type { SetupPlugins } from '../../../plugin';
 import type { CompleteRule, RuleParams } from '../schemas/rule_schemas';
@@ -58,7 +59,6 @@ export interface RunOpts<TParams extends RuleParams> {
     from: Moment;
     maxSignals: number;
   };
-  exceptionItems: ExceptionListItemSchema[];
   ruleExecutionLogger: IRuleExecutionLogForExecutors;
   listClient: ListClient;
   searchAfterSize: number;
@@ -72,6 +72,8 @@ export interface RunOpts<TParams extends RuleParams> {
   primaryTimestamp: string;
   secondaryTimestamp?: string;
   aggregatableTimestampField: string;
+  unprocessedExceptions: ExceptionListItemSchema[];
+  exceptionFilter: Filter | undefined;
 }
 
 export type SecurityAlertType<

x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts

@@ -7,6 +7,8 @@
 
 import { buildEqlSearchRequest, buildEventsSearchQuery } from './build_events_query';
 import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
+import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock';
+import { buildExceptionFilter } from '@kbn/lists-plugin/server/services/exception_lists';
 
 const emptyFilter = {
   bool: {
@@ -564,9 +566,9 @@ describe('create_signals', () => {
         filters: undefined,
         primaryTimestamp: '@timestamp',
         secondaryTimestamp: undefined,
-        exceptionLists: [],
         runtimeMappings: undefined,
         eventCategoryOverride: undefined,
+        exceptionFilter: undefined,
       });
       expect(request).toEqual({
         allow_no_indices: true,
@@ -615,10 +617,10 @@ describe('create_signals', () => {
         filters: undefined,
         primaryTimestamp: 'event.ingested',
         secondaryTimestamp: '@timestamp',
-        exceptionLists: [],
         runtimeMappings: undefined,
         eventCategoryOverride: 'event.other_category',
         timestampField: undefined,
+        exceptionFilter: undefined,
       });
       expect(request).toEqual({
         allow_no_indices: true,
@@ -703,10 +705,10 @@ describe('create_signals', () => {
         filters: undefined,
         primaryTimestamp: 'event.ingested',
         secondaryTimestamp: undefined,
-        exceptionLists: [],
         runtimeMappings: undefined,
         eventCategoryOverride: 'event.other_category',
         timestampField: undefined,
+        exceptionFilter: undefined,
       });
       expect(request).toEqual({
         allow_no_indices: true,
@@ -746,7 +748,14 @@ describe('create_signals', () => {
       });
     });
 
-    test('should build a request with exceptions', () => {
+    test('should build a request with exceptions', async () => {
+      const { filter } = await buildExceptionFilter({
+        listClient: getListClientMock(),
+        lists: [getExceptionListItemSchemaMock()],
+        alias: null,
+        chunkSize: 1024,
+        excludeExceptions: true,
+      });
       const request = buildEqlSearchRequest({
         query: 'process where true',
         index: ['testindex1', 'testindex2'],
@@ -756,95 +765,103 @@ describe('create_signals', () => {
         filters: undefined,
         primaryTimestamp: '@timestamp',
         secondaryTimestamp: undefined,
-        exceptionLists: [getExceptionListItemSchemaMock()],
         runtimeMappings: undefined,
         eventCategoryOverride: undefined,
+        exceptionFilter: filter,
       });
-      expect(request).toEqual({
-        allow_no_indices: true,
-        index: ['testindex1', 'testindex2'],
-        body: {
-          size: 100,
-          query: 'process where true',
-          runtime_mappings: undefined,
-          filter: {
-            bool: {
-              filter: [
-                {
-                  range: {
-                    '@timestamp': {
-                      gte: 'now-5m',
-                      lte: 'now',
-                      format: 'strict_date_optional_time',
+      expect(request).toMatchInlineSnapshot(`
+        Object {
+          "allow_no_indices": true,
+          "body": Object {
+            "event_category_field": undefined,
+            "fields": Array [
+              Object {
+                "field": "*",
+                "include_unmapped": true,
+              },
+              Object {
+                "field": "@timestamp",
+                "format": "strict_date_optional_time",
+              },
+            ],
+            "filter": Object {
+              "bool": Object {
+                "filter": Array [
+                  Object {
+                    "range": Object {
+                      "@timestamp": Object {
+                        "format": "strict_date_optional_time",
+                        "gte": "now-5m",
+                        "lte": "now",
+                      },
                     },
                   },
-                },
-                {
-                  bool: {
-                    must: [],
-                    filter: [],
-                    should: [],
-                    must_not: [
-                      {
-                        bool: {
-                          should: [
-                            {
-                              bool: {
-                                filter: [
-                                  {
-                                    nested: {
-                                      path: 'some.parentField',
-                                      query: {
-                                        bool: {
-                                          minimum_should_match: 1,
-                                          should: [
-                                            {
-                                              match_phrase: {
-                                                'some.parentField.nested.field': 'some value',
+                  Object {
+                    "bool": Object {
+                      "filter": Array [],
+                      "must": Array [],
+                      "must_not": Array [
+                        Object {
+                          "bool": Object {
+                            "should": Array [
+                              Object {
+                                "bool": Object {
+                                  "filter": Array [
+                                    Object {
+                                      "nested": Object {
+                                        "path": "some.parentField",
+                                        "query": Object {
+                                          "bool": Object {
+                                            "minimum_should_match": 1,
+                                            "should": Array [
+                                              Object {
+                                                "match_phrase": Object {
+                                                  "some.parentField.nested.field": "some value",
+                                                },
                                               },
-                                            },
-                                          ],
-                                        },
-                                      },
-                                      score_mode: 'none',
-                                    },
-                                  },
-                                  {
-                                    bool: {
-                                      minimum_should_match: 1,
-                                      should: [
-                                        {
-                                          match_phrase: {
-                                            'some.not.nested.field': 'some value',
+                                            ],
                                           },
                                         },
-                                      ],
+                                        "score_mode": "none",
+                                      },
                                     },
-                                  },
-                                ],
+                                    Object {
+                                      "bool": Object {
+                                        "minimum_should_match": 1,
+                                        "should": Array [
+                                          Object {
+                                            "match_phrase": Object {
+                                              "some.not.nested.field": "some value",
+                                            },
+                                          },
+                                        ],
+                                      },
+                                    },
+                                  ],
+                                },
                               },
-                            },
-                          ],
+                            ],
+                          },
                         },
-                      },
-                    ],
+                      ],
+                      "should": Array [],
+                    },
                   },
-                },
-              ],
+                ],
+              },
             },
+            "query": "process where true",
+            "runtime_mappings": undefined,
+            "size": 100,
+            "tiebreaker_field": undefined,
+            "timestamp_field": undefined,
           },
-          fields: [
-            {
-              field: '*',
-              include_unmapped: true,
-            },
-            {
-              field: '@timestamp',
-              format: 'strict_date_optional_time',
-            },
+          "index": Array [
+            "testindex1",
+            "testindex2",
           ],
-        },
-      });
+        }
+      `);
     });
 
     test('should build a request with filters', () => {
@@ -891,8 +908,8 @@ describe('create_signals', () => {
         filters,
         primaryTimestamp: '@timestamp',
         secondaryTimestamp: undefined,
-        exceptionLists: [],
         runtimeMappings: undefined,
+        exceptionFilter: undefined,
       });
       expect(request).toEqual({
         allow_no_indices: true,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts

@@ -5,14 +5,14 @@
  * 2.0.
  */
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
 import { isEmpty } from 'lodash';
+import type { Filter } from '@kbn/es-query';
 import type {
   FiltersOrUndefined,
   TimestampOverrideOrUndefined,
   TimestampOverride,
 } from '../../../../common/detection_engine/schemas/common/schemas';
-import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter';
+import { getQueryFilter } from './get_query_filter';
 
 interface BuildEventsSearchQuery {
   aggregations?: Record<string, estypes.AggregationsAggregationContainer>;
@@ -38,11 +38,11 @@ interface BuildEqlSearchRequestParams {
   filters: FiltersOrUndefined;
   primaryTimestamp: TimestampOverride;
   secondaryTimestamp: TimestampOverrideOrUndefined;
-  exceptionLists: ExceptionListItemSchema[];
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
   eventCategoryOverride?: string;
   timestampField?: string;
   tiebreakerField?: string;
+  exceptionFilter: Filter | undefined;
 }
 
 const buildTimeRangeFilter = ({
@@ -217,11 +217,11 @@ export const buildEqlSearchRequest = ({
   filters,
   primaryTimestamp,
   secondaryTimestamp,
-  exceptionLists,
   runtimeMappings,
   eventCategoryOverride,
   timestampField,
   tiebreakerField,
+  exceptionFilter,
 }: BuildEqlSearchRequestParams): estypes.EqlSearchRequest => {
   const timestamps = secondaryTimestamp
     ? [primaryTimestamp, secondaryTimestamp]
@@ -231,7 +231,13 @@ export const buildEqlSearchRequest = ({
     format: 'strict_date_optional_time',
   }));
 
-  const esFilter = getQueryFilter('', 'eql', filters || [], index, exceptionLists);
+  const esFilter = getQueryFilter({
+    query: '',
+    language: 'eql',
+    filters: filters || [],
+    index,
+    exceptionFilter,
+  });
 
   const rangeFilter = buildTimeRangeFilter({
     to,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/enrichments/search_enrichments.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter';
+import { getQueryFilter } from '../get_query_filter';
 import type { SearchEnrichments } from './types';
 
 export const searchEnrichments: SearchEnrichments = async ({ index, services, query, fields }) => {
@@ -15,7 +15,13 @@ export const searchEnrichments: SearchEnrichments = async ({ index, services, qu
       body: {
         _source: '',
         fields,
-        query: getQueryFilter('', 'kuery', [query], index, []),
+        query: getQueryFilter({
+          query: '',
+          language: 'kuery',
+          filters: [query],
+          index,
+          exceptionFilter: undefined,
+        }),
       },
       track_total_hits: false,
     });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts

@@ -9,7 +9,6 @@ import dateMath from '@kbn/datemath';
 import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks';
 import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
 import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
-import { getEntryListMock } from '@kbn/lists-plugin/common/schemas/types/entry_list.mock';
 import { DEFAULT_INDEX_PATTERN } from '../../../../../common/constants';
 import { getIndexVersion } from '../../routes/index/get_index_version';
 import { SIGNALS_TEMPLATE_VERSION } from '../../routes/index/get_signals_template';
@@ -46,13 +45,11 @@ describe('eql_executor', () => {
 
   describe('eqlExecutor', () => {
     it('should set a warning when exception list for eql rule contains value list exceptions', async () => {
-      const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })];
-      const response = await eqlExecutor({
+      await eqlExecutor({
         inputIndex: DEFAULT_INDEX_PATTERN,
         runtimeMappings: {},
         completeRule: eqlCompleteRule,
         tuple,
-        exceptionItems,
         ruleExecutionLogger,
         services: alertServices,
         version,
@@ -60,8 +57,13 @@ describe('eql_executor', () => {
         wrapHits: jest.fn(),
         wrapSequences: jest.fn(),
         primaryTimestamp: '@timestamp',
+        exceptionFilter: undefined,
+        unprocessedExceptions: [getExceptionListItemSchemaMock()],
       });
-      expect(response.warningMessages.length).toEqual(1);
+      expect(ruleExecutionLogger.warn).toHaveBeenCalled();
+      expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain(
+        "The following exceptions won't be applied to rule execution"
+      );
     });
   });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts

@@ -13,9 +13,8 @@ import type {
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-
+import type { Filter } from '@kbn/es-query';
 import { buildEqlSearchRequest } from '../build_events_query';
-import { hasLargeValueItem } from '../../../../../common/detection_engine/utils';
 import { createEnrichEventsFunction } from '../enrichments';
 
 import type {
@@ -26,7 +25,12 @@ import type {
   SearchAfterAndBulkCreateReturnType,
   SignalSource,
 } from '../types';
-import { addToSearchAfterReturn, createSearchAfterReturnType, makeFloatString } from '../utils';
+import {
+  addToSearchAfterReturn,
+  createSearchAfterReturnType,
+  makeFloatString,
+  logUnprocessedExceptionsWarnings,
+} from '../utils';
 import { buildReasonMessageForEqlAlert } from '../reason_formatters';
 import type { CompleteRule, EqlRuleParams } from '../../schemas/rule_schemas';
 import { withSecuritySpan } from '../../../../utils/with_security_span';
@@ -41,7 +45,6 @@ export const eqlExecutor = async ({
   runtimeMappings,
   completeRule,
   tuple,
-  exceptionItems,
   ruleExecutionLogger,
   services,
   version,
@@ -50,12 +53,13 @@ export const eqlExecutor = async ({
   wrapSequences,
   primaryTimestamp,
   secondaryTimestamp,
+  exceptionFilter,
+  unprocessedExceptions,
 }: {
   inputIndex: string[];
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
   completeRule: CompleteRule<EqlRuleParams>;
   tuple: RuleRangeTuple;
-  exceptionItems: ExceptionListItemSchema[];
   ruleExecutionLogger: IRuleExecutionLogForExecutors;
   services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
   version: string;
@@ -64,35 +68,32 @@ export const eqlExecutor = async ({
   wrapSequences: WrapSequences;
   primaryTimestamp: string;
   secondaryTimestamp?: string;
+  exceptionFilter: Filter | undefined;
+  unprocessedExceptions: ExceptionListItemSchema[];
 }): Promise<SearchAfterAndBulkCreateReturnType> => {
   const ruleParams = completeRule.ruleParams;
 
   return withSecuritySpan('eqlExecutor', async () => {
     const result = createSearchAfterReturnType();
-    if (hasLargeValueItem(exceptionItems)) {
-      result.warningMessages.push(
-        'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules'
-      );
-      result.warning = true;
-    }
 
     const request = buildEqlSearchRequest({
       query: ruleParams.query,
       index: inputIndex,
       from: tuple.from.toISOString(),
       to: tuple.to.toISOString(),
-      size: completeRule.ruleParams.maxSignals,
+      size: ruleParams.maxSignals,
       filters: ruleParams.filters,
       primaryTimestamp,
       secondaryTimestamp,
-      exceptionLists: exceptionItems,
       runtimeMappings,
       eventCategoryOverride: ruleParams.eventCategoryOverride,
       timestampField: ruleParams.timestampField,
       tiebreakerField: ruleParams.tiebreakerField,
+      exceptionFilter,
     });
 
     ruleExecutionLogger.debug(`EQL query request: ${JSON.stringify(request)}`);
+    logUnprocessedExceptionsWarnings(unprocessedExceptions, ruleExecutionLogger);
 
     const eqlSignalSearchStart = performance.now();
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts

@@ -9,7 +9,6 @@ import dateMath from '@kbn/datemath';
 import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks';
 import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
 import { mlExecutor } from './ml';
-import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
 import { getCompleteRuleMock, getMlRuleParams } from '../../schemas/rule_schemas.mock';
 import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock';
 import { findMlSignals } from '../find_ml_signals';
@@ -28,12 +27,12 @@ describe('ml_executor', () => {
   let ruleExecutionLogger: ReturnType<typeof ruleExecutionLogMock.forExecutors.create>;
   const params = getMlRuleParams();
   const mlCompleteRule = getCompleteRuleMock<MachineLearningRuleParams>(params);
-  const exceptionItems = [getExceptionListItemSchemaMock()];
   const tuple = {
     from: dateMath.parse(params.from)!,
     to: dateMath.parse(params.to)!,
     maxSignals: params.maxSignals,
   };
+  const listClient = getListClientMock();
 
   beforeEach(() => {
     jobsSummaryMock = jest.fn();
@@ -69,12 +68,13 @@ describe('ml_executor', () => {
         completeRule: mlCompleteRule,
         tuple,
         ml: undefined,
-        exceptionItems,
         services: alertServices,
         ruleExecutionLogger,
-        listClient: getListClientMock(),
+        listClient,
         bulkCreate: jest.fn(),
         wrapHits: jest.fn(),
+        exceptionFilter: undefined,
+        unprocessedExceptions: [],
       })
     ).rejects.toThrow('ML plugin unavailable during rule execution');
   });
@@ -85,12 +85,13 @@ describe('ml_executor', () => {
       completeRule: mlCompleteRule,
       tuple,
       ml: mlMock,
-      exceptionItems,
       services: alertServices,
       ruleExecutionLogger,
-      listClient: getListClientMock(),
+      listClient,
       bulkCreate: jest.fn(),
       wrapHits: jest.fn(),
+      exceptionFilter: undefined,
+      unprocessedExceptions: [],
     });
     expect(ruleExecutionLogger.warn).toHaveBeenCalled();
     expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain(
@@ -112,12 +113,13 @@ describe('ml_executor', () => {
       completeRule: mlCompleteRule,
       tuple,
       ml: mlMock,
-      exceptionItems,
       services: alertServices,
       ruleExecutionLogger,
-      listClient: getListClientMock(),
+      listClient,
       bulkCreate: jest.fn(),
       wrapHits: jest.fn(),
+      exceptionFilter: undefined,
+      unprocessedExceptions: [],
     });
     expect(ruleExecutionLogger.warn).toHaveBeenCalled();
     expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain(

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts

@@ -13,6 +13,7 @@ import type {
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
 import type { ListClient } from '@kbn/lists-plugin/server';
+import type { Filter } from '@kbn/es-query';
 import { isJobStarted } from '../../../../../common/machine_learning/helpers';
 import type { CompleteRule, MachineLearningRuleParams } from '../../schemas/rule_schemas';
 import { bulkCreateMlSignals } from '../bulk_create_ml_signals';
@@ -34,21 +35,23 @@ export const mlExecutor = async ({
   tuple,
   ml,
   listClient,
-  exceptionItems,
   services,
   ruleExecutionLogger,
   bulkCreate,
   wrapHits,
+  exceptionFilter,
+  unprocessedExceptions,
 }: {
   completeRule: CompleteRule<MachineLearningRuleParams>;
   tuple: RuleRangeTuple;
   ml: SetupPlugins['ml'];
   listClient: ListClient;
-  exceptionItems: ExceptionListItemSchema[];
   services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
   ruleExecutionLogger: IRuleExecutionLogForExecutors;
   bulkCreate: BulkCreate;
   wrapHits: WrapHits;
+  exceptionFilter: Filter | undefined;
+  unprocessedExceptions: ExceptionListItemSchema[];
 }) => {
   const result = createSearchAfterReturnType();
   const ruleParams = completeRule.ruleParams;
@@ -98,13 +101,13 @@ export const mlExecutor = async ({
       anomalyThreshold: ruleParams.anomalyThreshold,
       from: tuple.from.toISOString(),
       to: tuple.to.toISOString(),
-      exceptionItems,
+      exceptionFilter,
     });
 
     const [filteredAnomalyHits, _] = await filterEventsAgainstList({
       listClient,
-      exceptionsList: exceptionItems,
       ruleExecutionLogger,
+      exceptionsList: unprocessedExceptions,
       events: anomalyResults.hits.hits,
     });
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts

@@ -14,6 +14,7 @@ import type {
 import type { ListClient } from '@kbn/lists-plugin/server';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
+import type { Filter } from '@kbn/es-query';
 import { getFilter } from '../get_filter';
 import { searchAfterAndBulkCreate } from '../search_after_bulk_create';
 import type { RuleRangeTuple, BulkCreate, WrapHits } from '../types';
@@ -29,7 +30,6 @@ export const queryExecutor = async ({
   runtimeMappings,
   completeRule,
   tuple,
-  exceptionItems,
   listClient,
   experimentalFeatures,
   ruleExecutionLogger,
@@ -41,12 +41,13 @@ export const queryExecutor = async ({
   wrapHits,
   primaryTimestamp,
   secondaryTimestamp,
+  unprocessedExceptions,
+  exceptionFilter,
 }: {
   inputIndex: string[];
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
   completeRule: CompleteRule<UnifiedQueryRuleParams>;
   tuple: RuleRangeTuple;
-  exceptionItems: ExceptionListItemSchema[];
   listClient: ListClient;
   experimentalFeatures: ExperimentalFeatures;
   ruleExecutionLogger: IRuleExecutionLogForExecutors;
@@ -58,6 +59,8 @@ export const queryExecutor = async ({
   wrapHits: WrapHits;
   primaryTimestamp: string;
   secondaryTimestamp?: string;
+  unprocessedExceptions: ExceptionListItemSchema[];
+  exceptionFilter: Filter | undefined;
 }) => {
   const ruleParams = completeRule.ruleParams;
 
@@ -70,14 +73,14 @@ export const queryExecutor = async ({
       savedId: ruleParams.savedId,
       services,
       index: inputIndex,
-      lists: exceptionItems,
+      exceptionFilter,
     });
 
     return searchAfterAndBulkCreate({
       tuple,
+      exceptionsList: unprocessedExceptions,
       services,
       listClient,
-      exceptionsList: exceptionItems,
       ruleExecutionLogger,
       eventsTelemetry,
       inputIndexPattern: inputIndex,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts

@@ -14,6 +14,7 @@ import type {
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
 import type { ListClient } from '@kbn/lists-plugin/server';
+import type { Filter } from '@kbn/es-query';
 import type { RuleRangeTuple, BulkCreate, WrapHits } from '../types';
 import type { ITelemetryEventsSender } from '../../../telemetry/sender';
 import { createThreatSignals } from '../threat_mapping/create_threat_signals';
@@ -27,7 +28,6 @@ export const threatMatchExecutor = async ({
   runtimeMappings,
   completeRule,
   tuple,
-  exceptionItems,
   listClient,
   services,
   version,
@@ -38,12 +38,13 @@ export const threatMatchExecutor = async ({
   wrapHits,
   primaryTimestamp,
   secondaryTimestamp,
+  exceptionFilter,
+  unprocessedExceptions,
 }: {
   inputIndex: string[];
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
   completeRule: CompleteRule<ThreatRuleParams>;
   tuple: RuleRangeTuple;
-  exceptionItems: ExceptionListItemSchema[];
   listClient: ListClient;
   services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
   version: string;
@@ -54,6 +55,8 @@ export const threatMatchExecutor = async ({
   wrapHits: WrapHits;
   primaryTimestamp: string;
   secondaryTimestamp?: string;
+  exceptionFilter: Filter | undefined;
+  unprocessedExceptions: ExceptionListItemSchema[];
 }) => {
   const ruleParams = completeRule.ruleParams;
 
@@ -64,7 +67,6 @@ export const threatMatchExecutor = async ({
       completeRule,
       concurrentSearches: ruleParams.concurrentSearches ?? 1,
       eventsTelemetry,
-      exceptionItems,
       filters: ruleParams.filters ?? [],
       inputIndex,
       itemsPerSearch: ruleParams.itemsPerSearch ?? 9000,
@@ -88,6 +90,8 @@ export const threatMatchExecutor = async ({
       runtimeMappings,
       primaryTimestamp,
       secondaryTimestamp,
+      exceptionFilter,
+      unprocessedExceptions,
     });
   });
 };

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts

@@ -7,23 +7,21 @@
 
 import dateMath from '@kbn/datemath';
 import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
+import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
 import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks';
 import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
 import { thresholdExecutor } from './threshold';
-import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
-import { getEntryListMock } from '@kbn/lists-plugin/common/schemas/types/entry_list.mock';
 import { getThresholdRuleParams, getCompleteRuleMock } from '../../schemas/rule_schemas.mock';
 import { sampleEmptyAggsSearchResults } from '../__mocks__/es_results';
 import { getThresholdTermsHash } from '../utils';
 import type { ThresholdRuleParams } from '../../schemas/rule_schemas';
 import { createRuleDataClientMock } from '@kbn/rule-registry-plugin/server/rule_data_client/rule_data_client.mock';
 import { TIMESTAMP } from '@kbn/rule-data-utils';
-import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
 import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
 
 describe('threshold_executor', () => {
   let alertServices: RuleExecutorServicesMock;
-  let ruleExecutionLogger: IRuleExecutionLogForExecutors;
+  let ruleExecutionLogger: ReturnType<typeof ruleExecutionLogMock.forExecutors.create>;
 
   const version = '8.0.0';
   const params = getThresholdRuleParams();
@@ -53,35 +51,6 @@ describe('threshold_executor', () => {
   });
 
   describe('thresholdExecutor', () => {
-    it('should set a warning when exception list for threshold rule contains value list exceptions', async () => {
-      const ruleDataClientMock = createRuleDataClientMock();
-      const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })];
-      const response = await thresholdExecutor({
-        completeRule: thresholdCompleteRule,
-        tuple,
-        exceptionItems,
-        services: alertServices,
-        state: { initialized: true, signalHistory: {} },
-        version,
-        ruleExecutionLogger,
-        startedAt: new Date(),
-        bulkCreate: jest.fn().mockImplementation((hits) => ({
-          errors: [],
-          success: true,
-          bulkCreateDuration: '0',
-          createdItemsCount: 0,
-          createdItems: [],
-        })),
-        wrapHits: jest.fn(),
-        ruleDataReader: ruleDataClientMock.getReader({ namespace: 'default' }),
-        runtimeMappings: {},
-        inputIndex: ['auditbeat-*'],
-        primaryTimestamp: TIMESTAMP,
-        aggregatableTimestampField: TIMESTAMP,
-      });
-      expect(response.warningMessages.length).toEqual(1);
-    });
-
     it('should clean up any signal history that has fallen outside the window when state is initialized', async () => {
       const ruleDataClientMock = createRuleDataClientMock();
       const terms1 = [
@@ -114,7 +83,6 @@ describe('threshold_executor', () => {
       const response = await thresholdExecutor({
         completeRule: thresholdCompleteRule,
         tuple,
-        exceptionItems: [],
         services: alertServices,
         state,
         version,
@@ -133,6 +101,8 @@ describe('threshold_executor', () => {
         inputIndex: ['auditbeat-*'],
         primaryTimestamp: TIMESTAMP,
         aggregatableTimestampField: TIMESTAMP,
+        exceptionFilter: undefined,
+        unprocessedExceptions: [],
       });
       expect(response.state).toEqual({
         initialized: true,
@@ -141,5 +111,64 @@ describe('threshold_executor', () => {
         },
       });
     });
+
+    it('should log a warning if unprocessedExceptions is not empty', async () => {
+      const ruleDataClientMock = createRuleDataClientMock();
+      const terms1 = [
+        {
+          field: 'host.name',
+          value: 'elastic-pc-1',
+        },
+      ];
+      const signalHistoryRecord1 = {
+        terms: terms1,
+        lastSignalTimestamp: tuple.from.valueOf() - 60 * 1000,
+      };
+      const terms2 = [
+        {
+          field: 'host.name',
+          value: 'elastic-pc-2',
+        },
+      ];
+      const signalHistoryRecord2 = {
+        terms: terms2,
+        lastSignalTimestamp: tuple.from.valueOf() + 60 * 1000,
+      };
+      const state = {
+        initialized: true,
+        signalHistory: {
+          [`${getThresholdTermsHash(terms1)}`]: signalHistoryRecord1,
+          [`${getThresholdTermsHash(terms2)}`]: signalHistoryRecord2,
+        },
+      };
+      await thresholdExecutor({
+        completeRule: thresholdCompleteRule,
+        tuple,
+        services: alertServices,
+        state,
+        version,
+        ruleExecutionLogger,
+        startedAt: new Date(),
+        bulkCreate: jest.fn().mockImplementation((hits) => ({
+          errors: [],
+          success: true,
+          bulkCreateDuration: '0',
+          createdItemsCount: 0,
+          createdItems: [],
+        })),
+        wrapHits: jest.fn(),
+        ruleDataReader: ruleDataClientMock.getReader({ namespace: 'default' }),
+        runtimeMappings: {},
+        inputIndex: ['auditbeat-*'],
+        primaryTimestamp: TIMESTAMP,
+        aggregatableTimestampField: TIMESTAMP,
+        exceptionFilter: undefined,
+        unprocessedExceptions: [getExceptionListItemSchemaMock()],
+      });
+      expect(ruleExecutionLogger.warn).toHaveBeenCalled();
+      expect(ruleExecutionLogger.warn.mock.calls[0][0]).toContain(
+        "The following exceptions won't be applied to rule execution"
+      );
+    });
   });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts

@@ -15,7 +15,7 @@ import type {
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
 import type { IRuleDataReader } from '@kbn/rule-registry-plugin/server';
-import { hasLargeValueItem } from '../../../../../common/detection_engine/utils';
+import type { Filter } from '@kbn/es-query';
 import type { CompleteRule, ThresholdRuleParams } from '../../schemas/rule_schemas';
 import { getFilter } from '../get_filter';
 import {
@@ -31,7 +31,11 @@ import type {
   ThresholdAlertState,
   WrapHits,
 } from '../types';
-import { addToSearchAfterReturn, createSearchAfterReturnType } from '../utils';
+import {
+  addToSearchAfterReturn,
+  createSearchAfterReturnType,
+  logUnprocessedExceptionsWarnings,
+} from '../utils';
 import { withSecuritySpan } from '../../../../utils/with_security_span';
 import { buildThresholdSignalHistory } from '../threshold/build_signal_history';
 import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
@@ -41,7 +45,6 @@ export const thresholdExecutor = async ({
   runtimeMappings,
   completeRule,
   tuple,
-  exceptionItems,
   ruleExecutionLogger,
   services,
   version,
@@ -53,12 +56,13 @@ export const thresholdExecutor = async ({
   primaryTimestamp,
   secondaryTimestamp,
   aggregatableTimestampField,
+  exceptionFilter,
+  unprocessedExceptions,
 }: {
   inputIndex: string[];
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
   completeRule: CompleteRule<ThresholdRuleParams>;
   tuple: RuleRangeTuple;
-  exceptionItems: ExceptionListItemSchema[];
   services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
   ruleExecutionLogger: IRuleExecutionLogForExecutors;
   version: string;
@@ -70,11 +74,15 @@ export const thresholdExecutor = async ({
   primaryTimestamp: string;
   secondaryTimestamp?: string;
   aggregatableTimestampField: string;
+  exceptionFilter: Filter | undefined;
+  unprocessedExceptions: ExceptionListItemSchema[];
 }): Promise<SearchAfterAndBulkCreateReturnType & { state: ThresholdAlertState }> => {
   const result = createSearchAfterReturnType();
   const ruleParams = completeRule.ruleParams;
 
   return withSecuritySpan('thresholdExecutor', async () => {
+    logUnprocessedExceptionsWarnings(unprocessedExceptions, ruleExecutionLogger);
+
     // Get state or build initial state (on upgrade)
     const { signalHistory, searchErrors: previousSearchErrors } = state.initialized
       ? { signalHistory: state.signalHistory, searchErrors: [] }
@@ -99,13 +107,6 @@ export const thresholdExecutor = async ({
       }
     }
 
-    if (hasLargeValueItem(exceptionItems)) {
-      result.warningMessages.push(
-        'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules'
-      );
-      result.warning = true;
-    }
-
     // Eliminate dupes
     const bucketFilters = await getThresholdBucketFilters({
       signalHistory,
@@ -121,7 +122,7 @@ export const thresholdExecutor = async ({
       savedId: ruleParams.savedId,
       services,
       index: inputIndex,
-      lists: exceptionItems,
+      exceptionFilter,
     });
 
     // Look for new events over threshold

x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts

@@ -6,10 +6,9 @@
  */
 
 import dateMath from '@kbn/datemath';
-import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
-
 import type { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server';
 import type { MlPluginSetup } from '@kbn/ml-plugin/server';
+import type { Filter } from '@kbn/es-query';
 import type { AnomalyResults } from '../../machine_learning';
 import { getAnomalies } from '../../machine_learning';
 
@@ -21,7 +20,7 @@ export const findMlSignals = async ({
   anomalyThreshold,
   from,
   to,
-  exceptionItems,
+  exceptionFilter,
 }: {
   ml: MlPluginSetup;
   request: KibanaRequest;
@@ -30,7 +29,7 @@ export const findMlSignals = async ({
   anomalyThreshold: number;
   from: string;
   to: string;
-  exceptionItems: ExceptionListItemSchema[];
+  exceptionFilter: Filter | undefined;
 }): Promise<AnomalyResults> => {
   const { mlAnomalySearch } = ml.mlSystemProvider(request, savedObjectsClient);
   const params = {
@@ -38,7 +37,7 @@ export const findMlSignals = async ({
     threshold: anomalyThreshold,
     earliestMs: dateMath.parse(from)?.valueOf() ?? 0,
     latestMs: dateMath.parse(to)?.valueOf() ?? 0,
-    exceptionItems,
+    exceptionFilter,
   };
   return getAnomalies(params, mlAnomalySearch);
 };

x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts

@@ -9,6 +9,8 @@ import { getFilter } from './get_filter';
 import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks';
 import { alertsMock } from '@kbn/alerting-plugin/server/mocks';
 import { getExceptionListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/response/exception_list_item_schema.mock';
+import { getListClientMock } from '@kbn/lists-plugin/server/services/lists/list_client.mock';
+import { buildExceptionFilter } from '@kbn/lists-plugin/server/services/exception_lists';
 
 describe('get_filter', () => {
   let servicesMock: RuleExecutorServicesMock;
@@ -37,7 +39,7 @@ describe('get_filter', () => {
 
   describe('getFilter', () => {
     test('returns a query if given a type of query', async () => {
-      const filter = await getFilter({
+      const esFilter = await getFilter({
         type: 'query',
         filters: undefined,
         language: 'kuery',
@@ -45,9 +47,9 @@ describe('get_filter', () => {
         savedId: undefined,
         services: servicesMock,
         index: ['auditbeat-*'],
-        lists: [],
+        exceptionFilter: undefined,
       });
-      expect(filter).toEqual({
+      expect(esFilter).toEqual({
         bool: {
           must: [],
           filter: [
@@ -80,7 +82,7 @@ describe('get_filter', () => {
           savedId: undefined,
           services: servicesMock,
           index: ['auditbeat-*'],
-          lists: [],
+          exceptionFilter: undefined,
         })
       ).rejects.toThrow('query, filters, and index parameter should be defined');
     });
@@ -95,7 +97,7 @@ describe('get_filter', () => {
           savedId: undefined,
           services: servicesMock,
           index: ['auditbeat-*'],
-          lists: [],
+          exceptionFilter: undefined,
         })
       ).rejects.toThrow('query, filters, and index parameter should be defined');
     });
@@ -110,13 +112,13 @@ describe('get_filter', () => {
           savedId: undefined,
           services: servicesMock,
           index: undefined,
-          lists: [],
+          exceptionFilter: undefined,
         })
       ).rejects.toThrow('query, filters, and index parameter should be defined');
     });
 
     test('returns a saved query if given a type of query', async () => {
-      const filter = await getFilter({
+      const esFilter = await getFilter({
         type: 'saved_query',
         filters: undefined,
         language: undefined,
@@ -124,9 +126,9 @@ describe('get_filter', () => {
         savedId: 'some-id',
         services: servicesMock,
         index: ['auditbeat-*'],
-        lists: [],
+        exceptionFilter: undefined,
       });
-      expect(filter).toEqual({
+      expect(esFilter).toEqual({
         bool: {
           filter: [
             { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } },
@@ -139,7 +141,7 @@ describe('get_filter', () => {
     });
 
     test('returns the query persisted to the threat_match rule, despite saved_id being specified', async () => {
-      const filter = await getFilter({
+      const esFilter = await getFilter({
         type: 'threat_match',
         filters: undefined,
         language: 'kuery',
@@ -147,9 +149,9 @@ describe('get_filter', () => {
         savedId: 'some-id',
         services: servicesMock,
         index: ['auditbeat-*'],
-        lists: [],
+        exceptionFilter: undefined,
       });
-      expect(filter).toEqual({
+      expect(esFilter).toEqual({
         bool: {
           filter: [
             { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'siem' } }] } },
@@ -162,7 +164,7 @@ describe('get_filter', () => {
     });
 
     test('returns the query persisted to the threshold rule, despite saved_id being specified', async () => {
-      const filter = await getFilter({
+      const esFilter = await getFilter({
         type: 'threat_match',
         filters: undefined,
         language: 'kuery',
@@ -170,9 +172,9 @@ describe('get_filter', () => {
         savedId: 'some-id',
         services: servicesMock,
         index: ['auditbeat-*'],
-        lists: [],
+        exceptionFilter: undefined,
       });
-      expect(filter).toEqual({
+      expect(esFilter).toEqual({
         bool: {
           filter: [
             { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'siem' } }] } },
@@ -194,7 +196,7 @@ describe('get_filter', () => {
           savedId: undefined,
           services: servicesMock,
           index: ['auditbeat-*'],
-          lists: [],
+          exceptionFilter: undefined,
         })
       ).rejects.toThrow('savedId parameter should be defined');
     });
@@ -209,7 +211,7 @@ describe('get_filter', () => {
           savedId: 'some-id',
           services: servicesMock,
           index: undefined,
-          lists: [],
+          exceptionFilter: undefined,
         })
       ).rejects.toThrow('savedId parameter should be defined');
     });
@@ -224,13 +226,20 @@ describe('get_filter', () => {
           savedId: 'some-id',
           services: servicesMock,
           index: undefined,
-          lists: [],
+          exceptionFilter: undefined,
         })
       ).rejects.toThrow('Unsupported Rule of type "machine_learning" supplied to getFilter');
     });
 
     test('returns a query when given a list', async () => {
-      const filter = await getFilter({
+      const { filter } = await buildExceptionFilter({
+        listClient: getListClientMock(),
+        lists: [getExceptionListItemSchemaMock()],
+        alias: null,
+        chunkSize: 1024,
+        excludeExceptions: true,
+      });
+      const esFilter = await getFilter({
         type: 'query',
         filters: undefined,
         language: 'kuery',
@@ -238,73 +247,75 @@ describe('get_filter', () => {
         savedId: undefined,
         services: servicesMock,
         index: ['auditbeat-*'],
-        lists: [getExceptionListItemSchemaMock()],
+        exceptionFilter: filter,
       });
 
-      expect(filter).toEqual({
-        bool: {
-          must: [],
-          filter: [
-            {
-              bool: {
-                should: [
-                  {
-                    match: {
-                      'host.name': 'siem',
+      expect(esFilter).toMatchInlineSnapshot(`
+        Object {
+          "bool": Object {
+            "filter": Array [
+              Object {
+                "bool": Object {
+                  "minimum_should_match": 1,
+                  "should": Array [
+                    Object {
+                      "match": Object {
+                        "host.name": "siem",
+                      },
                     },
-                  },
-                ],
-                minimum_should_match: 1,
+                  ],
+                },
               },
-            },
-          ],
-          must_not: [
-            {
-              bool: {
-                should: [
-                  {
-                    bool: {
-                      filter: [
-                        {
-                          nested: {
-                            path: 'some.parentField',
-                            query: {
-                              bool: {
-                                should: [
-                                  {
-                                    match_phrase: {
-                                      'some.parentField.nested.field': 'some value',
+            ],
+            "must": Array [],
+            "must_not": Array [
+              Object {
+                "bool": Object {
+                  "should": Array [
+                    Object {
+                      "bool": Object {
+                        "filter": Array [
+                          Object {
+                            "nested": Object {
+                              "path": "some.parentField",
+                              "query": Object {
+                                "bool": Object {
+                                  "minimum_should_match": 1,
+                                  "should": Array [
+                                    Object {
+                                      "match_phrase": Object {
+                                        "some.parentField.nested.field": "some value",
+                                      },
                                     },
-                                  },
-                                ],
-                                minimum_should_match: 1,
-                              },
-                            },
-                            score_mode: 'none',
-                          },
-                        },
-                        {
-                          bool: {
-                            should: [
-                              {
-                                match_phrase: {
-                                  'some.not.nested.field': 'some value',
+                                  ],
                                 },
                               },
-                            ],
-                            minimum_should_match: 1,
+                              "score_mode": "none",
+                            },
                           },
-                        },
-                      ],
+                          Object {
+                            "bool": Object {
+                              "minimum_should_match": 1,
+                              "should": Array [
+                                Object {
+                                  "match_phrase": Object {
+                                    "some.not.nested.field": "some value",
+                                  },
+                                },
+                              ],
+                            },
+                          },
+                        ],
+                      },
                     },
-                  },
-                ],
+                  ],
+                },
               },
-            },
-          ],
-          should: [],
-        },
-      });
+            ],
+            "should": Array [],
+          },
+        }
+      `);
     });
   });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts

@@ -11,14 +11,13 @@ import type {
   LanguageOrUndefined,
   Language,
 } from '@kbn/securitysolution-io-ts-alerting-types';
-import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
 import type {
   AlertInstanceContext,
   AlertInstanceState,
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
+import type { Filter } from '@kbn/es-query';
 import { assertUnreachable } from '../../../../common/utility_types';
-import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter';
 import type {
   QueryOrUndefined,
   SavedIdOrUndefined,
@@ -27,6 +26,7 @@ import type {
 import type { PartialFilter } from '../types';
 import { withSecuritySpan } from '../../../utils/with_security_span';
 import type { ESBoolQuery } from '../../../../common/typed_json';
+import { getQueryFilter } from './get_query_filter';
 
 interface GetFilterArgs {
   type: Type;
@@ -36,7 +36,7 @@ interface GetFilterArgs {
   savedId: SavedIdOrUndefined;
   services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
   index: IndexOrUndefined;
-  lists: ExceptionListItemSchema[];
+  exceptionFilter: Filter | undefined;
 }
 
 interface QueryAttributes {
@@ -56,11 +56,17 @@ export const getFilter = async ({
   services,
   type,
   query,
-  lists,
+  exceptionFilter,
 }: GetFilterArgs): Promise<ESBoolQuery> => {
   const queryFilter = () => {
     if (query != null && language != null && index != null) {
-      return getQueryFilter(query, language, filters || [], index, lists);
+      return getQueryFilter({
+        query,
+        language,
+        filters: filters || [],
+        index,
+        exceptionFilter,
+      });
     } else {
       throw new BadRequestError('query, filters, and index parameter should be defined');
     }
@@ -73,18 +79,24 @@ export const getFilter = async ({
         const savedObject = await withSecuritySpan('getSavedFilter', () =>
           services.savedObjectsClient.get<QueryAttributes>('query', savedId)
         );
-        return getQueryFilter(
-          savedObject.attributes.query.query,
-          savedObject.attributes.query.language,
-          savedObject.attributes.filters,
+        return getQueryFilter({
+          query: savedObject.attributes.query.query,
+          language: savedObject.attributes.query.language,
+          filters: savedObject.attributes.filters,
           index,
-          lists
-        );
+          exceptionFilter,
+        });
       } catch (err) {
         // saved object does not exist, so try and fall back if the user pushed
         // any additional language, query, filters, etc...
         if (query != null && language != null && index != null) {
-          return getQueryFilter(query, language, filters || [], index, lists);
+          return getQueryFilter({
+            query,
+            language,
+            filters: filters || [],
+            index,
+            exceptionFilter,
+          });
         } else if (savedId && index != null) {
           // if savedId present and we ending up here, then saved query failed to be fetched
           // and we also didn't fall back to saved in rule query

x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_query_filter.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Language } from '@kbn/securitysolution-io-ts-alerting-types';
+import type { Filter, EsQueryConfig, DataViewBase } from '@kbn/es-query';
+import { buildEsQuery } from '@kbn/es-query';
+import type { ESBoolQuery } from '../../../../common/typed_json';
+import type { Index, Query } from '../../../../common/detection_engine/schemas/common';
+
+export const getQueryFilter = ({
+  query,
+  language,
+  filters,
+  index,
+  exceptionFilter,
+}: {
+  query: Query;
+  language: Language;
+  filters: unknown;
+  index: Index;
+  exceptionFilter: Filter | undefined;
+}): ESBoolQuery => {
+  const indexPattern: DataViewBase = {
+    fields: [],
+    title: index.join(),
+  };
+
+  const config: EsQueryConfig = {
+    allowLeadingWildcards: true,
+    queryStringOptions: { analyze_wildcard: true },
+    ignoreFilterIfFieldNotInIndex: false,
+    dateFormatTZ: 'Zulu',
+  };
+
+  const initialQuery = { query, language };
+  const allFilters = getAllFilters(filters as Filter[], exceptionFilter);
+
+  return buildEsQuery(indexPattern, initialQuery, allFilters, config);
+};
+
+export const getAllFilters = (filters: Filter[], exceptionFilter: Filter | undefined): Filter[] => {
+  if (exceptionFilter != null) {
+    return [...filters, exceptionFilter];
+  } else {
+    return [...filters];
+  }
+};

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts

@@ -11,7 +11,6 @@ import type { BuildThreatEnrichmentOptions, GetMatchedThreats } from './types';
 import { getThreatList } from './get_threat_list';
 
 export const buildThreatEnrichment = ({
-  exceptionItems,
   ruleExecutionLogger,
   services,
   threatFilters,
@@ -22,6 +21,7 @@ export const buildThreatEnrichment = ({
   pitId,
   reassignPitId,
   listClient,
+  exceptionFilter,
 }: BuildThreatEnrichmentOptions): SignalsEnrichment => {
   const getMatchedThreats: GetMatchedThreats = async (ids) => {
     const matchedThreatsFilter = {
@@ -35,7 +35,6 @@ export const buildThreatEnrichment = ({
     };
     const threatResponse = await getThreatList({
       esClient: services.scopedClusterClient.asCurrentUser,
-      exceptionItems,
       index: threatIndex,
       language: threatLanguage,
       perPage: undefined,
@@ -51,6 +50,7 @@ export const buildThreatEnrichment = ({
       reassignPitId,
       runtimeMappings: undefined,
       listClient,
+      exceptionFilter,
     });
 
     return threatResponse.hits.hits;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_event_signal.ts

@@ -24,7 +24,6 @@ export const createEventSignal = async ({
   currentResult,
   currentEventList,
   eventsTelemetry,
-  exceptionItems,
   filters,
   inputIndex,
   language,
@@ -49,6 +48,8 @@ export const createEventSignal = async ({
   runtimeMappings,
   primaryTimestamp,
   secondaryTimestamp,
+  exceptionFilter,
+  unprocessedExceptions,
 }: CreateEventSignalOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
   const threatFilter = buildThreatMappingFilter({
     threatMapping,
@@ -66,7 +67,6 @@ export const createEventSignal = async ({
   } else {
     const threatListHits = await getAllThreatListHits({
       esClient: services.scopedClusterClient.asCurrentUser,
-      exceptionItems,
       threatFilters: [...threatFilters, threatFilter],
       query: threatQuery,
       language: threatLanguage,
@@ -80,6 +80,7 @@ export const createEventSignal = async ({
       reassignPitId: reassignThreatPitId,
       runtimeMappings,
       listClient,
+      exceptionFilter,
     });
 
     const signalMatches = getSignalMatchesFromThreatList(threatListHits);
@@ -104,7 +105,7 @@ export const createEventSignal = async ({
       savedId,
       services,
       index: inputIndex,
-      lists: exceptionItems,
+      exceptionFilter,
     });
 
     ruleExecutionLogger.debug(
@@ -124,7 +125,7 @@ export const createEventSignal = async ({
       bulkCreate,
       enrichment: threatEnrichment,
       eventsTelemetry,
-      exceptionsList: exceptionItems,
+      exceptionsList: unprocessedExceptions,
       filter: esFilter,
       inputIndexPattern: inputIndex,
       listClient,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts

@@ -20,7 +20,6 @@ export const createThreatSignal = async ({
   currentResult,
   currentThreatList,
   eventsTelemetry,
-  exceptionItems,
   filters,
   inputIndex,
   language,
@@ -39,6 +38,8 @@ export const createThreatSignal = async ({
   runtimeMappings,
   primaryTimestamp,
   secondaryTimestamp,
+  exceptionFilter,
+  unprocessedExceptions,
 }: CreateThreatSignalOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
   const threatFilter = buildThreatMappingFilter({
     threatMapping,
@@ -62,7 +63,7 @@ export const createThreatSignal = async ({
       savedId,
       services,
       index: inputIndex,
-      lists: exceptionItems,
+      exceptionFilter,
     });
 
     ruleExecutionLogger.debug(
@@ -74,7 +75,7 @@ export const createThreatSignal = async ({
       bulkCreate,
       enrichment: threatEnrichment,
       eventsTelemetry,
-      exceptionsList: exceptionItems,
+      exceptionsList: unprocessedExceptions,
       filter: esFilter,
       inputIndexPattern: inputIndex,
       listClient,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts

@@ -29,7 +29,6 @@ export const createThreatSignals = async ({
   completeRule,
   concurrentSearches,
   eventsTelemetry,
-  exceptionItems,
   filters,
   inputIndex,
   itemsPerSearch,
@@ -53,6 +52,8 @@ export const createThreatSignals = async ({
   runtimeMappings,
   primaryTimestamp,
   secondaryTimestamp,
+  exceptionFilter,
+  unprocessedExceptions,
 }: CreateThreatSignalsOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
   const params = completeRule.ruleParams;
   ruleExecutionLogger.debug('Indicator matching rule starting');
@@ -80,13 +81,13 @@ export const createThreatSignals = async ({
   const eventCount = await getEventCount({
     esClient: services.scopedClusterClient.asCurrentUser,
     index: inputIndex,
-    exceptionItems,
     tuple,
     query,
     language,
     filters: allEventFilters,
     primaryTimestamp,
     secondaryTimestamp,
+    exceptionFilter,
   });
 
   ruleExecutionLogger.debug(`Total event count: ${eventCount}`);
@@ -108,11 +109,11 @@ export const createThreatSignals = async ({
 
   const threatListCount = await getThreatListCount({
     esClient: services.scopedClusterClient.asCurrentUser,
-    exceptionItems,
     threatFilters: allThreatFilters,
     query: threatQuery,
     language: threatLanguage,
     index: threatIndex,
+    exceptionFilter,
   });
 
   ruleExecutionLogger.debug(`Total indicator items: ${threatListCount}`);
@@ -123,7 +124,6 @@ export const createThreatSignals = async ({
   };
 
   const threatEnrichment = buildThreatEnrichment({
-    exceptionItems,
     ruleExecutionLogger,
     services,
     threatFilters: allThreatFilters,
@@ -134,6 +134,7 @@ export const createThreatSignals = async ({
     pitId: threatPitId,
     reassignPitId: reassignThreatPitId,
     listClient,
+    exceptionFilter,
   });
 
   const createSignals = async ({
@@ -184,7 +185,6 @@ export const createThreatSignals = async ({
         getEventList({
           services,
           ruleExecutionLogger,
-          exceptionItems,
           filters: allEventFilters,
           query,
           language,
@@ -195,6 +195,7 @@ export const createThreatSignals = async ({
           runtimeMappings,
           primaryTimestamp,
           secondaryTimestamp,
+          exceptionFilter,
         }),
 
       createSignal: (slicedChunk) =>
@@ -205,7 +206,6 @@ export const createThreatSignals = async ({
           currentEventList: slicedChunk,
           currentResult: results,
           eventsTelemetry,
-          exceptionItems,
           filters: allEventFilters,
           inputIndex,
           language,
@@ -231,6 +231,8 @@ export const createThreatSignals = async ({
           runtimeMappings,
           primaryTimestamp,
           secondaryTimestamp,
+          exceptionFilter,
+          unprocessedExceptions,
         }),
     });
   } else {
@@ -239,7 +241,6 @@ export const createThreatSignals = async ({
       getDocumentList: async ({ searchAfter }) =>
         getThreatList({
           esClient: services.scopedClusterClient.asCurrentUser,
-          exceptionItems,
           threatFilters: allThreatFilters,
           query: threatQuery,
           language: threatLanguage,
@@ -252,6 +253,7 @@ export const createThreatSignals = async ({
           reassignPitId: reassignThreatPitId,
           runtimeMappings,
           listClient,
+          exceptionFilter,
         }),
 
       createSignal: (slicedChunk) =>
@@ -262,7 +264,6 @@ export const createThreatSignals = async ({
           currentResult: results,
           currentThreatList: slicedChunk,
           eventsTelemetry,
-          exceptionItems,
           filters: allEventFilters,
           inputIndex,
           language,
@@ -281,6 +282,8 @@ export const createThreatSignals = async ({
           runtimeMappings,
           primaryTimestamp,
           secondaryTimestamp,
+          exceptionFilter,
+          unprocessedExceptions,
         }),
     });
   }

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.test.ts

@@ -15,16 +15,16 @@ describe('getEventCount', () => {
     jest.clearAllMocks();
   });
 
-  it('can respect tuple', () => {
-    getEventCount({
+  it('can respect tuple', async () => {
+    await getEventCount({
       esClient,
       query: '*:*',
       language: 'kuery',
       filters: [],
-      exceptionItems: [],
       index: ['test-index'],
       tuple: { to: moment('2022-01-14'), from: moment('2022-01-13'), maxSignals: 1337 },
       primaryTimestamp: '@timestamp',
+      exceptionFilter: undefined,
     });
 
     expect(esClient.count).toHaveBeenCalledWith({
@@ -52,17 +52,17 @@ describe('getEventCount', () => {
     });
   });
 
-  it('can override timestamp', () => {
-    getEventCount({
+  it('can override timestamp', async () => {
+    await getEventCount({
       esClient,
       query: '*:*',
       language: 'kuery',
       filters: [],
-      exceptionItems: [],
       index: ['test-index'],
       tuple: { to: moment('2022-01-14'), from: moment('2022-01-13'), maxSignals: 1337 },
       primaryTimestamp: 'event.ingested',
       secondaryTimestamp: '@timestamp',
+      exceptionFilter: undefined,
     });
 
     expect(esClient.count).toHaveBeenCalledWith({
@@ -113,16 +113,16 @@ describe('getEventCount', () => {
     });
   });
 
-  it('can override timestamp without fallback to @timestamp', () => {
-    getEventCount({
+  it('can override timestamp without fallback to @timestamp', async () => {
+    await getEventCount({
       esClient,
       query: '*:*',
       language: 'kuery',
       filters: [],
-      exceptionItems: [],
       index: ['test-index'],
       tuple: { to: moment('2022-01-14'), from: moment('2022-01-13'), maxSignals: 1337 },
       primaryTimestamp: 'event.ingested',
+      exceptionFilter: undefined,
     });
 
     expect(esClient.count).toHaveBeenCalledWith({

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_event_count.ts

@@ -7,7 +7,7 @@
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { EventCountOptions, EventsOptions, EventDoc } from './types';
-import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter';
+import { getQueryFilter } from '../get_query_filter';
 import { singleSearchAfter } from '../single_search_after';
 import { buildEventsSearchQuery } from '../build_events_query';
 
@@ -21,12 +21,12 @@ export const getEventList = async ({
   index,
   perPage,
   searchAfter,
-  exceptionItems,
   filters,
   tuple,
   primaryTimestamp,
   secondaryTimestamp,
   runtimeMappings,
+  exceptionFilter,
 }: EventsOptions): Promise<estypes.SearchResponse<EventDoc>> => {
   const calculatedPerPage = perPage ?? MAX_PER_PAGE;
   if (calculatedPerPage > 10000) {
@@ -37,7 +37,13 @@ export const getEventList = async ({
     `Querying the events items from the index: "${index}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items`
   );
 
-  const filter = getQueryFilter(query, language ?? 'kuery', filters, index, exceptionItems);
+  const queryFilter = getQueryFilter({
+    query,
+    language: language ?? 'kuery',
+    filters,
+    index,
+    exceptionFilter,
+  });
 
   const { searchResult } = await singleSearchAfter({
     searchAfterSortIds: searchAfter,
@@ -46,8 +52,8 @@ export const getEventList = async ({
     to: tuple.to.toISOString(),
     services,
     ruleExecutionLogger,
-    filter,
     pageSize: calculatedPerPage,
+    filter: queryFilter,
     primaryTimestamp,
     secondaryTimestamp,
     sortOrder: 'desc',
@@ -65,17 +71,23 @@ export const getEventCount = async ({
   language,
   filters,
   index,
-  exceptionItems,
   tuple,
   primaryTimestamp,
   secondaryTimestamp,
+  exceptionFilter,
 }: EventCountOptions): Promise<number> => {
-  const filter = getQueryFilter(query, language ?? 'kuery', filters, index, exceptionItems);
+  const queryFilter = getQueryFilter({
+    query,
+    language: language ?? 'kuery',
+    filters,
+    index,
+    exceptionFilter,
+  });
   const eventSearchQueryBodyQuery = buildEventsSearchQuery({
     index,
     from: tuple.from.toISOString(),
     to: tuple.to.toISOString(),
-    filter,
+    filter: queryFilter,
     size: 0,
     primaryTimestamp,
     secondaryTimestamp,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts

@@ -6,7 +6,7 @@
  */
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter';
+import { getQueryFilter } from '../get_query_filter';
 import type {
   GetThreatListOptions,
   ThreatListCountOptions,
@@ -22,7 +22,6 @@ export const INDICATOR_PER_PAGE = 1000;
 
 export const getThreatList = async ({
   esClient,
-  exceptionItems,
   index,
   language,
   perPage,
@@ -35,18 +34,19 @@ export const getThreatList = async ({
   reassignPitId,
   runtimeMappings,
   listClient,
+  exceptionFilter,
 }: GetThreatListOptions): Promise<estypes.SearchResponse<ThreatListDoc>> => {
   const calculatedPerPage = perPage ?? INDICATOR_PER_PAGE;
   if (calculatedPerPage > 10000) {
     throw new TypeError('perPage cannot exceed the size of 10000');
   }
-  const queryFilter = getQueryFilter(
+  const queryFilter = getQueryFilter({
     query,
-    language ?? 'kuery',
-    threatFilters,
+    language: language ?? 'kuery',
+    filters: threatFilters,
     index,
-    exceptionItems
-  );
+    exceptionFilter,
+  });
 
   ruleExecutionLogger.debug(
     `Querying the indicator items from the index: "${index}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items`
@@ -96,15 +96,15 @@ export const getThreatListCount = async ({
   language,
   threatFilters,
   index,
-  exceptionItems,
+  exceptionFilter,
 }: ThreatListCountOptions): Promise<number> => {
-  const queryFilter = getQueryFilter(
+  const queryFilter = getQueryFilter({
     query,
-    language ?? 'kuery',
-    threatFilters,
+    language: language ?? 'kuery',
+    filters: threatFilters,
     index,
-    exceptionItems
-  );
+    exceptionFilter,
+  });
   const response = await esClient.count({
     body: {
       query: queryFilter,