[Security Solution][Detection Engine] - Improve DE query build times for large lists (#85051)

## Summary

This PR addresses the following issues:
- https://github.com/elastic/kibana/issues/76979
- https://github.com/elastic/kibana/issues/82267
- removal of unused lucene exceptions logic

x-pack/plugins/lists/common/constants.mock.ts

@@ -36,6 +36,7 @@ export const TYPE = 'ip';
 export const VALUE = '127.0.0.1';
 export const VALUE_2 = '255.255.255';
 export const NAMESPACE_TYPE = 'single';
+export const NESTED_FIELD = 'parent.field';
 
 // Exception List specific
 export const ID = 'uuid_here';

x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts

@@ -43,6 +43,10 @@ export const getExceptionListItemSchemaMock = (): ExceptionListItemSchema => ({
   updated_by: USER,
 });
 
+export const getExceptionListItemSchemaXMock = (count = 1): ExceptionListItemSchema[] => {
+  return new Array(count).fill(null).map(() => getExceptionListItemSchemaMock());
+};
+
 /**
  * This is useful for end to end tests where we remove the auto generated parts for comparisons
  * such as created_at, updated_at, and id.

x-pack/plugins/lists/common/schemas/types/entry_exists.mock.ts

@@ -13,3 +13,8 @@ export const getEntryExistsMock = (): EntryExists => ({
   operator: OPERATOR,
   type: EXISTS,
 });
+
+export const getEntryExistsExcludedMock = (): EntryExists => ({
+  ...getEntryExistsMock(),
+  operator: 'excluded',
+});

x-pack/plugins/lists/common/schemas/types/entry_match.mock.ts

@@ -14,3 +14,8 @@ export const getEntryMatchMock = (): EntryMatch => ({
   type: MATCH,
   value: ENTRY_VALUE,
 });
+
+export const getEntryMatchExcludeMock = (): EntryMatch => ({
+  ...getEntryMatchMock(),
+  operator: 'excluded',
+});

x-pack/plugins/lists/common/schemas/types/entry_match_any.mock.ts

@@ -14,3 +14,9 @@ export const getEntryMatchAnyMock = (): EntryMatchAny => ({
   type: MATCH_ANY,
   value: [ENTRY_VALUE],
 });
+
+export const getEntryMatchAnyExcludeMock = (): EntryMatchAny => ({
+  ...getEntryMatchAnyMock(),
+  operator: 'excluded',
+  value: [ENTRY_VALUE, 'some other host name'],
+});

x-pack/plugins/lists/common/schemas/types/entry_nested.mock.ts

@@ -4,14 +4,25 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { FIELD, NESTED } from '../../constants.mock';
+import { NESTED, NESTED_FIELD } from '../../constants.mock';
 
 import { EntryNested } from './entry_nested';
-import { getEntryMatchMock } from './entry_match.mock';
-import { getEntryMatchAnyMock } from './entry_match_any.mock';
+import { getEntryMatchExcludeMock, getEntryMatchMock } from './entry_match.mock';
+import { getEntryMatchAnyExcludeMock, getEntryMatchAnyMock } from './entry_match_any.mock';
+import { getEntryExistsMock } from './entry_exists.mock';
 
 export const getEntryNestedMock = (): EntryNested => ({
   entries: [getEntryMatchMock(), getEntryMatchAnyMock()],
-  field: FIELD,
+  field: NESTED_FIELD,
   type: NESTED,
 });
+
+export const getEntryNestedExcludeMock = (): EntryNested => ({
+  ...getEntryNestedMock(),
+  entries: [getEntryMatchExcludeMock(), getEntryMatchAnyExcludeMock()],
+});
+
+export const getEntryNestedMixedEntries = (): EntryNested => ({
+  ...getEntryNestedMock(),
+  entries: [getEntryMatchMock(), getEntryMatchAnyExcludeMock(), getEntryExistsMock()],
+});

x-pack/plugins/lists/common/schemas/types/entry_nested.test.ts

@@ -86,7 +86,7 @@ describe('entriesNested', () => {
           value: ['some host name'],
         },
       ],
-      field: 'host.name',
+      field: 'parent.field',
       type: 'nested',
     });
   });
@@ -105,7 +105,7 @@ describe('entriesNested', () => {
           type: 'exists',
         },
       ],
-      field: 'host.name',
+      field: 'parent.field',
       type: 'nested',
     });
   });

x-pack/plugins/security_solution/common/detection_engine/build_exceptions_filter.ts

@@ -0,0 +1,293 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { chunk } from 'lodash/fp';
+
+import { Filter } from '../../../../../src/plugins/data/common';
+import {
+  ExceptionListItemSchema,
+  CreateExceptionListItemSchema,
+  EntryMatch,
+  EntryMatchAny,
+  EntryNested,
+  entriesMatch,
+  entriesMatchAny,
+  entriesExists,
+  entriesNested,
+  EntryExists,
+} from '../../../lists/common';
+import { BooleanFilter, NestedFilter } from './types';
+import { hasLargeValueList } from './utils';
+
+type NonListEntry = EntryMatch | EntryMatchAny | EntryNested | EntryExists;
+interface ExceptionListItemNonLargeList extends ExceptionListItemSchema {
+  entries: NonListEntry[];
+}
+
+interface CreateExceptionListItemNonLargeList extends CreateExceptionListItemSchema {
+  entries: NonListEntry[];
+}
+
+export type ExceptionItemSansLargeValueLists =
+  | ExceptionListItemNonLargeList
+  | CreateExceptionListItemNonLargeList;
+
+export const chunkExceptions = (
+  exceptions: ExceptionItemSansLargeValueLists[],
+  chunkSize: number
+): ExceptionItemSansLargeValueLists[][] => {
+  return chunk(chunkSize, exceptions);
+};
+
+export const buildExceptionItemFilter = (
+  exceptionItem: ExceptionItemSansLargeValueLists
+): BooleanFilter | NestedFilter => {
+  const { entries } = exceptionItem;
+
+  if (entries.length === 1) {
+    return createInnerAndClauses(entries[0]);
+  } else {
+    return {
+      bool: {
+        filter: entries.map((entry) => createInnerAndClauses(entry)),
+      },
+    };
+  }
+};
+
+export const createOrClauses = (
+  exceptionItems: ExceptionItemSansLargeValueLists[]
+): Array<BooleanFilter | NestedFilter> => {
+  return exceptionItems.map((exceptionItem) => buildExceptionItemFilter(exceptionItem));
+};
+
+export const buildExceptionFilter = ({
+  lists,
+  excludeExceptions,
+  chunkSize,
+}: {
+  lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
+  excludeExceptions: boolean;
+  chunkSize: number;
+}): Filter | undefined => {
+  // Remove exception items with large value lists. These are evaluated
+  // elsewhere for the moment being.
+  const exceptionsWithoutLargeValueLists = lists.filter(
+    (item): item is ExceptionItemSansLargeValueLists => !hasLargeValueList(item.entries)
+  );
+
+  const exceptionFilter: Filter = {
+    meta: {
+      alias: null,
+      negate: excludeExceptions,
+      disabled: false,
+    },
+    query: {
+      bool: {
+        should: undefined,
+      },
+    },
+  };
+
+  if (exceptionsWithoutLargeValueLists.length === 0) {
+    return undefined;
+  } else if (exceptionsWithoutLargeValueLists.length <= chunkSize) {
+    const clause = createOrClauses(exceptionsWithoutLargeValueLists);
+    exceptionFilter.query.bool.should = clause;
+    return exceptionFilter;
+  } else {
+    const chunks = chunkExceptions(exceptionsWithoutLargeValueLists, chunkSize);
+
+    const filters = chunks.map<Filter>((exceptionsChunk) => {
+      const orClauses = createOrClauses(exceptionsChunk);
+
+      return {
+        meta: {
+          alias: null,
+          negate: false,
+          disabled: false,
+        },
+        query: {
+          bool: {
+            should: orClauses,
+          },
+        },
+      };
+    });
+
+    const clauses = filters.map<BooleanFilter>(({ query }) => query);
+
+    return {
+      meta: {
+        alias: null,
+        negate: excludeExceptions,
+        disabled: false,
+      },
+      query: {
+        bool: {
+          should: clauses,
+        },
+      },
+    };
+  }
+};
+
+export const buildExclusionClause = (booleanFilter: BooleanFilter): BooleanFilter => {
+  return {
+    bool: {
+      must_not: booleanFilter,
+    },
+  };
+};
+
+export const buildMatchClause = (entry: EntryMatch): BooleanFilter => {
+  const { field, operator, value } = entry;
+  const matchClause = {
+    bool: {
+      should: [
+        {
+          match_phrase: {
+            [field]: value,
+          },
+        },
+      ],
+      minimum_should_match: 1,
+    },
+  };
+
+  if (operator === 'excluded') {
+    return buildExclusionClause(matchClause);
+  } else {
+    return matchClause;
+  }
+};
+
+export const getBaseMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
+  const { field, value } = entry;
+
+  if (value.length === 1) {
+    return {
+      bool: {
+        should: [
+          {
+            match_phrase: {
+              [field]: value[0],
+            },
+          },
+        ],
+        minimum_should_match: 1,
+      },
+    };
+  }
+
+  return {
+    bool: {
+      should: value.map((val) => {
+        return {
+          bool: {
+            should: [
+              {
+                match_phrase: {
+                  [field]: val,
+                },
+              },
+            ],
+            minimum_should_match: 1,
+          },
+        };
+      }),
+      minimum_should_match: 1,
+    },
+  };
+};
+
+export const buildMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
+  const { operator } = entry;
+  const matchAnyClause = getBaseMatchAnyClause(entry);
+
+  if (operator === 'excluded') {
+    return buildExclusionClause(matchAnyClause);
+  } else {
+    return matchAnyClause;
+  }
+};
+
+export const buildExistsClause = (entry: EntryExists): BooleanFilter => {
+  const { field, operator } = entry;
+  const existsClause = {
+    bool: {
+      should: [
+        {
+          exists: {
+            field,
+          },
+        },
+      ],
+      minimum_should_match: 1,
+    },
+  };
+
+  if (operator === 'excluded') {
+    return buildExclusionClause(existsClause);
+  } else {
+    return existsClause;
+  }
+};
+
+const isBooleanFilter = (clause: object): clause is BooleanFilter => {
+  const keys = Object.keys(clause);
+  return keys.includes('bool') != null;
+};
+
+export const getBaseNestedClause = (
+  entries: NonListEntry[],
+  parentField: string
+): BooleanFilter => {
+  if (entries.length === 1) {
+    const [singleNestedEntry] = entries;
+    const innerClause = createInnerAndClauses(singleNestedEntry, parentField);
+    return isBooleanFilter(innerClause) ? innerClause : { bool: {} };
+  }
+
+  return {
+    bool: {
+      filter: entries.map((nestedEntry) => createInnerAndClauses(nestedEntry, parentField)),
+    },
+  };
+};
+
+export const buildNestedClause = (entry: EntryNested): NestedFilter => {
+  const { field, entries } = entry;
+
+  const baseNestedClause = getBaseNestedClause(entries, field);
+
+  return {
+    nested: {
+      path: field,
+      query: baseNestedClause,
+      score_mode: 'none',
+    },
+  };
+};
+
+export const createInnerAndClauses = (
+  entry: NonListEntry,
+  parent?: string
+): BooleanFilter | NestedFilter => {
+  if (entriesExists.is(entry)) {
+    const field = parent != null ? `${parent}.${entry.field}` : entry.field;
+    return buildExistsClause({ ...entry, field });
+  } else if (entriesMatch.is(entry)) {
+    const field = parent != null ? `${parent}.${entry.field}` : entry.field;
+    return buildMatchClause({ ...entry, field });
+  } else if (entriesMatchAny.is(entry)) {
+    const field = parent != null ? `${parent}.${entry.field}` : entry.field;
+    return buildMatchAnyClause({ ...entry, field });
+  } else if (entriesNested.is(entry)) {
+    return buildNestedClause(entry);
+  } else {
+    throw new TypeError(`Unexpected exception entry: ${entry}`);
+  }
+};

x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts

@@ -1,788 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import {
-  buildExceptionListQueries,
-  buildExceptionItem,
-  operatorBuilder,
-  buildExists,
-  buildMatch,
-  buildMatchAny,
-  buildEntry,
-  getLanguageBooleanOperator,
-  buildNested,
-} from './build_exceptions_query';
-import { EntryNested, EntryMatchAny, EntriesArray } from '../../../lists/common/schemas';
-import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
-import { getEntryMatchMock } from '../../../lists/common/schemas/types/entry_match.mock';
-import { getEntryMatchAnyMock } from '../../../lists/common/schemas/types/entry_match_any.mock';
-import { getEntryExistsMock } from '../../../lists/common/schemas/types/entry_exists.mock';
-
-describe('build_exceptions_query', () => {
-  describe('getLanguageBooleanOperator', () => {
-    test('it returns value as uppercase if language is "lucene"', () => {
-      const result = getLanguageBooleanOperator({ language: 'lucene', value: 'not' });
-
-      expect(result).toEqual('NOT');
-    });
-
-    test('it returns value as is if language is "kuery"', () => {
-      const result = getLanguageBooleanOperator({ language: 'kuery', value: 'not' });
-
-      expect(result).toEqual('not');
-    });
-  });
-
-  describe('operatorBuilder', () => {
-    describe('and language is kuery', () => {
-      test('it returns empty string when operator is "included"', () => {
-        const operator = operatorBuilder({ operator: 'included', language: 'kuery' });
-        expect(operator).toEqual('');
-      });
-      test('it returns "not " when operator is "excluded"', () => {
-        const operator = operatorBuilder({ operator: 'excluded', language: 'kuery' });
-        expect(operator).toEqual('not ');
-      });
-    });
-
-    describe('and language is lucene', () => {
-      test('it returns empty string when operator is "included"', () => {
-        const operator = operatorBuilder({ operator: 'included', language: 'lucene' });
-        expect(operator).toEqual('');
-      });
-      test('it returns "NOT " when operator is "excluded"', () => {
-        const operator = operatorBuilder({ operator: 'excluded', language: 'lucene' });
-        expect(operator).toEqual('NOT ');
-      });
-    });
-  });
-
-  describe('buildExists', () => {
-    describe('kuery', () => {
-      test('it returns formatted wildcard string when operator is "excluded"', () => {
-        const query = buildExists({
-          entry: { ...getEntryExistsMock(), operator: 'excluded' },
-          language: 'kuery',
-        });
-        expect(query).toEqual('not host.name:*');
-      });
-      test('it returns formatted wildcard string when operator is "included"', () => {
-        const query = buildExists({
-          entry: { ...getEntryExistsMock(), operator: 'included' },
-          language: 'kuery',
-        });
-        expect(query).toEqual('host.name:*');
-      });
-    });
-
-    describe('lucene', () => {
-      test('it returns formatted wildcard string when operator is "excluded"', () => {
-        const query = buildExists({
-          entry: { ...getEntryExistsMock(), operator: 'excluded' },
-          language: 'lucene',
-        });
-        expect(query).toEqual('NOT _exists_host.name');
-      });
-      test('it returns formatted wildcard string when operator is "included"', () => {
-        const query = buildExists({
-          entry: { ...getEntryExistsMock(), operator: 'included' },
-          language: 'lucene',
-        });
-        expect(query).toEqual('_exists_host.name');
-      });
-    });
-  });
-
-  describe('buildMatch', () => {
-    describe('kuery', () => {
-      test('it returns formatted string when operator is "included"', () => {
-        const query = buildMatch({
-          entry: { ...getEntryMatchMock(), operator: 'included' },
-          language: 'kuery',
-        });
-        expect(query).toEqual('host.name:"some host name"');
-      });
-      test('it returns formatted string when operator is "excluded"', () => {
-        const query = buildMatch({
-          entry: { ...getEntryMatchMock(), operator: 'excluded' },
-          language: 'kuery',
-        });
-        expect(query).toEqual('not host.name:"some host name"');
-      });
-    });
-
-    describe('lucene', () => {
-      test('it returns formatted string when operator is "included"', () => {
-        const query = buildMatch({
-          entry: { ...getEntryMatchMock(), operator: 'included' },
-          language: 'lucene',
-        });
-        expect(query).toEqual('host.name:"some host name"');
-      });
-      test('it returns formatted string when operator is "excluded"', () => {
-        const query = buildMatch({
-          entry: { ...getEntryMatchMock(), operator: 'excluded' },
-          language: 'lucene',
-        });
-        expect(query).toEqual('NOT host.name:"some host name"');
-      });
-    });
-  });
-
-  describe('buildMatchAny', () => {
-    const entryWithIncludedAndNoValues: EntryMatchAny = {
-      ...getEntryMatchAnyMock(),
-      field: 'host.name',
-      value: [],
-    };
-    const entryWithIncludedAndOneValue: EntryMatchAny = {
-      ...getEntryMatchAnyMock(),
-      field: 'host.name',
-      value: ['some host name'],
-    };
-    const entryWithExcludedAndTwoValues: EntryMatchAny = {
-      ...getEntryMatchAnyMock(),
-      field: 'host.name',
-      value: ['some host name', 'auditd'],
-      operator: 'excluded',
-    };
-
-    describe('kuery', () => {
-      test('it returns empty string if given an empty array for "values"', () => {
-        const exceptionSegment = buildMatchAny({
-          entry: entryWithIncludedAndNoValues,
-          language: 'kuery',
-        });
-        expect(exceptionSegment).toEqual('');
-      });
-
-      test('it returns formatted string when "values" includes only one item', () => {
-        const exceptionSegment = buildMatchAny({
-          entry: entryWithIncludedAndOneValue,
-          language: 'kuery',
-        });
-
-        expect(exceptionSegment).toEqual('host.name:("some host name")');
-      });
-
-      test('it returns formatted string when operator is "included"', () => {
-        const exceptionSegment = buildMatchAny({
-          entry: { ...getEntryMatchAnyMock(), value: ['some host name', 'auditd'] },
-          language: 'kuery',
-        });
-
-        expect(exceptionSegment).toEqual('host.name:("some host name" or "auditd")');
-      });
-
-      test('it returns formatted string when operator is "excluded"', () => {
-        const exceptionSegment = buildMatchAny({
-          entry: entryWithExcludedAndTwoValues,
-          language: 'kuery',
-        });
-
-        expect(exceptionSegment).toEqual('not host.name:("some host name" or "auditd")');
-      });
-    });
-
-    describe('lucene', () => {
-      test('it returns formatted string when operator is "included"', () => {
-        const exceptionSegment = buildMatchAny({
-          entry: { ...getEntryMatchAnyMock(), value: ['some host name', 'auditd'] },
-          language: 'lucene',
-        });
-
-        expect(exceptionSegment).toEqual('host.name:("some host name" OR "auditd")');
-      });
-      test('it returns formatted string when operator is "excluded"', () => {
-        const exceptionSegment = buildMatchAny({
-          entry: entryWithExcludedAndTwoValues,
-          language: 'lucene',
-        });
-
-        expect(exceptionSegment).toEqual('NOT host.name:("some host name" OR "auditd")');
-      });
-      test('it returns formatted string when "values" includes only one item', () => {
-        const exceptionSegment = buildMatchAny({
-          entry: entryWithIncludedAndOneValue,
-          language: 'lucene',
-        });
-
-        expect(exceptionSegment).toEqual('host.name:("some host name")');
-      });
-    });
-  });
-
-  describe('buildNested', () => {
-    // NOTE: Only KQL supports nested
-    describe('kuery', () => {
-      test('it returns formatted query when one item in nested entry', () => {
-        const entry: EntryNested = {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            {
-              ...getEntryMatchMock(),
-              field: 'nestedField',
-              operator: 'included',
-              value: 'value-1',
-            },
-          ],
-        };
-        const result = buildNested({ entry, language: 'kuery' });
-
-        expect(result).toEqual('parent:{ nestedField:"value-1" }');
-      });
-
-      test('it returns formatted query when entry item is "exists"', () => {
-        const entry: EntryNested = {
-          field: 'parent',
-          type: 'nested',
-          entries: [{ ...getEntryExistsMock(), field: 'nestedField', operator: 'included' }],
-        };
-        const result = buildNested({ entry, language: 'kuery' });
-
-        expect(result).toEqual('parent:{ nestedField:* }');
-      });
-
-      test('it returns formatted query when entry item is "exists" and operator is "excluded"', () => {
-        const entry: EntryNested = {
-          field: 'parent',
-          type: 'nested',
-          entries: [{ ...getEntryExistsMock(), field: 'nestedField', operator: 'excluded' }],
-        };
-        const result = buildNested({ entry, language: 'kuery' });
-
-        expect(result).toEqual('parent:{ not nestedField:* }');
-      });
-
-      test('it returns formatted query when entry item is "match_any"', () => {
-        const entry: EntryNested = {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            {
-              ...getEntryMatchAnyMock(),
-              field: 'nestedField',
-              operator: 'included',
-              value: ['value1', 'value2'],
-            },
-          ],
-        };
-        const result = buildNested({ entry, language: 'kuery' });
-
-        expect(result).toEqual('parent:{ nestedField:("value1" or "value2") }');
-      });
-
-      test('it returns formatted query when entry item is "match_any" and operator is "excluded"', () => {
-        const entry: EntryNested = {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            {
-              ...getEntryMatchAnyMock(),
-              field: 'nestedField',
-              operator: 'excluded',
-              value: ['value1', 'value2'],
-            },
-          ],
-        };
-        const result = buildNested({ entry, language: 'kuery' });
-
-        expect(result).toEqual('parent:{ not nestedField:("value1" or "value2") }');
-      });
-
-      test('it returns formatted query when multiple items in nested entry', () => {
-        const entry: EntryNested = {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            {
-              ...getEntryMatchMock(),
-              field: 'nestedField',
-              operator: 'included',
-              value: 'value-1',
-            },
-            {
-              ...getEntryMatchMock(),
-              field: 'nestedFieldB',
-              operator: 'included',
-              value: 'value-2',
-            },
-          ],
-        };
-        const result = buildNested({ entry, language: 'kuery' });
-
-        expect(result).toEqual('parent:{ nestedField:"value-1" and nestedFieldB:"value-2" }');
-      });
-    });
-  });
-
-  describe('buildEntry', () => {
-    describe('kuery', () => {
-      test('it returns formatted wildcard string when "type" is "exists"', () => {
-        const result = buildEntry({
-          entry: { ...getEntryExistsMock(), operator: 'included' },
-          language: 'kuery',
-        });
-        expect(result).toEqual('host.name:*');
-      });
-
-      test('it returns formatted string when "type" is "match"', () => {
-        const result = buildEntry({
-          entry: { ...getEntryMatchMock(), operator: 'included' },
-          language: 'kuery',
-        });
-        expect(result).toEqual('host.name:"some host name"');
-      });
-
-      test('it returns formatted string when "type" is "match_any"', () => {
-        const result = buildEntry({
-          entry: { ...getEntryMatchAnyMock(), value: ['some host name', 'auditd'] },
-          language: 'kuery',
-        });
-        expect(result).toEqual('host.name:("some host name" or "auditd")');
-      });
-    });
-
-    describe('lucene', () => {
-      test('it returns formatted wildcard string when "type" is "exists"', () => {
-        const result = buildEntry({
-          entry: { ...getEntryExistsMock(), operator: 'included' },
-          language: 'lucene',
-        });
-        expect(result).toEqual('_exists_host.name');
-      });
-
-      test('it returns formatted string when "type" is "match"', () => {
-        const result = buildEntry({
-          entry: { ...getEntryMatchMock(), operator: 'included' },
-          language: 'lucene',
-        });
-        expect(result).toEqual('host.name:"some host name"');
-      });
-
-      test('it returns formatted string when "type" is "match_any"', () => {
-        const result = buildEntry({
-          entry: { ...getEntryMatchAnyMock(), value: ['some host name', 'auditd'] },
-          language: 'lucene',
-        });
-        expect(result).toEqual('host.name:("some host name" OR "auditd")');
-      });
-    });
-  });
-
-  describe('buildExceptionItem', () => {
-    test('it returns empty string if empty lists array passed in', () => {
-      const query = buildExceptionItem({
-        language: 'kuery',
-        entries: [],
-      });
-
-      expect(query).toEqual('');
-    });
-
-    test('it returns expected query when more than one item in exception item', () => {
-      const payload: EntriesArray = [
-        { ...getEntryMatchAnyMock(), field: 'b' },
-        { ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'value-3' },
-      ];
-      const query = buildExceptionItem({
-        language: 'kuery',
-        entries: payload,
-      });
-      const expectedQuery = 'b:("some host name") and not c:"value-3"';
-
-      expect(query).toEqual(expectedQuery);
-    });
-
-    test('it returns expected query when exception item includes nested value', () => {
-      const entries: EntriesArray = [
-        { ...getEntryMatchAnyMock(), field: 'b' },
-        {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            {
-              ...getEntryMatchMock(),
-              field: 'nestedField',
-              operator: 'included',
-              value: 'value-3',
-            },
-          ],
-        },
-      ];
-      const query = buildExceptionItem({
-        language: 'kuery',
-        entries,
-      });
-      const expectedQuery = 'b:("some host name") and parent:{ nestedField:"value-3" }';
-
-      expect(query).toEqual(expectedQuery);
-    });
-
-    test('it returns expected query when exception item includes multiple items and nested "and" values', () => {
-      const entries: EntriesArray = [
-        { ...getEntryMatchAnyMock(), field: 'b' },
-        {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            {
-              ...getEntryMatchMock(),
-              field: 'nestedField',
-              operator: 'included',
-              value: 'value-3',
-            },
-          ],
-        },
-        { ...getEntryExistsMock(), field: 'd' },
-      ];
-      const query = buildExceptionItem({
-        language: 'kuery',
-        entries,
-      });
-      const expectedQuery = 'b:("some host name") and parent:{ nestedField:"value-3" } and d:*';
-      expect(query).toEqual(expectedQuery);
-    });
-
-    test('it returns expected query when language is "lucene"', () => {
-      const entries: EntriesArray = [
-        { ...getEntryMatchAnyMock(), field: 'b' },
-        {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            {
-              ...getEntryMatchMock(),
-              field: 'nestedField',
-              operator: 'excluded',
-              value: 'value-3',
-            },
-          ],
-        },
-        { ...getEntryExistsMock(), field: 'e', operator: 'excluded' },
-      ];
-      const query = buildExceptionItem({
-        language: 'lucene',
-        entries,
-      });
-      const expectedQuery =
-        'b:("some host name") AND parent:{ NOT nestedField:"value-3" } AND NOT _exists_e';
-      expect(query).toEqual(expectedQuery);
-    });
-
-    describe('exists', () => {
-      test('it returns expected query when list includes single list item with operator of "included"', () => {
-        const entries: EntriesArray = [{ ...getEntryExistsMock(), field: 'b' }];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'b:*';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when list includes single list item with operator of "excluded"', () => {
-        const entries: EntriesArray = [
-          { ...getEntryExistsMock(), field: 'b', operator: 'excluded' },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'not b:*';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when exception item includes entry item with "and" values', () => {
-        const entries: EntriesArray = [
-          { ...getEntryExistsMock(), field: 'b', operator: 'excluded' },
-          {
-            field: 'parent',
-            type: 'nested',
-            entries: [
-              { ...getEntryMatchMock(), field: 'c', operator: 'included', value: 'value-1' },
-            ],
-          },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'not b:* and parent:{ c:"value-1" }';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when list includes multiple items', () => {
-        const entries: EntriesArray = [
-          { ...getEntryExistsMock(), field: 'b' },
-          {
-            field: 'parent',
-            type: 'nested',
-            entries: [
-              { ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'value-1' },
-              { ...getEntryMatchMock(), field: 'd', value: 'value-2' },
-            ],
-          },
-          { ...getEntryExistsMock(), field: 'e' },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'b:* and parent:{ not c:"value-1" and d:"value-2" } and e:*';
-
-        expect(query).toEqual(expectedQuery);
-      });
-    });
-
-    describe('match', () => {
-      test('it returns expected query when list includes single list item with operator of "included"', () => {
-        const entries: EntriesArray = [{ ...getEntryMatchMock(), field: 'b', value: 'value' }];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'b:"value"';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when list includes single list item with operator of "excluded"', () => {
-        const entries: EntriesArray = [
-          { ...getEntryMatchMock(), field: 'b', operator: 'excluded', value: 'value' },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'not b:"value"';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when list includes list item with "and" values', () => {
-        const entries: EntriesArray = [
-          { ...getEntryMatchMock(), field: 'b', operator: 'excluded', value: 'value' },
-          {
-            field: 'parent',
-            type: 'nested',
-            entries: [
-              { ...getEntryMatchMock(), field: 'c', operator: 'included', value: 'valueC' },
-            ],
-          },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'not b:"value" and parent:{ c:"valueC" }';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when list includes multiple items', () => {
-        const entries: EntriesArray = [
-          { ...getEntryMatchMock(), field: 'b', value: 'value' },
-          {
-            field: 'parent',
-            type: 'nested',
-            entries: [
-              { ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'valueC' },
-              { ...getEntryMatchMock(), field: 'd', operator: 'excluded', value: 'valueD' },
-            ],
-          },
-          { ...getEntryMatchMock(), field: 'e', value: 'valueE' },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery =
-          'b:"value" and parent:{ not c:"valueC" and not d:"valueD" } and e:"valueE"';
-
-        expect(query).toEqual(expectedQuery);
-      });
-    });
-
-    describe('match_any', () => {
-      test('it returns expected query when list includes single list item with operator of "included"', () => {
-        const entries: EntriesArray = [{ ...getEntryMatchAnyMock(), field: 'b' }];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'b:("some host name")';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when list includes single list item with operator of "excluded"', () => {
-        const entries: EntriesArray = [
-          { ...getEntryMatchAnyMock(), field: 'b', operator: 'excluded' },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'not b:("some host name")';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when list includes list item with nested values', () => {
-        const entries: EntriesArray = [
-          { ...getEntryMatchAnyMock(), field: 'b', operator: 'excluded' },
-          {
-            field: 'parent',
-            type: 'nested',
-            entries: [
-              { ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'valueC' },
-            ],
-          },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'not b:("some host name") and parent:{ not c:"valueC" }';
-
-        expect(query).toEqual(expectedQuery);
-      });
-
-      test('it returns expected query when list includes multiple items', () => {
-        const entries: EntriesArray = [
-          { ...getEntryMatchAnyMock(), field: 'b' },
-          { ...getEntryMatchAnyMock(), field: 'c' },
-        ];
-        const query = buildExceptionItem({
-          language: 'kuery',
-          entries,
-        });
-        const expectedQuery = 'b:("some host name") and c:("some host name")';
-
-        expect(query).toEqual(expectedQuery);
-      });
-    });
-  });
-
-  describe('buildExceptionListQueries', () => {
-    test('it returns empty array if lists is empty array', () => {
-      const query = buildExceptionListQueries({ language: 'kuery', lists: [] });
-
-      expect(query).toEqual([]);
-    });
-
-    test('it returns empty array if lists is undefined', () => {
-      const query = buildExceptionListQueries({ language: 'kuery', lists: undefined });
-
-      expect(query).toEqual([]);
-    });
-
-    test('it returns expected query when lists exist and language is "kuery"', () => {
-      const payload = getExceptionListItemSchemaMock();
-      const payload2 = getExceptionListItemSchemaMock();
-      payload2.entries = [
-        { ...getEntryMatchAnyMock(), field: 'b' },
-        {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            { ...getEntryMatchMock(), field: 'c', operator: 'included', value: 'valueC' },
-            { ...getEntryMatchMock(), field: 'd', operator: 'included', value: 'valueD' },
-          ],
-        },
-        { ...getEntryMatchAnyMock(), field: 'e', operator: 'excluded' },
-      ];
-      const queries = buildExceptionListQueries({
-        language: 'kuery',
-        lists: [payload, payload2],
-      });
-      const expectedQueries = [
-        {
-          query:
-            'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value"',
-          language: 'kuery',
-        },
-        {
-          query:
-            'b:("some host name") and parent:{ c:"valueC" and d:"valueD" } and not e:("some host name")',
-          language: 'kuery',
-        },
-      ];
-
-      expect(queries).toEqual(expectedQueries);
-    });
-
-    test('it returns expected query when lists exist and language is "lucene"', () => {
-      const payload = getExceptionListItemSchemaMock();
-      payload.entries = [
-        { ...getEntryMatchAnyMock(), field: 'a' },
-        { ...getEntryMatchAnyMock(), field: 'b' },
-      ];
-      const payload2 = getExceptionListItemSchemaMock();
-      payload2.entries = [
-        { ...getEntryMatchAnyMock(), field: 'c' },
-        { ...getEntryMatchAnyMock(), field: 'd' },
-      ];
-      const queries = buildExceptionListQueries({
-        language: 'lucene',
-        lists: [payload, payload2],
-      });
-      const expectedQueries = [
-        {
-          query: 'a:("some host name") AND b:("some host name")',
-          language: 'lucene',
-        },
-        {
-          query: 'c:("some host name") AND d:("some host name")',
-          language: 'lucene',
-        },
-      ];
-
-      expect(queries).toEqual(expectedQueries);
-    });
-
-    test('it builds correct queries for nested excluded fields', () => {
-      const payload = getExceptionListItemSchemaMock();
-      const payload2 = getExceptionListItemSchemaMock();
-      payload2.entries = [
-        { ...getEntryMatchAnyMock(), field: 'b' },
-        {
-          field: 'parent',
-          type: 'nested',
-          entries: [
-            // TODO: these operators are not being respected. buildNested needs to be updated
-            { ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'valueC' },
-            { ...getEntryMatchMock(), field: 'd', operator: 'excluded', value: 'valueD' },
-          ],
-        },
-        { ...getEntryMatchAnyMock(), field: 'e' },
-      ];
-      const queries = buildExceptionListQueries({
-        language: 'kuery',
-        lists: [payload, payload2],
-      });
-      const expectedQueries = [
-        {
-          query:
-            'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value"',
-          language: 'kuery',
-        },
-        {
-          query:
-            'b:("some host name") and parent:{ not c:"valueC" and not d:"valueD" } and e:("some host name")',
-          language: 'kuery',
-        },
-      ];
-
-      expect(queries).toEqual(expectedQueries);
-    });
-  });
-});

x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts

@@ -1,200 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-import { Query as DataQuery } from '../../../../../src/plugins/data/common';
-import {
-  Entry,
-  EntryMatch,
-  EntryMatchAny,
-  EntryNested,
-  EntryExists,
-  EntriesArray,
-  Operator,
-  entriesMatchAny,
-  entriesExists,
-  entriesMatch,
-  entriesNested,
-  ExceptionListItemSchema,
-  CreateExceptionListItemSchema,
-} from '../shared_imports';
-import { Language } from './schemas/common/schemas';
-import { hasLargeValueList } from './utils';
-
-type Operators = 'and' | 'or' | 'not';
-type LuceneOperators = 'AND' | 'OR' | 'NOT';
-
-export const getLanguageBooleanOperator = ({
-  language,
-  value,
-}: {
-  language: Language;
-  value: Operators;
-}): Operators | LuceneOperators => {
-  switch (language) {
-    case 'lucene':
-      const luceneValues: Record<Operators, LuceneOperators> = { and: 'AND', or: 'OR', not: 'NOT' };
-
-      return luceneValues[value];
-    case 'kuery':
-      return value;
-    default:
-      return value;
-  }
-};
-
-export const operatorBuilder = ({
-  operator,
-  language,
-}: {
-  operator: Operator;
-  language: Language;
-}): string => {
-  const not = getLanguageBooleanOperator({
-    language,
-    value: 'not',
-  });
-
-  if (operator === 'excluded') {
-    return `${not} `;
-  } else {
-    return '';
-  }
-};
-
-export const buildExists = ({
-  entry,
-  language,
-}: {
-  entry: EntryExists;
-  language: Language;
-}): string => {
-  const { operator, field } = entry;
-  const exceptionOperator = operatorBuilder({ operator, language });
-
-  switch (language) {
-    case 'kuery':
-      return `${exceptionOperator}${field}:*`;
-    case 'lucene':
-      return `${exceptionOperator}_exists_${field}`;
-    default:
-      return '';
-  }
-};
-
-export const buildMatch = ({
-  entry,
-  language,
-}: {
-  entry: EntryMatch;
-  language: Language;
-}): string => {
-  const { value, operator, field } = entry;
-  const exceptionOperator = operatorBuilder({ operator, language });
-
-  return `${exceptionOperator}${field}:"${value}"`;
-};
-
-export const buildMatchAny = ({
-  entry,
-  language,
-}: {
-  entry: EntryMatchAny;
-  language: Language;
-}): string => {
-  const { value, operator, field } = entry;
-
-  switch (value.length) {
-    case 0:
-      return '';
-    default:
-      const or = getLanguageBooleanOperator({ language, value: 'or' });
-      const exceptionOperator = operatorBuilder({ operator, language });
-      const matchAnyValues = value.map((v) => `"${v}"`);
-
-      return `${exceptionOperator}${field}:(${matchAnyValues.join(` ${or} `)})`;
-  }
-};
-
-export const buildNested = ({
-  entry,
-  language,
-}: {
-  entry: EntryNested;
-  language: Language;
-}): string => {
-  const { field, entries: subentries } = entry;
-  const and = getLanguageBooleanOperator({ language, value: 'and' });
-  const values = subentries.map((subentry) => buildEntry({ entry: subentry, language }));
-
-  return `${field}:{ ${values.join(` ${and} `)} }`;
-};
-
-export const buildEntry = ({
-  entry,
-  language,
-}: {
-  entry: Entry | EntryNested;
-  language: Language;
-}): string => {
-  if (entriesExists.is(entry)) {
-    return buildExists({ entry, language });
-  } else if (entriesMatch.is(entry)) {
-    return buildMatch({ entry, language });
-  } else if (entriesMatchAny.is(entry)) {
-    return buildMatchAny({ entry, language });
-  } else if (entriesNested.is(entry)) {
-    return buildNested({ entry, language });
-  } else {
-    return '';
-  }
-};
-
-export const buildExceptionItem = ({
-  entries,
-  language,
-}: {
-  entries: EntriesArray;
-  language: Language;
-}): string => {
-  const and = getLanguageBooleanOperator({ language, value: 'and' });
-  const exceptionItemEntries = entries.map((entry) => {
-    return buildEntry({ entry, language });
-  });
-
-  return exceptionItemEntries.join(` ${and} `);
-};
-
-export const buildExceptionListQueries = ({
-  language,
-  lists,
-}: {
-  language: Language;
-  lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema> | undefined;
-}): DataQuery[] => {
-  if (lists == null || (lists != null && lists.length === 0)) {
-    return [];
-  }
-
-  const exceptionItems = lists.reduce<string[]>((acc, exceptionItem) => {
-    const { entries } = exceptionItem;
-
-    if (entries != null && entries.length > 0 && !hasLargeValueList(entries)) {
-      return [...acc, buildExceptionItem({ entries, language })];
-    } else {
-      return acc;
-    }
-  }, []);
-
-  if (exceptionItems.length === 0) {
-    return [];
-  } else {
-    return exceptionItems.map((exceptionItem) => {
-      return {
-        query: exceptionItem,
-        language,
-      };
-    });
-  }
-};

x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts

@@ -7,7 +7,6 @@
 import {
   Filter,
   IIndexPattern,
-  isFilterDisabled,
   buildEsQuery,
   EsQueryConfig,
 } from '../../../../../src/plugins/data/common';
@@ -16,7 +15,7 @@ import {
   CreateExceptionListItemSchema,
 } from '../../../lists/common/schemas';
 import { ESBoolQuery } from '../typed_json';
-import { buildExceptionListQueries } from './build_exceptions_query';
+import { buildExceptionFilter } from './build_exceptions_filter';
 import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas';
 
 export const getQueryFilter = (
@@ -38,32 +37,27 @@ export const getQueryFilter = (
     ignoreFilterIfFieldNotInIndex: false,
     dateFormatTZ: 'Zulu',
   };
-
-  const enabledFilters = ((filters as unknown) as Filter[]).filter((f) => !isFilterDisabled(f));
-  /*
-   * Pinning exceptions to 'kuery' because lucene
-   * does not support nested queries, while our exceptions
-   * UI does, since we can pass both lucene and kql into
-   * buildEsQuery, this allows us to offer nested queries
-   * regardless
-   */
   // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
   // allowing us to make 1024-item chunks of exception list items.
   // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
   // very conservative value.
   const exceptionFilter = buildExceptionFilter({
     lists,
-    config,
     excludeExceptions,
     chunkSize: 1024,
-    indexPattern,
   });
-  if (exceptionFilter !== undefined) {
-    enabledFilters.push(exceptionFilter);
-  }
   const initialQuery = { query, language };
+  const allFilters = getAllFilters((filters as unknown) as Filter[], exceptionFilter);
 
-  return buildEsQuery(indexPattern, initialQuery, enabledFilters, config);
+  return buildEsQuery(indexPattern, initialQuery, allFilters, config);
+};
+
+export const getAllFilters = (filters: Filter[], exceptionFilter: Filter | undefined): Filter[] => {
+  if (exceptionFilter != null) {
+    return [...filters, exceptionFilter];
+  } else {
+    return [...filters];
+  }
 };
 
 interface EqlSearchRequest {
@@ -84,26 +78,14 @@ export const buildEqlSearchRequest = (
   eventCategoryOverride: string | undefined
 ): EqlSearchRequest => {
   const timestamp = timestampOverride ?? '@timestamp';
-  const indexPattern: IIndexPattern = {
-    fields: [],
-    title: index.join(),
-  };
-  const config: EsQueryConfig = {
-    allowLeadingWildcards: true,
-    queryStringOptions: { analyze_wildcard: true },
-    ignoreFilterIfFieldNotInIndex: false,
-    dateFormatTZ: 'Zulu',
-  };
   // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
   // allowing us to make 1024-item chunks of exception list items.
   // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
   // very conservative value.
   const exceptionFilter = buildExceptionFilter({
     lists: exceptionLists,
-    config,
     excludeExceptions: true,
     chunkSize: 1024,
-    indexPattern,
   });
   const indexString = index.join();
   const requestFilter: unknown[] = [
@@ -148,69 +130,3 @@ export const buildEqlSearchRequest = (
     return baseRequest;
   }
 };
-
-export const buildExceptionFilter = ({
-  lists,
-  config,
-  excludeExceptions,
-  chunkSize,
-  indexPattern,
-}: {
-  lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
-  config: EsQueryConfig;
-  excludeExceptions: boolean;
-  chunkSize: number;
-  indexPattern?: IIndexPattern;
-}) => {
-  const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
-  if (exceptionQueries.length === 0) {
-    return undefined;
-  }
-  const exceptionFilter: Filter = {
-    meta: {
-      alias: null,
-      negate: excludeExceptions,
-      disabled: false,
-    },
-    query: {
-      bool: {
-        should: undefined,
-      },
-    },
-  };
-  if (exceptionQueries.length <= chunkSize) {
-    const query = buildEsQuery(indexPattern, exceptionQueries, [], config);
-    exceptionFilter.query.bool.should = query.bool.filter;
-  } else {
-    const chunkedFilters: Filter[] = [];
-    for (let index = 0; index < exceptionQueries.length; index += chunkSize) {
-      const exceptionQueriesChunk = exceptionQueries.slice(index, index + chunkSize);
-      const esQueryChunk = buildEsQuery(indexPattern, exceptionQueriesChunk, [], config);
-      const filterChunk: Filter = {
-        meta: {
-          alias: null,
-          negate: false,
-          disabled: false,
-        },
-        query: {
-          bool: {
-            should: esQueryChunk.bool.filter,
-          },
-        },
-      };
-      chunkedFilters.push(filterChunk);
-    }
-    // Here we build a query with only the exceptions: it will put them all in the `filter` array
-    // of the resulting object, which would AND the exceptions together. When creating exceptionFilter,
-    // we move the `filter` array to `should` so they are OR'd together instead.
-    // This gets around the problem with buildEsQuery not allowing callers to specify whether queries passed in
-    // should be ANDed or ORed together.
-    exceptionFilter.query.bool.should = buildEsQuery(
-      indexPattern,
-      [],
-      chunkedFilters,
-      config
-    ).bool.filter;
-  }
-  return exceptionFilter;
-};

x-pack/plugins/security_solution/common/detection_engine/types.ts

@@ -55,3 +55,21 @@ export interface EqlSearchResponse<T> {
     events?: Array<BaseHit<T>>;
   };
 }
+
+export interface BooleanFilter {
+  bool: {
+    must?: unknown | unknown[];
+    must_not?: unknown | unknown[];
+    should?: unknown[];
+    filter?: unknown | unknown[];
+    minimum_should_match?: number;
+  };
+}
+
+export interface NestedFilter {
+  nested: {
+    path: string;
+    query: unknown | unknown[];
+    score_mode: string;
+  };
+}

x-pack/plugins/security_solution/public/common/components/exceptions/viewer/helpers.test.tsx

@@ -100,7 +100,7 @@ describe('Exception viewer helpers', () => {
           value: undefined,
         },
         {
-          fieldName: 'host.name',
+          fieldName: 'parent.field',
           isNested: false,
           operator: undefined,
           value: undefined,

x-pack/plugins/security_solution/server/lib/machine_learning/index.ts

@@ -6,8 +6,8 @@
 
 import { RequestParams } from '@elastic/elasticsearch';
 
+import { buildExceptionFilter } from '../../../common/detection_engine/build_exceptions_filter';
 import { ExceptionListItemSchema } from '../../../../lists/common';
-import { buildExceptionFilter } from '../../../common/detection_engine/get_query_filter';
 import { AnomalyRecordDoc as Anomaly } from '../../../../ml/server';
 import { SearchResponse } from '../types';
 
@@ -54,12 +54,6 @@ export const getAnomalies = async (
             ],
             must_not: buildExceptionFilter({
               lists: params.exceptionItems,
-              config: {
-                allowLeadingWildcards: true,
-                queryStringOptions: { analyze_wildcard: true },
-                ignoreFilterIfFieldNotInIndex: false,
-                dateFormatTZ: 'Zulu',
-              },
               excludeExceptions: true,
               chunkSize: 1024,
             })?.query,