[SIEM] Escape Query Bug (#37390) (#37445)

* Fix Escape query bug on timeline query

* just realized with andrewG that updateColumns action is not called as before and also some attribute is not saved anymore in the columns attributes

* cleanup

x-pack/plugins/siem/public/components/draggables/index.tsx

@@ -9,7 +9,6 @@ import * as React from 'react';
 import { pure } from 'recompose';
 import styled from 'styled-components';
 
-import { escapeQueryValue } from '../../lib/keury';
 import { DragEffects, DraggableWrapper } from '../drag_and_drop/draggable_wrapper';
 import { escapeDataProviderId } from '../drag_and_drop/helpers';
 import { getEmptyStringTag } from '../empty_value';
@@ -95,7 +94,7 @@ export const DefaultDraggable = pure<DefaultDraggableType>(
           kqlQuery: '',
           queryMatch: {
             field,
-            value: escapeQueryValue(queryValue ? queryValue : value),
+            value: queryValue ? queryValue : value,
             operator: IS_OPERATOR,
           },
         }}

x-pack/plugins/siem/public/components/formatted_ip/index.tsx

@@ -8,7 +8,6 @@ import { isArray, isEmpty, isString, uniq } from 'lodash/fp';
 import * as React from 'react';
 import { pure } from 'recompose';
 
-import { escapeQueryValue } from '../../lib/keury';
 import { DragEffects, DraggableWrapper } from '../drag_and_drop/draggable_wrapper';
 import { escapeDataProviderId } from '../drag_and_drop/helpers';
 import { getOrEmptyTagFromValue } from '../empty_value';
@@ -54,7 +53,7 @@ const getDataProvider = ({
   name: `${fieldName}: ${parseQueryValue(address)}`,
   queryMatch: {
     field: fieldName,
-    value: escapeQueryValue(parseQueryValue(address)),
+    value: parseQueryValue(address),
     operator: IS_OPERATOR,
   },
   excluded: false,

x-pack/plugins/siem/public/components/page/hosts/hosts_table/columns.tsx

@@ -63,7 +63,7 @@ export const getHostsColumns = (
               ) : (
                 <AddToKql
                   indexPattern={indexPattern}
-                  expression={`host.name: ${escapeQueryValue(hostName[0])}`}
+                  expression={`host.name: "${escapeQueryValue(hostName[0])}"`}
                   componentFilterType="hosts"
                   type={type}
                 >
@@ -112,7 +112,7 @@ export const getHostsColumns = (
         return (
           <AddToKql
             indexPattern={indexPattern}
-            expression={`host.os.name: ${escapeQueryValue(hostOsName)}`}
+            expression={`host.os.name: "${escapeQueryValue(hostOsName)}"`}
             componentFilterType="hosts"
             type={type}
           >
@@ -134,7 +134,7 @@ export const getHostsColumns = (
         return (
           <AddToKql
             indexPattern={indexPattern}
-            expression={`host.os.version: ${escapeQueryValue(hostOsVersion)}`}
+            expression={`host.os.version: "${escapeQueryValue(hostOsVersion)}"`}
             componentFilterType="hosts"
             type={type}
           >

x-pack/plugins/siem/public/components/page/network/domains_table/columns.tsx

@@ -103,7 +103,7 @@ export const getDomainsColumns = (
               key={escapeDataProviderId(
                 `${tableId}-table-${flowTarget}-${flowDirection}-direction-${direction}`
               )}
-              expression={`network.direction: ${escapeQueryValue(direction)}`}
+              expression={`network.direction: "${escapeQueryValue(direction)}"`}
               type={type}
               componentFilterType={'network'}
             >

x-pack/plugins/siem/public/components/page/network/network_dns_table/columns.tsx

@@ -8,7 +8,6 @@ import numeral from '@elastic/numeral';
 import React from 'react';
 
 import { NetworkDnsFields, NetworkDnsItem } from '../../../../graphql/types';
-import { escapeQueryValue } from '../../../../lib/keury';
 import { networkModel } from '../../../../store';
 import { DragEffects, DraggableWrapper } from '../../../drag_and_drop/draggable_wrapper';
 import { escapeDataProviderId } from '../../../drag_and_drop/helpers';
@@ -49,7 +48,7 @@ export const getNetworkDnsColumns = (
               kqlQuery: '',
               queryMatch: {
                 field: 'dns.question.etld_plus_one',
-                value: escapeQueryValue(dnsName),
+                value: dnsName,
                 operator: IS_OPERATOR,
               },
             }}

x-pack/plugins/siem/public/components/page/network/network_top_n_flow_table/columns.tsx

@@ -119,7 +119,7 @@ export const getNetworkTopNFlowColumns = (
               key={escapeDataProviderId(
                 `${tableId}-table-${flowTarget}-${flowDirection}-direction-${direction}`
               )}
-              expression={`network.direction: ${escapeQueryValue(direction)}`}
+              expression={`network.direction: "${escapeQueryValue(direction)}"`}
               componentFilterType="network"
               type={type}
             >

x-pack/plugins/siem/public/components/timeline/body/renderers/empty_column_renderer.tsx

@@ -10,7 +10,6 @@ import { ColumnHeader } from '../column_headers/column_header';
 import { ColumnRenderer } from './column_renderer';
 import { DraggableWrapper, DragEffects } from '../../../drag_and_drop/draggable_wrapper';
 import { escapeDataProviderId } from '../../../drag_and_drop/helpers';
-import { escapeQueryValue } from '../../../../lib/keury';
 import { parseQueryValue } from './parse_query_value';
 import { IS_OPERATOR } from '../../data_providers/data_provider';
 import { Provider } from '../../data_providers/provider';
@@ -44,7 +43,7 @@ export const emptyColumnRenderer: ColumnRenderer = {
         name: `${columnName}: ${parseQueryValue(null)}`,
         queryMatch: {
           field: field.id,
-          value: escapeQueryValue(parseQueryValue(null)),
+          value: parseQueryValue(null),
           operator: IS_OPERATOR,
         },
         excluded: false,

x-pack/plugins/siem/public/components/timeline/body/renderers/plain_column_renderer.tsx

@@ -8,7 +8,6 @@ import { isNumber } from 'lodash/fp';
 import React from 'react';
 
 import { TimelineNonEcsData } from '../../../../graphql/types';
-import { escapeQueryValue } from '../../../../lib/keury';
 import { DragEffects, DraggableWrapper } from '../../../drag_and_drop/draggable_wrapper';
 import { escapeDataProviderId } from '../../../drag_and_drop/helpers';
 import { getEmptyTagValue } from '../../../empty_value';
@@ -52,7 +51,7 @@ export const plainColumnRenderer: ColumnRenderer = {
             name: `${columnName}: ${parseQueryValue(value)}`,
             queryMatch: {
               field: field.id,
-              value: escapeQueryValue(parseQueryValue(value)),
+              value: parseQueryValue(value),
               operator: IS_OPERATOR,
             },
             excluded: false,

x-pack/plugins/siem/public/components/timeline/body/renderers/zeek/zeek_signature.tsx

@@ -11,7 +11,6 @@ import { pure } from 'recompose';
 import styled from 'styled-components';
 
 import { Ecs } from '../../../../../graphql/types';
-import { escapeQueryValue } from '../../../../../lib/keury';
 import { DragEffects, DraggableWrapper } from '../../../../drag_and_drop/draggable_wrapper';
 import { escapeDataProviderId } from '../../../../drag_and_drop/helpers';
 import { ExternalLinkIcon } from '../../../../external_link_icon';
@@ -75,7 +74,7 @@ export const DraggableZeekElement = pure<{
           kqlQuery: '',
           queryMatch: {
             field,
-            value: escapeQueryValue(value),
+            value,
             operator: IS_OPERATOR,
           },
         }}

x-pack/plugins/siem/public/lib/keury/index.test.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import { escapeKuery } from '.';
+
+describe('Kuery escape', () => {
+  it('should escape quotes', () => {
+    const value = 'I said, "Hello."';
+    const expected = 'I said, \\"Hello.\\"';
+    expect(escapeKuery(value)).to.be(expected);
+  });
+
+  it('should escape special characters', () => {
+    const value = `This \\ has (a lot of) <special> characters, don't you *think*? "Yes."`;
+    const expected = `This \\\\ has \\(a lot of\\) \\<special\\> characters, don't you \\*think\\*? \\"Yes.\\"`;
+    expect(escapeKuery(value)).to.be(expected);
+  });
+
+  it('should escape keywords', () => {
+    const value = 'foo and bar or baz not qux';
+    const expected = 'foo \\and bar \\or baz \\not qux';
+    expect(escapeKuery(value)).to.be(expected);
+  });
+
+  it('should escape keywords next to each other', () => {
+    const value = 'foo and bar or not baz';
+    const expected = 'foo \\and bar \\or \\not baz';
+    expect(escapeKuery(value)).to.be(expected);
+  });
+
+  it('should not escape keywords without surrounding spaces', () => {
+    const value = 'And this has keywords, or does it not?';
+    const expected = 'And this has keywords, \\or does it not?';
+    expect(escapeKuery(value)).to.be(expected);
+  });
+
+  it('should escape uppercase keywords', () => {
+    const value = 'foo AND bar';
+    const expected = 'foo \\AND bar';
+    expect(escapeKuery(value)).to.be(expected);
+  });
+
+  it('should escape both keywords and special characters', () => {
+    const value = 'Hello, world, and <nice> to meet you!';
+    const expected = 'Hello, world, \\and \\<nice\\> to meet you!';
+    expect(escapeKuery(value)).to.be(expected);
+  });
+
+  it('should escape newlines and tabs', () => {
+    const value = 'This\nhas\tnewlines\r\nwith\ttabs';
+    const expected = 'This\\nhas\\tnewlines\\r\\nwith\\ttabs';
+    expect(escapeKuery(value)).to.be(expected);
+  });
+});

x-pack/plugins/siem/public/lib/keury/index.ts

@@ -5,7 +5,7 @@
  */
 
 import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
-import { isString } from 'lodash/fp';
+import { isString, flow } from 'lodash/fp';
 import { StaticIndexPattern } from 'ui/index_patterns';
 
 import { KueryFilterQuery } from '../../store';
@@ -25,7 +25,7 @@ export const convertKueryToElasticSearchQuery = (
 
 export const escapeQueryValue = (val: number | string = ''): string | number => {
   if (isString(val)) {
-    return val.replace(/"/g, '\\"');
+    return escapeKuery(val);
   }
 
   return val;
@@ -41,3 +41,24 @@ export const isFromKueryExpressionValid = (kqlFilterQuery: KueryFilterQuery | nu
   }
   return true;
 };
+
+const escapeWhitespace = (val: string) =>
+  val
+    .replace(/\t/g, '\\t')
+    .replace(/\r/g, '\\r')
+    .replace(/\n/g, '\\n');
+
+// See the SpecialCharacter rule in kuery.peg
+const escapeSpecialCharacters = (val: string) => val.replace(/[\\():<>"*]/g, '\\$&'); // $& means the whole matched string
+
+// See the Keyword rule in kuery.peg
+const escapeAndOr = (val: string) => val.replace(/(\s+)(and|or)(\s+)/gi, '$1\\$2$3');
+
+const escapeNot = (val: string) => val.replace(/not(\s+)/gi, '\\$&');
+
+export const escapeKuery = flow(
+  escapeSpecialCharacters,
+  escapeAndOr,
+  escapeNot,
+  escapeWhitespace
+);

x-pack/plugins/siem/public/pages/hosts/host_details.tsx

@@ -230,7 +230,7 @@ const getFilterQuery = (
       : ''
     : convertKueryToElasticSearchQuery(
         `${filterQueryExpression} ${
-          hostName ? `and host.name: ${escapeQueryValue(hostName)}` : ''
+          hostName ? `and host.name: "${escapeQueryValue(hostName)}"` : ''
         }`,
         indexPattern
       );

x-pack/plugins/siem/public/store/timeline/epic.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { get, has, merge as mergeObject, set, omit, getOr } from 'lodash/fp';
+import { get, has, merge as mergeObject, set, omit } from 'lodash/fp';
 import { Action } from 'redux';
 import { Epic } from 'redux-observable';
 import { from, Observable, Subject, empty, merge } from 'rxjs';
@@ -151,7 +151,6 @@ export const createTimelineEpic = <State>(): Epic<
       withLatestFrom(timeline$),
       filter(([action, timeline]) => {
         const timelineId: TimelineModel = timeline[get('payload.id', action)];
-        const columns: ColumnHeader[] = getOr([], 'columns', timelineId);
         if (action.type === createTimeline.type) {
           myEpicTimelineId.setTimelineId(null);
           myEpicTimelineId.setTimelineVersion(null);
@@ -160,12 +159,6 @@ export const createTimelineEpic = <State>(): Epic<
           myEpicTimelineId.setTimelineId(addNewTimeline.savedObjectId);
           myEpicTimelineId.setTimelineVersion(addNewTimeline.version);
         } else if (timelineActionsType.includes(action.type) && !timelineId.isLoading) {
-          if (
-            action.type === addProvider.type &&
-            columns.filter(col => col.type != null).length === 0
-          ) {
-            return false;
-          }
           return true;
         }
         return false;