[Security Solution] Insight creation form bug fixes (#151097)

x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx

@@ -7,6 +7,7 @@
 
 import { pickBy, isEmpty } from 'lodash';
 import type { Plugin } from 'unified';
+import moment from 'moment';
 import React, { useContext, useMemo, useCallback, useState } from 'react';
 import type { RemarkTokenizer } from '@elastic/eui';
 import {
@@ -35,17 +36,7 @@ import type { DataView } from '@kbn/data-views-plugin/common';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import type { Filter } from '@kbn/es-query';
-import {
-  FILTERS,
-  isCombinedFilter,
-  isRangeFilter,
-  isPhraseFilter,
-  isPhrasesFilter,
-  isExistsFilter,
-  BooleanRelation,
-  FilterStateStore,
-} from '@kbn/es-query';
-import type { PhraseFilterValue } from '@kbn/es-query/src/filters/build_filters';
+import { FilterStateStore } from '@kbn/es-query';
 import { useForm, FormProvider, useController } from 'react-hook-form';
 import { useAppToasts } from '../../../../hooks/use_app_toasts';
 import { useKibana } from '../../../../lib/kibana';
@@ -63,6 +54,7 @@ import type { TimeRange } from '../../../../store/inputs/model';
 import { DEFAULT_TIMEPICKER_QUICK_RANGES } from '../../../../../../common/constants';
 import { useSourcererDataView } from '../../../../containers/sourcerer';
 import { SourcererScopeName } from '../../../../store/sourcerer/model';
+import { filtersToInsightProviders } from './provider';
 
 interface InsightComponentProps {
   label?: string;
@@ -143,78 +135,6 @@ export const parser: Plugin = function () {
   methods.splice(methods.indexOf('text'), 0, 'insight');
 };
 
-const buildPrimitiveProvider = (filter: Filter): Provider => {
-  const field = filter.meta?.key ?? '';
-  const excluded = filter.meta?.negate ?? false;
-  const queryType = filter.meta?.type ?? FILTERS.PHRASE;
-  const baseFilter = {
-    field,
-    excluded,
-    queryType,
-  };
-  if (isRangeFilter(filter)) {
-    const gte = filter.query.range[field].gte;
-    const lt = filter.query.range[field].lt;
-    const value = JSON.stringify({ gte, lt });
-    return {
-      ...baseFilter,
-      value,
-      queryType: filter.meta.type ?? FILTERS.RANGE,
-    };
-  } else if (isPhrasesFilter(filter)) {
-    const typeOfParams: PhraseFilterValue = typeof filter.meta?.params[0];
-    return {
-      ...baseFilter,
-      value: JSON.stringify(filter.meta?.params ?? []),
-      valueType: typeOfParams,
-      queryType: filter.meta.type ?? FILTERS.PHRASES,
-    };
-  } else if (isExistsFilter(filter)) {
-    return {
-      ...baseFilter,
-      value: '',
-      queryType: filter.meta.type ?? FILTERS.EXISTS,
-    };
-  } else if (isPhraseFilter(filter)) {
-    const valueType: PhraseFilterValue = typeof filter.meta?.params?.query;
-    return {
-      ...baseFilter,
-      value: filter.meta?.params?.query ?? '',
-      valueType,
-      queryType: filter.meta.type ?? FILTERS.PHRASE,
-    };
-  } else {
-    return {
-      ...baseFilter,
-      value: '',
-      queryType: FILTERS.PHRASE,
-    };
-  }
-};
-
-const filtersToInsightProviders = (filters: Filter[]): Provider[][] => {
-  const providers = [];
-  for (let index = 0; index < filters.length; index++) {
-    const filter = filters[index];
-    if (isCombinedFilter(filter)) {
-      if (filter.meta.relation === BooleanRelation.AND) {
-        return filtersToInsightProviders(filter.meta?.params);
-      } else {
-        return filter.meta?.params.map((innerFilter) => {
-          if (isCombinedFilter(innerFilter)) {
-            return filtersToInsightProviders([innerFilter]).map(([provider]) => provider);
-          } else {
-            return [buildPrimitiveProvider(innerFilter)];
-          }
-        });
-      }
-    } else {
-      providers.push([buildPrimitiveProvider(filter)]);
-    }
-  }
-  return providers;
-};
-
 const resultFormat = '0,0.[000]a';
 
 // receives the configuration from the parser and renders
@@ -238,28 +158,46 @@ const InsightComponent = ({
       }),
     });
   }
-  const { data: alertData } = useContext(BasicAlertDataContext);
+  const { data: alertData, timestamp } = useContext(BasicAlertDataContext);
   const { dataProviders, filters } = useInsightDataProviders({
     providers: parsedProviders,
     alertData,
   });
+  const relativeTimerange: TimeRange | null = useMemo(() => {
+    if (relativeFrom && relativeTo) {
+      const alertRelativeDate = timestamp ? moment(timestamp) : moment();
+      const from = parseDateWithDefault(
+        relativeFrom,
+        DEFAULT_FROM_MOMENT,
+        false,
+        moment,
+        alertRelativeDate.toDate()
+      ).toISOString();
+      const to = parseDateWithDefault(
+        relativeTo,
+        DEFAULT_TO_MOMENT,
+        true,
+        moment,
+        alertRelativeDate.toDate()
+      ).toISOString();
+      return {
+        kind: 'absolute',
+        from,
+        to,
+      };
+    } else {
+      return null;
+    }
+  }, [relativeFrom, relativeTo, timestamp]);
+
   const { totalCount, isQueryLoading, oldestTimestamp, hasError } = useInsightQuery({
     dataProviders,
     filters,
+    relativeTimerange,
   });
   const timerange: TimeRange = useMemo(() => {
-    if (relativeFrom && relativeTo) {
-      const fromStr = relativeFrom;
-      const toStr = relativeTo;
-      const from = parseDateWithDefault(fromStr, DEFAULT_FROM_MOMENT).toISOString();
-      const to = parseDateWithDefault(toStr, DEFAULT_TO_MOMENT, true).toISOString();
-      return {
-        kind: 'relative',
-        from,
-        to,
-        fromStr,
-        toStr,
-      };
+    if (relativeTimerange) {
+      return relativeTimerange;
     } else if (oldestTimestamp != null) {
       return {
         kind: 'absolute',
@@ -276,7 +214,8 @@ const InsightComponent = ({
         toStr,
       };
     }
-  }, [oldestTimestamp, relativeFrom, relativeTo]);
+  }, [oldestTimestamp, relativeTimerange]);
+
   if (isQueryLoading) {
     return <EuiLoadingSpinner size="l" />;
   } else {
@@ -416,6 +355,10 @@ const InsightEditorComponent = ({
     },
     [relativeTimerangeController.field]
   );
+  const disableSubmit = useMemo(() => {
+    const labelOrEmpty = labelController.field.value ? labelController.field.value : '';
+    return labelOrEmpty.trim() === '' || providers.length === 0;
+  }, [labelController.field.value, providers]);
   const filtersStub = useMemo(() => {
     const index = indexPattern && indexPattern.getName ? indexPattern.getName() : '*';
     return [
@@ -524,7 +467,7 @@ const InsightEditorComponent = ({
             defaultMessage: 'Cancel',
           })}
         </EuiButtonEmpty>
-        <EuiButton onClick={formMethods.handleSubmit(onSubmit)} fill>
+        <EuiButton onClick={formMethods.handleSubmit(onSubmit)} fill disabled={disableSubmit}>
           {isEditMode ? (
             <FormattedMessage
               id="xpack.securitySolution.markdown.insight.addModalConfirmButtonLabel"

x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/provider.test.ts

@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { CombinedFilter } from '@kbn/es-query';
+import { FILTERS, BooleanRelation, FilterStateStore } from '@kbn/es-query';
+import { filtersToInsightProviders } from './provider';
+
+const flatValueFilters = [
+  {
+    query: {
+      exists: {
+        field: 'Memory_protection.thread_count',
+      },
+    },
+    meta: {
+      negate: false,
+      index: 'security-solution-default',
+      key: 'Memory_protection.thread_count',
+      field: 'Memory_protection.thread_count',
+      value: 'exists',
+      type: 'exists',
+    },
+  },
+  {
+    meta: {
+      negate: false,
+      index: 'security-solution-default',
+      key: 'process.session_leader.entity_id',
+      field: 'process.session_leader.entity_id',
+      params: {
+        query: 'fbqfoee0al',
+      },
+      type: 'phrase',
+    },
+    query: {
+      match_phrase: {
+        'process.session_leader.entity_id': 'fbqfoee0al',
+      },
+    },
+  },
+];
+
+const combinedFilter: CombinedFilter = {
+  $state: {
+    store: FilterStateStore.APP_STATE,
+  },
+  meta: {
+    type: FILTERS.COMBINED,
+    relation: BooleanRelation.OR,
+    params: flatValueFilters,
+    index: 'security-solution-default',
+    disabled: false,
+    negate: false,
+  },
+};
+
+const combined = [
+  {
+    $state: {
+      store: FilterStateStore.APP_STATE,
+    },
+    meta: {
+      type: 'combined',
+      relation: 'OR',
+      params: [
+        {
+          query: {
+            exists: {
+              field: '_index',
+            },
+          },
+          meta: {
+            negate: false,
+            index: 'logs-*',
+            key: '_index',
+            field: '_index',
+            value: 'exists',
+            type: 'exists',
+          },
+        },
+        {
+          $state: {
+            store: FilterStateStore.APP_STATE,
+          },
+          meta: {
+            type: 'combined',
+            relation: 'AND',
+            params: [
+              {
+                query: {
+                  exists: {
+                    field: 'Endpoint.policy.applied.artifacts.global.identifiers.sha256',
+                  },
+                },
+                meta: {
+                  negate: false,
+                  index: 'security-solution-default',
+                  key: 'Endpoint.policy.applied.artifacts.global.identifiers.sha256',
+                  field: 'Endpoint.policy.applied.artifacts.global.identifiers.sha256',
+                  value: 'exists',
+                  type: 'exists',
+                },
+              },
+              {
+                meta: {
+                  negate: true,
+                  index: 'security-solution-default',
+                  key: 'Ransomware.child_processes.files.entropy',
+                  field: 'Ransomware.child_processes.files.entropy',
+                  params: {
+                    query: '0',
+                  },
+                  type: 'phrase',
+                },
+                query: {
+                  match_phrase: {
+                    'Ransomware.child_processes.files.entropy': '0',
+                  },
+                },
+              },
+            ],
+            index: 'security-solution-default',
+            disabled: false,
+            negate: false,
+          },
+        },
+      ],
+      index: 'security-solution-default',
+      disabled: false,
+      negate: false,
+    },
+  },
+];
+
+describe('filter to provider conversion', () => {
+  it('should return a single array for ANDed top level values', () => {
+    const result = filtersToInsightProviders(flatValueFilters);
+    expect(result.length).toBe(1);
+  });
+
+  it('should return a 2d array for a top level OR', () => {
+    const result = filtersToInsightProviders([combinedFilter]);
+    expect(result.length).toBe(2);
+  });
+
+  it('should return an array with 2 top level providers, and one with multiple', () => {
+    const result = filtersToInsightProviders(combined);
+    const [first, second] = result;
+    expect(result.length).toBe(2);
+    expect(first.length).toBe(1);
+    expect(second.length).toBe(2);
+  });
+});

x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/provider.ts

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { flatten } from 'lodash';
+import type { Filter } from '@kbn/es-query';
+import {
+  FILTERS,
+  isCombinedFilter,
+  isRangeFilter,
+  isPhraseFilter,
+  isPhrasesFilter,
+  isExistsFilter,
+  BooleanRelation,
+} from '@kbn/es-query';
+import type { PhraseFilterValue } from '@kbn/es-query/src/filters/build_filters';
+
+export interface Provider {
+  field: string;
+  excluded: boolean;
+  queryType: string;
+  value: string | number | boolean;
+  valueType?: string;
+}
+
+const buildPrimitiveProvider = (filter: Filter): Provider => {
+  const field = filter.meta?.key ?? '';
+  const excluded = filter.meta?.negate ?? false;
+  const queryType = filter.meta?.type ?? FILTERS.PHRASE;
+  const baseFilter = {
+    field,
+    excluded,
+    queryType,
+  };
+  if (isRangeFilter(filter)) {
+    const { gte, lt } = filter.query.range[field];
+    const value = JSON.stringify({ gte, lt });
+    return {
+      ...baseFilter,
+      value,
+      queryType: filter.meta.type ?? FILTERS.RANGE,
+    };
+  } else if (isPhrasesFilter(filter)) {
+    const typeOfParams: PhraseFilterValue = typeof filter.meta?.params[0];
+    return {
+      ...baseFilter,
+      value: JSON.stringify(filter.meta?.params ?? []),
+      valueType: typeOfParams,
+      queryType: filter.meta.type ?? FILTERS.PHRASES,
+    };
+  } else if (isExistsFilter(filter)) {
+    return {
+      ...baseFilter,
+      value: '',
+      queryType: filter.meta.type ?? FILTERS.EXISTS,
+    };
+  } else if (isPhraseFilter(filter)) {
+    const valueType: PhraseFilterValue = typeof filter.meta?.params?.query;
+    return {
+      ...baseFilter,
+      value: filter.meta?.params?.query ?? '',
+      valueType,
+      queryType: filter.meta.type ?? FILTERS.PHRASE,
+    };
+  } else {
+    return {
+      ...baseFilter,
+      value: '',
+      queryType: FILTERS.PHRASE,
+    };
+  }
+};
+
+const nonCombinedToProvider = (filters: Filter[]): Provider[] => {
+  return filters.map((filter) => {
+    return buildPrimitiveProvider(filter);
+  });
+};
+
+/**
+ * This function takes an array of Filter types and returns a 2d array
+ * of an intermediate data structure called a Provider, which can map from
+ * Filter <-> DataProvider. Items in each inner array of the Provider[][]
+ * return value are AND'ed together, items in the outer arrays are OR'ed.
+ */
+export const filtersToInsightProviders = (filters: Filter[]): Provider[][] => {
+  const hasCombined = filters.some(isCombinedFilter);
+  if (hasCombined === false) {
+    return [nonCombinedToProvider(filters)];
+  } else {
+    const combinedFilterToProviders: Provider[][] = filters.reduce(
+      (outerFilters: Provider[][], filter) => {
+        if (isCombinedFilter(filter)) {
+          const innerFilters = filter.meta.params;
+          if (filter.meta.relation === BooleanRelation.OR) {
+            const oredFilters = innerFilters.map((f) => {
+              const oredFilter = filtersToInsightProviders([f]);
+              return flatten(oredFilter);
+            });
+            return [...outerFilters, ...oredFilters];
+          } else {
+            const innerFiltersToProviders = filtersToInsightProviders(innerFilters);
+            return [...outerFilters, ...innerFiltersToProviders];
+          }
+        } else {
+          return [...outerFilters, [buildPrimitiveProvider(filter)]];
+        }
+      },
+      []
+    );
+    return combinedFilterToProviders;
+  }
+};

x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/replace_params_query.ts

@@ -19,7 +19,7 @@ export const replaceParamsQuery = (
     };
   }
   const regex = /\{{([^}]+)\}}/g;
-  const matchedBrackets = query.match(regex);
+  const matchedBrackets = query.match(new RegExp(regex));
   let resultQuery = query;
 
   if (matchedBrackets && data) {
@@ -37,7 +37,7 @@ export const replaceParamsQuery = (
     });
   }
 
-  const skipped = regex.test(resultQuery);
+  const skipped = new RegExp(regex).test(resultQuery);
 
   return {
     result: resultQuery,

x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/use_insight_data_providers.ts

@@ -45,6 +45,32 @@ const dataProviderQueryType = (type: string): QueryOperator => {
   }
 };
 
+const filterStub = {
+  $state: {
+    store: FilterStateStore.APP_STATE,
+  },
+  meta: {
+    disabled: false,
+    negate: false,
+    alias: null,
+    index: undefined,
+  },
+};
+
+const dataProviderStub: DataProvider = {
+  and: [],
+  enabled: true,
+  id: '',
+  name: '',
+  excluded: false,
+  kqlQuery: '',
+  type: DataProviderType.default,
+  queryMatch: {
+    field: '',
+    value: '',
+    operator: EXISTS_OPERATOR,
+  },
+};
 const buildDataProviders = (
   providers: Provider[][],
   alertData?: TimelineEventsDetailsItem[] | null
@@ -71,7 +97,6 @@ const buildDataProviders = (
         };
       } else {
         const newProvider = {
-          and: [],
           enabled: true,
           id: JSON.stringify(field + value),
           name: field,
@@ -87,20 +112,10 @@ const buildDataProviders = (
         prev.and.push(newProvider);
       }
       return prev;
-    }, {} as DataProvider);
+    }, dataProviderStub);
   });
 };
-const filterStub = {
-  $state: {
-    store: FilterStateStore.APP_STATE,
-  },
-  meta: {
-    disabled: false,
-    negate: false,
-    alias: null,
-    index: undefined,
-  },
-};
+
 const buildPrimitiveFilter = (provider: Provider): Filter => {
   const baseFilter = {
     ...filterStub,

x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/use_insight_query.test.ts

@@ -34,6 +34,7 @@ describe('useInsightQuery', () => {
         useInsightQuery({
           dataProviders: [mockProvider],
           filters: [],
+          relativeTimerange: null,
         }),
       {
         wrapper: TestProviders,

x-pack/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/use_insight_query.ts

@@ -15,10 +15,12 @@ import { combineQueries } from '../../../../lib/kuery';
 import { useTimelineEvents } from '../../../../../timelines/containers';
 import { useSourcererDataView } from '../../../../containers/sourcerer';
 import { SourcererScopeName } from '../../../../store/sourcerer/model';
+import type { TimeRange } from '../../../../store/inputs/model';
 
 export interface UseInsightQuery {
   dataProviders: DataProvider[];
   filters: Filter[];
+  relativeTimerange: TimeRange | null;
 }
 
 export interface UseInsightQueryResult {
@@ -31,6 +33,7 @@ export interface UseInsightQueryResult {
 export const useInsightQuery = ({
   dataProviders,
   filters,
+  relativeTimerange,
 }: UseInsightQuery): UseInsightQueryResult => {
   const { uiSettings } = useKibana().services;
   const esQueryConfig = useMemo(() => getEsQueryConfig(uiSettings), [uiSettings]);
@@ -70,6 +73,9 @@ export const useInsightQuery = ({
     language: 'kuery',
     limit: 1,
     runtimeMappings: {},
+    ...(relativeTimerange
+      ? { startDate: relativeTimerange?.from, endDate: relativeTimerange?.to }
+      : {}),
   });
   const [oldestEvent] = events;
   const timestamp =

x-pack/plugins/security_solution/public/common/utils/default_date_settings.ts

@@ -79,14 +79,19 @@ export const getIntervalSettings = (uiSettings = true): Policy => {
  * Parses a date and returns the default if the date string is not valid.
  * @param dateString The date string to parse
  * @param defaultDate The defaultDate if we cannot parse the dateMath
+ * @param {Boolean} roundUp should relative timeranges be rounded up or down
+ * @param momentInstance A moment instance to use in place of the library's, will use independent locale settings.
+ * @param {Date} forceNow A valid date object to use in place of Date.now()
  * @returns The moment of the date time parsed
  */
 export const parseDateWithDefault = (
   dateString: string,
   defaultDate: moment.Moment,
-  roundUp: boolean = false
+  roundUp: boolean = false,
+  momentInstance?: typeof moment,
+  forceNow?: Date
 ): moment.Moment => {
-  const date = dateMath.parse(dateString, { roundUp });
+  const date = dateMath.parse(dateString, { roundUp, momentInstance, forceNow });
   if (date != null && date.isValid()) {
     return date;
   } else {

x-pack/plugins/security_solution/server/lib/timeline/saved_object/timelines/index.ts

@@ -381,7 +381,6 @@ export const persistTimeline = async (
     if (timelineId == null) {
       return await createTimeline({ timelineId, timeline, userInfo, savedObjectsClient });
     }
-
     return await updateTimeline({
       request,
       timelineId,