[Visualize] Duplicated query filters in es request (#60106) (#60519)

* [Visualize] Duplicated query filters in es request

Closes: #59630

* Fix CI

* fix CI

* move uniq_filters to common

* fix scripts/check_published_api_changes

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esfilters.md

@@ -44,8 +44,8 @@ esFilters: {
     getPhraseFilterField: (filter: import("../common").PhraseFilter) => string;
     getPhraseFilterValue: (filter: import("../common").PhraseFilter) => string | number | boolean;
     getDisplayValueFromFilter: typeof getDisplayValueFromFilter;
-    compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions) => boolean;
-    COMPARE_ALL_OPTIONS: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions;
+    compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("../common").FilterCompareOptions) => boolean;
+    COMPARE_ALL_OPTIONS: import("../common").FilterCompareOptions;
     generateFilters: typeof generateFilters;
     onlyDisabledFiltersChanged: (newFilters?: import("../common").Filter[] | undefined, oldFilters?: import("../common").Filter[] | undefined) => boolean;
     changeTimeFilter: typeof changeTimeFilter;

src/plugins/data/common/query/filter_manager/compare_filters.test.ts

@@ -18,7 +18,7 @@
  */
 
 import { compareFilters, COMPARE_ALL_OPTIONS } from './compare_filters';
-import { buildEmptyFilter, buildQueryFilter, FilterStateStore } from '../../../../common';
+import { buildEmptyFilter, buildQueryFilter, FilterStateStore } from '../../es_query';
 
 describe('filter manager utilities', () => {
   describe('compare filters', () => {

src/plugins/data/common/query/filter_manager/compare_filters.ts

@@ -18,7 +18,7 @@
  */
 
 import { defaults, isEqual, omit, map } from 'lodash';
-import { FilterMeta, Filter } from '../../../../common';
+import { FilterMeta, Filter } from '../../es_query';
 
 export interface FilterCompareOptions {
   disabled?: boolean;

src/plugins/data/common/query/filter_manager/dedup_filters.test.ts

@@ -18,14 +18,8 @@
  */
 
 import { dedupFilters } from './dedup_filters';
-import {
-  Filter,
-  IIndexPattern,
-  IFieldType,
-  buildRangeFilter,
-  buildQueryFilter,
-  FilterStateStore,
-} from '../../../../common';
+import { Filter, buildRangeFilter, buildQueryFilter, FilterStateStore } from '../../es_query';
+import { IIndexPattern, IFieldType } from '../../index_patterns';
 
 describe('filter manager utilities', () => {
   let indexPattern: IIndexPattern;

src/plugins/data/common/query/filter_manager/dedup_filters.ts

@@ -19,7 +19,7 @@
 
 import { filter, find } from 'lodash';
 import { compareFilters, FilterCompareOptions } from './compare_filters';
-import { Filter } from '../../../../common';
+import { Filter } from '../../es_query';
 
 /**
  * Combine 2 filter collections, removing duplicates

src/plugins/data/common/query/filter_manager/index.ts

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { dedupFilters } from './dedup_filters';
+export { uniqFilters } from './uniq_filters';
+export { compareFilters, COMPARE_ALL_OPTIONS, FilterCompareOptions } from './compare_filters';

src/plugins/data/common/query/filter_manager/uniq_filters.test.ts

@@ -18,7 +18,7 @@
  */
 
 import { uniqFilters } from './uniq_filters';
-import { buildQueryFilter, Filter, FilterStateStore } from '../../../../common';
+import { buildQueryFilter, Filter, FilterStateStore } from '../../es_query';
 
 describe('filter manager utilities', () => {
   describe('niqFilter', () => {

src/plugins/data/common/query/filter_manager/uniq_filters.ts

@@ -17,8 +17,8 @@
  * under the License.
  */
 import { each, union } from 'lodash';
+import { Filter } from '../../es_query';
 import { dedupFilters } from './dedup_filters';
-import { Filter } from '../../../../common';
 
 /**
  * Remove duplicate filters from an array of filters

src/plugins/data/common/query/index.ts

@@ -17,4 +17,5 @@
  * under the License.
  */
 
+export * from './filter_manager';
 export * from './types';

src/plugins/data/public/index.ts

@@ -47,13 +47,13 @@ import {
   isQueryStringFilter,
   isRangeFilter,
   toggleFilterNegated,
+  compareFilters,
+  COMPARE_ALL_OPTIONS,
 } from '../common';
 
 import { FilterLabel } from './ui/filter_bar';
 
 import {
-  compareFilters,
-  COMPARE_ALL_OPTIONS,
   generateFilters,
   onlyDisabledFiltersChanged,
   changeTimeFilter,

src/plugins/data/public/public.api.md

@@ -362,8 +362,8 @@ export const esFilters: {
     getPhraseFilterField: (filter: import("../common").PhraseFilter) => string;
     getPhraseFilterValue: (filter: import("../common").PhraseFilter) => string | number | boolean;
     getDisplayValueFromFilter: typeof getDisplayValueFromFilter;
-    compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions) => boolean;
-    COMPARE_ALL_OPTIONS: import("./query/filter_manager/lib/compare_filters").FilterCompareOptions;
+    compareFilters: (first: import("../common").Filter | import("../common").Filter[], second: import("../common").Filter | import("../common").Filter[], comparatorOptions?: import("../common").FilterCompareOptions) => boolean;
+    COMPARE_ALL_OPTIONS: import("../common").FilterCompareOptions;
     generateFilters: typeof generateFilters;
     onlyDisabledFiltersChanged: (newFilters?: import("../common").Filter[] | undefined, oldFilters?: import("../common").Filter[] | undefined) => boolean;
     changeTimeFilter: typeof changeTimeFilter;
@@ -1843,8 +1843,8 @@ export type TSearchStrategyProvider<T extends TStrategyTypes> = (context: ISearc
 // src/plugins/data/public/index.ts:405:1 - (ae-forgotten-export) The symbol "parseInterval" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:406:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/index.ts:409:1 - (ae-forgotten-export) The symbol "toAbsoluteDates" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/query/state_sync/connect_to_query_state.ts:34:33 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts
-// src/plugins/data/public/query/state_sync/connect_to_query_state.ts:38:1 - (ae-forgotten-export) The symbol "QueryStateChange" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/query/state_sync/connect_to_query_state.ts:33:33 - (ae-forgotten-export) The symbol "FilterStateStore" needs to be exported by the entry point index.d.ts
+// src/plugins/data/public/query/state_sync/connect_to_query_state.ts:37:1 - (ae-forgotten-export) The symbol "QueryStateChange" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/types.ts:52:5 - (ae-forgotten-export) The symbol "createFiltersFromEvent" needs to be exported by the entry point index.d.ts
 // src/plugins/data/public/types.ts:60:5 - (ae-forgotten-export) The symbol "IndexPatternSelectProps" needs to be exported by the entry point index.d.ts
 

src/plugins/data/public/query/filter_manager/filter_manager.ts

@@ -22,13 +22,19 @@ import { Subject } from 'rxjs';
 
 import { IUiSettingsClient } from 'src/core/public';
 
-import { COMPARE_ALL_OPTIONS, compareFilters } from './lib/compare_filters';
 import { sortFilters } from './lib/sort_filters';
 import { mapAndFlattenFilters } from './lib/map_and_flatten_filters';
-import { uniqFilters } from './lib/uniq_filters';
 import { onlyDisabledFiltersChanged } from './lib/only_disabled';
 import { PartitionedFilters } from './types';
-import { FilterStateStore, Filter, isFilterPinned } from '../../../common';
+
+import {
+  FilterStateStore,
+  Filter,
+  uniqFilters,
+  isFilterPinned,
+  compareFilters,
+  COMPARE_ALL_OPTIONS,
+} from '../../../common';
 
 export class FilterManager {
   private filters: Filter[] = [];

src/plugins/data/public/query/filter_manager/index.ts

@@ -19,8 +19,6 @@
 
 export { FilterManager } from './filter_manager';
 
-export { uniqFilters } from './lib/uniq_filters';
 export { mapAndFlattenFilters } from './lib/map_and_flatten_filters';
 export { onlyDisabledFiltersChanged } from './lib/only_disabled';
 export { generateFilters } from './lib/generate_filters';
-export { compareFilters, COMPARE_ALL_OPTIONS } from './lib/compare_filters';

src/plugins/data/public/query/filter_manager/lib/only_disabled.ts

@@ -18,8 +18,7 @@
  */
 
 import { filter } from 'lodash';
-import { Filter } from '../../../../common';
-import { compareFilters, COMPARE_ALL_OPTIONS } from './compare_filters';
+import { Filter, compareFilters, COMPARE_ALL_OPTIONS } from '../../../../common';
 
 const isEnabled = (f: Filter) => f && f.meta && !f.meta.disabled;
 

src/plugins/data/public/query/state_sync/connect_to_query_state.ts

@@ -21,10 +21,9 @@ import { Subscription } from 'rxjs';
 import { filter, map } from 'rxjs/operators';
 import _ from 'lodash';
 import { BaseStateContainer } from '../../../../kibana_utils/public';
-import { COMPARE_ALL_OPTIONS, compareFilters } from '../filter_manager/lib/compare_filters';
 import { QuerySetup, QueryStart } from '../query_service';
 import { QueryState, QueryStateChange } from './types';
-import { FilterStateStore } from '../../../common/es_query/filters';
+import { FilterStateStore, COMPARE_ALL_OPTIONS, compareFilters } from '../../../common';
 
 /**
  * Helper to setup two-way syncing of global data and a state container

src/plugins/data/public/query/state_sync/create_global_query_observable.ts

@@ -20,10 +20,10 @@
 import { Observable, Subscription } from 'rxjs';
 import { map, tap } from 'rxjs/operators';
 import { TimefilterSetup } from '../timefilter';
-import { COMPARE_ALL_OPTIONS, compareFilters, FilterManager } from '../filter_manager';
+import { FilterManager } from '../filter_manager';
 import { QueryState, QueryStateChange } from './index';
 import { createStateContainer } from '../../../../kibana_utils/public';
-import { isFilterPinned } from '../../../common/es_query/filters';
+import { isFilterPinned, compareFilters, COMPARE_ALL_OPTIONS } from '../../../common';
 
 export function createQueryStateObservable({
   timefilter: { timefilter },

src/plugins/expressions/common/expression_functions/specs/kibana_context.ts

@@ -16,10 +16,11 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
+import { uniq } from 'lodash';
 import { i18n } from '@kbn/i18n';
 import { ExpressionFunctionDefinition } from '../../expression_functions';
 import { KibanaContext } from '../../expression_types';
+import { Query, uniqFilters } from '../../../../data/common';
 
 interface Arguments {
   q?: string | null;
@@ -35,6 +36,15 @@ export type ExpressionFunctionKibanaContext = ExpressionFunctionDefinition<
   Promise<KibanaContext>
 >;
 
+const getParsedValue = (data: any, defaultValue: any) =>
+  typeof data === 'string' && data.length ? JSON.parse(data) || defaultValue : defaultValue;
+
+const mergeQueries = (first: Query | Query[] = [], second: Query | Query[]) =>
+  uniq<Query>(
+    [...(Array.isArray(first) ? first : [first]), ...(Array.isArray(second) ? second : [second])],
+    (n: any) => JSON.stringify(n.query)
+  );
+
 export const kibanaContextFunction: ExpressionFunctionKibanaContext = {
   name: 'kibana_context',
   type: 'kibana_context',
@@ -75,9 +85,9 @@ export const kibanaContextFunction: ExpressionFunctionKibanaContext = {
   },
 
   async fn(input, args, { getSavedObject }) {
-    const queryArg = args.q ? JSON.parse(args.q) : [];
-    let queries = Array.isArray(queryArg) ? queryArg : [queryArg];
-    let filters = args.filters ? JSON.parse(args.filters) : [];
+    const timeRange = getParsedValue(args.timeRange, input?.timeRange);
+    let queries = mergeQueries(input?.query, getParsedValue(args?.q, []));
+    let filters = [...(input?.filters || []), ...getParsedValue(args?.filters, [])];
 
     if (args.savedSearchId) {
       if (typeof getSavedObject !== 'function') {
@@ -89,29 +99,20 @@ export const kibanaContextFunction: ExpressionFunctionKibanaContext = {
       }
       const obj = await getSavedObject('search', args.savedSearchId);
       const search = obj.attributes.kibanaSavedObjectMeta as { searchSourceJSON: string };
-      const data = JSON.parse(search.searchSourceJSON) as { query: string; filter: any[] };
-      queries = queries.concat(data.query);
-      filters = filters.concat(data.filter);
-    }
+      const { query, filter } = getParsedValue(search.searchSourceJSON, {});
 
-    if (input && input.query) {
-      queries = queries.concat(input.query);
+      if (query) {
+        queries = mergeQueries(queries, query);
+      }
+      if (filter) {
+        filters = [...filters, ...(Array.isArray(filter) ? filter : [filter])];
+      }
     }
 
-    if (input && input.filters) {
-      filters = filters.concat(input.filters).filter((f: any) => !f.meta.disabled);
-    }
-
-    const timeRange = args.timeRange
-      ? JSON.parse(args.timeRange)
-      : input
-      ? input.timeRange
-      : undefined;
-
     return {
       type: 'kibana_context',
       query: queries,
-      filters,
+      filters: uniqFilters(filters).filter((f: any) => !f.meta?.disabled),
       timeRange,
     };
   },