kql, lucene and timerange functions (#93043)

api_docs/data.json

@@ -27594,4 +27594,4 @@
       }
     ]
   }
-}
+}

api_docs/data_search.json

@@ -19470,4 +19470,4 @@
       }
     ]
   }
-}
+}

api_docs/expressions.json

@@ -33883,4 +33883,4 @@
       }
     ]
   }
-}
+}

src/plugins/data/common/search/expressions/index.ts

@@ -8,6 +8,11 @@
 
 export * from './kibana';
 export * from './kibana_context';
+export * from './kql';
+export * from './lucene';
+export * from './query_to_ast';
+export * from './timerange_to_ast';
 export * from './kibana_context_type';
 export * from './esaggs';
 export * from './utils';
+export * from './timerange';

src/plugins/data/common/search/expressions/kibana_context.ts

@@ -12,11 +12,13 @@ import { ExpressionFunctionDefinition, ExecutionContext } from 'src/plugins/expr
 import { Adapters } from 'src/plugins/inspector/common';
 import { Query, uniqFilters } from '../../query';
 import { ExecutionContextSearch, KibanaContext } from './kibana_context_type';
+import { KibanaQueryOutput } from './kibana_context_type';
+import { KibanaTimerangeOutput } from './timerange';
 
 interface Arguments {
-  q?: string | null;
+  q?: KibanaQueryOutput | null;
   filters?: string | null;
-  timeRange?: string | null;
+  timeRange?: KibanaTimerangeOutput | null;
   savedSearchId?: string | null;
 }
 
@@ -46,7 +48,7 @@ export const kibanaContextFunction: ExpressionFunctionKibanaContext = {
   }),
   args: {
     q: {
-      types: ['string', 'null'],
+      types: ['kibana_query', 'null'],
       aliases: ['query', '_'],
       default: null,
       help: i18n.translate('data.search.functions.kibana_context.q.help', {
@@ -61,7 +63,7 @@ export const kibanaContextFunction: ExpressionFunctionKibanaContext = {
       }),
     },
     timeRange: {
-      types: ['string', 'null'],
+      types: ['timerange', 'null'],
       default: null,
       help: i18n.translate('data.search.functions.kibana_context.timeRange.help', {
         defaultMessage: 'Specify Kibana time range filter',
@@ -77,8 +79,8 @@ export const kibanaContextFunction: ExpressionFunctionKibanaContext = {
   },
 
   async fn(input, args, { getSavedObject }) {
-    const timeRange = getParsedValue(args.timeRange, input?.timeRange);
-    let queries = mergeQueries(input?.query, getParsedValue(args?.q, []));
+    const timeRange = args.timeRange || input?.timeRange;
+    let queries = mergeQueries(input?.query, args?.q || []);
     let filters = [...(input?.filters || []), ...getParsedValue(args?.filters, [])];
 
     if (args.savedSearchId) {

src/plugins/data/common/search/expressions/kibana_context_type.ts

@@ -22,6 +22,8 @@ export type ExpressionValueSearchContext = ExpressionValueBoxed<
   ExecutionContextSearch
 >;
 
+export type KibanaQueryOutput = ExpressionValueBoxed<'kibana_query', Query>;
+
 // TODO: These two are exported for legacy reasons - remove them eventually.
 export type KIBANA_CONTEXT_NAME = 'kibana_context';
 export type KibanaContext = ExpressionValueSearchContext;

src/plugins/data/common/search/expressions/kql.test.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ExecutionContext } from 'src/plugins/expressions/common';
+import { ExpressionValueSearchContext } from './kibana_context_type';
+import { functionWrapper } from './utils';
+import { kqlFunction } from './kql';
+
+describe('interpreter/functions#kql', () => {
+  const fn = functionWrapper(kqlFunction);
+  let input: Partial<ExpressionValueSearchContext>;
+  let context: ExecutionContext;
+
+  beforeEach(() => {
+    input = { timeRange: { from: '0', to: '1' } };
+    context = {
+      getSearchContext: () => ({}),
+      getSearchSessionId: () => undefined,
+      types: {},
+      variables: {},
+      abortSignal: {} as any,
+      inspectorAdapters: {} as any,
+    };
+  });
+
+  it('returns an object with the correct structure', () => {
+    const actual = fn(input, { q: 'test' }, context);
+    expect(actual).toMatchInlineSnapshot(
+      `
+      Object {
+        "language": "kuery",
+        "query": "test",
+        "type": "kibana_query",
+      }
+    `
+    );
+  });
+});

src/plugins/data/common/search/expressions/kql.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { ExpressionFunctionDefinition } from 'src/plugins/expressions/common';
+import { KibanaQueryOutput } from './kibana_context_type';
+
+interface Arguments {
+  q: string;
+}
+
+export type ExpressionFunctionKql = ExpressionFunctionDefinition<
+  'kql',
+  null,
+  Arguments,
+  KibanaQueryOutput
+>;
+
+export const kqlFunction: ExpressionFunctionKql = {
+  name: 'kql',
+  type: 'kibana_query',
+  inputTypes: ['null'],
+  help: i18n.translate('data.search.functions.kql.help', {
+    defaultMessage: 'Create kibana kql query',
+  }),
+  args: {
+    q: {
+      types: ['string'],
+      required: true,
+      aliases: ['query', '_'],
+      help: i18n.translate('data.search.functions.kql.q.help', {
+        defaultMessage: 'Specify Kibana KQL free form text query',
+      }),
+    },
+  },
+
+  fn(input, args) {
+    return {
+      type: 'kibana_query',
+      language: 'kuery',
+      query: args.q,
+    };
+  },
+};

src/plugins/data/common/search/expressions/lucene.test.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ExecutionContext } from 'src/plugins/expressions/common';
+import { ExpressionValueSearchContext } from './kibana_context_type';
+import { functionWrapper } from './utils';
+import { luceneFunction } from './lucene';
+
+describe('interpreter/functions#lucene', () => {
+  const fn = functionWrapper(luceneFunction);
+  let input: Partial<ExpressionValueSearchContext>;
+  let context: ExecutionContext;
+
+  beforeEach(() => {
+    input = { timeRange: { from: '0', to: '1' } };
+    context = {
+      getSearchContext: () => ({}),
+      getSearchSessionId: () => undefined,
+      types: {},
+      variables: {},
+      abortSignal: {} as any,
+      inspectorAdapters: {} as any,
+    };
+  });
+
+  it('returns an object with the correct structure', () => {
+    const actual = fn(input, { q: '{ "test": 1 }' }, context);
+    expect(actual).toMatchInlineSnapshot(`
+      Object {
+        "language": "lucene",
+        "query": Object {
+          "test": 1,
+        },
+        "type": "kibana_query",
+      }
+    `);
+  });
+});

src/plugins/data/common/search/expressions/lucene.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { ExpressionFunctionDefinition } from 'src/plugins/expressions/common';
+import { KibanaQueryOutput } from './kibana_context_type';
+
+interface Arguments {
+  q: string;
+}
+
+export type ExpressionFunctionLucene = ExpressionFunctionDefinition<
+  'lucene',
+  null,
+  Arguments,
+  KibanaQueryOutput
+>;
+
+export const luceneFunction: ExpressionFunctionLucene = {
+  name: 'lucene',
+  type: 'kibana_query',
+  inputTypes: ['null'],
+  help: i18n.translate('data.search.functions.lucene.help', {
+    defaultMessage: 'Create kibana lucene query',
+  }),
+  args: {
+    q: {
+      types: ['string'],
+      required: true,
+      aliases: ['query', '_'],
+      help: i18n.translate('data.search.functions.lucene.q.help', {
+        defaultMessage: 'Specify Lucene free form text query',
+      }),
+    },
+  },
+
+  fn(input, args) {
+    return {
+      type: 'kibana_query',
+      language: 'lucene',
+      query: JSON.parse(args.q),
+    };
+  },
+};

src/plugins/data/common/search/expressions/query_to_ast.test.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { queryToAst } from './query_to_ast';
+
+describe('queryToAst', () => {
+  it('returns an object with the correct structure for lucene queies', () => {
+    const actual = queryToAst({ language: 'lucene', query: { country: 'US' } });
+    expect(actual).toHaveProperty('functions');
+    expect(actual.functions[0]).toHaveProperty('name', 'lucene');
+    expect(actual.functions[0]).toHaveProperty('arguments', {
+      q: ['{"country":"US"}'],
+    });
+  });
+
+  it('returns an object with the correct structure for kql queies', () => {
+    const actual = queryToAst({ language: 'kuery', query: 'country:US' });
+    expect(actual).toHaveProperty('functions');
+    expect(actual.functions[0]).toHaveProperty('name', 'kql');
+    expect(actual.functions[0]).toHaveProperty('arguments', {
+      q: ['country:US'],
+    });
+  });
+});

src/plugins/data/common/search/expressions/query_to_ast.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { buildExpression, buildExpressionFunction } from '../../../../expressions/common';
+import { Query } from '../../query';
+import { ExpressionFunctionKql } from './kql';
+import { ExpressionFunctionLucene } from './lucene';
+
+export const queryToAst = (query: Query) => {
+  if (query.language === 'kuery') {
+    return buildExpression([
+      buildExpressionFunction<ExpressionFunctionKql>('kql', { q: query.query as string }),
+    ]);
+  }
+  return buildExpression([
+    buildExpressionFunction<ExpressionFunctionLucene>('lucene', { q: JSON.stringify(query.query) }),
+  ]);
+};

src/plugins/data/common/search/expressions/timerange.test.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ExecutionContext } from 'src/plugins/expressions/common';
+import { ExpressionValueSearchContext } from './kibana_context_type';
+import { functionWrapper } from './utils';
+import { kibanaTimerangeFunction } from './timerange';
+
+describe('interpreter/functions#timerange', () => {
+  const fn = functionWrapper(kibanaTimerangeFunction);
+  let input: Partial<ExpressionValueSearchContext>;
+  let context: ExecutionContext;
+
+  beforeEach(() => {
+    input = { timeRange: { from: '0', to: '1' } };
+    context = {
+      getSearchContext: () => ({}),
+      getSearchSessionId: () => undefined,
+      types: {},
+      variables: {},
+      abortSignal: {} as any,
+      inspectorAdapters: {} as any,
+    };
+  });
+
+  it('returns an object with the correct structure', () => {
+    const actual = fn(input, { from: 'now', to: 'now-7d', mode: 'absolute' }, context);
+    expect(actual).toMatchInlineSnapshot(
+      `
+      Object {
+        "from": "now",
+        "mode": "absolute",
+        "to": "now-7d",
+        "type": "timerange",
+      }
+    `
+    );
+  });
+});

src/plugins/data/common/search/expressions/timerange.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { ExpressionFunctionDefinition, ExpressionValueBoxed } from 'src/plugins/expressions/common';
+import { TimeRange } from '../../query';
+
+export type KibanaTimerangeOutput = ExpressionValueBoxed<'timerange', TimeRange>;
+
+export type ExpressionFunctionKibanaTimerange = ExpressionFunctionDefinition<
+  'timerange',
+  null,
+  TimeRange,
+  KibanaTimerangeOutput
+>;
+
+export const kibanaTimerangeFunction: ExpressionFunctionKibanaTimerange = {
+  name: 'timerange',
+  type: 'timerange',
+  inputTypes: ['null'],
+  help: i18n.translate('data.search.functions.timerange.help', {
+    defaultMessage: 'Create kibana timerange',
+  }),
+  args: {
+    from: {
+      types: ['string'],
+      required: true,
+      help: i18n.translate('data.search.functions.timerange.from.help', {
+        defaultMessage: 'Specify the start date',
+      }),
+    },
+    to: {
+      types: ['string'],
+      required: true,
+      help: i18n.translate('data.search.functions.timerange.to.help', {
+        defaultMessage: 'Specify the end date',
+      }),
+    },
+    mode: {
+      types: ['string'],
+      options: ['absolute', 'relative'],
+      help: i18n.translate('data.search.functions.timerange.mode.help', {
+        defaultMessage: 'Specify the mode (absolute or relative)',
+      }),
+    },
+  },
+
+  fn(input, args) {
+    return {
+      type: 'timerange',
+      from: args.from,
+      to: args.to,
+      mode: args.mode,
+    };
+  },
+};

src/plugins/data/common/search/expressions/timerange_to_ast.test.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { timerangeToAst } from './timerange_to_ast';
+
+describe('timerangeToAst', () => {
+  it('returns an object with the correct structure', () => {
+    const actual = timerangeToAst({ from: 'now', to: 'now-7d', mode: 'absolute' });
+    expect(actual).toHaveProperty('name', 'timerange');
+    expect(actual).toHaveProperty('arguments', {
+      from: ['now'],
+      mode: ['absolute'],
+      to: ['now-7d'],
+    });
+  });
+});

src/plugins/data/common/search/expressions/timerange_to_ast.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { buildExpressionFunction } from '../../../../expressions/common';
+import { TimeRange } from '../../query';
+import { ExpressionFunctionKibanaTimerange } from './timerange';
+
+export const timerangeToAst = (timerange: TimeRange) => {
+  return buildExpressionFunction<ExpressionFunctionKibanaTimerange>('timerange', {
+    from: timerange.from,
+    to: timerange.to,
+    mode: timerange.mode,
+  });
+};

src/plugins/data/public/search/search_service.ts

@@ -25,6 +25,9 @@ import {
   ISearchGeneric,
   SearchSourceDependencies,
   SearchSourceService,
+  kibanaTimerangeFunction,
+  luceneFunction,
+  kqlFunction,
 } from '../../common/search';
 import { getCallMsearch } from './legacy';
 import { AggsService, AggsStartDependencies } from './aggs';
@@ -102,6 +105,9 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
     );
     expressions.registerFunction(kibana);
     expressions.registerFunction(kibanaContextFunction);
+    expressions.registerFunction(luceneFunction);
+    expressions.registerFunction(kqlFunction);
+    expressions.registerFunction(kibanaTimerangeFunction);
     expressions.registerType(kibanaContext);
 
     expressions.registerFunction(esdsl);

src/plugins/data/server/search/search_service.ts

@@ -51,6 +51,9 @@ import {
   kibana,
   kibanaContext,
   kibanaContextFunction,
+  kibanaTimerangeFunction,
+  kqlFunction,
+  luceneFunction,
   SearchSourceDependencies,
   searchSourceRequiredUiSettings,
   SearchSourceService,
@@ -142,6 +145,9 @@ export class SearchService implements Plugin<ISearchSetup, ISearchStart> {
 
     expressions.registerFunction(getEsaggs({ getStartServices: core.getStartServices }));
     expressions.registerFunction(kibana);
+    expressions.registerFunction(luceneFunction);
+    expressions.registerFunction(kqlFunction);
+    expressions.registerFunction(kibanaTimerangeFunction);
     expressions.registerFunction(kibanaContextFunction);
     expressions.registerType(kibanaContext);
 

src/plugins/visualizations/public/embeddable/to_ast.ts

@@ -10,6 +10,7 @@ import { ExpressionFunctionKibana, ExpressionFunctionKibanaContext } from '../..
 import { buildExpression, buildExpressionFunction } from '../../../expressions/public';
 
 import { VisToExpressionAst } from '../types';
+import { queryToAst } from '../../../data/common';
 
 /**
  * Creates an ast expression for a visualization based on kibana context (query, filters, timerange)
@@ -25,7 +26,7 @@ export const toExpressionAst: VisToExpressionAst = async (vis, params) => {
 
   const kibana = buildExpressionFunction<ExpressionFunctionKibana>('kibana', {});
   const kibanaContext = buildExpressionFunction<ExpressionFunctionKibanaContext>('kibana_context', {
-    q: query && JSON.stringify(query),
+    q: query && queryToAst(query),
     filters: filters && JSON.stringify(filters),
     savedSearchId,
   });

test/interpreter_functional/test_suites/run_pipeline/esaggs.ts

@@ -49,7 +49,7 @@ export default function ({
           to: '2015-09-22T00:00:00Z',
         };
         const expression = `
-          kibana_context timeRange='${JSON.stringify(timeRange)}'
+          kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
           | esaggs index={indexPatternLoad id='logstash-*'}
           aggs={aggCount id="1" enabled=true schema="metric"}
         `;
@@ -63,7 +63,7 @@ export default function ({
           to: '2015-09-22T00:00:00Z',
         };
         const expression = `
-          kibana_context timeRange='${JSON.stringify(timeRange)}'
+          kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
           | esaggs index={indexPatternLoad id='logstash-*'}
           timeFields='relatedContent.article:published_time'
           aggs={aggCount id="1" enabled=true schema="metric"}
@@ -78,7 +78,7 @@ export default function ({
           to: '2015-09-22T00:00:00Z',
         };
         const expression = `
-          kibana_context timeRange='${JSON.stringify(timeRange)}'
+          kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'}
           | esaggs index={indexPatternLoad id='logstash-*'}
           timeFields='relatedContent.article:published_time'
           timeFields='@timestamp'