github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ES|QL] Enabling the timepicker for indices without the @timestamp field (#184361)

Browse source 
## Summary

Based on https://github.com/elastic/elasticsearch/pull/108421

Closes https://github.com/elastic/kibana/issues/180805

It allows the users to add an `?earliest` and `?latest` variable in the
ES|QL editor.

When we are detecting this variable, we are also sending the values to
ES.

- Earliest is the from value of the date picker
- Latest is the to value of the date picker


![meow](177ceafe-558d-43ea-809c-f0ec2833ec17)

Usage in bucket


![meow](8dea8188-b4e0-43e6-894e-236811374030)

This enables 2 very important features:

- I can use the values of the time picker in the bucket function
- I can use the time picker for indices without `@timestamp` field

### For reviewers
- Although it seems as a big PR, the majority of the changes happen due
to the signature change of the `getESQLAdHocDataview`
- The ML changes are mostly because the ML code has a lot of repetition.
I think the code needs to be refactored to have a central point
(preferably the `getESQLResults` from the esql-utils. I will create an
issue for the ML team.
- I am not proposing this in bucket autocomplete because it doesnt work
great in general for the date histogram case. We are working on
autocomplete improvements so I am expecting this to be part of a follow
up PR.
- I want to talk to the docs team to add it in the docs.


### Follow ups
- Change the histogram to use the bucket instead of the date_trunc
(needs investigation first)
- Speak with the docs team about adding an example on our official docs


### Flaky test runner 
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6521

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Julia Rechkunova <julia.rechkunova@gmail.com>
This commit is contained in:
Stratoula Kalafateli 2024-07-17 16:23:16 -07:00 • committed by GitHub
parent 661c25133d
commit fcf2702c0e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
58 changed files with 616 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

1
packages/kbn-es-types/src/search.ts
View file

@ -691,4 +691,5 @@ export interface ESQLSearchParams {
filter?: unknown;
locale?: string;
dropNullColumns?: boolean;
params?: Array<Record<string, string | undefined>>;
}

22
packages/kbn-esql-ast/src/walker/walker.test.ts
View file

@ -87,6 +87,28 @@ test('can find assignment expression', () => {
expect((functions[0].args[0] as any).name).toBe('var0');
});
test('can collect all params from grouping functions', () => {
const query =
'ROW x=1, time=2024-07-10 | stats z = avg(x) by bucket(time, 20, ?earliest,?latest)';
const { ast } = getAstAndSyntaxErrors(query);
const params = Walker.params(ast);
expect(params).toMatchObject([
{
type: 'literal',
literalType: 'param',
paramType: 'named',
value: 'earliest',
},
{
type: 'literal',
literalType: 'param',
paramType: 'named',
value: 'latest',
},
]);
});
describe('Walker.hasFunction()', () => {
test('can find assignment expression', () => {
const query1 = 'METRICS source bucket(bytes, 1 hour)';

4
packages/kbn-esql-ast/src/walker/walker.ts
View file

@ -139,6 +139,10 @@ export class Walker {
this.walkFunction(node as ESQLFunction);
break;
}
case 'option': {
this.walkAstItem(node.args);
break;
}
case 'column': {
options.visitColumn?.(node);
break;

3
packages/kbn-esql-utils/index.ts
View file

@ -20,6 +20,9 @@ export {
getESQLQueryColumns,
getESQLQueryColumnsRaw,
getESQLResults,
getTimeFieldFromESQLQuery,
getEarliestLatestParams,
hasEarliestLatestParams,
TextBasedLanguages,
} from './src';

9
packages/kbn-esql-utils/src/index.ts
View file

@ -15,6 +15,13 @@ export {
getLimitFromESQLQuery,
removeDropCommandsFromESQLQuery,
hasTransformationalCommand,
getTimeFieldFromESQLQuery,
} from './utils/query_parsing_helpers';
export { appendToESQLQuery, appendWhereClauseToESQLQuery } from './utils/append_to_query';
export { getESQLQueryColumns, getESQLQueryColumnsRaw, getESQLResults } from './utils/run_query';
export {
getESQLQueryColumns,
getESQLQueryColumnsRaw,
getESQLResults,
getEarliestLatestParams,
hasEarliestLatestParams,
} from './utils/run_query';

9
packages/kbn-esql-utils/src/utils/get_esql_adhoc_dataview.ts
View file

@ -7,6 +7,7 @@
*/
import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
import { ESQL_TYPE } from '@kbn/data-view-utils';
import { getTimeFieldFromESQLQuery, getIndexPatternFromESQLQuery } from './query_parsing_helpers';
// uses browser sha256 method with fallback if unavailable
async function sha256(str: string) {
@ -28,18 +29,22 @@ async function sha256(str: string) {
// the same adhoc dataview can be constructed/used. This comes with great advantages such
// as solving the problem described here https://github.com/elastic/kibana/issues/168131
export async function getESQLAdHocDataview(
indexPattern: string,
query: string,
dataViewsService: DataViewsPublicPluginStart
) {
const timeField = getTimeFieldFromESQLQuery(query);
const indexPattern = getIndexPatternFromESQLQuery(query);
const dataView = await dataViewsService.create({
title: indexPattern,
type: ESQL_TYPE,
id: await sha256(`esql-${indexPattern}`),
});
dataView.timeFieldName = timeField;
// If the indexPattern is empty string means that the user used either the ROW or META FUNCTIONS / SHOW INFO commands
// we don't want to add the @timestamp field in this case https://github.com/elastic/kibana/issues/163417
if (indexPattern && dataView?.fields?.getByName?.('@timestamp')?.type === 'date') {
if (!timeField && indexPattern && dataView?.fields?.getByName?.('@timestamp')?.type === 'date') {
dataView.timeFieldName = '@timestamp';
}

33
packages/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts
View file

@ -11,6 +11,7 @@ import {
getLimitFromESQLQuery,
removeDropCommandsFromESQLQuery,
hasTransformationalCommand,
getTimeFieldFromESQLQuery,
} from './query_parsing_helpers';
describe('esql query helpers', () => {
@ -142,4 +143,36 @@ describe('esql query helpers', () => {
expect(hasTransformationalCommand('metrics a var = avg(b)')).toBeTruthy();
});
});
describe('getTimeFieldFromESQLQuery', () => {
it('should return undefined if there are no time params', () => {
expect(getTimeFieldFromESQLQuery('from a | eval b = 1')).toBeUndefined();
});
it('should return the time field if there is at least one time param', () => {
expect(getTimeFieldFromESQLQuery('from a | eval b = 1 | where time >= ?earliest')).toBe(
'time'
);
});
it('should return undefined if there is one named param but is not ?earliest or ?latest', () => {
expect(
getTimeFieldFromESQLQuery('from a | eval b = 1 | where time >= ?late')
).toBeUndefined();
});
it('should return undefined if there is one named param but is used without a time field', () => {
expect(
getTimeFieldFromESQLQuery('from a | eval b = DATE_TRUNC(1 day, ?earliest)')
).toBeUndefined();
});
it('should return the time field if there is at least one time param in the bucket function', () => {
expect(
getTimeFieldFromESQLQuery(
'from a | stats meow = avg(bytes) by bucket(event.timefield, 200, ?earliest, ?latest)'
)
).toBe('event.timefield');
});
});
});

41
packages/kbn-esql-utils/src/utils/query_parsing_helpers.ts
View file

@ -5,7 +5,8 @@
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { type ESQLSource, getAstAndSyntaxErrors } from '@kbn/esql-ast';
import type { ESQLSource, ESQLFunction, ESQLColumn, ESQLSingleAstItem } from '@kbn/esql-ast';
import { getAstAndSyntaxErrors, Walker, walk } from '@kbn/esql-ast';
const DEFAULT_ESQL_LIMIT = 500;
@ -53,3 +54,41 @@ export function removeDropCommandsFromESQLQuery(esql?: string): string {
const pipes = (esql || '').split('|');
return pipes.filter((statement) => !/DROP\s/i.test(statement)).join('|');
}
/**
* When the ?earliest and ?latest params are used, we want to retrieve the timefield from the query.
* @param esql:string
* @returns string
*/
export const getTimeFieldFromESQLQuery = (esql: string) => {
const { ast } = getAstAndSyntaxErrors(esql);
const functions: ESQLFunction[] = [];
walk(ast, {
visitFunction: (node) => functions.push(node),
});
const params = Walker.params(ast);
const timeNamedParam = params.find(
(param) => param.value === 'earliest' || param.value === 'latest'
);
if (!timeNamedParam || !functions.length) {
return undefined;
}
const allFunctionsWithNamedParams = functions.filter(
({ location }) =>
location.min <= timeNamedParam.location.min && location.max >= timeNamedParam.location.max
);
if (!allFunctionsWithNamedParams.length) {
return undefined;
}
const lowLevelFunction = allFunctionsWithNamedParams[allFunctionsWithNamedParams.length - 1];
const column = lowLevelFunction.args.find((arg) => {
const argument = arg as ESQLSingleAstItem;
return argument.type === 'column';
}) as ESQLColumn;
return column?.name;
};

42
packages/kbn-esql-utils/src/utils/run_query.test.ts Normal file
View file

@ -0,0 +1,42 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { getEarliestLatestParams } from './run_query';
describe('getEarliestLatestParams', () => {
it('should return an empty array if there are no time params', () => {
const time = { from: 'now-15m', to: 'now' };
const query = 'FROM foo';
const params = getEarliestLatestParams(query, time);
expect(params).toEqual([]);
});
it('should return an array with the earliest param if exists at the query', () => {
const time = { from: 'Jul 5, 2024 @ 08:03:56.849', to: 'Jul 5, 2024 @ 10:03:56.849' };
const query = 'FROM foo | where time > ?earliest';
const params = getEarliestLatestParams(query, time);
expect(params).toHaveLength(1);
expect(params[0]).toHaveProperty('earliest');
});
it('should return an array with the latest param if exists at the query', () => {
const time = { from: 'Jul 5, 2024 @ 08:03:56.849', to: 'Jul 5, 2024 @ 10:03:56.849' };
const query = 'FROM foo | where time < ?latest';
const params = getEarliestLatestParams(query, time);
expect(params).toHaveLength(1);
expect(params[0]).toHaveProperty('latest');
});
it('should return an array with the latest and earliest params if exist at the query', () => {
const time = { from: 'Jul 5, 2024 @ 08:03:56.849', to: 'Jul 5, 2024 @ 10:03:56.849' };
const query = 'FROM foo | where time < ?latest amd time > ?earliest';
const params = getEarliestLatestParams(query, time);
expect(params).toHaveLength(2);
expect(params[0]).toHaveProperty('earliest');
expect(params[1]).toHaveProperty('latest');
});
});

36
packages/kbn-esql-utils/src/utils/run_query.ts
View file

@ -6,12 +6,36 @@
* Side Public License, v 1.
*/
import { i18n } from '@kbn/i18n';
import dateMath from '@kbn/datemath';
import type { DatatableColumn } from '@kbn/expressions-plugin/common';
import type { ISearchGeneric } from '@kbn/search-types';
import type { TimeRange } from '@kbn/es-query';
import { esFieldTypeToKibanaFieldType } from '@kbn/field-types';
import type { ESQLColumn, ESQLSearchResponse, ESQLSearchParams } from '@kbn/es-types';
import { lastValueFrom } from 'rxjs';
export const hasEarliestLatestParams = (query: string) => /\?earliest|\?latest/i.test(query);
export const getEarliestLatestParams = (query: string, time?: TimeRange) => {
const earliestNamedParams = /\?earliest/i.test(query);
const latestNamedParams = /\?latest/i.test(query);
if (time && (earliestNamedParams || latestNamedParams)) {
const timeParams = {
earliest: earliestNamedParams ? dateMath.parse(time.from)?.toISOString() : undefined,
latest: latestNamedParams ? dateMath.parse(time.to)?.toISOString() : undefined,
};
const namedParams = [];
if (timeParams?.earliest) {
namedParams.push({ earliest: timeParams.earliest });
}
if (timeParams?.latest) {
namedParams.push({ latest: timeParams.latest });
}
return namedParams;
}
return [];
};
export function formatESQLColumns(columns: ESQLColumn[]): DatatableColumn[] {
return columns.map(({ name, type }) => {
const kibanaType = esFieldTypeToKibanaFieldType(type);
@ -29,17 +53,21 @@ export async function getESQLQueryColumnsRaw({
esqlQuery,
search,
signal,
timeRange,
}: {
esqlQuery: string;
search: ISearchGeneric;
signal?: AbortSignal;
timeRange?: TimeRange;
}): Promise<ESQLColumn[]> {
try {
const namedParams = getEarliestLatestParams(esqlQuery, timeRange);
const response = await lastValueFrom(
search(
{
params: {
query: `${esqlQuery} | limit 0`,
...(namedParams.length ? { params: namedParams } : {}),
},
},
{
@ -66,13 +94,15 @@ export async function getESQLQueryColumns({
esqlQuery,
search,
signal,
timeRange,
}: {
esqlQuery: string;
search: ISearchGeneric;
signal?: AbortSignal;
timeRange?: TimeRange;
}): Promise<DatatableColumn[]> {
try {
const rawColumns = await getESQLQueryColumnsRaw({ esqlQuery, search, signal });
const rawColumns = await getESQLQueryColumnsRaw({ esqlQuery, search, signal, timeRange });
const columns = formatESQLColumns(rawColumns) ?? [];
return columns;
} catch (error) {
@ -93,16 +123,19 @@ export async function getESQLResults({
signal,
filter,
dropNullColumns,
timeRange,
}: {
esqlQuery: string;
search: ISearchGeneric;
signal?: AbortSignal;
filter?: unknown;
dropNullColumns?: boolean;
timeRange?: TimeRange;
}): Promise<{
response: ESQLSearchResponse;
params: ESQLSearchParams;
}> {
const namedParams = getEarliestLatestParams(esqlQuery, timeRange);
const result = await lastValueFrom(
search(
{
@ -110,6 +143,7 @@ export async function getESQLResults({
...(filter ? { filter } : {}),
query: esqlQuery,
...(dropNullColumns ? { dropNullColumns: true } : {}),
...(namedParams.length ? { params: namedParams } : {}),
},
},
{

4
packages/kbn-esql-utils/tsconfig.json
View file

@ -24,6 +24,8 @@
"@kbn/expressions-plugin",
"@kbn/field-types",
"@kbn/es-types",
"@kbn/i18n"
"@kbn/i18n",
"@kbn/datemath",
"@kbn/es-query"
]
}

6
packages/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/helpers.ts
View file

@ -16,6 +16,7 @@ import { groupingFunctionDefinitions } from '../../definitions/grouping';
import * as autocomplete from '../autocomplete';
import type { ESQLCallbacks } from '../../shared/types';
import type { EditorContext } from '../types';
import { TIME_SYSTEM_PARAMS } from '../factories';
export interface Integration {
name: string;
@ -221,6 +222,11 @@ export function getLiteralsByType(_type: string | string[]) {
return [];
}
export function getDateLiteralsByFieldType(_requestedType: string | string[]) {
const requestedType = Array.isArray(_requestedType) ? _requestedType : [_requestedType];
return requestedType.includes('date') ? TIME_SYSTEM_PARAMS : [];
}
export function createCustomCallbackMocks(
customFields?: Array<{ name: string; type: string }>,
customSources?: Array<{ name: string; hidden: boolean }>,

10
packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts
View file

@ -10,7 +10,7 @@ import { suggest } from './autocomplete';
import { evalFunctionDefinitions } from '../definitions/functions';
import { timeUnitsToSuggest } from '../definitions/literals';
import { commandDefinitions } from '../definitions/commands';
import { getUnitDuration, TRIGGER_SUGGESTION_COMMAND } from './factories';
import { getUnitDuration, TRIGGER_SUGGESTION_COMMAND, TIME_SYSTEM_PARAMS } from './factories';
import { camelCase, partition } from 'lodash';
import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
import { FunctionParameter } from '../definitions/types';
@ -21,6 +21,7 @@ import {
getFunctionSignaturesByReturnType,
getFieldNamesByType,
getLiteralsByType,
getDateLiteralsByFieldType,
createCustomCallbackMocks,
createSuggestContext,
getPolicyFields,
@ -730,6 +731,9 @@ describe('autocomplete', () => {
suggestedConstants?.length
? suggestedConstants.map((option) => `"${option}"${requiresMoreArgs ? ',' : ''}`)
: [
...getDateLiteralsByFieldType(
getTypesFromParamDefs(acceptsFieldParamDefs)
).map((l) => (requiresMoreArgs ? `${l},` : l)),
...getFieldNamesByType(getTypesFromParamDefs(acceptsFieldParamDefs)).map(
(f) => (requiresMoreArgs ? `${f},` : f)
),
@ -752,6 +756,9 @@ describe('autocomplete', () => {
suggestedConstants?.length
? suggestedConstants.map((option) => `"${option}"${requiresMoreArgs ? ',' : ''}`)
: [
...getDateLiteralsByFieldType(
getTypesFromParamDefs(acceptsFieldParamDefs)
).map((l) => (requiresMoreArgs ? `${l},` : l)),
...getFieldNamesByType(getTypesFromParamDefs(acceptsFieldParamDefs)).map(
(f) => (requiresMoreArgs ? `${f},` : f)
),
@ -810,6 +817,7 @@ describe('autocomplete', () => {
testSuggestions(
'from a | eval var0=date_trunc()',
[
...TIME_SYSTEM_PARAMS.map((t) => `${t},`),
...getLiteralsByType('time_literal').map((t) => `${t},`),
...getFunctionSignaturesByReturnType('eval', 'date', { evalMath: true }, undefined, [
'date_trunc',

7
packages/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.ts
View file

@ -70,6 +70,7 @@ import {
buildOptionDefinition,
buildSettingDefinitions,
buildValueDefinitions,
getDateLiterals,
buildFieldsDefinitionsWithMetadata,
} from './factories';
import { EDITOR_MARKER, SINGLE_BACKTICK, METADATA_FIELDS } from '../shared/constants';
@ -1086,7 +1087,11 @@ async function getFieldsOrFunctionsSuggestions(
}
}
// could also be in stats (bucket) but our autocomplete is not great yet
const displayDateSuggestions = types.includes('date') && ['where', 'eval'].includes(commandName);
const suggestions = filteredFieldsByType.concat(
displayDateSuggestions ? getDateLiterals() : [],
functions ? getCompatibleFunctionDefinition(commandName, optionName, types, ignoreFn) : [],
variables
? pushItUpInTheList(buildVariablesDefinitions(filteredVariablesByType), functions)
@ -1326,7 +1331,7 @@ async function getFunctionArgsSuggestions(
return suggestions.map(({ text, ...rest }) => ({
...rest,
text: addCommaIf(hasMoreMandatoryArgs && fnDefinition.type !== 'builtin', text),
text: addCommaIf(hasMoreMandatoryArgs && fnDefinition.type !== 'builtin' && text !== '', text),
}));
}

17
packages/kbn-esql-validation-autocomplete/src/autocomplete/factories.ts
View file

@ -28,6 +28,8 @@ const allFunctions = statsAggregationFunctionDefinitions
.concat(evalFunctionDefinitions)
.concat(groupingFunctionDefinitions);
export const TIME_SYSTEM_PARAMS = ['?earliest', '?latest'];
export const TRIGGER_SUGGESTION_COMMAND = {
title: 'Trigger Suggestion Dialog',
id: 'editor.action.triggerSuggest',
@ -198,7 +200,8 @@ export const buildSourcesDefinitions = (
export const buildConstantsDefinitions = (
userConstants: string[],
detail?: string
detail?: string,
sortText?: string
): SuggestionRawDefinition[] =>
userConstants.map((label) => ({
label,
@ -209,7 +212,7 @@ export const buildConstantsDefinitions = (
i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.constantDefinition', {
defaultMessage: `Constant`,
}),
sortText: 'A',
sortText: sortText ?? 'A',
}));
export const buildValueDefinitions = (
@ -389,3 +392,13 @@ export function getCompatibleLiterals(commandName: string, types: string[], name
}
return suggestions;
}
export function getDateLiterals() {
return buildConstantsDefinitions(
TIME_SYSTEM_PARAMS,
i18n.translate('kbn-esql-validation-autocomplete.esql.autocomplete.namedParamDefinition', {
defaultMessage: 'Named parameter',
}),
'1A'
);
}

2
packages/kbn-monaco/src/esql/lib/esql_theme.ts
View file

@ -132,7 +132,7 @@ export const buildESQlTheme = (): monaco.editor.IStandaloneThemeData => ({
// values
...buildRuleGroup(
['quoted_string', 'integer_literal', 'decimal_literal'],
['quoted_string', 'integer_literal', 'decimal_literal', 'named_or_positional_param'],
euiThemeVars.euiColorSuccessText
),
],

11
packages/kbn-text-based-editor/src/editor_footer.tsx
View file

@ -101,7 +101,7 @@ interface EditorFooterProps {
};
errors?: MonacoMessage[];
warnings?: MonacoMessage[];
detectTimestamp: boolean;
detectedTimestamp?: string;
onErrorClick: (error: MonacoMessage) => void;
runQuery: () => void;
updateQuery: (qs: string) => void;
@ -126,7 +126,7 @@ export const EditorFooter = memo(function EditorFooter({
styles,
errors,
warnings,
detectTimestamp,
detectedTimestamp,
onErrorClick,
runQuery,
updateQuery,
@ -195,7 +195,7 @@ export const EditorFooter = memo(function EditorFooter({
</EuiText>
</EuiFlexItem>
{/* If there is no space and no @timestamp detected hide the information */}
{(detectTimestamp || !isSpaceReduced) && !hideTimeFilterInfo && (
{(detectedTimestamp || !isSpaceReduced) && !hideTimeFilterInfo && (
<EuiFlexItem grow={false} style={{ marginRight: '16px' }}>
<EuiFlexGroup gutterSize="xs" responsive={false} alignItems="center">
<EuiFlexItem grow={false}>
@ -207,11 +207,12 @@ export const EditorFooter = memo(function EditorFooter({
<p>
{isSpaceReduced
? '@timestamp'
: detectTimestamp
: detectedTimestamp
? i18n.translate(
'textBasedEditor.query.textBasedLanguagesEditor.timestampDetected',
{
defaultMessage: '@timestamp found',
defaultMessage: '{detectedTimestamp} found',
values: { detectedTimestamp },
}
)
: i18n.translate(

4
packages/kbn-text-based-editor/src/text_based_languages_editor.test.tsx
View file

@ -113,11 +113,11 @@ describe('TextBasedLanguagesEditor', () => {
expect(component.find('[data-test-subj="TextBasedLangEditor-date-info"]').length).toBe(0);
});
it('should render the date info with @timestamp found if detectTimestamp is true', async () => {
it('should render the date info with @timestamp found if detectedTimestamp is given', async () => {
const newProps = {
...props,
isCodeEditorExpanded: true,
detectTimestamp: true,
detectedTimestamp: '@timestamp',
};
const component = mount(renderTextBasedLanguagesEditorComponent({ ...newProps }));
expect(

8
packages/kbn-text-based-editor/src/text_based_languages_editor.tsx
View file

@ -79,7 +79,7 @@ export interface TextBasedLanguagesEditorProps {
* The text based queries are relying on adhoc dataviews which
* can have an @timestamp timefield or nothing
*/
detectTimestamp?: boolean;
detectedTimestamp?: string;
/** Array of errors */
errors?: Error[];
/** Warning string as it comes from ES */
@ -150,7 +150,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
onTextLangQuerySubmit,
expandCodeEditor,
isCodeEditorExpanded,
detectTimestamp = false,
detectedTimestamp,
errors: serverErrors,
warning: serverWarning,
isLoading,
@ -984,7 +984,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
onErrorClick={onErrorClick}
runQuery={onQuerySubmit}
updateQuery={onQueryUpdate}
detectTimestamp={detectTimestamp}
detectedTimestamp={detectedTimestamp}
editorIsInline={editorIsInline}
disableSubmitAction={disableSubmitAction}
hideRunQueryText={hideRunQueryText}
@ -1083,7 +1083,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
onErrorClick={onErrorClick}
runQuery={onQuerySubmit}
updateQuery={onQueryUpdate}
detectTimestamp={detectTimestamp}
detectedTimestamp={detectedTimestamp}
hideRunQueryText={hideRunQueryText}
editorIsInline={editorIsInline}
disableSubmitAction={disableSubmitAction}

1
packages/kbn-unified-field-list/src/services/field_stats_text_based/load_field_stats_text_based.ts
View file

@ -67,6 +67,7 @@ export const loadFieldStatsTextBased: LoadFieldStatsTextBasedHandler = async ({
filter,
search: data.search.search,
signal: abortController?.signal,
timeRange: { from: fromDate, to: toDate },
});
return result.response;
};

8
src/plugins/data/common/search/expressions/esql.ts
View file

@ -12,6 +12,7 @@ import { i18n } from '@kbn/i18n';
import type { IKibanaSearchResponse, IKibanaSearchRequest } from '@kbn/search-types';
import type { Datatable, ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
import { RequestAdapter } from '@kbn/inspector-plugin/common';
import { getEarliestLatestParams } from '@kbn/esql-utils';
import { zipObject } from 'lodash';
import { Observable, defer, throwError } from 'rxjs';
@ -152,6 +153,13 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
const esQueryConfigs = getEsQueryConfig(
uiSettings as Parameters<typeof getEsQueryConfig>[0]
);
const namedParams = getEarliestLatestParams(query, input.timeRange);
if (namedParams.length) {
params.params = namedParams;
}
const timeFilter =
input.timeRange &&
getTime(undefined, input.timeRange, {

2
src/plugins/data/tsconfig.json
View file

@ -55,7 +55,7 @@
"@kbn/react-kibana-mount",
"@kbn/search-types",
"@kbn/safer-lodash-set",
"@kbn/esql-utils",
"@kbn/esql-utils"
],
"exclude": [
"target/**/*",

24
src/plugins/discover/public/application/main/state_management/utils/get_esql_data_view.test.ts
View file

@ -5,7 +5,7 @@
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { DataView } from '@kbn/data-views-plugin/public';
import { getEsqlDataView } from './get_esql_data_view';
import { dataViewAdHoc } from '../../../../__mocks__/data_view_complex';
import { dataViewMock } from '@kbn/discover-utils/src/__mocks__';
@ -18,21 +18,33 @@ describe('getEsqlDataView', () => {
id: 'ad-hoc-id',
title: 'test',
});
const dataViewAdHocNoAtTimestamp = {
...dataViewAdHoc,
timeFieldName: undefined,
} as DataView;
const services = discoverServiceMock;
it('returns the current dataview if is adhoc and query has not changed', async () => {
it('returns the current dataview if it is adhoc with no named params and query index pattern is the same as the dataview index pattern', async () => {
const query = { esql: 'from data-view-ad-hoc-title' };
const dataView = await getEsqlDataView(query, dataViewAdHoc, services);
expect(dataView).toStrictEqual(dataViewAdHoc);
const dataView = await getEsqlDataView(query, dataViewAdHocNoAtTimestamp, services);
expect(dataView).toStrictEqual(dataViewAdHocNoAtTimestamp);
});
it('creates an adhoc dataview if the current dataview is persistent and query has not changed', async () => {
it('returns an adhoc dataview if it is adhoc with named params and query index pattern is the same as the dataview index pattern', async () => {
const query = { esql: 'from data-view-ad-hoc-title | where time >= ?earliest' };
const dataView = await getEsqlDataView(query, dataViewAdHocNoAtTimestamp, services);
expect(dataView.timeFieldName).toBe('time');
});
it('creates an adhoc dataview if the current dataview is persistent and query index pattern is the same as the dataview index pattern', async () => {
const query = { esql: 'from the-data-view-title' };
const dataView = await getEsqlDataView(query, dataViewMock, services);
expect(dataView.isPersisted()).toEqual(false);
expect(dataView.timeFieldName).toBe('@timestamp');
});
it('creates an adhoc dataview if the current dataview is ad hoc and query has changed', async () => {
it('creates an adhoc dataview if the current dataview is ad hoc and query index pattern is different from the dataview index pattern', async () => {
discoverServiceMock.dataViews.create = jest.fn().mockReturnValue({
...dataViewAdHoc,
isPersisted: () => false,

15
src/plugins/discover/public/application/main/state_management/utils/get_esql_data_view.ts
View file

@ -6,7 +6,11 @@
* Side Public License, v 1.
*/
import type { AggregateQuery } from '@kbn/es-query';
import { getESQLAdHocDataview, getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import {
getESQLAdHocDataview,
getIndexPatternFromESQLQuery,
getTimeFieldFromESQLQuery,
} from '@kbn/esql-utils';
import { DataView } from '@kbn/data-views-plugin/common';
import { DiscoverServices } from '../../../../build_services';
@ -16,12 +20,15 @@ export async function getEsqlDataView(
services: DiscoverServices
) {
const indexPatternFromQuery = getIndexPatternFromESQLQuery(query.esql);
const newTimeField = getTimeFieldFromESQLQuery(query.esql);
if (
currentDataView?.isPersisted() ||
indexPatternFromQuery !== currentDataView?.getIndexPattern()
indexPatternFromQuery !== currentDataView?.getIndexPattern() ||
// here the pattern hasn't changed but the time field has
(newTimeField !== currentDataView?.timeFieldName &&
indexPatternFromQuery === currentDataView?.getIndexPattern())
) {
return await getESQLAdHocDataview(indexPatternFromQuery, services.dataViews);
return await getESQLAdHocDataview(query.esql, services.dataViews);
}
return currentDataView;
}

5
src/plugins/esql_datagrid/README.md
View file

@ -24,7 +24,7 @@ Contains a Discover-like table specifically for ES|QL queries:
### How to use it
```tsx
import { getIndexPatternFromESQLQuery, getESQLAdHocDataview, formatESQLColumns } from '@kbn/esql-utils';
import { getESQLAdHocDataview, formatESQLColumns } from '@kbn/esql-utils';
import { ESQLDataGrid } from '@kbn/esql-datagrid/public';
/**
@ -32,8 +32,7 @@ import { ESQLDataGrid } from '@kbn/esql-datagrid/public';
This will return a response with columns and values
**/
const indexPattern = getIndexPatternFromESQLQuery(query);
const adHocDataView = getESQLAdHocDataview(indexPattern, dataViewService);
const adHocDataView = getESQLAdHocDataview(query, dataViewService);
const formattedColumns = formatESQLColumns(columns);
<ESQLDataGrid

8
src/plugins/unified_histogram/public/services/lens_vis_service.ts
View file

@ -455,13 +455,7 @@ export class LensVisService {
queryParams: QueryParams;
}): Suggestion | undefined => {
const { dataView, query, timeRange } = queryParams;
if (
dataView.isTimeBased() &&
query &&
isOfAggregateQueryType(query) &&
getAggregateQueryMode(query) === 'esql' &&
timeRange
) {
if (dataView.isTimeBased() && query && isOfAggregateQueryType(query) && timeRange) {
const isOnHistogramMode = shouldDisplayHistogram(query);
if (!isOnHistogramMode) return undefined;

4
src/plugins/unified_search/public/query_string_input/query_bar_top_row.test.tsx
View file

@ -310,7 +310,7 @@ describe('QueryBarTopRowTopRow', () => {
expect(component.find(QUERY_INPUT_SELECTOR).length).toBe(0);
expect(component.find(TEXT_BASED_EDITOR).length).toBe(1);
expect(component.find(TEXT_BASED_EDITOR).prop('detectTimestamp')).toBe(true);
expect(component.find(TEXT_BASED_EDITOR).prop('detectedTimestamp')).toBe('@timestamp');
expect(component.find(TIMEPICKER_SELECTOR).prop('isDisabled')).toBe(false);
});
@ -335,7 +335,7 @@ describe('QueryBarTopRowTopRow', () => {
expect(component.find(QUERY_INPUT_SELECTOR).length).toBe(0);
expect(component.find(TEXT_BASED_EDITOR).length).toBe(1);
expect(component.find(TEXT_BASED_EDITOR).prop('detectTimestamp')).toBe(false);
expect(component.find(TEXT_BASED_EDITOR).prop('detectedTimestamp')).toBeUndefined();
expect(component.find(TIMEPICKER_SELECTOR).prop('isDisabled')).toMatchInlineSnapshot(`
Object {
"display": <span

6
src/plugins/unified_search/public/query_string_input/query_bar_top_row.tsx
View file

@ -723,9 +723,9 @@ export const QueryBarTopRow = React.memo(
function renderTextLangEditor() {
const adHocDataview = props.indexPatterns?.[0];
let detectTimestamp = false;
let detectedTimestamp;
if (adHocDataview && typeof adHocDataview !== 'string') {
detectTimestamp = Boolean(adHocDataview?.timeFieldName);
detectedTimestamp = adHocDataview?.timeFieldName;
}
return (
isQueryLangSelected &&
@ -738,7 +738,7 @@ export const QueryBarTopRow = React.memo(
isCodeEditorExpanded={codeEditorIsExpanded}
errors={props.textBasedLanguageModeErrors}
warning={props.textBasedLanguageModeWarning}
detectTimestamp={detectTimestamp}
detectedTimestamp={detectedTimestamp}
onTextLangQuerySubmit={async () =>
onSubmit({
query: queryRef.current,

42
test/functional/apps/discover/esql/_esql_view.ts
View file

@ -46,12 +46,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover');
// and load a set of makelogs data
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
await esArchiver.load('test/functional/fixtures/es_archiver/kibana_sample_data_flights');
await kibanaServer.importExport.load(
'test/functional/fixtures/kbn_archiver/kibana_sample_data_flights_index_pattern'
);
await kibanaServer.uiSettings.replace(defaultSettings);
await PageObjects.common.navigateToApp('discover');
await PageObjects.timePicker.setDefaultAbsoluteRange();
});
describe('test', () => {
describe('ES|QL in Discover', () => {
it('should render esql view correctly', async function () {
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
@ -93,7 +97,43 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
expect(await testSubjects.exists('discoverFieldListPanelEditItem')).to.be(false);
});
it('should not render the histogram for indices with no @timestamp field', async function () {
await PageObjects.discover.selectTextBaseLang();
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
const testQuery = `from kibana_sample_data_flights | limit 10`;
await monacoEditor.setCodeEditorValue(testQuery);
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await testSubjects.exists('TextBasedLangEditor')).to.be(true);
// I am not rendering the histogram for indices with no @timestamp field
expect(await testSubjects.exists('unifiedHistogramChart')).to.be(false);
});
it('should render the histogram for indices with no @timestamp field when the ?earliest, ?latest params are in the query', async function () {
await PageObjects.discover.selectTextBaseLang();
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
const testQuery = `from kibana_sample_data_flights | limit 10 | where timestamp >= ?earliest and timestamp <= ?latest`;
await monacoEditor.setCodeEditorValue(testQuery);
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
const fromTime = 'Apr 10, 2018 @ 00:00:00.000';
const toTime = 'Nov 15, 2018 @ 00:00:00.000';
await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
expect(await testSubjects.exists('TextBasedLangEditor')).to.be(true);
expect(await testSubjects.exists('unifiedHistogramChart')).to.be(true);
});
it('should perform test query correctly', async function () {
await PageObjects.timePicker.setDefaultAbsoluteRange();
await PageObjects.discover.selectTextBaseLang();
const testQuery = `from logstash-* | limit 10 | stats countB = count(bytes) by geo.dest | sort countB`;

4
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_esql.tsx
View file

@ -116,7 +116,7 @@ export const IndexDataVisualizerESQL: FC<IndexDataVisualizerESQLProps> = (dataVi
if (!indexPattern) return;
const dv = await getOrCreateDataViewByIndexPattern(
data.dataViews,
indexPattern,
query.esql,
currentDataView
);
@ -266,7 +266,7 @@ export const IndexDataVisualizerESQL: FC<IndexDataVisualizerESQLProps> = (dataVi
onTextLangQuerySubmit={onTextLangQuerySubmit}
expandCodeEditor={expandCodeEditor}
isCodeEditorExpanded={true}
detectTimestamp={true}
detectedTimestamp={currentDataView?.timeFieldName}
hideMinimizeButton={true}
hideRunQueryText={false}
isLoading={queryHistoryStatus ?? false}

7
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/field_stats/field_stats_factory.tsx
View file

@ -43,7 +43,7 @@ import { css } from '@emotion/react';
import type { ActionExecutionContext } from '@kbn/ui-actions-plugin/public';
import type { Filter } from '@kbn/es-query';
import { FilterStateStore } from '@kbn/es-query';
import { getESQLAdHocDataview, getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import { getESQLAdHocDataview } from '@kbn/esql-utils';
import { ACTION_GLOBAL_APPLY_FILTER } from '@kbn/unified-search-plugin/public';
import type { DataVisualizerTableState } from '../../../../../common/types';
import type { DataVisualizerPluginStart } from '../../../../plugin';
@ -164,10 +164,7 @@ export const getFieldStatsChartEmbeddableFactory = (
let initialDataView: DataView[] | undefined;
try {
const dataView = isESQLQuery(state.query)
? await getESQLAdHocDataview(
getIndexPatternFromESQLQuery(state.query.esql),
deps.data.dataViews
)
? await getESQLAdHocDataview(state.query.esql, deps.data.dataViews)
: await deps.data.dataViews.get(validDataViewId);
initialDataView = [dataView];
} catch (error) {

7
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/field_stats/field_stats_initializer.tsx
View file

@ -25,7 +25,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
import type { FC } from 'react';
import { useEffect } from 'react';
import React, { useMemo, useState, useCallback } from 'react';
import { ENABLE_ESQL, getESQLAdHocDataview, getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import { ENABLE_ESQL, getESQLAdHocDataview } from '@kbn/esql-utils';
import type { AggregateQuery } from '@kbn/es-query';
import { css } from '@emotion/react';
import { euiThemeVars } from '@kbn/ui-theme';
@ -96,10 +96,7 @@ export const FieldStatisticsInitializer: FC<FieldStatsInitializerProps> = ({
}, [dataViewId, viewType, esqlQuery.esql, isEsqlMode]);
const onESQLQuerySubmit = useCallback(
async (query: AggregateQuery, abortController: AbortController) => {
const adhocDataView = await getESQLAdHocDataview(
getIndexPatternFromESQLQuery(query.esql),
dataViews
);
const adhocDataView = await getESQLAdHocDataview(query.esql, dataViews);
if (adhocDataView && adhocDataView.id) {
setDataViewId(adhocDataView.id);
}

4
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/embeddables/field_stats/resolve_field_stats_embeddable_input.tsx
View file

@ -9,7 +9,6 @@ import { tracksOverlays } from '@kbn/presentation-containers';
import { toMountPoint } from '@kbn/react-kibana-mount';
import React from 'react';
import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import { isDefined } from '@kbn/ml-is-defined';
import { FieldStatisticsInitializer } from './field_stats_initializer';
import type { DataVisualizerStartDependencies } from '../../../common/types/data_visualizer_plugin';
@ -53,10 +52,9 @@ export async function resolveEmbeddableFieldStatsUserInput(
const update = async (nextUpdate: FieldStatsInitialState) => {
const esqlQuery = nextUpdate?.query?.esql;
if (isDefined(esqlQuery)) {
const indexPatternFromQuery = getIndexPatternFromESQLQuery(esqlQuery);
const dv = await getOrCreateDataViewByIndexPattern(
pluginStart.data.dataViews,
indexPatternFromQuery,
esqlQuery,
undefined
);
if (dv?.id && nextUpdate.dataViewId !== dv.id) {

2
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_data_visualizer_esql_data.tsx
View file

@ -234,6 +234,7 @@ export const useESQLDataVisualizerData = (
runtimeFieldMap: currentDataView?.getRuntimeMappings(),
lastRefresh,
filter,
timeRange,
};
},
// eslint-disable-next-line react-hooks/exhaustive-deps
@ -374,6 +375,7 @@ export const useESQLDataVisualizerData = (
columns: fieldStatFieldsToFetch,
filter: fieldStatsRequest?.filter,
limit: fieldStatsRequest?.limit ?? DEFAULT_ESQL_LIMIT,
timeRange: fieldStatsRequest?.timeRange,
});
useEffect(

8
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_field_stats_data.ts
View file

@ -6,7 +6,7 @@
*/
import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import type { AggregateQuery } from '@kbn/es-query';
import type { AggregateQuery, TimeRange } from '@kbn/es-query';
import { i18n } from '@kbn/i18n';
import { useEffect, useReducer, useState } from 'react';
import { chunk } from 'lodash';
@ -27,11 +27,13 @@ export const useESQLFieldStatsData = <T extends Column>({
columns: allColumns,
filter,
limit,
timeRange,
}: {
searchQuery?: AggregateQuery;
columns?: T[];
filter?: QueryDslQueryContainer;
limit: number;
timeRange?: TimeRange;
}) => {
const [fieldStats, setFieldStats] = useState<Map<string, FieldStats>>();
@ -93,6 +95,7 @@ export const useESQLFieldStatsData = <T extends Column>({
filter,
runRequest,
esqlBaseQuery,
timeRange,
}).then(addToProcessedFieldStats);
// GETTING STATS FOR KEYWORD FIELDS
@ -103,6 +106,7 @@ export const useESQLFieldStatsData = <T extends Column>({
filter,
runRequest,
esqlBaseQuery,
timeRange,
}).then(addToProcessedFieldStats);
// GETTING STATS FOR BOOLEAN FIELDS
@ -111,6 +115,7 @@ export const useESQLFieldStatsData = <T extends Column>({
filter,
runRequest,
esqlBaseQuery,
timeRange,
}).then(addToProcessedFieldStats);
// GETTING STATS FOR DATE FIELDS
@ -119,6 +124,7 @@ export const useESQLFieldStatsData = <T extends Column>({
filter,
runRequest,
esqlBaseQuery,
timeRange,
}).then(addToProcessedFieldStats);
}
setFetchState({

17
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/esql/use_esql_overall_stats_data.ts
View file

@ -7,8 +7,9 @@
import { ESQL_ASYNC_SEARCH_STRATEGY, KBN_FIELD_TYPES } from '@kbn/data-plugin/common';
import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import type { AggregateQuery } from '@kbn/es-query';
import type { AggregateQuery, TimeRange } from '@kbn/es-query';
import { i18n } from '@kbn/i18n';
import { getEarliestLatestParams } from '@kbn/esql-utils';
import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from 'react';
import { type UseCancellableSearch, useCancellableSearch } from '@kbn/ml-cancellable-search';
import type { estypes } from '@elastic/elasticsearch';
@ -69,7 +70,8 @@ const getESQLDocumentCountStats = async (
timeFieldName?: string,
intervalMs?: number,
searchOptions?: ISearchOptions,
onError?: HandleErrorCallback
onError?: HandleErrorCallback,
timeRange?: TimeRange
): Promise<{ documentCountStats?: DocumentCountStats; totalCount: number; request?: object }> => {
if (!isESQLQuery(query)) {
throw Error(
@ -81,6 +83,7 @@ const getESQLDocumentCountStats = async (
const esqlBaseQuery = query.esql;
let earliestMs = Infinity;
let latestMs = -Infinity;
const namedParams = getEarliestLatestParams(esqlBaseQuery, timeRange);
if (timeFieldName) {
const aggQuery = appendToESQLQuery(
@ -95,6 +98,7 @@ const getESQLDocumentCountStats = async (
params: {
query: aggQuery,
...(filter ? { filter } : {}),
...(namedParams.length ? { params: namedParams } : {}),
},
};
try {
@ -143,6 +147,7 @@ const getESQLDocumentCountStats = async (
params: {
query: appendToESQLQuery(esqlBaseQuery, ' | STATS _count_ = COUNT(*) | LIMIT 1'),
...(filter ? { filter } : {}),
...(namedParams.length ? { params: namedParams } : {}),
},
};
try {
@ -204,6 +209,7 @@ export const useESQLOverallStatsData = (
limit: number;
filter?: QueryDslQueryContainer;
totalCount?: number;
timeRange?: TimeRange;
}
| undefined
) => {
@ -273,6 +279,7 @@ export const useESQLOverallStatsData = (
filter: filter,
limit,
totalCount: knownTotalCount,
timeRange,
} = fieldStatsRequest;
if (!isESQLQuery(searchQuery)) {
@ -291,12 +298,14 @@ export const useESQLOverallStatsData = (
// And use this one query to
// 1) identify populated/empty fields
// 2) gather examples for populated text fields
const namedParams = getEarliestLatestParams(esqlBaseQuery, timeRange);
const columnsResp = (await runRequest(
{
params: {
// Doing this to match with the default limit
query: getESQLWithSafeLimit(esqlBaseQuery, ESQL_SAFE_LIMIT),
...(filter ? { filter } : {}),
...(namedParams.length ? { params: namedParams } : {}),
dropNullColumns: true,
},
},
@ -362,7 +371,8 @@ export const useESQLOverallStatsData = (
timeFieldName,
intervalInMs,
undefined,
onError
onError,
timeRange
);
totalCount = results.totalCount;
@ -446,6 +456,7 @@ export const useESQLOverallStatsData = (
limitSize: limit,
totalCount,
onError,
timeRange,
});
if (!stats) return;
stats.aggregatableNotExistsFields = aggregatableNotExistsFields;

9
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_boolean_field_stats.ts
View file

@ -4,12 +4,12 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { TimeRange } from '@kbn/es-query';
import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
import pLimit from 'p-limit';
import { appendToESQLQuery } from '@kbn/esql-utils';
import { appendToESQLQuery, getEarliestLatestParams } from '@kbn/esql-utils';
import type { Column } from '../../hooks/esql/use_esql_overall_stats_data';
import { getSafeESQLName } from '../requests/esql_utils';
import { isFulfilled, isRejected } from '../../../common/util/promise_all_settled_utils';
@ -22,6 +22,7 @@ interface Params {
columns: Column[];
esqlBaseQuery: string;
filter?: QueryDslQueryContainer;
timeRange?: TimeRange;
}
export const getESQLBooleanFieldStats = async ({
@ -29,9 +30,10 @@ export const getESQLBooleanFieldStats = async ({
columns,
esqlBaseQuery,
filter,
timeRange,
}: Params): Promise<Array<BooleanFieldStats | FieldStatsError | undefined>> => {
const limiter = pLimit(MAX_CONCURRENT_REQUESTS);
const namedParams = getEarliestLatestParams(esqlBaseQuery, timeRange);
const booleanFields = columns
.filter((f) => f.secondaryType === 'boolean')
.map((field) => {
@ -49,6 +51,7 @@ export const getESQLBooleanFieldStats = async ({
params: {
query,
...(filter ? { filter } : {}),
...(namedParams.length ? { params: namedParams } : {}),
},
},
};

10
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_count_and_cardinality.ts
View file

@ -8,8 +8,9 @@ import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
import pLimit from 'p-limit';
import { chunk } from 'lodash';
import { isDefined } from '@kbn/ml-is-defined';
import type { TimeRange } from '@kbn/es-query';
import type { ESQLSearchResponse } from '@kbn/es-types';
import { appendToESQLQuery } from '@kbn/esql-utils';
import { appendToESQLQuery, getEarliestLatestParams } from '@kbn/esql-utils';
import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { i18n } from '@kbn/i18n';
@ -33,6 +34,7 @@ const getESQLOverallStatsInChunk = async ({
limitSize,
totalCount,
onError,
timeRange,
}: {
runRequest: UseCancellableSearch['runRequest'];
fields: Field[];
@ -41,6 +43,7 @@ const getESQLOverallStatsInChunk = async ({
limitSize: number;
totalCount: number;
onError?: HandleErrorCallback;
timeRange?: TimeRange;
}) => {
if (fields.length > 0) {
const aggToIndex = { count: 0, cardinality: 1 };
@ -91,11 +94,13 @@ const getESQLOverallStatsInChunk = async ({
let countQuery = fieldsToFetch.length > 0 ? '| STATS ' : '';
countQuery += fieldsToFetch.map((field) => field.query).join(',');
const query = appendToESQLQuery(esqlBaseQueryWithLimit, countQuery);
const namedParams = getEarliestLatestParams(esqlBaseQueryWithLimit, timeRange);
const request = {
params: {
query,
...(filter ? { filter } : {}),
...(namedParams.length ? { params: namedParams } : {}),
},
};
@ -194,6 +199,7 @@ export const getESQLOverallStats = async ({
limitSize,
totalCount,
onError,
timeRange,
}: {
runRequest: UseCancellableSearch['runRequest'];
fields: Column[];
@ -202,6 +208,7 @@ export const getESQLOverallStats = async ({
limitSize: number;
totalCount: number;
onError?: HandleErrorCallback;
timeRange?: TimeRange;
}) => {
const limiter = pLimit(MAX_CONCURRENT_REQUESTS);
@ -218,6 +225,7 @@ export const getESQLOverallStats = async ({
filter,
totalCount,
onError,
timeRange,
})
)
)

7
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_date_field_stats.ts
View file

@ -7,8 +7,9 @@
import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import type { TimeRange } from '@kbn/es-query';
import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
import { appendToESQLQuery } from '@kbn/esql-utils';
import { appendToESQLQuery, getEarliestLatestParams } from '@kbn/esql-utils';
import type { Column } from '../../hooks/esql/use_esql_overall_stats_data';
import { getSafeESQLName } from '../requests/esql_utils';
import type { DateFieldStats, FieldStatsError } from '../../../../../common/types/field_stats';
@ -18,6 +19,7 @@ interface Params {
columns: Column[];
esqlBaseQuery: string;
filter?: QueryDslQueryContainer;
timeRange?: TimeRange;
}
export const getESQLDateFieldStats = async ({
@ -25,6 +27,7 @@ export const getESQLDateFieldStats = async ({
columns,
esqlBaseQuery,
filter,
timeRange,
}: Params) => {
const dateFields = columns.map((field) => {
return {
@ -36,12 +39,14 @@ export const getESQLDateFieldStats = async ({
});
if (dateFields.length > 0) {
const namedParams = getEarliestLatestParams(esqlBaseQuery, timeRange);
const dateStatsQuery = ' | STATS ' + dateFields.map(({ query }) => query).join(',');
const query = appendToESQLQuery(esqlBaseQuery, dateStatsQuery);
const request = {
params: {
query,
...(filter ? { filter } : {}),
...(namedParams.length ? { params: namedParams } : {}),
},
};
try {

9
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_keyword_fields.ts
View file

@ -4,12 +4,12 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { TimeRange } from '@kbn/es-query';
import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
import pLimit from 'p-limit';
import { appendToESQLQuery } from '@kbn/esql-utils';
import { appendToESQLQuery, getEarliestLatestParams } from '@kbn/esql-utils';
import type { Column } from '../../hooks/esql/use_esql_overall_stats_data';
import { getSafeESQLName } from '../requests/esql_utils';
import { isFulfilled, isRejected } from '../../../common/util/promise_all_settled_utils';
@ -22,15 +22,17 @@ interface Params {
columns: Column[];
esqlBaseQuery: string;
filter?: QueryDslQueryContainer;
timeRange?: TimeRange;
}
export const getESQLKeywordFieldStats = async ({
runRequest,
columns,
esqlBaseQuery,
filter,
timeRange,
}: Params) => {
const limiter = pLimit(MAX_CONCURRENT_REQUESTS);
const namedParams = getEarliestLatestParams(esqlBaseQuery, timeRange);
const keywordFields = columns.map((field) => {
const query = appendToESQLQuery(
esqlBaseQuery,
@ -47,6 +49,7 @@ export const getESQLKeywordFieldStats = async ({
params: {
query,
...(filter ? { filter } : {}),
...(namedParams.length ? { params: namedParams } : {}),
},
},
};

10
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/esql_requests/get_numeric_field_stats.ts
View file

@ -4,11 +4,11 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { TimeRange } from '@kbn/es-query';
import type { UseCancellableSearch } from '@kbn/ml-cancellable-search';
import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import { ESQL_ASYNC_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
import { appendToESQLQuery } from '@kbn/esql-utils';
import { appendToESQLQuery, getEarliestLatestParams } from '@kbn/esql-utils';
import { chunk } from 'lodash';
import pLimit from 'p-limit';
import type { Column } from '../../hooks/esql/use_esql_overall_stats_data';
@ -28,12 +28,14 @@ interface Params {
columns: Column[];
esqlBaseQuery: string;
filter?: QueryDslQueryContainer;
timeRange?: TimeRange;
}
const getESQLNumericFieldStatsInChunk = async ({
runRequest,
columns,
esqlBaseQuery,
filter,
timeRange,
}: Params): Promise<Array<NonSampledNumericFieldStats | FieldStatsError>> => {
// Hashmap of agg to index/order of which is made in the ES|QL query
// {min: 0, max: 1, p0: 2, p5: 3, ..., p100: 22}
@ -67,10 +69,12 @@ const getESQLNumericFieldStatsInChunk = async ({
const numericStatsQuery = '| STATS ' + numericFields.map(({ query }) => query).join(',');
const query = appendToESQLQuery(esqlBaseQuery, numericStatsQuery);
const namedParams = getEarliestLatestParams(esqlBaseQuery, timeRange);
const request = {
params: {
query,
...(filter ? { filter } : {}),
...(namedParams.length ? { params: namedParams } : {}),
},
};
try {
@ -127,6 +131,7 @@ export const getESQLNumericFieldStats = async ({
columns,
esqlBaseQuery,
filter,
timeRange,
}: Params): Promise<Array<NonSampledNumericFieldStats | FieldStatsError>> => {
const limiter = pLimit(MAX_CONCURRENT_REQUESTS);
@ -142,6 +147,7 @@ export const getESQLNumericFieldStats = async ({
filter,
runRequest,
esqlBaseQuery,
timeRange,
})
)
)

28
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_data_view_by_index_pattern.ts
View file

@ -6,7 +6,11 @@
*/
import type { DataView, DataViewsContract } from '@kbn/data-views-plugin/public';
import { getESQLAdHocDataview } from '@kbn/esql-utils';
import {
getESQLAdHocDataview,
getIndexPatternFromESQLQuery,
getTimeFieldFromESQLQuery,
} from '@kbn/esql-utils';
/**
* Get a saved data view that matches the index pattern (as close as possible)
@ -18,24 +22,20 @@ import { getESQLAdHocDataview } from '@kbn/esql-utils';
*/
export async function getOrCreateDataViewByIndexPattern(
dataViews: DataViewsContract,
indexPatternFromQuery: string | undefined,
query: string,
currentDataView: DataView | undefined
) {
if (indexPatternFromQuery) {
const matched = await dataViews.find(indexPatternFromQuery);
// Only returns persisted data view if it matches index pattern exactly
// Because * in pattern can result in misleading matches (i.e. "kibana*" will return data view with pattern "kibana_1")
// which is not neccessarily the one we want to use
if (matched.length > 0 && matched[0].getIndexPattern() === indexPatternFromQuery)
return matched[0];
}
const indexPatternFromQuery = getIndexPatternFromESQLQuery(query);
const newTimeField = getTimeFieldFromESQLQuery(query);
if (
indexPatternFromQuery &&
(currentDataView?.isPersisted() || indexPatternFromQuery !== currentDataView?.getIndexPattern())
currentDataView?.isPersisted() ||
indexPatternFromQuery !== currentDataView?.getIndexPattern() ||
// here the pattern hasn't changed but the time field has
(newTimeField !== currentDataView?.timeFieldName &&
indexPatternFromQuery === currentDataView?.getIndexPattern())
) {
return await getESQLAdHocDataview(indexPatternFromQuery, dataViews);
return await getESQLAdHocDataview(query, dataViews);
}
return currentDataView;
}

4
x-pack/plugins/data_visualizer/public/application/index_data_visualizer/ui_actions/create_field_stats_table.tsx
View file

@ -12,7 +12,6 @@ import type { EmbeddableApiContext } from '@kbn/presentation-publishing';
import type { UiActionsActionDefinition } from '@kbn/ui-actions-plugin/public';
import { IncompatibleActionError } from '@kbn/ui-actions-plugin/public';
import type { CoreStart } from '@kbn/core-lifecycle-browser';
import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import { toMountPoint } from '@kbn/react-kibana-mount';
import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
import React from 'react';
@ -80,10 +79,9 @@ async function updatePanelFromFlyoutEdits({
const update = async (nextUpdate: FieldStatsInitialState) => {
const esqlQuery = nextUpdate?.query?.esql;
if (isDefined(esqlQuery)) {
const indexPatternFromQuery = getIndexPatternFromESQLQuery(esqlQuery);
const dv = await getOrCreateDataViewByIndexPattern(
pluginStart.data.dataViews,
indexPatternFromQuery,
esqlQuery,
undefined
);
if (dv?.id && nextUpdate.dataViewId !== dv.id) {

3
x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts
View file

@ -34,12 +34,13 @@ export const getSuggestions = async (
const dataView = dataViewSpec
? await deps.dataViews.create(dataViewSpec)
: await getESQLAdHocDataview(indexPattern, deps.dataViews);
: await getESQLAdHocDataview(query.esql, deps.dataViews);
const columns = await getESQLQueryColumns({
esqlQuery: 'esql' in query ? query.esql : '',
search: deps.data.search.search,
signal: abortController?.signal,
timeRange: deps.data.query.timefilter.timefilter.getAbsoluteTime(),
});
const context = {
dataViewSpec: dataView?.toSpec(false),

2
x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/lens_configuration_flyout.tsx
View file

@ -453,7 +453,7 @@ export function LensEditConfigurationFlyout({
}}
expandCodeEditor={(status: boolean) => {}}
isCodeEditorExpanded
detectTimestamp={Boolean(adHocDataViews?.[0]?.timeFieldName)}
detectedTimestamp={adHocDataViews?.[0]?.timeFieldName}
hideTimeFilterInfo={hideTimeFilterInfo}
errors={errors}
warning={

106
x-pack/plugins/lens/public/datasources/text_based/utils.test.ts
View file

@ -247,7 +247,7 @@ describe('Text based languages utils', () => {
},
{
id: '4',
timeField: 'timeField',
timeField: undefined,
title: 'my-adhoc-index-pattern',
},
],
@ -259,7 +259,111 @@ describe('Text based languages utils', () => {
query: {
esql: 'FROM my-fake-index-pattern',
},
timeField: undefined,
},
},
});
});
it('should return the correct state for query with named params', async () => {
const state = {
layers: {
first: {
columns: [],
query: undefined,
index: '',
},
},
indexPatternRefs: [],
initialContext: {
textBasedColumns: textBasedQueryColumns,
query: { esql: 'from foo' },
fieldName: '',
dataViewSpec: {
title: 'foo',
id: '1',
name: 'Foo',
},
},
};
const dataViewsMock = dataViewPluginMocks.createStartContract();
const dataMock = dataPluginMock.createStartContract();
const expressionsMock = expressionsPluginMock.createStartContract();
const updatedState = await getStateFromAggregateQuery(
state,
{ esql: 'FROM my-fake-index-pattern | WHERE time <= ?latest' },
{
...dataViewsMock,
getIdsWithTitle: jest.fn().mockReturnValue(
Promise.resolve([
{ id: '1', title: 'my-fake-index-pattern' },
{ id: '2', title: 'my-fake-restricted-pattern' },
{ id: '3', title: 'my-compatible-pattern' },
])
),
get: jest.fn().mockReturnValue(
Promise.resolve({
id: '1',
title: 'my-fake-index-pattern',
timeFieldName: 'timeField',
})
),
create: jest.fn().mockReturnValue(
Promise.resolve({
id: '4',
title: 'my-adhoc-index-pattern',
name: 'my-adhoc-index-pattern',
timeFieldName: 'timeField',
isPersisted: () => false,
})
),
},
dataMock,
expressionsMock
);
expect(updatedState).toStrictEqual({
initialContext: {
textBasedColumns: textBasedQueryColumns,
query: { esql: 'from foo' },
fieldName: '',
dataViewSpec: {
title: 'foo',
id: '1',
name: 'Foo',
},
},
indexPatternRefs: [
{
id: '3',
timeField: 'timeField',
title: 'my-compatible-pattern',
},
{
id: '1',
timeField: 'timeField',
title: 'my-fake-index-pattern',
},
{
id: '2',
timeField: 'timeField',
title: 'my-fake-restricted-pattern',
},
{
id: '4',
timeField: 'time',
title: 'my-adhoc-index-pattern',
},
],
layers: {
first: {
columns: [],
errors: [],
index: '4',
query: {
esql: 'FROM my-fake-index-pattern | WHERE time <= ?latest',
},
timeField: 'time',
},
},
});

2
x-pack/plugins/lens/public/datasources/text_based/utils.ts
View file

@ -85,7 +85,7 @@ export async function getStateFromAggregateQuery(
let columnsFromQuery: DatatableColumn[] = [];
let timeFieldName;
try {
const dataView = await getESQLAdHocDataview(indexPattern, dataViews);
const dataView = await getESQLAdHocDataview(query.esql, dataViews);
if (dataView && dataView.id) {
dataViewId = dataView?.id;

3
x-pack/plugins/lens/public/trigger_actions/open_lens_config/create_action_helpers.ts
View file

@ -49,7 +49,7 @@ export async function executeCreateAction({
const getFallbackDataView = async () => {
const indexName = await getIndexForESQLQuery({ dataViews: deps.dataViews });
if (!indexName) return null;
const dataView = await getESQLAdHocDataview(indexName, deps.dataViews);
const dataView = await getESQLAdHocDataview(`from ${indexName}`, deps.dataViews);
return dataView;
};
@ -75,6 +75,7 @@ export async function executeCreateAction({
esqlQuery: `from ${defaultIndex}`,
search: deps.data.search.search,
signal: abortController.signal,
timeRange: deps.data.query.timefilter.timefilter.getAbsoluteTime(),
});
const context = {

5
x-pack/plugins/maps/public/classes/sources/esql_source/create_source_editor.tsx
View file

@ -56,7 +56,10 @@ export function CreateSourceEditor(props: Props) {
getDataView()
.then(async (dataView) => {
const adhocDataView = dataView
? await getESQLAdHocDataview(dataView.getIndexPattern(), getIndexPatternService())
? await getESQLAdHocDataview(
`from ${dataView.getIndexPattern()}`,
getIndexPatternService()
)
: undefined;
if (ignore) {
return;

32
x-pack/plugins/maps/public/classes/sources/esql_source/esql_source.tsx
View file

@ -11,7 +11,12 @@ import { lastValueFrom } from 'rxjs';
import { tap } from 'rxjs';
import { v4 as uuidv4 } from 'uuid';
import { Adapters } from '@kbn/inspector-plugin/common/adapters';
import { getIndexPatternFromESQLQuery, getLimitFromESQLQuery } from '@kbn/esql-utils';
import {
getIndexPatternFromESQLQuery,
getLimitFromESQLQuery,
getEarliestLatestParams,
hasEarliestLatestParams,
} from '@kbn/esql-utils';
import { buildEsQuery } from '@kbn/es-query';
import type { Filter, Query } from '@kbn/es-query';
import type { ESQLSearchParams, ESQLSearchResponse } from '@kbn/es-types';
@ -112,11 +117,11 @@ export class ESQLSource
}
getApplyGlobalQuery() {
return this._descriptor.narrowByGlobalSearch;
return this._descriptor.narrowByGlobalSearch || hasEarliestLatestParams(this._descriptor.esql);
}
async isTimeAware() {
return this._descriptor.narrowByGlobalTime;
return this._descriptor.narrowByGlobalTime || hasEarliestLatestParams(this._descriptor.esql);
}
getApplyGlobalTime() {
@ -183,6 +188,14 @@ export class ESQLSource
filters.push(extentFilter);
}
const timeRange = requestMeta.timeslice
? {
from: new Date(requestMeta.timeslice.from).toISOString(),
to: new Date(requestMeta.timeslice.to).toISOString(),
mode: 'absolute' as 'absolute',
}
: requestMeta.timeFilters;
if (requestMeta.applyGlobalTime) {
if (!this._descriptor.dateField) {
throw new Error(
@ -192,21 +205,20 @@ export class ESQLSource
})
);
}
const timeRange = requestMeta.timeslice
? {
from: new Date(requestMeta.timeslice.from).toISOString(),
to: new Date(requestMeta.timeslice.to).toISOString(),
mode: 'absolute' as 'absolute',
}
: requestMeta.timeFilters;
const timeFilter = getTime(undefined, timeRange, {
fieldName: this._descriptor.dateField,
});
if (timeFilter) {
filters.push(timeFilter);
}
}
const namedParams = getEarliestLatestParams(this._descriptor.esql, timeRange);
if (namedParams.length) {
params.params = namedParams;
}
params.filter = buildEsQuery(undefined, query, filters, getEsQueryConfig(getUiSettings()));
const requestResponder = inspectorAdapters.requests!.start(

12
x-pack/plugins/maps/public/classes/sources/esql_source/esql_utils.ts
View file

@ -7,11 +7,7 @@
import { i18n } from '@kbn/i18n';
import type { DataView } from '@kbn/data-plugin/common';
import {
getESQLAdHocDataview,
getIndexPatternFromESQLQuery,
getESQLQueryColumnsRaw,
} from '@kbn/esql-utils';
import { getESQLAdHocDataview, getESQLQueryColumnsRaw } from '@kbn/esql-utils';
import type { ESQLColumn } from '@kbn/es-types';
import { ES_GEO_FIELD_TYPE } from '../../../../common/constants';
import { getData, getIndexPatternService } from '../../../kibana_services';
@ -53,14 +49,12 @@ export function verifyGeometryColumn(columns: ESQLColumn[]) {
}
export async function getESQLMeta(esql: string) {
const adhocDataView = await getESQLAdHocDataview(
getIndexPatternFromESQLQuery(esql),
getIndexPatternService()
);
const adhocDataView = await getESQLAdHocDataview(esql, getIndexPatternService());
return {
columns: await getESQLQueryColumnsRaw({
esqlQuery: esql,
search: getData().search.search,
timeRange: getData().query.timefilter.timefilter.getAbsoluteTime(),
}),
adhocDataViewId: adhocDataView.id!,
...getFields(adhocDataView),

8
x-pack/plugins/observability_solution/observability_ai_assistant_app/public/functions/visualize_esql.tsx
View file

@ -18,7 +18,7 @@ import {
import type { ESQLRow } from '@kbn/es-types';
import { ESQLDataGrid } from '@kbn/esql-datagrid/public';
import type { DataViewsServicePublic } from '@kbn/data-views-plugin/public/types';
import { getESQLAdHocDataview, getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import { getESQLAdHocDataview } from '@kbn/esql-utils';
import type { DatatableColumn } from '@kbn/expressions-plugin/common';
import { i18n } from '@kbn/i18n';
import type {
@ -117,15 +117,13 @@ export function VisualizeESQL({
ObservabilityAIAssistantMultipaneFlyoutContext,
errorMessages,
}: VisualizeESQLProps) {
// fetch the pattern from the query
const indexPattern = getIndexPatternFromESQLQuery(query);
const lensHelpersAsync = useAsync(() => {
return lens.stateHelperApi();
}, [lens]);
const dataViewAsync = useAsync(() => {
return getESQLAdHocDataview(indexPattern, dataViews);
}, [indexPattern, dataViews]);
return getESQLAdHocDataview(query, dataViews);
}, [query, dataViews]);
const chatFlyoutSecondSlotHandler = useContext(ObservabilityAIAssistantMultipaneFlyoutContext);

20
x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/esql_query_expression.tsx
View file

@ -6,7 +6,6 @@
*/
import React, { useState, Fragment, useEffect, useCallback } from 'react';
import { get } from 'lodash';
import { FormattedMessage } from '@kbn/i18n-react';
import {
EuiFieldNumber,
@ -19,7 +18,7 @@ import {
import { getFields, RuleTypeParamsExpressionProps } from '@kbn/triggers-actions-ui-plugin/public';
import { TextBasedLangEditor } from '@kbn/esql/public';
import { fetchFieldsFromESQL } from '@kbn/text-based-editor';
import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import { getESQLAdHocDataview } from '@kbn/esql-utils';
import type { AggregateQuery } from '@kbn/es-query';
import { parseDuration } from '@kbn/alerting-plugin/common';
import {
@ -39,7 +38,7 @@ import { rowToDocument, toEsQueryHits, transformDatatableToEsqlTable } from '../
export const EsqlQueryExpression: React.FC<
RuleTypeParamsExpressionProps<EsQueryRuleParams<SearchType.esqlQuery>, EsQueryRuleMetaData>
> = ({ ruleParams, setRuleParams, setRuleProperty, errors }) => {
const { expressions, http, fieldFormats, isServerless } = useTriggerUiActionServices();
const { expressions, http, fieldFormats, isServerless, dataViews } = useTriggerUiActionServices();
const { esqlQuery, timeWindowSize, timeWindowUnit, timeField } = ruleParams;
const [currentRuleParams, setCurrentRuleParams] = useState<
@ -63,7 +62,7 @@ export const EsqlQueryExpression: React.FC<
});
const [query, setQuery] = useState<AggregateQuery>({ esql: '' });
const [timeFieldOptions, setTimeFieldOptions] = useState([firstFieldOption]);
const [detectTimestamp, setDetectTimestamp] = useState<boolean>(false);
const [detectedTimestamp, setDetectedTimestamp] = useState<string | undefined>(undefined);
const [isLoading, setIsLoading] = useState<boolean>(false);
const setParam = useCallback(
@ -161,19 +160,18 @@ export const EsqlQueryExpression: React.FC<
]);
const refreshTimeFields = async (q: AggregateQuery) => {
let hasTimestamp = false;
const indexPattern: string = getIndexPatternFromESQLQuery(get(q, 'esql'));
const esqlDataView = await getESQLAdHocDataview(q.esql, dataViews);
const indexPattern: string = esqlDataView.getIndexPattern();
const currentEsFields = await getFields(http, [indexPattern]);
const timeFields = getTimeFieldOptions(currentEsFields);
setTimeFieldOptions([firstFieldOption, ...timeFields]);
const timestampField = timeFields.find((field) => field.value === '@timestamp');
const timestampField = esqlDataView.timeFieldName;
if (timestampField) {
setParam('timeField', timestampField.value);
hasTimestamp = true;
setParam('timeField', timestampField);
}
setDetectTimestamp(hasTimestamp);
setDetectedTimestamp(timestampField);
};
return (
@ -199,7 +197,7 @@ export const EsqlQueryExpression: React.FC<
expandCodeEditor={() => true}
isCodeEditorExpanded={true}
onTextLangQuerySubmit={async () => {}}
detectTimestamp={detectTimestamp}
detectedTimestamp={detectedTimestamp}
hideMinimizeButton={true}
hideRunQueryText={true}
isLoading={isLoading}

8
x-pack/plugins/stack_alerts/public/rule_types/es_query/expression/expression.test.tsx
View file

@ -4,7 +4,7 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { DataView } from '@kbn/data-views-plugin/public';
import { mountWithIntl } from '@kbn/test-jest-helpers';
import 'brace';
import React, { useState } from 'react';
@ -154,6 +154,12 @@ const dataViewEditorMock = dataViewEditorPluginMock.createStartContract();
(dataViewsMock.getIds as jest.Mock) = jest.fn().mockImplementation(() => Promise.resolve([]));
dataViewsMock.getDefaultDataView = jest.fn(() => Promise.resolve(null));
dataViewsMock.get = jest.fn();
dataViewsMock.create.mockResolvedValue({
title: 'test-index',
type: 'esql',
id: 'test-index',
getIndexPattern: () => 'test-index',
} as DataView);
(dataMock.query.savedQueries.getSavedQuery as jest.Mock).mockImplementation(() =>
Promise.resolve(savedQueryMock)
);

1
x-pack/plugins/translations/translations/fr-FR.json
View file

@ -6653,7 +6653,6 @@
"textBasedEditor.query.textBasedLanguagesEditor.runQuery": "Exécuter la requête",
"textBasedEditor.query.textBasedLanguagesEditor.sourceCommands": "Commandes sources",
"textBasedEditor.query.textBasedLanguagesEditor.submitFeedback": "Soumettre un commentaire",
"textBasedEditor.query.textBasedLanguagesEditor.timestampDetected": "@timestamp trouvé",
"textBasedEditor.query.textBasedLanguagesEditor.timestampNotDetected": "@timestamp non trouvé",
"textBasedEditor.query.textBasedLanguagesEditor.warningCount": "{count} {count, plural, one {avertissement} other {avertissements}}",
"textBasedEditor.query.textBasedLanguagesEditor.warningsTitle": "Avertissements",

2
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -6629,7 +6629,7 @@
"textBasedEditor.query.textBasedLanguagesEditor.runQuery": "クエリを実行",
"textBasedEditor.query.textBasedLanguagesEditor.sourceCommands": "ソースコマンド",
"textBasedEditor.query.textBasedLanguagesEditor.submitFeedback": "フィードバックを送信",
"textBasedEditor.query.textBasedLanguagesEditor.timestampDetected": "@timestampが見つかりました",
"": "@timestampが見つかりました",
"textBasedEditor.query.textBasedLanguagesEditor.timestampNotDetected": "@timestampが見つかりません",
"textBasedEditor.query.textBasedLanguagesEditor.warningCount": "{count} {count, plural, other {件の警告}}",
"textBasedEditor.query.textBasedLanguagesEditor.warningsTitle": "警告",

1
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -6662,7 +6662,6 @@
"textBasedEditor.query.textBasedLanguagesEditor.runQuery": "运行查询",
"textBasedEditor.query.textBasedLanguagesEditor.sourceCommands": "源命令",
"textBasedEditor.query.textBasedLanguagesEditor.submitFeedback": "提交反馈",
"textBasedEditor.query.textBasedLanguagesEditor.timestampDetected": "找到 @timestamp",
"textBasedEditor.query.textBasedLanguagesEditor.timestampNotDetected": "未找到 @timestamp",
"textBasedEditor.query.textBasedLanguagesEditor.warningCount": "{count} 个{count, plural, other {警告}}",
"textBasedEditor.query.textBasedLanguagesEditor.warningsTitle": "警告",

42
x-pack/test_serverless/functional/test_suites/common/discover/esql/_esql_view.ts
View file

@ -42,6 +42,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover');
// and load a set of makelogs data
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
await esArchiver.load('test/functional/fixtures/es_archiver/kibana_sample_data_flights');
await kibanaServer.importExport.load(
'test/functional/fixtures/kbn_archiver/kibana_sample_data_flights_index_pattern'
);
await kibanaServer.uiSettings.replace(defaultSettings);
await PageObjects.svlCommonPage.loginAsAdmin();
await PageObjects.common.navigateToApp('discover');
@ -49,7 +53,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.timePicker.setDefaultAbsoluteRange();
});
describe('test', () => {
describe('ES|QL in Discover', () => {
it('should render esql view correctly', async function () {
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
@ -91,7 +95,43 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await testSubjects.missingOrFail('discoverFieldListPanelEditItem');
});
it('should not render the histogram for indices with no @timestamp field', async function () {
await PageObjects.discover.selectTextBaseLang();
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
const testQuery = `from kibana_sample_data_flights | limit 10`;
await monacoEditor.setCodeEditorValue(testQuery);
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await testSubjects.exists('TextBasedLangEditor')).to.be(true);
// I am not rendering the histogram for indices with no @timestamp field
expect(await testSubjects.exists('unifiedHistogramChart')).to.be(false);
});
it('should render the histogram for indices with no @timestamp field when the ?earliest, ?latest params are in the query', async function () {
await PageObjects.discover.selectTextBaseLang();
await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
const testQuery = `from kibana_sample_data_flights | limit 10 | where timestamp >= ?earliest and timestamp <= ?latest`;
await monacoEditor.setCodeEditorValue(testQuery);
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
const fromTime = 'Apr 10, 2018 @ 00:00:00.000';
const toTime = 'Nov 15, 2018 @ 00:00:00.000';
await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
expect(await testSubjects.exists('TextBasedLangEditor')).to.be(true);
expect(await testSubjects.exists('unifiedHistogramChart')).to.be(true);
});
it('should perform test query correctly', async function () {
await PageObjects.timePicker.setDefaultAbsoluteRange();
await PageObjects.discover.selectTextBaseLang();
const testQuery = `from logstash-* | limit 10 | stats countB = count(bytes) by geo.dest | sort countB`;