[Discover][ES|QL] Reset selected fields when modifying the ES|QL query (#185997)

- Closes https://github.com/elastic/kibana/issues/183961

## Summary

This PR updates the logic of resetting columns when modifying the ES|QL
query. The previous implementation was added in
https://github.com/elastic/kibana/pull/167492 and then changed in
https://github.com/elastic/kibana/pull/177241.


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

---------

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
Co-authored-by: Matthias Wilhelm <ankertal@gmail.com>
This commit is contained in:
Julia Rechkunova 2024-06-18 10:07:07 +02:00 committed by GitHub
parent d3b81237ee
commit 250c729087
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 383 additions and 33 deletions

View file

@ -115,6 +115,7 @@ enabled:
- test/functional/apps/discover/ccs_compatibility/config.ts
- test/functional/apps/discover/classic/config.ts
- test/functional/apps/discover/embeddable/config.ts
- test/functional/apps/discover/esql/config.ts
- test/functional/apps/discover/group1/config.ts
- test/functional/apps/discover/group2_data_grid1/config.ts
- test/functional/apps/discover/group2_data_grid2/config.ts

View file

@ -150,10 +150,11 @@ describe('useEsqlMode', () => {
});
});
test('changing an ES|QL query with same result columns should not change state when loading and finished', async () => {
test('changing an ES|QL query with same result columns but a different index pattern should change state when loading and finished', async () => {
const { replaceUrlState, stateContainer } = renderHookWithContext(false);
const documents$ = stateContainer.dataState.data$.documents$;
stateContainer.dataState.data$.documents$.next(msgComplete);
replaceUrlState.mockReset();
documents$.next({
fetchStatus: FetchStatus.PARTIAL,
@ -166,7 +167,54 @@ describe('useEsqlMode', () => {
],
query: { esql: 'from the-data-view-2' },
});
await waitFor(() => expect(replaceUrlState).toHaveBeenCalledTimes(1));
await waitFor(() => {
expect(replaceUrlState).toHaveBeenCalledWith({
columns: [],
});
});
});
test('changing a ES|QL query with no transformational commands should not change state when loading and finished if index pattern is the same', async () => {
const { replaceUrlState, stateContainer } = renderHookWithContext(false);
const documents$ = stateContainer.dataState.data$.documents$;
stateContainer.dataState.data$.documents$.next(msgComplete);
await waitFor(() => expect(replaceUrlState).toHaveBeenCalledTimes(0));
replaceUrlState.mockReset();
documents$.next({
fetchStatus: FetchStatus.PARTIAL,
result: [
{
id: '1',
raw: { field1: 1 },
flattened: { field1: 1 },
} as unknown as DataTableRecord,
],
// non transformational command
query: { esql: 'from the-data-view-title | where field1 > 0' },
});
await waitFor(() => expect(replaceUrlState).toHaveBeenCalledTimes(0));
replaceUrlState.mockReset();
documents$.next({
fetchStatus: FetchStatus.PARTIAL,
result: [
{
id: '1',
raw: { field1: 1 },
flattened: { field1: 1 },
} as unknown as DataTableRecord,
],
// non transformational command
query: { esql: 'from the-data-view-title2 | where field1 > 0' },
});
await waitFor(() => {
expect(replaceUrlState).toHaveBeenCalledWith({
columns: [],
});
});
});
test('only changing an ES|QL query with same result columns should not change columns', async () => {
@ -268,7 +316,13 @@ describe('useEsqlMode', () => {
query: { esql: 'from the-data-view-title | keep field 1 | WHERE field1=1' },
});
expect(replaceUrlState).toHaveBeenCalledTimes(0);
await waitFor(() => expect(replaceUrlState).toHaveBeenCalledTimes(1));
await waitFor(() => {
expect(replaceUrlState).toHaveBeenCalledWith({
columns: ['field1', 'field2'],
});
});
replaceUrlState.mockReset();
documents$.next({
fetchStatus: FetchStatus.PARTIAL,

View file

@ -7,8 +7,8 @@
*/
import { isEqual } from 'lodash';
import { isOfAggregateQueryType, getAggregateQueryMode } from '@kbn/es-query';
import { hasTransformationalCommand } from '@kbn/esql-utils';
import { isOfAggregateQueryType } from '@kbn/es-query';
import { hasTransformationalCommand, getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
import { useCallback, useEffect, useRef } from 'react';
import type { DataViewsContract } from '@kbn/data-views-plugin/public';
import { switchMap } from 'rxjs';
@ -31,9 +31,9 @@ export function useEsqlMode({
}) {
const prev = useRef<{
query: string;
columns: string[];
recentlyUpdatedToColumns: string[];
}>({
columns: [],
recentlyUpdatedToColumns: [],
query: '',
});
const initialFetch = useRef<boolean>(true);
@ -43,9 +43,10 @@ export function useEsqlMode({
if (prev.current.query) {
// cleanup when it's not an ES|QL query
prev.current = {
columns: [],
recentlyUpdatedToColumns: [],
query: '',
};
initialFetch.current = true;
}
}, []);
@ -57,55 +58,60 @@ export function useEsqlMode({
if (!query || next.fetchStatus === FetchStatus.ERROR) {
return;
}
const sendComplete = () => {
stateContainer.dataState.data$.documents$.next({
...next,
fetchStatus: FetchStatus.COMPLETE,
});
};
const { viewMode } = stateContainer.appState.getState();
let nextColumns: string[] = [];
const isEsqlQuery = isOfAggregateQueryType(query);
const hasResults = Boolean(next.result?.length);
let queryHasTransformationalCommands = false;
if ('esql' in query) {
if (hasTransformationalCommand(query.esql)) {
queryHasTransformationalCommands = true;
}
}
if (isEsqlQuery) {
const language = getAggregateQueryMode(query);
const hasResults = Boolean(next.result?.length);
if (next.fetchStatus !== FetchStatus.PARTIAL) {
return;
}
let nextColumns: string[] = prev.current.recentlyUpdatedToColumns;
if (hasResults) {
// check if state needs to contain column transformation due to a different columns in the resultset
const firstRow = next.result![0];
const firstRowColumns = Object.keys(firstRow.raw).slice(0, MAX_NUM_OF_COLUMNS);
if (!queryHasTransformationalCommands) {
nextColumns = [];
initialFetch.current = false;
const firstRowColumns = Object.keys(firstRow.raw);
if (hasTransformationalCommand(query.esql)) {
nextColumns = firstRowColumns.slice(0, MAX_NUM_OF_COLUMNS);
} else {
nextColumns = firstRowColumns;
if (initialFetch.current && !prev.current.columns.length) {
prev.current.columns = firstRowColumns;
}
nextColumns = [];
}
}
const addColumnsToState = !isEqual(nextColumns, prev.current.columns);
const queryChanged = query[language] !== prev.current.query;
if (initialFetch.current) {
initialFetch.current = false;
prev.current.query = query.esql;
prev.current.recentlyUpdatedToColumns = nextColumns;
}
const indexPatternChanged =
getIndexPatternFromESQLQuery(query.esql) !==
getIndexPatternFromESQLQuery(prev.current.query);
const addColumnsToState =
indexPatternChanged || !isEqual(nextColumns, prev.current.recentlyUpdatedToColumns);
const changeViewMode = viewMode !== getValidViewMode({ viewMode, isEsqlMode: true });
if (!queryChanged || (!addColumnsToState && !changeViewMode)) {
if (!indexPatternChanged && !addColumnsToState && !changeViewMode) {
sendComplete();
return;
}
if (queryChanged) {
prev.current.query = query[language];
prev.current.columns = nextColumns;
}
prev.current.query = query.esql;
prev.current.recentlyUpdatedToColumns = nextColumns;
// just change URL state if necessary
if (addColumnsToState || changeViewMode) {
const nextState = {

View file

@ -0,0 +1,245 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../ftr_provider_context';
const SAVED_SEARCH_NON_TRANSFORMATIONAL_INITIAL_COLUMNS = 'nonTransformationalInitialColumns';
const SAVED_SEARCH_NON_TRANSFORMATIONAL_CUSTOM_COLUMNS = 'nonTransformationalCustomColumns';
const SAVED_SEARCH_TRANSFORMATIONAL_INITIAL_COLUMNS = 'transformationalInitialColumns';
const SAVED_SEARCH_TRANSFORMATIONAL_CUSTOM_COLUMNS = 'transformationalCustomColumns';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
const security = getService('security');
const dataGrid = getService('dataGrid');
const browser = getService('browser');
const monacoEditor = getService('monacoEditor');
const testSubjects = getService('testSubjects');
const PageObjects = getPageObjects([
'common',
'discover',
'dashboard',
'header',
'timePicker',
'unifiedFieldList',
]);
const defaultSettings = {
defaultIndex: 'logstash-*',
};
describe('discover esql columns', async function () {
before(async () => {
await kibanaServer.savedObjects.cleanStandardList();
await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover');
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
await kibanaServer.uiSettings.replace(defaultSettings);
await PageObjects.common.navigateToApp('discover');
await PageObjects.timePicker.setDefaultAbsoluteRange();
await PageObjects.discover.waitUntilSearchingHasFinished();
await PageObjects.discover.selectTextBaseLang();
await PageObjects.discover.waitUntilSearchingHasFinished();
});
beforeEach(async () => {
await PageObjects.discover.clickNewSearchButton();
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
});
it('should render initial columns for non-transformational commands correctly', async () => {
const columns = ['@timestamp', 'Document'];
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await browser.refresh();
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await PageObjects.discover.saveSearch(SAVED_SEARCH_NON_TRANSFORMATIONAL_INITIAL_COLUMNS);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
});
it('should render custom columns for non-transformational commands correctly', async () => {
const columns = ['bytes', 'extension'];
await PageObjects.unifiedFieldList.clickFieldListItemAdd('bytes');
await PageObjects.unifiedFieldList.clickFieldListItemAdd('extension');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await browser.refresh();
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await PageObjects.discover.saveSearch(SAVED_SEARCH_NON_TRANSFORMATIONAL_CUSTOM_COLUMNS);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
});
it('should reset columns only if index pattern changes in non-transformational query', async () => {
const columns = ['@timestamp', 'Document'];
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await monacoEditor.setCodeEditorValue('from logstash-* | limit 1');
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await monacoEditor.setCodeEditorValue('from logs* | limit 1');
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await PageObjects.unifiedFieldList.clickFieldListItemAdd('bytes');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['bytes']);
// different index pattern => reset columns
await monacoEditor.setCodeEditorValue('from logstash-* | limit 1');
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await PageObjects.unifiedFieldList.clickFieldListItemAdd('extension');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['extension']);
// same index pattern => don't reset columns
await monacoEditor.setCodeEditorValue(
`${await monacoEditor.getCodeEditorValue()} | where bytes > 0`
);
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['extension']);
});
it('should render initial columns for a transformational command correctly', async () => {
const columns = ['ip', '@timestamp'];
await monacoEditor.setCodeEditorValue(
`${await monacoEditor.getCodeEditorValue()} | keep ip, @timestamp`
);
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await browser.refresh();
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await PageObjects.discover.saveSearch(SAVED_SEARCH_TRANSFORMATIONAL_INITIAL_COLUMNS);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
});
it('should render custom columns for a transformational command correctly', async () => {
const columns = ['ip', 'bytes'];
await monacoEditor.setCodeEditorValue(
`${await monacoEditor.getCodeEditorValue()} | keep ip, @timestamp, bytes`
);
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['ip', '@timestamp', 'bytes']);
await PageObjects.unifiedFieldList.clickFieldListItemRemove('@timestamp');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await browser.refresh();
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
await PageObjects.discover.saveSearch(SAVED_SEARCH_TRANSFORMATIONAL_CUSTOM_COLUMNS);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
});
it('should reset columns if available fields or index pattern are different in transformational query', async () => {
await monacoEditor.setCodeEditorValue('from logstash-* | keep ip, @timestamp');
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['ip', '@timestamp']);
// reset columns if available fields are different
await monacoEditor.setCodeEditorValue('from logstash-* | keep ip, @timestamp, bytes');
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['ip', '@timestamp', 'bytes']);
// don't reset columns if available fields and index pattern are the same
await monacoEditor.setCodeEditorValue(
'from logstash-* | keep ip, @timestamp, bytes | limit 1'
);
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['ip', '@timestamp', 'bytes']);
await PageObjects.unifiedFieldList.clickFieldListItemRemove('@timestamp');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['ip', 'bytes']);
// reset columns if index pattern is different
await monacoEditor.setCodeEditorValue('from logs* | keep ip, @timestamp, bytes | limit 1');
await testSubjects.click('querySubmitButton');
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['ip', '@timestamp', 'bytes']);
});
it('should restore columns correctly when switching between saved searches', async () => {
await PageObjects.discover.loadSavedSearch(SAVED_SEARCH_NON_TRANSFORMATIONAL_INITIAL_COLUMNS);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['@timestamp', 'Document']);
await PageObjects.discover.loadSavedSearch(SAVED_SEARCH_NON_TRANSFORMATIONAL_CUSTOM_COLUMNS);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['bytes', 'extension']);
await PageObjects.discover.loadSavedSearch(SAVED_SEARCH_TRANSFORMATIONAL_INITIAL_COLUMNS);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['ip', '@timestamp']);
await PageObjects.discover.loadSavedSearch(SAVED_SEARCH_TRANSFORMATIONAL_CUSTOM_COLUMNS);
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['ip', 'bytes']);
await PageObjects.discover.clickNewSearchButton();
await PageObjects.header.waitUntilLoadingHasFinished();
await PageObjects.discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(['@timestamp', 'Document']);
});
});
}

View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../../config.base.js'));
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
};
}

View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { FtrProviderContext } from '../ftr_provider_context';
export default function ({ getService, loadTestFile }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const browser = getService('browser');
describe('discover/esql', function () {
before(async function () {
await browser.setWindowSize(1600, 1200);
});
after(async function unloadMakelogs() {
await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
});
loadTestFile(require.resolve('./_esql_columns'));
loadTestFile(require.resolve('./_esql_view'));
});
}

View file

@ -22,7 +22,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./_discover_fields_api'));
loadTestFile(require.resolve('./_adhoc_data_views'));
loadTestFile(require.resolve('./_esql_view'));
loadTestFile(require.resolve('./_date_nested'));
loadTestFile(require.resolve('./_chart_hidden'));
loadTestFile(require.resolve('./_context_encoded_url_params'));

View file

@ -18,6 +18,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
testFiles: [
require.resolve('../apps/discover/classic'),
require.resolve('../apps/discover/esql'),
require.resolve('../apps/discover/group1'),
require.resolve('../apps/discover/group2_data_grid1'),
require.resolve('../apps/discover/group2_data_grid2'),