github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ES|QL] Batch of fixes and new features for autocomplete and validation (#175986)

Browse source 
## Summary

This PR is a batch of fixes and (tiny) new features for validation and
autocomplete items in ES|QL client side area.
Content of the PR:
* 🐛 remove array/multi-value definition annotation for `mv_*`
functions - fix #174709
<img width="305" alt="Screenshot 2024-01-31 at 13 43 37"
src="2a458e74-9c87-4a67-8b42-0071e7a04f48">
<img width="287" alt="Screenshot 2024-01-31 at 13 43 25"
src="7205d984-d0c9-410b-8b39-c1d530c3f54f">
* while the type check has been relaxed for lists, the actual
non-multivalued argument type is retained and used for validation
<img width="758" alt="Screenshot 2024-01-31 at 13 44 06"
src="384e9dfa-aaf8-4e0f-953e-a2d656776ea7">

* 🐛 Remove array/multi-value definition annotation for `mv_expand`
command - fix #174709

<img width="556" alt="Screenshot 2024-01-31 at 14 36 19"
src="0debbccc-da9a-4e6a-bfc9-3fded61300b5">
<img width="277" alt="Screenshot 2024-01-31 at 14 36 13"
src="9438832b-d8d5-48c5-a030-b8983057546e">


* ✨ add metadata fields validation + autocomplete (fields are
retrieved via Kibana `uiSettings`) ( part of #172646 )
  

![metadata_fix](958e6609-ab73-4949-8652-f368c6604a09)
  
<img width="760" alt="Screenshot 2024-01-31 at 13 45 43"
src="b3f13d99-3e12-4586-b39c-f4ea3691a2f1">
  * ✨ Quick fixes now available also for metadata fields
  

![metadata_fixes](b273418f-a754-4c50-9539-a128f35fc4cd)

* 🐛 fixed autocomplete for `NOT` operation

<img width="549" alt="Screenshot 2024-01-31 at 13 46 43"
src="29b7aa7c-db66-485a-a5b0-329e66df842f">

* fixed autocomplete for `in` operation, with or without `not`:
* ✨ now a missing list bracket is correctly detected and
suggested after `in`
* ✨ the content for the list is suggested and takes into
account both the left side of the `in`/`not in` operator, but also the
current content of the list


![in_list_fix_2](50ea8db5-23da-411c-b81e-5ba3397b65c1)

![in_list_fix](cbb73ad2-4073-4eb6-9c12-61835858fd5e)

![not_in_list_fix](9d7fa756-3d34-49fe-ac7a-28e2edbe7fee)

* fixed `grok` and `dissect` in various ways
*✨ the pattern provided by autocomplete for both was not valid
in `dissect` nor useful at all, so I've changed to something more useful
like `'"%{WORD:firstWord}"'` for `grok` and `'"%{firstWord}"'` for
`dissect` to match the first word in the text field.
* 🐛 there was a bug in the validation engine as both `grok` and
`dissect` could generate new columns based on matches, so now a new
field query is fired (only when either a `GROK` or a `DISSECT` command
is detected) which enriches the set of fields available (similar to the
enrich fields)
  * ✅ Added tons of tests here


![grok_fix](a23c8edc-0702-4531-aaa5-6126e375913b)

![dissect_fix](d5591753-775e-4768-be00-31c4371be330)

* 🐛 fixed an issue with proposing an assign (`=`) operator when it
should not
  * `... EVAL var0 = round( ...) <here>` not within another assignment
  * `... EVAL var0 = 1 <here>` not within another assignment
  * `... EVAL var0 = 1 year + 2 <here>` not within another assignment
  * `... | WHERE field` should not shadow a field name in this case

* 🐛 fix an annoying auto trigger suggestion on field selection, and
kept it only for functions

![auto_trigger_fix](31d0c296-6338-450d-8f81-e08cd1c7526d)

* 🐛 fixed an error in console with autocomplete when a typed
function does not exists
* ✅  Add tests for the hover feature
* 🔥 Removed some unused functions detected via code coverage
analysis


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
This commit is contained in:
Marco Liberati 2024-02-01 17:18:06 +01:00 • committed by GitHub
parent 2258b7ec9e
commit af9951fb26
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 1023 additions and 326 deletions
Download patch file Download diff file Expand all files Collapse all files

3
packages/kbn-monaco/src/esql/lib/ast/ast_factory.ts
View file

@ -120,8 +120,7 @@ export class AstListener implements ESQLParserListener {
if (metadataContext) {
const option = createOption(metadataContext.METADATA().text.toLowerCase(), metadataContext);
commandAst.args.push(option);
// skip for the moment as there's no easy way to get meta fields right now
// option.args.push(...collectAllColumnIdentifiers(metadataContext));
option.args.push(...collectAllColumnIdentifiers(metadataContext));
}
}

9
packages/kbn-monaco/src/esql/lib/ast/ast_helpers.ts
View file

@ -100,7 +100,7 @@ export function createLiteralString(token: Token): ESQLLiteral {
};
}
function isMissingText(text: string) {
export function isMissingText(text: string) {
return /<missing /.test(text);
}
@ -180,6 +180,13 @@ export function computeLocationExtends(fn: ESQLFunction) {
location.min = walkFunctionStructure(fn.args, location, 'min', () => 0);
// get max location navigating in depth keeping the right/last arg
location.max = walkFunctionStructure(fn.args, location, 'max', (args) => args.length - 1);
// in case of empty array as last arg, bump the max location by 3 chars (empty brackets)
if (
Array.isArray(fn.args[fn.args.length - 1]) &&
!(fn.args[fn.args.length - 1] as ESQLAstItem[]).length
) {
location.max += 3;
}
}
return location;
}

31
packages/kbn-monaco/src/esql/lib/ast/ast_walker.ts
View file

@ -76,6 +76,7 @@ import {
createPolicy,
createSettingTuple,
createLiteralString,
isMissingText,
} from './ast_helpers';
import { getPosition } from './ast_position_utils';
import type {
@ -206,11 +207,16 @@ function visitLogicalAndsOrs(ctx: LogicalBinaryContext) {
function visitLogicalIns(ctx: LogicalInContext) {
const fn = createFunction(ctx.NOT() ? 'not_in' : 'in', ctx);
const [left, ...list] = ctx.valueExpression();
const values = [visitValueExpression(left), list.map((ve) => visitValueExpression(ve))];
for (const arg of values) {
if (arg) {
const filteredArgs = Array.isArray(arg) ? arg.filter(nonNullable) : [arg];
fn.args.push(filteredArgs);
const leftArg = visitValueExpression(left);
if (leftArg) {
fn.args.push(...(Array.isArray(leftArg) ? leftArg : [leftArg]));
const values = list.map((ve) => visitValueExpression(ve));
const listArgs = values
.filter(nonNullable)
.flatMap((arg) => (Array.isArray(arg) ? arg.filter(nonNullable) : arg));
// distinguish between missing brackets (missing text error) and an empty list
if (!isMissingText(ctx.text)) {
fn.args.push(listArgs);
}
}
// update the location of the assign based on arguments
@ -244,6 +250,9 @@ function getComparisonName(ctx: ComparisonOperatorContext) {
}
function visitValueExpression(ctx: ValueExpressionContext) {
if (isMissingText(ctx.text)) {
return [];
}
if (ctx instanceof ValueExpressionDefaultContext) {
return visitOperatorExpression(ctx.operatorExpression());
}
@ -538,16 +547,18 @@ export function visitDissect(ctx: DissectCommandContext) {
const pattern = ctx.string().tryGetToken(esql_parser.STRING, 0);
return [
visitPrimaryExpression(ctx.primaryExpression()),
createLiteral('string', pattern),
...visitDissectOptions(ctx.commandOptions()),
...(pattern && !isMissingText(pattern.text)
? [createLiteral('string', pattern), ...visitDissectOptions(ctx.commandOptions())]
: []),
].filter(nonNullable);
}
export function visitGrok(ctx: GrokCommandContext) {
const pattern = ctx.string().tryGetToken(esql_parser.STRING, 0);
return [visitPrimaryExpression(ctx.primaryExpression()), createLiteral('string', pattern)].filter(
nonNullable
);
return [
visitPrimaryExpression(ctx.primaryExpression()),
...(pattern && !isMissingText(pattern.text) ? [createLiteral('string', pattern)] : []),
].filter(nonNullable);
}
function visitDissectOptions(ctx: CommandOptionsContext | undefined) {

200
packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.test.ts
View file

@ -25,12 +25,8 @@ const fields: Array<{ name: string; type: string; suggestedAs?: string }> = [
name: `${type}Field`,
type,
})),
{ name: 'any#Char$ field', type: 'number', suggestedAs: '`any#Char$ field`' },
{ name: 'any#Char$Field', type: 'number', suggestedAs: '`any#Char$Field`' },
{ name: 'kubernetes.something.something', type: 'number' },
{
name: `listField`,
type: `list`,
},
];
const indexes = (
@ -77,7 +73,14 @@ const policies = [
function getFunctionSignaturesByReturnType(
command: string,
expectedReturnType: string,
{ agg, evalMath, builtin }: { agg?: boolean; evalMath?: boolean; builtin?: boolean } = {},
{
agg,
evalMath,
builtin,
// skipAssign here is used to communicate to not propose an assignment if it's not possible
// within the current context (the actual logic has it, but here we want a shortcut)
skipAssign,
}: { agg?: boolean; evalMath?: boolean; builtin?: boolean; skipAssign?: boolean } = {},
paramsTypes?: string[],
ignored?: string[]
) {
@ -90,27 +93,33 @@ function getFunctionSignaturesByReturnType(
list.push(...evalFunctionsDefinitions);
}
if (builtin) {
list.push(...builtinFunctions);
list.push(...builtinFunctions.filter(({ name }) => (skipAssign ? name !== '=' : true)));
}
return list
.filter(({ signatures, ignoreAsSuggestion, supportedCommands }) => {
.filter(({ signatures, ignoreAsSuggestion, supportedCommands, name }) => {
if (ignoreAsSuggestion) {
return false;
}
if (!supportedCommands.includes(command)) {
return false;
}
const filteredByReturnType = signatures.some(
const filteredByReturnType = signatures.filter(
({ returnType }) => expectedReturnType === 'any' || returnType === expectedReturnType
);
if (!filteredByReturnType) {
if (!filteredByReturnType.length) {
return false;
}
if (paramsTypes?.length) {
return signatures.some(({ params }) =>
paramsTypes.every(
(expectedType, i) => expectedType === 'any' || expectedType === params[i].type
)
return filteredByReturnType.some(
({ params }) =>
!params.length ||
(paramsTypes.length <= params.length &&
paramsTypes.every(
(expectedType, i) =>
expectedType === 'any' ||
params[i].type === 'any' ||
expectedType === params[i].type
))
);
}
return true;
@ -162,6 +171,7 @@ function createCustomCallbackMocks(
getFieldsFor: jest.fn(async () => finalFields),
getSources: jest.fn(async () => finalSources),
getPolicies: jest.fn(async () => finalPolicies),
getMetaFields: jest.fn(async () => ['_index', '_score']),
};
}
@ -315,6 +325,8 @@ describe('autocomplete', () => {
testSuggestions('from a, b ', ['[metadata $0 ]', '|', ',']);
testSuggestions('from *,', suggestedIndexes);
testSuggestions('from index', suggestedIndexes, 6 /* index index in from */);
testSuggestions('from a, b [metadata ]', ['_index', '_score'], 20);
testSuggestions('from a, b [metadata _index, ]', ['_score'], 27);
});
describe('show', () => {
@ -389,7 +401,9 @@ describe('autocomplete', () => {
]);
}
testSuggestions('from a | stats a=avg(numberField) | where a ', [
...getFunctionSignaturesByReturnType('where', 'any', { builtin: true }, ['number']),
...getFunctionSignaturesByReturnType('where', 'any', { builtin: true, skipAssign: true }, [
'number',
]),
]);
// Mind this test: suggestion is aware of previous commands when checking for fields
// in this case the numberField has been wiped by the STATS command and suggest cannot find it's type
@ -426,8 +440,78 @@ describe('autocomplete', () => {
],
','
);
testSuggestions('from index | WHERE stringField not ', ['like $0', 'rlike $0', 'in $0']);
testSuggestions('from index | WHERE stringField NOT ', ['like $0', 'rlike $0', 'in $0']);
testSuggestions('from index | WHERE not ', [
...getFieldNamesByType('boolean'),
...getFunctionSignaturesByReturnType('eval', 'boolean', { evalMath: true }),
]);
testSuggestions('from index | WHERE numberField in ', ['( $0 )']);
testSuggestions('from index | WHERE numberField not in ', ['( $0 )']);
testSuggestions(
'from index | WHERE numberField not in ( )',
[
...getFieldNamesByType('number').filter((name) => name !== 'numberField'),
...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }),
],
'('
);
testSuggestions(
'from index | WHERE numberField in ( `any#Char$Field`, )',
[
...getFieldNamesByType('number').filter(
(name) => name !== '`any#Char$Field`' && name !== 'numberField'
),
...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }),
],
54 // after the first suggestions
);
testSuggestions(
'from index | WHERE numberField not in ( `any#Char$Field`, )',
[
...getFieldNamesByType('number').filter(
(name) => name !== '`any#Char$Field`' && name !== 'numberField'
),
...getFunctionSignaturesByReturnType('where', 'number', { evalMath: true }),
],
58 // after the first suggestions
);
});
for (const command of ['grok', 'dissect']) {
describe(command, () => {
const constantPattern = command === 'grok' ? '"%{WORD:firstWord}"' : '"%{firstWord}"';
const subExpressions = [
'',
`${command} stringField |`,
`${command} stringField ${constantPattern} |`,
`dissect stringField ${constantPattern} append_separator = ":" |`,
];
if (command === 'grok') {
subExpressions.push(`dissect stringField ${constantPattern} |`);
}
for (const subExpression of subExpressions) {
testSuggestions(`from a | ${subExpression} ${command} `, getFieldNamesByType('string'));
testSuggestions(`from a | ${subExpression} ${command} stringField `, [constantPattern]);
testSuggestions(
`from a | ${subExpression} ${command} stringField ${constantPattern} `,
(command === 'dissect' ? ['append_separator = $0'] : []).concat(['|'])
);
if (command === 'dissect') {
testSuggestions(
`from a | ${subExpression} ${command} stringField ${constantPattern} append_separator = `,
['":"', '";"']
);
testSuggestions(
`from a | ${subExpression} ${command} stringField ${constantPattern} append_separator = ":" `,
['|']
);
}
}
});
}
describe('sort', () => {
testSuggestions('from a | sort ', getFieldNamesByType('any'));
testSuggestions('from a | sort stringField ', ['asc', 'desc', '|', ',']);
@ -442,7 +526,7 @@ describe('autocomplete', () => {
});
describe('mv_expand', () => {
testSuggestions('from a | mv_expand ', ['listField']);
testSuggestions('from a | mv_expand ', getFieldNamesByType('any'));
testSuggestions('from a | mv_expand a ', ['|']);
});
@ -569,9 +653,8 @@ describe('autocomplete', () => {
'dateField',
'booleanField',
'ipField',
'any#Char$ field',
'any#Char$Field',
'kubernetes.something.something',
'listField',
]);
testSuggestions(`from a ${prevCommand}| enrich policy on b `, ['with', '|', ',']);
testSuggestions(`from a ${prevCommand}| enrich policy on b with `, [
@ -614,10 +697,28 @@ describe('autocomplete', () => {
...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
]);
testSuggestions('from a | eval numberField ', [
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true }, ['number']),
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
'number',
]),
'|',
',',
]);
testSuggestions('from index | EVAL stringField not ', ['like $0', 'rlike $0', 'in $0']);
testSuggestions('from index | EVAL stringField NOT ', ['like $0', 'rlike $0', 'in $0']);
testSuggestions('from index | EVAL numberField in ', ['( $0 )']);
testSuggestions(
'from index | EVAL numberField in ( )',
[
...getFieldNamesByType('number').filter((name) => name !== 'numberField'),
...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
],
'('
);
testSuggestions('from index | EVAL numberField not in ', ['( $0 )']);
testSuggestions('from index | EVAL not ', [
...getFieldNamesByType('boolean'),
...getFunctionSignaturesByReturnType('eval', 'boolean', { evalMath: true }),
]);
testSuggestions('from a | eval a=', [
...getFunctionSignaturesByReturnType('eval', 'any', { evalMath: true }),
]);
@ -657,11 +758,33 @@ describe('autocomplete', () => {
'from a | eval raund(', // note the typo in round
[]
);
testSuggestions(
'from a | eval raund(5, ', // note the typo in round
[]
);
testSuggestions(
'from a | eval var0 = raund(5, ', // note the typo in round
[]
);
testSuggestions('from a | eval a=round(numberField) ', [
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true }, ['number']),
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
'number',
]),
'|',
',',
]);
testSuggestions('from a | eval a=round(numberField, ', [
...getFieldNamesByType('number'),
...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
'round',
]),
]);
testSuggestions('from a | eval round(numberField, ', [
...getFieldNamesByType('number'),
...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }, undefined, [
'round',
]),
]);
testSuggestions('from a | eval a=round(numberField),', [
'var0 =',
...getFieldNamesByType('any'),
@ -673,6 +796,21 @@ describe('autocomplete', () => {
...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
'a', // @TODO remove this
]);
testSuggestions('from a | eval a=round(numberField)+ ', [
...getFieldNamesByType('number'),
...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
'a', // @TODO remove this
]);
testSuggestions('from a | eval a=numberField+ ', [
...getFieldNamesByType('number'),
...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
'a', // @TODO remove this
]);
testSuggestions('from a | eval a=`any#Char$Field`+ ', [
...getFieldNamesByType('number'),
...getFunctionSignaturesByReturnType('eval', 'number', { evalMath: true }),
'a', // @TODO remove this
]);
testSuggestions(
'from a | stats avg(numberField) by stringField | eval ',
[
@ -758,7 +896,9 @@ describe('autocomplete', () => {
testSuggestions(
'from a | eval var0 = abs(numberField) | eval abs(var0)',
[
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true }, ['number']),
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
'number',
]),
'|',
',',
],
@ -824,22 +964,34 @@ describe('autocomplete', () => {
const dateSuggestions = timeLiterals.map(({ name }) => name);
// If a literal number is detected then suggest also date period keywords
testSuggestions('from a | eval a = 1 ', [
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true }, ['number']),
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
'number',
]),
...dateSuggestions,
'|',
',',
]);
testSuggestions('from a | eval a = 1 year ', [
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true }, ['time_interval']),
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
'time_interval',
]),
'|',
',',
]);
testSuggestions('from a | eval a = 1 day + 2 ', [
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true }, ['number']),
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
'number',
]),
...dateSuggestions,
'|',
',',
]);
testSuggestions('from a | eval 1 day + 2 ', [
...getFunctionSignaturesByReturnType('eval', 'any', { builtin: true, skipAssign: true }, [
'number',
]),
...dateSuggestions,
]);
testSuggestions(
'from a | eval var0=date_trunc()',
[...getLiteralsByType('time_literal').map((t) => `${t},`)],

329
packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.ts
View file

@ -18,6 +18,7 @@ import {
getCommandOption,
getFunctionDefinition,
getLastCharFromTrimmed,
isArrayType,
isAssignment,
isAssignmentComplete,
isColumnItem,
@ -46,11 +47,15 @@ import type {
} from '../types';
import type { ESQLPolicy, ESQLRealField, ESQLVariable, ReferenceMaps } from '../validation/types';
import {
colonCompleteItem,
commaCompleteItem,
commandAutocompleteDefinitions,
getAssignmentDefinitionCompletitionItem,
getBuiltinCompatibleFunctionDefinition,
getNextTokenForNot,
listCompleteItem,
pipeCompleteItem,
semiColonCompleteItem,
} from './complete_items';
import {
buildFieldsDefinitions,
@ -64,7 +69,6 @@ import {
buildConstantsDefinitions,
buildVariablesDefinitions,
buildOptionDefinition,
TRIGGER_SUGGESTION_COMMAND,
buildSettingDefinitions,
buildSettingValueDefinitions,
} from './factories';
@ -86,6 +90,7 @@ type GetFieldsByTypeFn = (
type GetFieldsMapFn = () => Promise<Map<string, ESQLRealField>>;
type GetPoliciesFn = () => Promise<AutocompleteCommandDefinition[]>;
type GetPolicyMetadataFn = (name: string) => Promise<ESQLPolicy | undefined>;
type GetMetaFieldsFn = () => Promise<string[]>;
function hasSameArgBothSides(assignFn: ESQLFunction) {
if (assignFn.name === '=' && isColumnItem(assignFn.args[0]) && assignFn.args[1]) {
@ -166,7 +171,7 @@ export async function suggest(
(context.triggerKind === 0 && unclosedRoundBrackets === 0) ||
(context.triggerCharacter === ' ' &&
// make this more robust
(isMathFunction(innerText[offset - 2]) || isComma(innerText[offset - 2])))
(isMathFunction(innerText, offset) || isComma(innerText[offset - 2])))
) {
finalText = `${innerText.substring(0, offset)}${EDITOR_MARKER}${innerText.substring(offset)}`;
}
@ -194,6 +199,7 @@ export async function suggest(
);
const getSources = getSourcesRetriever(resourceRetriever);
const { getPolicies, getPolicyMetadata } = getPolicyRetriever(resourceRetriever);
const getMetaFields = getMetaFieldsRetriever(resourceRetriever);
if (astContext.type === 'newCommand') {
// propose main commands here
@ -243,7 +249,8 @@ export async function suggest(
{ option, ...rest },
getFieldsByType,
getFieldsMap,
getPolicyMetadata
getPolicyMetadata,
getMetaFields
);
}
}
@ -257,6 +264,16 @@ export async function suggest(
getPolicyMetadata
);
}
if (astContext.type === 'list') {
return getListArgsSuggestions(
innerText,
ast,
astContext,
getFieldsByType,
getFieldsMap,
getPolicyMetadata
);
}
return [];
}
@ -271,6 +288,13 @@ function getFieldsByTypeRetriever(queryString: string, resourceRetriever?: ESQLC
};
}
function getMetaFieldsRetriever(resourceRetriever?: ESQLCallbacks): () => Promise<string[]> {
if (resourceRetriever?.getMetaFields == null) {
return async () => [];
}
return async () => resourceRetriever!.getMetaFields!();
}
function getPolicyRetriever(resourceRetriever?: ESQLCallbacks) {
const helpers = getPolicyHelper(resourceRetriever);
return {
@ -408,6 +432,9 @@ function isFunctionArgComplete(
if (!argLengthCheck) {
return { complete: false, reason: 'fewArgs' };
}
if (fnDefinition.name === 'in' && Array.isArray(arg.args[1]) && !arg.args[1].length) {
return { complete: false, reason: 'fewArgs' };
}
const hasCorrectTypes = fnDefinition.signatures.some((def) => {
return arg.args.every((a, index) => {
if (def.infiniteParams) {
@ -465,7 +492,18 @@ async function getExpressionSuggestionsByType(
// A new expression is considered either
// * just after a command name => i.e. ... | STATS <here>
// * or after a comma => i.e. STATS fieldA, <here>
const isNewExpression = isRestartingExpression(innerText) || argIndex === 0;
const isNewExpression =
isRestartingExpression(innerText) ||
(argIndex === 0 && (!isFunctionItem(nodeArg) || !nodeArg?.args.length));
// the not function is a special operator that can be used in different ways,
// and not all these are mapped within the AST data structure: in particular
// <COMMAND> <field> NOT <here>
// is an incomplete statement and it results in a missing AST node, so we need to detect
// from the query string itself
const endsWithNot =
/ not$/i.test(innerText.trimEnd()) &&
!command.args.some((arg) => isFunctionItem(arg) && arg.name === 'not');
// early exit in case of a missing function
if (isFunctionItem(lastArg) && !getFunctionDefinition(lastArg.name)) {
@ -531,19 +569,27 @@ async function getExpressionSuggestionsByType(
if (argDef) {
if (argDef.type === 'column' || argDef.type === 'any' || argDef.type === 'function') {
if (isNewExpression && canHaveAssignments) {
// i.e.
// ... | ROW <suggest>
// ... | STATS <suggest>
// ... | STATS ..., <suggest>
// ... | EVAL <suggest>
// ... | EVAL ..., <suggest>
suggestions.push(buildNewVarDefinition(findNewVariable(anyVariables)));
if (endsWithNot) {
// i.e.
// ... | ROW field NOT <suggest>
// ... | EVAL field NOT <suggest>
// there's not way to know the type of the field here, so suggest anything
suggestions.push(...getNextTokenForNot(command.name, option?.name, 'any'));
} else {
// i.e.
// ... | ROW <suggest>
// ... | STATS <suggest>
// ... | STATS ..., <suggest>
// ... | EVAL <suggest>
// ... | EVAL ..., <suggest>
suggestions.push(buildNewVarDefinition(findNewVariable(anyVariables)));
}
}
}
// Suggest fields or variables
if (argDef.type === 'column' || argDef.type === 'any') {
// ... | <COMMAND> <suggest>
if (!nodeArg || (isNewExpression && commandDef.signature.multipleParams)) {
if ((!nodeArg || isNewExpression) && !endsWithNot) {
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
[argDef.innerType || 'any'],
@ -577,7 +623,10 @@ async function getExpressionSuggestionsByType(
suggestions.push(getAssignmentDefinitionCompletitionItem());
}
}
if (isNewExpression || (isAssignment(nodeArg) && !isAssignmentComplete(nodeArg))) {
if (
(isNewExpression && !endsWithNot) ||
(isAssignment(nodeArg) && !isAssignmentComplete(nodeArg))
) {
// ... | STATS a = <suggest>
// ... | EVAL a = <suggest>
// ... | STATS a = ..., <suggest>
@ -631,24 +680,40 @@ async function getExpressionSuggestionsByType(
}
} else {
if (isFunctionItem(nodeArg)) {
const nodeArgType = extractFinalTypeFromArg(nodeArg, references);
suggestions.push(
...(await getBuiltinFunctionNextArgument(
command,
option,
argDef,
nodeArg,
nodeArgType || 'any',
references,
getFieldsByType
))
);
if (nodeArg.args.some(isTimeIntervalItem)) {
const lastFnArg = nodeArg.args[nodeArg.args.length - 1];
const lastFnArgType = extractFinalTypeFromArg(lastFnArg, references);
if (lastFnArgType === 'number' && isLiteralItem(lastFnArg))
// ... EVAL var = 1 year + 2 <suggest>
suggestions.push(...getCompatibleLiterals(command.name, ['time_literal_unit']));
if (nodeArg.name === 'not') {
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
['boolean'],
command.name,
option?.name,
getFieldsByType,
{
functions: true,
fields: true,
variables: anyVariables,
}
))
);
} else {
const nodeArgType = extractFinalTypeFromArg(nodeArg, references);
suggestions.push(
...(await getBuiltinFunctionNextArgument(
command,
option,
argDef,
nodeArg,
nodeArgType || 'any',
references,
getFieldsByType
))
);
if (nodeArg.args.some(isTimeIntervalItem)) {
const lastFnArg = nodeArg.args[nodeArg.args.length - 1];
const lastFnArgType = extractFinalTypeFromArg(lastFnArg, references);
if (lastFnArgType === 'number' && isLiteralItem(lastFnArg))
// ... EVAL var = 1 year + 2 <suggest>
suggestions.push(...getCompatibleLiterals(command.name, ['time_literal_unit']));
}
}
}
}
@ -669,21 +734,28 @@ async function getExpressionSuggestionsByType(
} else {
// or it can be anything else as long as it is of the right type and the end (i.e. column or function)
if (!nodeArg) {
// ... | <COMMAND> <suggest>
// In this case start suggesting something not strictly based on type
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
['any'],
command.name,
option?.name,
getFieldsByType,
{
functions: true,
fields: true,
variables: anyVariables,
}
))
);
if (endsWithNot) {
// i.e.
// ... | WHERE field NOT <suggest>
// there's not way to know the type of the field here, so suggest anything
suggestions.push(...getNextTokenForNot(command.name, option?.name, 'any'));
} else {
// ... | <COMMAND> <suggest>
// In this case start suggesting something not strictly based on type
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
['any'],
command.name,
option?.name,
getFieldsByType,
{
functions: true,
fields: true,
variables: anyVariables,
}
))
);
}
} else {
// if something is already present, leverage its type to suggest something in context
const nodeArgType = extractFinalTypeFromArg(nodeArg, references);
@ -696,17 +768,33 @@ async function getExpressionSuggestionsByType(
if (nodeArgType) {
if (isFunctionItem(nodeArg)) {
suggestions.push(
...(await getBuiltinFunctionNextArgument(
command,
option,
argDef,
nodeArg,
nodeArgType,
references,
getFieldsByType
))
);
if (nodeArg.name === 'not') {
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
['boolean'],
command.name,
option?.name,
getFieldsByType,
{
functions: true,
fields: true,
variables: anyVariables,
}
))
);
} else {
suggestions.push(
...(await getBuiltinFunctionNextArgument(
command,
option,
argDef,
nodeArg,
nodeArgType,
references,
getFieldsByType
))
);
}
} else {
// i.e. ... | <COMMAND> field <suggest>
suggestions.push(
@ -750,7 +838,9 @@ async function getExpressionSuggestionsByType(
) {
// suggest some command options
if (optionsAvailable.length) {
suggestions.push(...optionsAvailable.map(buildOptionDefinition));
suggestions.push(
...optionsAvailable.map((opt) => buildOptionDefinition(opt, command.name === 'dissect'))
);
}
if (!optionsAvailable.length || optionsAvailable.every(({ optional }) => optional)) {
@ -799,28 +889,33 @@ async function getBuiltinFunctionNextArgument(
const nestedType = extractFinalTypeFromArg(nodeArg.args[cleanedArgs.length - 1], references);
if (isFnComplete.reason === 'fewArgs') {
const finalType = nestedType || nodeArgType || 'any';
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
// this is a special case with AND/OR
// <COMMAND> expression AND/OR <suggest>
// technically another boolean value should be suggested, but it is a better experience
// to actually suggest a wider set of fields/functions
[
finalType === 'boolean' && getFunctionDefinition(nodeArg.name)?.type === 'builtin'
? 'any'
: finalType,
],
command.name,
option?.name,
getFieldsByType,
{
functions: true,
fields: true,
variables: references.variables,
}
))
);
const fnDef = getFunctionDefinition(nodeArg.name);
if (fnDef?.signatures.every(({ params }) => params.some(({ type }) => isArrayType(type)))) {
suggestions.push(listCompleteItem);
} else {
const finalType = nestedType || nodeArgType || 'any';
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
// this is a special case with AND/OR
// <COMMAND> expression AND/OR <suggest>
// technically another boolean value should be suggested, but it is a better experience
// to actually suggest a wider set of fields/functions
[
finalType === 'boolean' && getFunctionDefinition(nodeArg.name)?.type === 'builtin'
? 'any'
: finalType,
],
command.name,
option?.name,
getFieldsByType,
{
functions: true,
fields: true,
variables: references.variables,
}
))
);
}
}
if (isFnComplete.reason === 'wrongTypes') {
if (nestedType) {
@ -898,7 +993,6 @@ async function getFieldsOrFunctionsSuggestions(
...rest,
kind,
sortText: String.fromCharCode(97 - kind),
command: TRIGGER_SUGGESTION_COMMAND,
}));
}
@ -1023,6 +1117,61 @@ async function getFunctionArgsSuggestions(
}));
}
async function getListArgsSuggestions(
innerText: string,
commands: ESQLCommand[],
{
command,
node,
}: {
command: ESQLCommand;
node: ESQLSingleAstItem | undefined;
},
getFieldsByType: GetFieldsByTypeFn,
getFieldsMaps: GetFieldsMapFn,
getPolicyMetadata: GetPolicyMetadataFn
) {
const suggestions = [];
// node is supposed to be the function who support a list argument (like the "in" operator)
// so extract the type of the first argument and suggest fields of that type
if (node && isFunctionItem(node)) {
const fieldsMap: Map<string, ESQLRealField> = await getFieldsMaps();
const anyVariables = collectVariables(commands, fieldsMap);
// extract the current node from the variables inferred
anyVariables.forEach((values, key) => {
if (values.some((v) => v.location === node.location)) {
anyVariables.delete(key);
}
});
const [firstArg] = node.args;
if (isColumnItem(firstArg)) {
const argType = extractFinalTypeFromArg(firstArg, {
fields: fieldsMap,
variables: anyVariables,
});
if (argType) {
// do not propose existing columns again
const otherArgs = node.args.filter(Array.isArray).flat().filter(isColumnItem);
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
[argType],
command.name,
undefined,
getFieldsByType,
{
functions: true,
fields: true,
variables: anyVariables,
},
{ ignoreFields: [firstArg.name, ...otherArgs.map(({ name }) => name)] }
))
);
}
}
}
return suggestions;
}
async function getSettingArgsSuggestions(
innerText: string,
commands: ESQLCommand[],
@ -1078,7 +1227,8 @@ async function getOptionArgsSuggestions(
},
getFieldsByType: GetFieldsByTypeFn,
getFieldsMaps: GetFieldsMapFn,
getPolicyMetadata: GetPolicyMetadataFn
getPolicyMetadata: GetPolicyMetadataFn,
getMetaFields: GetMetaFieldsFn
) {
const optionDef = getCommandOption(option.name);
const { nodeArg, argIndex, lastArg } = extractArgMeta(option, node);
@ -1175,6 +1325,19 @@ async function getOptionArgsSuggestions(
}
}
if (command.name === 'dissect') {
if (option.args.length < 1 && optionDef) {
suggestions.push(colonCompleteItem, semiColonCompleteItem);
}
}
if (option.name === 'metadata') {
const existingFields = new Set(option.args.filter(isColumnItem).map(({ name }) => name));
const metaFields = await getMetaFields();
const filteredMetaFields = metaFields.filter((name) => !existingFields.has(name));
suggestions.push(...buildFieldsDefinitions(filteredMetaFields));
}
if (command.name === 'stats') {
suggestions.push(
...(await getFieldsOrFunctionsSuggestions(
@ -1228,7 +1391,7 @@ async function getOptionArgsSuggestions(
) {
suggestions.push(
...getFinalSuggestions({
comma: true,
comma: optionDef.signature.multipleParams,
})
);
} else if (isNewExpression || (isAssignment(nodeArg) && !isAssignmentComplete(nodeArg))) {

93
packages/kbn-monaco/src/esql/lib/ast/autocomplete/complete_items.ts
View file

@ -16,6 +16,8 @@ import {
getAutocompleteFunctionDefinition,
getAutocompleteBuiltinDefinition,
getAutocompleteCommandDefinition,
TRIGGER_SUGGESTION_COMMAND,
buildConstantsDefinitions,
} from './factories';
export const mathCommandDefinition: AutocompleteCommandDefinition[] = evalFunctionsDefinitions.map(
@ -30,6 +32,35 @@ export function getAssignmentDefinitionCompletitionItem() {
return getAutocompleteBuiltinDefinition(assignFn);
}
export const getNextTokenForNot = (
command: string,
option: string | undefined,
argType: string
): AutocompleteCommandDefinition[] => {
const compatibleFunctions = builtinFunctions.filter(
({ name, supportedCommands, supportedOptions, ignoreAsSuggestion }) =>
!ignoreAsSuggestion &&
!/not_/.test(name) &&
(option ? supportedOptions?.includes(option) : supportedCommands.includes(command))
);
if (argType === 'string' || argType === 'any') {
// suggest IS, LIKE, RLIKE and TRUE/FALSE
return compatibleFunctions
.filter(({ name }) => name === 'like' || name === 'rlike' || name === 'in')
.map(getAutocompleteBuiltinDefinition);
}
if (argType === 'boolean') {
// suggest IS, NOT and TRUE/FALSE
return [
...compatibleFunctions
.filter(({ name }) => name === 'in')
.map(getAutocompleteBuiltinDefinition),
...buildConstantsDefinitions(['true', 'false']),
];
}
return [];
};
export const getBuiltinCompatibleFunctionDefinition = (
command: string,
option: string | undefined,
@ -61,22 +92,58 @@ export const commandAutocompleteDefinitions: AutocompleteCommandDefinition[] = g
getAutocompleteCommandDefinition
);
export const pipeCompleteItem: AutocompleteCommandDefinition = {
label: '|',
insertText: '|',
kind: 1,
detail: i18n.translate('monaco.esql.autocomplete.pipeDoc', {
function buildCharCompleteItem(
label: string,
detail: string,
{ sortText, quoted }: { sortText?: string; quoted: boolean } = { quoted: false }
): AutocompleteCommandDefinition {
return {
label,
insertText: quoted ? `"${label}"` : label,
kind: 1,
detail,
sortText,
};
}
export const pipeCompleteItem = buildCharCompleteItem(
'|',
i18n.translate('monaco.esql.autocomplete.pipeDoc', {
defaultMessage: 'Pipe (|)',
}),
sortText: 'B',
};
{ sortText: 'B', quoted: false }
);
export const commaCompleteItem: AutocompleteCommandDefinition = {
label: ',',
insertText: ',',
kind: 1,
detail: i18n.translate('monaco.esql.autocomplete.commaDoc', {
export const commaCompleteItem = buildCharCompleteItem(
',',
i18n.translate('monaco.esql.autocomplete.commaDoc', {
defaultMessage: 'Comma (,)',
}),
sortText: 'C',
{ sortText: 'C', quoted: false }
);
export const colonCompleteItem = buildCharCompleteItem(
':',
i18n.translate('monaco.esql.autocomplete.colonDoc', {
defaultMessage: 'Colon (:)',
}),
{ sortText: 'A', quoted: true }
);
export const semiColonCompleteItem = buildCharCompleteItem(
';',
i18n.translate('monaco.esql.autocomplete.semiColonDoc', {
defaultMessage: 'Semi colon (;)',
}),
{ sortText: 'A', quoted: true }
);
export const listCompleteItem: AutocompleteCommandDefinition = {
label: '( ... )',
insertText: '( $0 )',
insertTextRules: 4,
kind: 1,
detail: i18n.translate('monaco.esql.autocomplete.listDoc', {
defaultMessage: 'List of items ( ...)',
}),
sortText: 'A',
command: TRIGGER_SUGGESTION_COMMAND,
};

28
packages/kbn-monaco/src/esql/lib/ast/autocomplete/factories.ts
View file

@ -44,6 +44,8 @@ export function getAutocompleteFunctionDefinition(fn: FunctionDefinition) {
value: buildFunctionDocumentation(fullSignatures),
},
sortText: 'C',
// trigger a suggestion follow up on selection
command: TRIGGER_SUGGESTION_COMMAND,
};
}
@ -63,13 +65,6 @@ export function getAutocompleteBuiltinDefinition(fn: FunctionDefinition) {
};
}
export const isCompatibleFunctionName = (fnName: string, command: string) => {
const fnSupportedByCommand = allFunctions.filter(({ supportedCommands }) =>
supportedCommands.includes(command)
);
return fnSupportedByCommand.some(({ name }) => name === fnName);
};
export const getCompatibleFunctionDefinition = (
command: string,
option: string | undefined,
@ -205,7 +200,10 @@ export const buildMatchingFieldsDefinition = (
sortText: 'D',
}));
export const buildOptionDefinition = (option: CommandOptionsDefinition) => {
export const buildOptionDefinition = (
option: CommandOptionsDefinition,
isAssignType: boolean = false
) => {
const completeItem: AutocompleteCommandDefinition = {
label: option.name,
insertText: option.name,
@ -217,6 +215,11 @@ export const buildOptionDefinition = (option: CommandOptionsDefinition) => {
completeItem.insertText = `${option.wrapped[0]}${option.name} $0 ${option.wrapped[1]}`;
completeItem.insertTextRules = 4; // monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet;
}
if (isAssignType) {
completeItem.insertText = `${option.name} = $0`;
completeItem.insertTextRules = 4; // monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet;
completeItem.command = TRIGGER_SUGGESTION_COMMAND;
}
return completeItem;
};
@ -305,7 +308,14 @@ export function getCompatibleLiterals(commandName: string, types: string[], name
if (names) {
const index = types.indexOf('string');
if (/pattern/.test(names[index])) {
suggestions.push(...buildConstantsDefinitions(['"a-pattern"'], 'A pattern string'));
suggestions.push(
...buildConstantsDefinitions(
[commandName === 'grok' ? '"%{WORD:firstWord}"' : '"%{firstWord}"'],
i18n.translate('monaco.esql.autocomplete.aPatternString', {
defaultMessage: 'A pattern string',
})
)
);
} else {
suggestions.push(...buildConstantsDefinitions(['string'], ''));
}

29
packages/kbn-monaco/src/esql/lib/ast/code_actions/index.test.ts → packages/kbn-monaco/src/esql/lib/ast/code_actions/actions.test.ts
View file

@ -8,7 +8,7 @@
import { EditorError } from '../../../../types';
import { CharStreams } from 'antlr4ts';
import { getActions } from '.';
import { getActions } from './actions';
import { getParser, ROOT_STATEMENT } from '../../antlr_facade';
import { ESQLErrorListener } from '../../monaco/esql_error_listener';
import { AstListener } from '../ast_factory';
@ -63,6 +63,7 @@ function getCallbackMocks() {
enrichFields: ['other-field', 'yetAnotherField'],
},
]),
getMetaFields: jest.fn(async () => ['_index', '_id', '_source', '_score']),
};
}
@ -181,6 +182,32 @@ describe('quick fixes logic', () => {
testQuickFixes(`FROM index | ENRICH policy ON stringField WITH yetAnotherField2`, [
'yetAnotherField',
]);
describe('metafields spellchecks', () => {
testQuickFixes(`FROM index [metadata _i_ndex]`, ['_index']);
testQuickFixes(`FROM index [metadata _id, _i_ndex]`, ['_index']);
testQuickFixes(`FROM index [METADATA _id, _i_ndex]`, ['_index']);
});
});
describe('fixing meta fields spellchecks', () => {
for (const command of ['KEEP', 'DROP', 'EVAL']) {
testQuickFixes(`FROM index | ${command} stringField`, []);
// strongField => stringField
testQuickFixes(`FROM index | ${command} strongField`, ['stringField']);
testQuickFixes(`FROM index | ${command} numberField, strongField`, ['stringField']);
}
testQuickFixes(`FROM index | EVAL round(strongField)`, ['stringField']);
testQuickFixes(`FROM index | EVAL var0 = round(strongField)`, ['stringField']);
testQuickFixes(`FROM index | WHERE round(strongField) > 0`, ['stringField']);
testQuickFixes(`FROM index | WHERE 0 < round(strongField)`, ['stringField']);
testQuickFixes(`FROM index | RENAME strongField as newField`, ['stringField']);
// This levarage the knowledge of the enrich policy fields to suggest the right field
testQuickFixes(`FROM index | ENRICH policy | KEEP yetAnotherField2`, ['yetAnotherField']);
testQuickFixes(`FROM index | ENRICH policy ON strongField`, ['stringField']);
testQuickFixes(`FROM index | ENRICH policy ON stringField WITH yetAnotherField2`, [
'yetAnotherField',
]);
});
describe('fixing policies spellchecks', () => {

41
packages/kbn-monaco/src/esql/lib/ast/code_actions/index.ts → packages/kbn-monaco/src/esql/lib/ast/code_actions/actions.ts
View file

@ -15,19 +15,21 @@ import {
} from '../shared/resources_helpers';
import { getAllFunctions, isSourceItem, shouldBeQuotedText } from '../shared/helpers';
import { ESQLCallbacks } from '../shared/types';
import { AstProviderFn, ESQLAst } from '../types';
import { AstProviderFn, ESQLAst, ESQLCommand } from '../types';
import { buildQueryForFieldsFromSource } from '../validation/helpers';
type GetSourceFn = () => Promise<string[]>;
type GetFieldsByTypeFn = (type: string | string[], ignored?: string[]) => Promise<string[]>;
type GetPoliciesFn = () => Promise<string[]>;
type GetPolicyFieldsFn = (name: string) => Promise<string[]>;
type GetMetaFieldsFn = () => Promise<string[]>;
interface Callbacks {
getSources: GetSourceFn;
getFieldsByType: GetFieldsByTypeFn;
getPolicies: GetPoliciesFn;
getPolicyFields: GetPolicyFieldsFn;
getMetaFields: GetMetaFieldsFn;
}
function getFieldsByTypeRetriever(queryString: string, resourceRetriever?: ESQLCallbacks) {
@ -64,6 +66,19 @@ function getSourcesRetriever(resourceRetriever?: ESQLCallbacks) {
};
}
export function getMetaFieldsRetriever(
queryString: string,
commands: ESQLCommand[],
callbacks?: ESQLCallbacks
) {
return async () => {
if (!callbacks || !callbacks.getMetaFields) {
return [];
}
return await callbacks.getMetaFields();
};
}
export const getCompatibleFunctionDefinitions = (
command: string,
option: string | undefined,
@ -264,6 +279,18 @@ async function getSpellingActionForFunctions(
);
}
async function getSpellingActionForMetadata(
error: monaco.editor.IMarkerData,
uri: monaco.Uri,
queryString: string,
ast: ESQLAst,
{ getMetaFields }: Callbacks
) {
const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1);
const possibleMetafields = await getSpellingPossibilities(getMetaFields, errorText);
return wrapIntoSpellingChangeAction(error, uri, possibleMetafields);
}
function wrapIntoSpellingChangeAction(
error: monaco.editor.IMarkerData,
uri: monaco.Uri,
@ -310,12 +337,14 @@ export async function getActions(
const { getFieldsByType } = getFieldsByTypeRetriever(queryForFields, resourceRetriever);
const getSources = getSourcesRetriever(resourceRetriever);
const { getPolicies, getPolicyFields } = getPolicyRetriever(resourceRetriever);
const getMetaFields = getMetaFieldsRetriever(innerText, ast, resourceRetriever);
const callbacks = {
getFieldsByType,
getSources,
getPolicies,
getPolicyFields,
getMetaFields,
};
// Markers are sent only on hover and are limited to the hovered area
@ -360,6 +389,16 @@ export async function getActions(
);
actions.push(...fnsSpellChanges);
break;
case 'unknownMetadataField':
const metadataSpellChanges = await getSpellingActionForMetadata(
error,
model.uri,
innerText,
ast,
callbacks
);
actions.push(...metadataSpellChanges);
break;
case 'wrongQuotes':
// it is a syntax error, so location won't be helpful here
const [, errorText] = error.message.split('at ');

2
packages/kbn-monaco/src/esql/lib/ast/definitions/commands.ts
View file

@ -275,7 +275,7 @@ export const commandDefinitions: CommandDefinition[] = [
modes: [],
signature: {
multipleParams: false,
params: [{ name: 'column', type: 'column', innerType: 'list' }],
params: [{ name: 'column', type: 'column', innerType: 'any' }],
},
},
{

20
packages/kbn-monaco/src/esql/lib/ast/definitions/functions.ts
View file

@ -221,7 +221,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
{ name: 'words', type: 'string' },
{ name: 'separator', type: 'string' },
],
returnType: 'string[]',
returnType: 'string',
examples: [`ROW words="foo;bar;baz;qux;quux;corge" | EVAL word = SPLIT(words, ";")`],
},
],
@ -873,7 +873,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
}),
signatures: [
{
params: [{ name: 'multivalue', type: 'number[]' }],
params: [{ name: 'multivalue', type: 'number' }],
returnType: 'number',
examples: ['row a = [1, 2, 3] | eval mv_avg(a)'],
},
@ -888,7 +888,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
signatures: [
{
params: [
{ name: 'multivalue', type: 'string[]' },
{ name: 'multivalue', type: 'string' },
{ name: 'delimeter', type: 'string' },
],
returnType: 'string',
@ -904,7 +904,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
}),
signatures: [
{
params: [{ name: 'multivalue', type: 'any[]' }],
params: [{ name: 'multivalue', type: 'any' }],
returnType: 'number',
examples: ['row a = [1, 2, 3] | eval mv_count(a)'],
},
@ -917,8 +917,8 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
}),
signatures: [
{
params: [{ name: 'multivalue', type: 'any[]' }],
returnType: 'any[]',
params: [{ name: 'multivalue', type: 'any' }],
returnType: 'any',
examples: ['row a = [2, 2, 3] | eval mv_dedupe(a)'],
},
],
@ -959,7 +959,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
}),
signatures: [
{
params: [{ name: 'multivalue', type: 'number[]' }],
params: [{ name: 'multivalue', type: 'number' }],
returnType: 'number',
examples: ['row a = [1, 2, 3] | eval mv_max(a)'],
},
@ -973,7 +973,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
}),
signatures: [
{
params: [{ name: 'multivalue', type: 'number[]' }],
params: [{ name: 'multivalue', type: 'number' }],
returnType: 'number',
examples: ['row a = [1, 2, 3] | eval mv_min(a)'],
},
@ -987,7 +987,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
}),
signatures: [
{
params: [{ name: 'multivalue', type: 'number[]' }],
params: [{ name: 'multivalue', type: 'number' }],
returnType: 'number',
examples: ['row a = [1, 2, 3] | eval mv_median(a)'],
},
@ -1001,7 +1001,7 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
}),
signatures: [
{
params: [{ name: 'multivalue', type: 'number[]' }],
params: [{ name: 'multivalue', type: 'number' }],
returnType: 'number',
examples: ['row a = [1, 2, 3] | eval mv_sum(a)'],
},

23
packages/kbn-monaco/src/esql/lib/ast/definitions/helpers.ts
View file

@ -6,28 +6,7 @@
* Side Public License, v 1.
*/
import { CommandDefinition, CommandOptionsDefinition, FunctionDefinition } from './types';
export function getCommandOrOptionsSignature({
name,
signature,
...rest
}: CommandDefinition | CommandOptionsDefinition): string {
const args = signature.params
.map(({ name: argName, type }) => {
return `<${argName}>`;
})
.join(' ');
const optionArgs =
'options' in rest ? rest.options.map(getCommandOrOptionsSignature).join(' ') : '';
const signatureString = `${name.toUpperCase()} ${args}${
signature.multipleParams ? `[, ${args}]` : ''
}${optionArgs ? ' ' + optionArgs : ''}`;
if ('wrapped' in rest && rest.wrapped) {
return `${rest.wrapped[0]}${signatureString}${rest.wrapped[1]}${rest.optional ? '?' : ''}`;
}
return signatureString;
}
import { CommandDefinition, FunctionDefinition } from './types';
export function getFunctionSignatures(
{ name, signatures }: FunctionDefinition,

28
packages/kbn-monaco/src/esql/lib/ast/definitions/options.ts
View file

@ -7,7 +7,7 @@
*/
import { i18n } from '@kbn/i18n';
import { isLiteralItem } from '../shared/helpers';
import { isColumnItem, isLiteralItem } from '../shared/helpers';
import { ESQLCommandOption, ESQLMessage } from '../types';
import { CommandOptionsDefinition } from './types';
@ -34,6 +34,32 @@ export const metadataOption: CommandOptionsDefinition = {
},
optional: true,
wrapped: ['[', ']'],
skipCommonValidation: true,
validate: (option, command, references) => {
const messages: ESQLMessage[] = [];
const fields = option.args.filter(isColumnItem);
const metadataFieldsAvailable = references as unknown as Set<string>;
if (metadataFieldsAvailable.size > 0) {
for (const field of fields) {
if (!metadataFieldsAvailable.has(field.name)) {
messages.push({
location: field.location,
text: i18n.translate('monaco.esql.validation.wrongMetadataArgumentType', {
defaultMessage:
'Metadata field [{value}] is not available. Available metadata fields are: [{availableFields}]',
values: {
value: field.name,
availableFields: Array.from(metadataFieldsAvailable).join(', '),
},
}),
type: 'error',
code: 'unknownMetadataField',
});
}
}
}
return messages;
},
};
export const asOption: CommandOptionsDefinition = {

6
packages/kbn-monaco/src/esql/lib/ast/definitions/types.ts
View file

@ -57,7 +57,11 @@ export interface CommandOptionsDefinition extends CommandBaseDefinition {
wrapped?: string[];
optional: boolean;
skipCommonValidation?: boolean;
validate?: (option: ESQLCommandOption, command: ESQLCommand) => ESQLMessage[];
validate?: (
option: ESQLCommandOption,
command: ESQLCommand,
references?: unknown
) => ESQLMessage[];
}
export interface CommandModeDefinition extends CommandBaseDefinition {

214
packages/kbn-monaco/src/esql/lib/ast/hover/hover.test.ts Normal file
View file

@ -0,0 +1,214 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { monaco } from '../../../../monaco_imports';
import { CharStreams } from 'antlr4ts';
import { getParser, ROOT_STATEMENT } from '../../antlr_facade';
import { ESQLErrorListener } from '../../monaco/esql_error_listener';
import { AstListener } from '../ast_factory';
import { getHoverItem } from './hover';
import { getFunctionDefinition } from '../shared/helpers';
import { getFunctionSignatures } from '../definitions/helpers';
const fields: Array<{ name: string; type: string; suggestedAs?: string }> = [
...['string', 'number', 'date', 'boolean', 'ip'].map((type) => ({
name: `${type}Field`,
type,
})),
{ name: 'any#Char$Field', type: 'number', suggestedAs: '`any#Char$Field`' },
{ name: 'kubernetes.something.something', type: 'number' },
];
const indexes = (
[] as Array<{ name: string; hidden: boolean; suggestedAs: string | undefined }>
).concat(
['a', 'index', 'otherIndex', '.secretIndex', 'my-index'].map((name) => ({
name,
hidden: name.startsWith('.'),
suggestedAs: undefined,
})),
['my-index[quoted]', 'my-index$', 'my_index{}'].map((name) => ({
name,
hidden: false,
suggestedAs: `\`${name}\``,
}))
);
const policies = [
{
name: 'policy',
sourceIndices: ['enrichIndex1'],
matchField: 'otherStringField',
enrichFields: ['otherField', 'yetAnotherField', 'yet-special-field'],
suggestedAs: undefined,
},
...['my-policy[quoted]', 'my-policy$', 'my_policy{}'].map((name) => ({
name,
sourceIndices: ['enrichIndex1'],
matchField: 'otherStringField',
enrichFields: ['otherField', 'yetAnotherField', 'yet-special-field'],
suggestedAs: `\`${name}\``,
})),
];
function createCustomCallbackMocks(
customFields: Array<{ name: string; type: string }> | undefined,
customSources: Array<{ name: string; hidden: boolean }> | undefined,
customPolicies:
| Array<{
name: string;
sourceIndices: string[];
matchField: string;
enrichFields: string[];
}>
| undefined
) {
const finalFields = customFields || fields;
const finalSources = customSources || indexes;
const finalPolicies = customPolicies || policies;
return {
getFieldsFor: jest.fn(async () => finalFields),
getSources: jest.fn(async () => finalSources),
getPolicies: jest.fn(async () => finalPolicies),
getMetaFields: jest.fn(async () => ['_index', '_score']),
};
}
function createModelAndPosition(text: string, string: string) {
return {
model: { getValue: () => text } as monaco.editor.ITextModel,
// bumo the column by one as the internal logic has a -1 offset when converting frmo monaco
position: { lineNumber: 1, column: text.lastIndexOf(string) + 1 } as monaco.Position,
};
}
describe('hover', () => {
const getAstAndErrors = async (text: string) => {
const errorListener = new ESQLErrorListener();
const parseListener = new AstListener();
const parser = getParser(CharStreams.fromString(text), errorListener, parseListener);
parser[ROOT_STATEMENT]();
return { ...parseListener.getAst(), errors: [] };
};
type TestArgs = [
string,
string,
(n: string) => string[],
Parameters<typeof createCustomCallbackMocks>?
];
const testSuggestionsFn = (
statement: string,
triggerString: string,
contentFn: (name: string) => string[],
customCallbacksArgs: Parameters<typeof createCustomCallbackMocks> = [
undefined,
undefined,
undefined,
],
{ only, skip }: { only?: boolean; skip?: boolean } = {}
) => {
const token: monaco.CancellationToken = {
isCancellationRequested: false,
onCancellationRequested: () => ({ dispose: () => {} }),
};
const { model, position } = createModelAndPosition(statement, triggerString);
const testFn = only ? test.only : skip ? test.skip : test;
const expected = contentFn(triggerString);
testFn(
`${statement} (hover: "${triggerString}" @ ${position.column} - ${
position.column + triggerString.length
})=> ["${expected.join('","')}"]`,
async () => {
const callbackMocks = createCustomCallbackMocks(...customCallbacksArgs);
const { contents } = await getHoverItem(
model,
position,
token,
async (text) => (text ? await getAstAndErrors(text) : { ast: [], errors: [] }),
callbackMocks
);
expect(contents.map(({ value }) => value)).toEqual(expected);
}
);
};
// Enrich the function to work with .only and .skip as regular test function
const testSuggestions = Object.assign(testSuggestionsFn, {
skip: (...args: TestArgs) => {
const paddingArgs = [[undefined, undefined, undefined]].slice(args.length - 1);
return testSuggestionsFn(
...((args.length > 1 ? [...args, ...paddingArgs] : args) as TestArgs),
{
skip: true,
}
);
},
only: (...args: TestArgs) => {
const paddingArgs = [[undefined, undefined, undefined]].slice(args.length - 1);
return testSuggestionsFn(
...((args.length > 1 ? [...args, ...paddingArgs] : args) as TestArgs),
{
only: true,
}
);
},
});
describe('policies', () => {
function createPolicyContent(
policyName: string,
customPolicies: Array<typeof policies[number]> = policies
) {
const policyHit = customPolicies.find((p) => p.name === policyName);
if (!policyHit) {
return [];
}
return [
`**Indexes**: ${policyHit.sourceIndices.join(', ')}`,
`**Matching field**: ${policyHit.matchField}`,
`**Fields**: ${policyHit.enrichFields.join(', ')}`,
];
}
testSuggestions(
`from a | enrich policy on b with var0 = stringField`,
'policy',
createPolicyContent
);
testSuggestions(`from a | enrich policy`, 'policy', createPolicyContent);
testSuggestions(`from a | enrich policy on b `, 'policy', createPolicyContent);
testSuggestions(`from a | enrich policy on b `, 'non-policy', createPolicyContent);
});
describe('functions', () => {
function createFunctionContent(fn: string) {
const fnDefinition = getFunctionDefinition(fn);
if (!fnDefinition) {
return [];
}
return [getFunctionSignatures(fnDefinition)[0].declaration, fnDefinition.description];
}
testSuggestions(`from a | eval round(numberField)`, 'round', createFunctionContent);
testSuggestions(
`from a | eval nonExistentFn(numberField)`,
'nonExistentFn',
createFunctionContent
);
testSuggestions(`from a | stats avg(round(numberField))`, 'round', createFunctionContent);
testSuggestions(`from a | stats avg(round(numberField))`, 'avg', createFunctionContent);
testSuggestions(
`from a | stats avg(nonExistentFn(numberField))`,
'nonExistentFn',
createFunctionContent
);
testSuggestions(`from a | where round(numberField) > 0`, 'round', createFunctionContent);
});
});

4
packages/kbn-monaco/src/esql/lib/ast/hover/index.ts → packages/kbn-monaco/src/esql/lib/ast/hover/hover.ts
View file

@ -59,7 +59,7 @@ export async function getHoverItem(
{
value: `${i18n.translate('monaco.esql.hover.policyIndexes', {
defaultMessage: '**Indexes**',
})}: ${policyMetadata.sourceIndices}`,
})}: ${policyMetadata.sourceIndices.join(', ')}`,
},
{
value: `${i18n.translate('monaco.esql.hover.policyMatchingField', {
@ -69,7 +69,7 @@ export async function getHoverItem(
{
value: `${i18n.translate('monaco.esql.hover.policyEnrichedFields', {
defaultMessage: '**Fields**',
})}: ${policyMetadata.enrichFields}`,
})}: ${policyMetadata.enrichFields.join(', ')}`,
},
],
};

2
packages/kbn-monaco/src/esql/lib/ast/shared/context.ts
View file

@ -139,7 +139,7 @@ export function getAstContext(innerText: string, ast: ESQLAst, offset: number) {
const { command, option, setting, node } = findAstPosition(ast, offset);
if (node) {
if (node.type === 'function') {
if (['in', 'not_in'].includes(node.name)) {
if (['in', 'not_in'].includes(node.name) && Array.isArray(node.args[1])) {
// command ... a in ( <here> )
return { type: 'list' as const, command, node, option, setting };
}

52
packages/kbn-monaco/src/esql/lib/ast/shared/helpers.ts
View file

@ -92,13 +92,24 @@ export function isIncompleteItem(arg: ESQLAstItem): boolean {
return !arg || (!Array.isArray(arg) && arg.incomplete);
}
export function isMathFunction(char: string) {
export function isMathFunction(query: string, offset: number) {
const queryTrimmed = query.substring(0, offset).trimEnd();
// try to get the full operation token (e.g. "+", "in", "like", etc...) but it requires the token
// to be spaced out from a field/function (e.g. "field + ") so it is subject to issues
const [opString] = queryTrimmed.split(' ').reverse();
// compare last char for all math functions
// limit only to 2 chars operators
return builtinFunctions
.filter(({ name }) => name.length < 3)
.map(({ name }) => name[name.length - 1])
.some((op) => char === op);
const fns = builtinFunctions.filter(({ name }) => name.length < 3).map(({ name }) => name);
const tokenMatch = fns.some((op) => opString === op);
// there's a match, that's good
if (tokenMatch) {
return true;
}
// either there's no match or it is the case where field/function and op are not spaced out
// e.g "field+" or "fn()+"
// so try to extract the last char and compare it with the single char math functions
const singleCharFns = fns.filter((name) => name.length === 1);
return singleCharFns.some((c) => c === opString[opString.length - 1]);
}
export function isComma(char: string) {
@ -359,11 +370,6 @@ export function isEqualType(
if (item.type === 'literal') {
return compareLiteralType(argType, item);
}
if (item.type === 'list') {
const listType = `${item.values[0].literalType}[]`;
// argType = 'list' means any list value is ok
return argType === item.type || argType === listType;
}
if (item.type === 'function') {
if (isSupportedFunction(item.name, parentCommand).supported) {
const fnDef = buildFunctionLookup().get(item.name)!;
@ -385,32 +391,6 @@ export function isEqualType(
const wrappedTypes = Array.isArray(hit.type) ? hit.type : [hit.type];
return wrappedTypes.some((ct) => argType === ct);
}
if (item.type === 'source') {
return item.sourceType === argType;
}
}
export function endsWithOpenBracket(text: string) {
return /\($/.test(text);
}
export function isDateFunction(fnName: string) {
// TODO: improve this and rely in signature in the future
return ['to_datetime', 'date_trunc', 'date_parse'].includes(fnName.toLowerCase());
}
export function getDateMathOperation() {
return builtinFunctions.filter(({ name }) => ['+', '-'].includes(name));
}
export function getDurationItemsWithQuantifier(quantifier: number = 1) {
return timeLiterals
.filter(({ name }) => !/s$/.test(name))
.map(({ name, ...rest }) => ({
label: `${quantifier} ${name}`,
insertText: `${quantifier} ${name}`,
...rest,
}));
}
function fuzzySearch(fuzzyName: string, resources: IterableIterator<string>) {

1
packages/kbn-monaco/src/esql/lib/ast/shared/types.ts
View file

@ -13,6 +13,7 @@ type CallbackFn<Options = {}, Result = string> = (ctx?: Options) => Result[] | P
export interface ESQLCallbacks {
getSources?: CallbackFn<{}, { name: string; hidden: boolean }>;
getFieldsFor?: CallbackFn<{ query: string }, { name: string; type: string }>;
getMetaFields?: CallbackFn;
getPolicies?: CallbackFn<
{},
{ name: string; sourceIndices: string[]; matchField: string; enrichFields: string[] }

13
packages/kbn-monaco/src/esql/lib/ast/validation/helpers.ts
View file

@ -19,3 +19,16 @@ export function buildQueryForFieldsInPolicies(policies: ESQLPolicy[]) {
.flatMap(({ sourceIndices }) => sourceIndices)
.join(', ')} | keep ${policies.flatMap(({ enrichFields }) => enrichFields).join(', ')}`;
}
export function buildQueryForFieldsForStringSources(queryString: string, ast: ESQLAst) {
// filter out the query until the last GROK or DISSECT command
const lastCommandIndex =
ast.length - [...ast].reverse().findIndex(({ name }) => ['grok', 'dissect'].includes(name));
// we're sure it's not -1 because we check the commands chain before calling this function
const nextCommandIndex = Math.min(lastCommandIndex + 1, ast.length - 1);
const customQuery = queryString.substring(0, ast[nextCommandIndex].location.min).trimEnd();
if (customQuery[customQuery.length - 1] === '|') {
return customQuery.substring(0, customQuery.length - 1);
}
return customQuery;
}

26
packages/kbn-monaco/src/esql/lib/ast/validation/resources.ts
View file

@ -15,7 +15,11 @@ import {
} from '../shared/resources_helpers';
import type { ESQLCallbacks } from '../shared/types';
import type { ESQLCommand } from '../types';
import { buildQueryForFieldsFromSource, buildQueryForFieldsInPolicies } from './helpers';
import {
buildQueryForFieldsForStringSources,
buildQueryForFieldsFromSource,
buildQueryForFieldsInPolicies,
} from './helpers';
import type { ESQLRealField, ESQLPolicy } from './types';
export async function retrieveFields(
@ -83,3 +87,23 @@ export async function retrievePoliciesFields(
);
return await getFieldsByTypeHelper(customQuery, callbacks).getFieldsMap();
}
export async function retrieveMetadataFields(callbacks?: ESQLCallbacks): Promise<Set<string>> {
if (!callbacks || !callbacks.getMetaFields) {
return new Set();
}
const fields = await callbacks.getMetaFields();
return new Set(fields);
}
export async function retrieveFieldsFromStringSources(
queryString: string,
commands: ESQLCommand[],
callbacks?: ESQLCallbacks
): Promise<Map<string, ESQLRealField>> {
if (!callbacks) {
return new Map();
}
const customQuery = buildQueryForFieldsForStringSources(queryString, commands);
return await getFieldsByTypeHelper(customQuery, callbacks).getFieldsMap();
}

1
packages/kbn-monaco/src/esql/lib/ast/validation/types.ts
View file

@ -32,6 +32,7 @@ export interface ReferenceMaps {
variables: Map<string, ESQLVariable[]>;
fields: Map<string, ESQLRealField>;
policies: Map<string, ESQLPolicy>;
metadataFields: Set<string>;
}
export interface ValidationErrors {

47
packages/kbn-monaco/src/esql/lib/ast/validation/validation.test.ts
View file

@ -39,10 +39,6 @@ function getCallbackMocks() {
...fieldTypes.map((type) => ({ name: `${camelCase(type)}Field`, type })),
{ name: 'any#Char$ field', type: 'number' },
{ name: 'kubernetes.something.something', type: 'number' },
{
name: `listField`,
type: `list`,
},
{ name: '@timestamp', type: 'date' },
]
),
@ -66,6 +62,7 @@ function getCallbackMocks() {
enrichFields: ['otherField', 'yetAnotherField'],
},
]),
getMetaFields: jest.fn(async () => ['_id', '_source']),
};
}
@ -113,7 +110,7 @@ function getFieldName(
: `${camelCase(typeString)}Field`;
}
function getMultiValue(type: 'string[]' | 'number[]' | 'boolean[]' | 'any[]') {
function getMultiValue(type: string) {
if (/string|any/.test(type)) {
return `["a", "b", "c"]`;
}
@ -163,9 +160,9 @@ function getFieldMapping(
...rest,
};
}
if (['string[]', 'number[]', 'boolean[]', 'any[]'].includes(typeString)) {
if (/[]$/.test(typeString)) {
return {
name: getMultiValue(typeString as 'string[]' | 'number[]' | 'boolean[]' | 'any[]'),
name: getMultiValue(typeString),
type,
...rest,
};
@ -269,6 +266,9 @@ describe('validation logic', () => {
testErrorsAndWarnings(`from index [metadata _id]`, []);
testErrorsAndWarnings(`from index [METADATA _id, _source]`, []);
testErrorsAndWarnings(`from index [METADATA _id, _source2]`, [
'Metadata field [_source2] is not available. Available metadata fields are: [_id, _source]',
]);
testErrorsAndWarnings(`from index [metadata _id, _source] [METADATA _id2]`, [
'SyntaxError: expected {<EOF>, PIPE} but found "["',
]);
@ -293,6 +293,8 @@ describe('validation logic', () => {
testErrorsAndWarnings(`from *:indexes [METADATA _id]`, []);
testErrorsAndWarnings('from .secretIndex', []);
testErrorsAndWarnings('from my-index', []);
testErrorsAndWarnings('from numberField', ['Unknown index [numberField]']);
testErrorsAndWarnings('from policy', ['Unknown index [policy]']);
});
describe('row', () => {
@ -314,6 +316,10 @@ describe('validation logic', () => {
'SyntaxError: expected {STRING, INTEGER_LITERAL, DECIMAL_LITERAL, FALSE, LP, NOT, NULL, PARAM, TRUE, PLUS, MINUS, OPENING_BRACKET, UNQUOTED_IDENTIFIER, QUOTED_IDENTIFIER} but found ","',
"SyntaxError: extraneous input ')' expecting <EOF>",
]);
for (const bool of ['true', 'false']) {
testErrorsAndWarnings(`row a=NOT ${bool}`, []);
testErrorsAndWarnings(`row NOT ${bool}`, []);
}
testErrorsAndWarnings('row var = 1 in (1, 2, 3)', []);
testErrorsAndWarnings('row var = 5 in (1, 2, 3)', []);
@ -512,6 +518,19 @@ describe('validation logic', () => {
"SyntaxError: token recognition error at: 'a'",
"SyntaxError: token recognition error at: 'h'",
]);
testErrorsAndWarnings('show numberField', [
"SyntaxError: token recognition error at: 'n'",
"SyntaxError: token recognition error at: 'u'",
"SyntaxError: token recognition error at: 'm'",
"SyntaxError: token recognition error at: 'b'",
"SyntaxError: token recognition error at: 'e'",
"SyntaxError: token recognition error at: 'r'",
"SyntaxError: token recognition error at: 'Fi'",
"SyntaxError: token recognition error at: 'e'",
"SyntaxError: token recognition error at: 'l'",
"SyntaxError: token recognition error at: 'd'",
'SyntaxError: expected {SHOW} but found "<EOF>"',
]);
});
describe('limit', () => {
@ -633,20 +652,16 @@ describe('validation logic', () => {
testErrorsAndWarnings('from a | mv_expand ', [
"SyntaxError: missing {UNQUOTED_IDENTIFIER, QUOTED_IDENTIFIER} at '<EOF>'",
]);
testErrorsAndWarnings('from a | mv_expand stringField', [
'MV_EXPAND only supports list type values, found [stringField] of type string',
]);
for (const type of ['string', 'number', 'date', 'boolean', 'ip']) {
testErrorsAndWarnings(`from a | mv_expand ${type}Field`, []);
}
testErrorsAndWarnings(`from a | mv_expand listField`, []);
testErrorsAndWarnings('from a | mv_expand listField, b', [
testErrorsAndWarnings('from a | mv_expand numberField, b', [
"SyntaxError: token recognition error at: ','",
"SyntaxError: extraneous input 'b' expecting <EOF>",
]);
testErrorsAndWarnings('row a = "a" | mv_expand a', [
'MV_EXPAND only supports list type values, found [a] of type string',
]);
testErrorsAndWarnings('row a = "a" | mv_expand a', []);
testErrorsAndWarnings('row a = [1, 2, 3] | mv_expand a', []);
});

142
packages/kbn-monaco/src/esql/lib/ast/validation/validation.ts
View file

@ -59,6 +59,8 @@ import {
retrieveFields,
retrievePolicies,
retrievePoliciesFields,
retrieveMetadataFields,
retrieveFieldsFromStringSources,
} from './resources';
function validateFunctionLiteralArg(
@ -391,11 +393,6 @@ function validateFunction(
return validateFn(astFunction, actualArg, argDef, references, parentCommand);
});
failingSignature.push(...argValidationMessages);
if (isSourceItem(actualArg)) {
// something went wrong with the AST translation
throw new Error('Source should not allowed as function argument');
}
}
});
if (failingSignature.length) {
@ -485,48 +482,16 @@ function validateOption(
}
// use dedicate validate fn if provided
if (optionDef.validate) {
messages.push(...optionDef.validate(option, command));
messages.push(...optionDef.validate(option, command, referenceMaps.metadataFields));
}
if (!optionDef.skipCommonValidation) {
option.args.forEach((arg, index) => {
option.args.forEach((arg) => {
if (!Array.isArray(arg)) {
if (!optionDef.signature.multipleParams) {
const argDef = optionDef.signature.params[index];
if (!isEqualType(arg, argDef, referenceMaps, command.name)) {
const value = 'value' in arg ? arg.value : arg.name;
messages.push(
getMessageFromId({
messageId: 'wrongArgumentType',
values: {
name: option.name,
argType: argDef.type,
value,
givenType: arg.type,
},
locations: arg.location,
})
);
}
if (isColumnItem(arg)) {
messages.push(...validateColumnForCommand(arg, command.name, referenceMaps));
}
} else {
const argDef = optionDef.signature.params[0];
if (!isEqualType(arg, argDef, referenceMaps, command.name)) {
const value = 'value' in arg ? arg.value : arg.name;
messages.push(
getMessageFromId({
messageId: 'wrongArgumentType',
values: {
name: argDef.name,
argType: argDef.type,
value,
givenType: arg.type,
},
locations: arg.location,
})
);
}
if (isColumnItem(arg)) {
messages.push(...validateColumnForCommand(arg, command.name, referenceMaps));
}
@ -551,55 +516,40 @@ function validateSource(
return messages;
}
const commandDef = getCommandDefinition(commandName);
if (commandDef.signature.params.every(({ type }) => type !== source.type)) {
const firstArg = commandDef.signature.params[0];
messages.push(
getMessageFromId({
messageId: 'wrongArgumentType',
values: {
name: firstArg.name,
argType: firstArg.type,
value: source.name,
givenType: source.type,
},
locations: source.location,
})
);
} else {
// give up on validate if CCS for now
const hasCCS = hasCCSSource(source.name);
if (!hasCCS) {
const isWildcardAndNotSupported =
hasWildcard(source.name) && !commandDef.signature.params.some(({ wildcards }) => wildcards);
if (isWildcardAndNotSupported) {
// give up on validate if CCS for now
const hasCCS = hasCCSSource(source.name);
if (!hasCCS) {
const isWildcardAndNotSupported =
hasWildcard(source.name) && !commandDef.signature.params.some(({ wildcards }) => wildcards);
if (isWildcardAndNotSupported) {
messages.push(
getMessageFromId({
messageId: 'wildcardNotSupportedForCommand',
values: { command: commandName.toUpperCase(), value: source.name },
locations: source.location,
})
);
} else {
if (source.sourceType === 'index' && !sourceExists(source.name, sources)) {
messages.push(
getMessageFromId({
messageId: 'wildcardNotSupportedForCommand',
values: { command: commandName.toUpperCase(), value: source.name },
messageId: 'unknownIndex',
values: { name: source.name },
locations: source.location,
})
);
} else if (source.sourceType === 'policy' && !policies.has(source.name)) {
messages.push(
getMessageFromId({
messageId: 'unknownPolicy',
values: { name: source.name },
locations: source.location,
})
);
} else {
if (source.sourceType === 'index' && !sourceExists(source.name, sources)) {
messages.push(
getMessageFromId({
messageId: 'unknownIndex',
values: { name: source.name },
locations: source.location,
})
);
} else if (source.sourceType === 'policy' && !policies.has(source.name)) {
messages.push(
getMessageFromId({
messageId: 'unknownPolicy',
values: { name: source.name },
locations: source.location,
})
);
}
}
}
}
return messages;
}
@ -610,9 +560,6 @@ function validateColumnForCommand(
): ESQLMessage[] {
const messages: ESQLMessage[] = [];
if (['from', 'show', 'limit'].includes(commandName)) {
return messages;
}
if (commandName === 'row') {
if (!references.variables.has(column.name)) {
messages.push(
@ -636,11 +583,10 @@ function validateColumnForCommand(
if (columnParamsWithInnerTypes.length) {
// this should be guaranteed by the columnCheck above
const columnRef = getColumnHit(nameHit, references)!;
if (
columnParamsWithInnerTypes.every(({ innerType }) => {
return innerType !== columnRef.type;
})
) {
const hasSomeWrongInnerTypes = columnParamsWithInnerTypes.every(({ innerType }) => {
return innerType !== 'any' && innerType !== columnRef.type;
});
if (hasSomeWrongInnerTypes) {
const supportedTypes = columnParamsWithInnerTypes.map(({ innerType }) => innerType);
messages.push(
@ -833,13 +779,15 @@ export async function validateAst(
const { ast, errors } = await astProvider(queryString);
const [sources, availableFields, availablePolicies] = await Promise.all([
const [sources, availableFields, availablePolicies, availableMetadataFields] = await Promise.all([
// retrieve the list of available sources
retrieveSources(ast, callbacks),
// retrieve available fields (if a source command has been defined)
retrieveFields(queryString, ast, callbacks),
// retrieve available policies (if an enrich command has been defined)
retrievePolicies(ast, callbacks),
// retrieve available metadata fields
retrieveMetadataFields(callbacks),
]);
if (availablePolicies.size && ast.filter(({ name }) => name === 'enrich')) {
@ -847,6 +795,21 @@ export async function validateAst(
fieldsFromPoliciesMap.forEach((value, key) => availableFields.set(key, value));
}
if (ast.some(({ name }) => ['grok', 'dissect'].includes(name))) {
const fieldsFromGrokOrDissect = await retrieveFieldsFromStringSources(
queryString,
ast,
callbacks
);
fieldsFromGrokOrDissect.forEach((value, key) => {
// if the field is already present, do not overwrite it
// Note: this can also overlap with some variables
if (!availableFields.has(key)) {
availableFields.set(key, value);
}
});
}
const variables = collectVariables(ast, availableFields);
// notify if the user is rewriting a column as variable with another type
messages.push(...validateFieldsShadowing(availableFields, variables));
@ -858,6 +821,7 @@ export async function validateAst(
fields: availableFields,
policies: availablePolicies,
variables,
metadataFields: availableMetadataFields,
});
messages.push(...commandMessages);
}

4
packages/kbn-monaco/src/esql/lib/monaco/esql_ast_provider.ts
View file

@ -10,10 +10,10 @@ import type { ESQLCallbacks } from '../ast/shared/types';
import { monaco } from '../../../monaco_imports';
import type { ESQLWorker } from '../../worker/esql_worker';
import { suggest } from '../ast/autocomplete/autocomplete';
import { getHoverItem } from '../ast/hover';
import { getHoverItem } from '../ast/hover/hover';
import { getSignatureHelp } from '../ast/signature';
import { validateAst } from '../ast/validation/validation';
import { getActions } from '../ast/code_actions';
import { getActions } from '../ast/code_actions/actions';
import { wrapAsMonacoMessage } from '../ast/shared/monaco_utils';
export class ESQLAstAdapter {

1
packages/kbn-text-based-editor/src/text_based_languages_editor.tsx
View file

@ -341,6 +341,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
}
return [];
},
getMetaFields: async () => ['_version', '_id', '_index', '_source'],
getPolicies: async () => {
const { data: policies, error } =
(await indexManagementApiService?.getAllEnrichPolicies()) || {};