[ES|QL] Add more functions to the validator (#180640)

## Summary

- [x]
[`SIGNUM`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-signum)
- [x] spatial functions
- [x]
[`ST_CENTROID_AGG`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-agg-st-centroid)
- [x]
[`ST_CONTAINS`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-st_contains)
- [x]
[`ST_DISJOINT`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-st_disjoint)
- [x]
[`ST_INTERSECTS`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-st_intersects)
- [x]
[`ST_WITHIN`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-st_within)
- [x]
[`ST_X`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-st_x)
- [x]
[`ST_Y`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-st_y)
- [x]
[`MV_SLICE`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-mv_slice)
- [x]
[`MV_ZIP`](https://www.elastic.co/guide/en/elasticsearch/reference/master/esql-functions-operators.html#esql-mv_zip)

### Checklist

- [x]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
This commit is contained in:
Drew Tate 2024-04-13 08:54:45 +02:00 committed by GitHub
parent fd9722cab0
commit bec1755030
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 2941 additions and 54 deletions

View file

@ -20,8 +20,18 @@ import { SuggestionRawDefinition } from './types';
const triggerCharacters = [',', '(', '=', ' '];
const fields: Array<{ name: string; type: string; suggestedAs?: string }> = [
...['string', 'number', 'date', 'boolean', 'ip'].map((type) => ({
name: `${type}Field`,
...[
'string',
'number',
'date',
'boolean',
'ip',
'geo_point',
'geo_shape',
'cartesian_point',
'cartesian_shape',
].map((type) => ({
name: `${camelCase(type)}Field`,
type,
})),
{ name: 'any#Char$Field', type: 'number', suggestedAs: '`any#Char$Field`' },
@ -71,7 +81,7 @@ const policies = [
*/
function getFunctionSignaturesByReturnType(
command: string,
expectedReturnType: string,
_expectedReturnType: string | string[],
{
agg,
evalMath,
@ -83,6 +93,10 @@ function getFunctionSignaturesByReturnType(
paramsTypes?: string[],
ignored?: string[]
) {
const expectedReturnType = Array.isArray(_expectedReturnType)
? _expectedReturnType
: [_expectedReturnType];
const list = [];
if (agg) {
list.push(...statsAggregationFunctionDefinitions);
@ -103,7 +117,8 @@ function getFunctionSignaturesByReturnType(
return false;
}
const filteredByReturnType = signatures.filter(
({ returnType }) => expectedReturnType === 'any' || returnType === expectedReturnType
({ returnType }) =>
expectedReturnType.includes('any') || expectedReturnType.includes(returnType)
);
if (!filteredByReturnType.length) {
return false;
@ -137,18 +152,20 @@ function getFunctionSignaturesByReturnType(
});
}
function getFieldNamesByType(requestedType: string) {
function getFieldNamesByType(_requestedType: string | string[]) {
const requestedType = Array.isArray(_requestedType) ? _requestedType : [_requestedType];
return fields
.filter(({ type }) => requestedType === 'any' || type === requestedType)
.filter(({ type }) => requestedType.includes('any') || requestedType.includes(type))
.map(({ name, suggestedAs }) => suggestedAs || name);
}
function getLiteralsByType(type: string) {
if (type === 'time_literal') {
function getLiteralsByType(_type: string | string[]) {
const type = Array.isArray(_type) ? _type : [_type];
if (type.includes('time_literal')) {
// return only singular
return timeLiterals.map(({ name }) => `1 ${name}`).filter((s) => !/s$/.test(s));
}
if (type === 'chrono_literal') {
if (type.includes('chrono_literal')) {
return chronoLiterals.map(({ name }) => name);
}
return [];
@ -755,6 +772,10 @@ describe('autocomplete', () => {
'dateField',
'booleanField',
'ipField',
'geoPointField',
'geoShapeField',
'cartesianPointField',
'cartesianShapeField',
'any#Char$Field',
'kubernetes.something.something',
]);
@ -1058,22 +1079,29 @@ describe('autocomplete', () => {
const canHaveMoreArgs =
i + 1 < (signature.minParams ?? 0) ||
signature.params.filter(({ optional }, j) => !optional && j > i).length > i;
const allPossibleParamTypes = Array.from(
new Set(fn.signatures.map((s) => s.params[i].type))
);
testSuggestions(
`from a | eval ${fn.name}(${Array(i).fill('field').join(', ')}${i ? ',' : ''} )`,
param.literalOptions?.length
? param.literalOptions.map((option) => `"${option}"${canHaveMoreArgs ? ',' : ''}`)
: [
...getFieldNamesByType(param.type).map((f) =>
...getFieldNamesByType(allPossibleParamTypes).map((f) =>
canHaveMoreArgs ? `${f},` : f
),
...getFunctionSignaturesByReturnType(
'eval',
param.type,
allPossibleParamTypes,
{ evalMath: true },
undefined,
[fn.name]
).map((l) => (canHaveMoreArgs ? `${l},` : l)),
...getLiteralsByType(param.type).map((d) => (canHaveMoreArgs ? `${d},` : d)),
...getLiteralsByType(allPossibleParamTypes).map((d) =>
canHaveMoreArgs ? `${d},` : d
),
]
);
testSuggestions(
@ -1083,17 +1111,19 @@ describe('autocomplete', () => {
param.literalOptions?.length
? param.literalOptions.map((option) => `"${option}"${canHaveMoreArgs ? ',' : ''}`)
: [
...getFieldNamesByType(param.type).map((f) =>
...getFieldNamesByType(allPossibleParamTypes).map((f) =>
canHaveMoreArgs ? `${f},` : f
),
...getFunctionSignaturesByReturnType(
'eval',
param.type,
allPossibleParamTypes,
{ evalMath: true },
undefined,
[fn.name]
).map((l) => (canHaveMoreArgs ? `${l},` : l)),
...getLiteralsByType(param.type).map((d) => (canHaveMoreArgs ? `${d},` : d)),
...getLiteralsByType(allPossibleParamTypes).map((d) =>
canHaveMoreArgs ? `${d},` : d
),
]
);
}

View file

@ -147,7 +147,7 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
],
},
{
name: 'st_centroid',
name: 'st_centroid_agg',
type: 'agg',
description: i18n.translate(
'kbn-esql-validation-autocomplete.esql.definitions.stCentroidDoc',
@ -161,16 +161,16 @@ export const statsAggregationFunctionDefinitions: FunctionDefinition[] = [
params: [{ name: 'column', type: 'cartesian_point', noNestingFunctions: true }],
returnType: 'cartesian_point',
examples: [
`from index | stats result = st_centroid(cartesian_field)`,
`from index | stats st_centroid(cartesian_field)`,
`from index | stats result = st_centroid_agg(cartesian_field)`,
`from index | stats st_centroid_agg(cartesian_field)`,
],
},
{
params: [{ name: 'column', type: 'geo_point', noNestingFunctions: true }],
returnType: 'geo_point',
examples: [
`from index | stats result = st_centroid(geo_field)`,
`from index | stats st_centroid(geo_field)`,
`from index | stats result = st_centroid_agg(geo_field)`,
`from index | stats st_centroid_agg(geo_field)`,
],
},
],

View file

@ -57,6 +57,20 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
},
],
},
{
name: 'signum',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.signumDoc', {
defaultMessage:
'Returns the sign of the given number. It returns -1 for negative numbers, 0 for 0 and 1 for positive numbers.',
}),
signatures: [
{
params: [{ name: 'field', type: 'number' }],
returnType: 'number',
examples: [`from index | eval s = signum(field)`],
},
],
},
{
name: 'abs',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.absDoc', {
@ -1084,6 +1098,44 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
},
],
},
{
name: 'mv_slice',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvSliceDoc', {
defaultMessage:
'Returns a subset of the multivalued field using the start and end index values.',
}),
signatures: [
{
params: [
{ name: 'multivalue', type: 'any' },
{ name: 'start', type: 'number' },
{ name: 'end', type: 'number' },
],
returnType: 'number',
examples: ['row a = [1, 2, 2, 3] | eval a1 = mv_slice(a, 1), a2 = mv_slice(a, 2, 3)'],
},
],
},
{
name: 'mv_zip',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.mvZipDoc', {
defaultMessage:
'Combines the values from two multivalued fields with a delimiter that joins them together.',
}),
signatures: [
{
params: [
{ name: 'mvLeft', type: 'string' },
{ name: 'mvRight', type: 'string' },
{ name: 'delim', type: 'string' },
],
returnType: 'string',
examples: [
'ROW a = ["x", "y", "z"], b = ["1", "2"] \n| EVAL c = mv_zip(a, b, "-") \n| KEEP a, b, c',
],
},
],
},
{
name: 'pi',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.piDoc', {
@ -1123,6 +1175,549 @@ export const evalFunctionsDefinitions: FunctionDefinition[] = [
},
],
},
// begin spatial functions
{
name: 'st_contains',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.stContainsDoc', {
defaultMessage: 'Returns whether the first geometry contains the second geometry.',
}),
signatures: [
{
params: [
{
name: 'geomA',
type: 'geo_point',
},
{
name: 'geomB',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_contains(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_point',
},
{
name: 'geomB',
type: 'geo_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_contains(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_shape',
},
{
name: 'geomB',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_contains(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_shape',
},
{
name: 'geomB',
type: 'geo_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_contains(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_point',
},
{
name: 'geomB',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_contains(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_point',
},
{
name: 'geomB',
type: 'cartesian_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_contains(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_shape',
},
{
name: 'geomB',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_contains(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_shape',
},
{
name: 'geomB',
type: 'cartesian_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_contains(geometryA, geometryB)'],
},
],
},
{
name: 'st_within',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.stWithinDoc', {
defaultMessage: 'Returns whether the first geometry is within the second geometry.',
}),
signatures: [
{
params: [
{
name: 'geomA',
type: 'geo_point',
},
{
name: 'geomB',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_within(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_point',
},
{
name: 'geomB',
type: 'geo_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_within(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_shape',
},
{
name: 'geomB',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_within(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_shape',
},
{
name: 'geomB',
type: 'geo_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_within(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_point',
},
{
name: 'geomB',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_within(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_point',
},
{
name: 'geomB',
type: 'cartesian_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_within(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_shape',
},
{
name: 'geomB',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_within(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_shape',
},
{
name: 'geomB',
type: 'cartesian_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_within(geometryA, geometryB)'],
},
],
},
{
name: 'st_disjoint',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.stDisjointDoc', {
defaultMessage: 'Returns whether the two geometries or geometry columns are disjoint.',
}),
signatures: [
{
params: [
{
name: 'geomA',
type: 'geo_point',
},
{
name: 'geomB',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_disjoint(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_point',
},
{
name: 'geomB',
type: 'geo_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_disjoint(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_shape',
},
{
name: 'geomB',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_disjoint(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_shape',
},
{
name: 'geomB',
type: 'geo_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_disjoint(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_point',
},
{
name: 'geomB',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_disjoint(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_point',
},
{
name: 'geomB',
type: 'cartesian_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_disjoint(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_shape',
},
{
name: 'geomB',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_disjoint(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_shape',
},
{
name: 'geomB',
type: 'cartesian_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_disjoint(geometryA, geometryB)'],
},
],
},
{
name: 'st_intersects',
description: i18n.translate(
'kbn-esql-validation-autocomplete.esql.definitions.stIntersectsDoc',
{
defaultMessage:
'Returns true if two geometries intersect. They intersect if they have any point in common, including their interior points (points along lines or within polygons).',
}
),
signatures: [
{
params: [
{
name: 'geomA',
type: 'geo_point',
},
{
name: 'geomB',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_intersects(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_point',
},
{
name: 'geomB',
type: 'geo_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_intersects(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_shape',
},
{
name: 'geomB',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_intersects(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'geo_shape',
},
{
name: 'geomB',
type: 'geo_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_intersects(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_point',
},
{
name: 'geomB',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_intersects(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_point',
},
{
name: 'geomB',
type: 'cartesian_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_intersects(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_shape',
},
{
name: 'geomB',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_intersects(geometryA, geometryB)'],
},
{
params: [
{
name: 'geomA',
type: 'cartesian_shape',
},
{
name: 'geomB',
type: 'cartesian_shape',
},
],
returnType: 'boolean',
examples: ['from index | eval st_intersects(geometryA, geometryB)'],
},
],
},
{
name: 'st_x',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.stXDoc', {
defaultMessage:
'Extracts the x coordinate from the supplied point. If the points is of type geo_point this is equivalent to extracting the longitude value.',
}),
signatures: [
{
params: [
{
name: 'point',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_x(point)'],
},
{
params: [
{
name: 'point',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_x(point)'],
},
],
},
{
name: 'st_y',
description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.stYDoc', {
defaultMessage:
'Extracts the y coordinate from the supplied point. If the points is of type geo_point this is equivalent to extracting the latitude value.',
}),
signatures: [
{
params: [
{
name: 'point',
type: 'geo_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_y(point)'],
},
{
params: [
{
name: 'point',
type: 'cartesian_point',
},
],
returnType: 'boolean',
examples: ['from index | eval st_y(point)'],
},
],
},
]
.sort(({ name: a }, { name: b }) => a.localeCompare(b))
.map((def) => ({

View file

@ -19,7 +19,17 @@ import { camelCase } from 'lodash';
import { getAstAndSyntaxErrors } from '@kbn/esql-ast';
import { nonNullable } from '../shared/helpers';
const fieldTypes = ['number', 'date', 'boolean', 'ip', 'string', 'cartesian_point', 'geo_point'];
const fieldTypes = [
'number',
'date',
'boolean',
'ip',
'string',
'cartesian_point',
'cartesian_shape',
'geo_point',
'geo_shape',
];
const fields = [
...fieldTypes.map((type) => ({ name: `${camelCase(type)}Field`, type })),
{ name: 'any#Char$Field', type: 'number' },
@ -538,7 +548,11 @@ describe('validation logic', () => {
.replace(/stringField/g, '"a"')
.replace(/dateField/g, 'now()')
.replace(/booleanField/g, 'true')
.replace(/ipField/g, 'to_ip("127.0.0.1")');
.replace(/ipField/g, 'to_ip("127.0.0.1")')
.replace(/geoPointField/g, 'to_geopoint("POINT (30 10)")')
.replace(/geoShapeField/g, 'to_geoshape("POINT (30 10)")')
.replace(/cartesianPointField/g, 'to_cartesianpoint("POINT (30 10)")')
.replace(/cartesianShapeField/g, 'to_cartesianshape("POINT (30 10)")');
}
for (const { name, alias, signatures, ...defRest } of evalFunctionsDefinitions) {

View file

@ -62,6 +62,9 @@ function createIndexRequest(
if (type === 'cartesian_point') {
esType = 'point';
}
if (type === 'cartesian_shape') {
esType = 'shape';
}
if (type === 'unsupported') {
esType = 'integer_range';
}