Remove square brackets in FROM METADATA declaration (#196991)

## Summary
Hello, this PR addresses the deprecation of square brackets in FROM
METADATA declarations in Elasticsearch queries, in preparation for
Elasticsearch 9.0. Closes #196988

The changes involve removing square brackets around metadata fields
(e.g., `[metadata _id]` becomes `metadata _id`) across various parts of
the codebase. No functional changes to the UE, only internal query
syntax updates.

### Checklist
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

### For maintainers

- [ ] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels)
- [ ] This will appear in the **Release Notes** and follow the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
This commit is contained in:
Kyra Cho 2024-10-31 01:38:22 -07:00 committed by GitHub
parent 7dea07f90d
commit e3b8ccf025
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 56 additions and 198 deletions

View file

@ -91,14 +91,12 @@ describe('autocomplete.suggest', () => {
test('on <kbd>SPACE</kbd> after "METADATA" keyword suggests all metadata fields', async () => {
const { assertSuggestions } = await setup();
await assertSuggestions('from a, b [METADATA /]', metadataFields);
await assertSuggestions('from a, b METADATA /', metadataFields);
});
test('on <kbd>SPACE</kbd> after "METADATA" column suggests command and pipe operators', async () => {
const { assertSuggestions } = await setup();
await assertSuggestions('from a, b [metadata _index /]', [',', '| ']);
await assertSuggestions('from a, b metadata _index /', [',', '| ']);
await assertSuggestions('from a, b metadata _index, _source /', [',', '| ']);
await assertSuggestions(`from a, b metadata ${METADATA_FIELDS.join(', ')} /`, ['| ']);
@ -107,7 +105,6 @@ describe('autocomplete.suggest', () => {
test('filters out already used metadata fields', async () => {
const { assertSuggestions } = await setup();
await assertSuggestions('from a, b [metadata _index, /]', metadataFieldsAndIndex);
await assertSuggestions('from a, b metadata _index, /', metadataFieldsAndIndex);
});
});

View file

@ -104,7 +104,7 @@ describe('autocomplete', () => {
.map(({ name }) => name.toUpperCase() + ' $0')
);
testSuggestions(
'from a [metadata _id] | /',
'from a metadata _id | /',
commandDefinitions
.filter(({ name }) => !sourceCommands.includes(name))
.map(({ name }) => name.toUpperCase() + ' $0')
@ -116,7 +116,7 @@ describe('autocomplete', () => {
.map(({ name }) => name.toUpperCase() + ' $0')
);
testSuggestions(
'from a [metadata _id] | eval var0 = a | /',
'from a metadata _id | eval var0 = a | /',
commandDefinitions
.filter(({ name }) => !sourceCommands.includes(name))
.map(({ name }) => name.toUpperCase() + ' $0')

View file

@ -38,16 +38,6 @@ export const metadataOption: CommandOptionsDefinition = {
skipCommonValidation: true,
validate: (option, command, references) => {
const messages: ESQLMessage[] = [];
// need to test the parent command here
if (/\[metadata/i.test(command.text)) {
messages.push(
getMessageFromId({
messageId: 'metadataBracketsDeprecation',
values: {},
locations: option.location,
})
);
}
const fields = option.args.filter(isColumnItem);
const metadataFieldsAvailable = references as unknown as Set<string>;
if (metadataFieldsAvailable.size > 0) {

View file

@ -106,7 +106,7 @@ export const validationFromCommandTestSuite = (setup: helpers.Setup) => {
await expectErrors('from index metadata _id, \t\n _index\n ', []);
});
test('errors when wrapped in brackets', async () => {
test('errors when wrapped in parentheses', async () => {
const { expectErrors } = await setup();
await expectErrors(`from index (metadata _id)`, [
@ -114,65 +114,20 @@ export const validationFromCommandTestSuite = (setup: helpers.Setup) => {
]);
});
for (const isWrapped of [true, false]) {
function setWrapping(option: string) {
return isWrapped ? `[${option}]` : option;
}
describe('validates fields', () => {
test('validates fields', async () => {
const { expectErrors } = await setup();
function addBracketsWarning() {
return isWrapped
? ["Square brackets '[]' need to be removed from FROM METADATA declaration"]
: [];
}
describe(`wrapped = ${isWrapped}`, () => {
test('no errors on correct usage, waning on square brackets', async () => {
const { expectErrors } = await setup();
await expectErrors(`from index ${setWrapping('METADATA _id')}`, []);
await expectErrors(
`from index ${setWrapping('METADATA _id')}`,
[],
addBracketsWarning()
);
await expectErrors(
`from index ${setWrapping('metadata _id')}`,
[],
addBracketsWarning()
);
await expectErrors(
`from index ${setWrapping('METADATA _id, _source')}`,
[],
addBracketsWarning()
);
});
test('validates fields', async () => {
const { expectErrors } = await setup();
await expectErrors(
`from index ${setWrapping('METADATA _id, _source2')}`,
[
`Metadata field [_source2] is not available. Available metadata fields are: [${METADATA_FIELDS.join(
', '
)}]`,
],
addBracketsWarning()
);
await expectErrors(
`from index ${setWrapping('metadata _id, _source')} ${setWrapping(
'METADATA _id2'
)}`,
[
isWrapped
? "SyntaxError: mismatched input '[' expecting <EOF>"
: "SyntaxError: mismatched input 'METADATA' expecting <EOF>",
],
addBracketsWarning()
);
});
await expectErrors(`from index METADATA _id, _source2`, [
`Metadata field [_source2] is not available. Available metadata fields are: [${METADATA_FIELDS.join(
', '
)}]`,
]);
await expectErrors(`from index metadata _id, _source METADATA _id2`, [
"SyntaxError: mismatched input 'METADATA' expecting <EOF>",
]);
});
}
});
});
});
});

View file

@ -24,33 +24,13 @@ describe('validation', () => {
});
describe('... METADATA <indices>', () => {
for (const isWrapped of [true, false]) {
function setWrapping(option: string) {
return isWrapped ? `[${option}]` : option;
}
function addBracketsWarning() {
return isWrapped
? ["Square brackets '[]' need to be removed from FROM METADATA declaration"]
: [];
}
describe(`wrapped = ${isWrapped}`, () => {
test('no errors on correct usage, waning on square brackets', async () => {
const { expectErrors } = await setup();
await expectErrors(
`from remote-ccs:indexes ${setWrapping('METADATA _id')}`,
['Unknown index [remote-ccs:indexes]'],
addBracketsWarning()
);
await expectErrors(
`from *:indexes ${setWrapping('METADATA _id')}`,
['Unknown index [*:indexes]'],
addBracketsWarning()
);
});
});
}
test('no errors on correct usage', async () => {
const { expectErrors } = await setup();
await expectErrors(`from remote-ccs:indexes METADATA _id`, [
'Unknown index [remote-ccs:indexes]',
]);
await expectErrors(`from *:indexes METADATA _id`, ['Unknown index [*:indexes]']);
});
});
});
});

View file

@ -9947,70 +9947,6 @@
],
"warning": []
},
{
"query": "from index [METADATA _id]",
"error": [],
"warning": []
},
{
"query": "from index [METADATA _id]",
"error": [],
"warning": [
"Square brackets '[]' need to be removed from FROM METADATA declaration"
]
},
{
"query": "from index [metadata _id]",
"error": [],
"warning": [
"Square brackets '[]' need to be removed from FROM METADATA declaration"
]
},
{
"query": "from index [METADATA _id, _source]",
"error": [],
"warning": [
"Square brackets '[]' need to be removed from FROM METADATA declaration"
]
},
{
"query": "from index [METADATA _id, _source2]",
"error": [
"Metadata field [_source2] is not available. Available metadata fields are: [_version, _id, _index, _source, _ignored, _index_mode]"
],
"warning": [
"Square brackets '[]' need to be removed from FROM METADATA declaration"
]
},
{
"query": "from index [metadata _id, _source] [METADATA _id2]",
"error": [
"SyntaxError: mismatched input '[' expecting <EOF>"
],
"warning": [
"Square brackets '[]' need to be removed from FROM METADATA declaration"
]
},
{
"query": "from index METADATA _id",
"error": [],
"warning": []
},
{
"query": "from index METADATA _id",
"error": [],
"warning": []
},
{
"query": "from index metadata _id",
"error": [],
"warning": []
},
{
"query": "from index METADATA _id, _source",
"error": [],
"warning": []
},
{
"query": "from index METADATA _id, _source2",
"error": [
@ -10019,11 +9955,11 @@
"warning": []
},
{
"query": "from index metadata _id, _source METADATA _id2",
"query": "from index METADATA _id, _source METADATA _id2",
"error": [
"SyntaxError: mismatched input 'METADATA' expecting <EOF>"
],
"warning": []
}
]
}
}

View file

@ -1741,7 +1741,7 @@ describe('validation logic', () => {
it('should basically work when all callbacks are passed', async () => {
const allErrors = await Promise.all(
fixtures.testCases
.filter(({ query }) => query === 'from index [METADATA _id, _source2]')
.filter(({ query }) => query === 'from index METADATA _id, _source2')
.map(({ query }) =>
validateQuery(
query,
@ -1753,7 +1753,7 @@ describe('validation logic', () => {
);
for (const [index, { errors }] of Object.entries(allErrors)) {
expect(errors.map((e) => ('severity' in e ? e.message : e.text))).toEqual(
fixtures.testCases.filter(({ query }) => query === 'from index [METADATA _id, _source2]')[
fixtures.testCases.filter(({ query }) => query === 'from index METADATA _id, _source2')[
Number(index)
].error
);

View file

@ -19,7 +19,7 @@ export const initialSection = (
<Markdown
markdownContent={i18n.translate('languageDocumentation.documentationESQL.markdown', {
defaultMessage: `
An ES|QL (Elasticsearch query language) query consists of a series of commands, separated by pipe characters: \`|\`. Each query starts with a **source command**, which produces a table, typically with data from Elasticsearch.
An ES|QL (Elasticsearch query language) query consists of a series of commands, separated by pipe characters: \`|\`. Each query starts with a **source command**, which produces a table, typically with data from Elasticsearch.
A source command can be followed by one or more **processing commands**. Processing commands can change the output table of the previous command by adding, removing, and changing rows and columns.
@ -29,7 +29,7 @@ source-command
| processing-command2
\`\`\`
The result of a query is the table produced by the final processing command.
The result of a query is the table produced by the final processing command.
`,
})}
/>
@ -77,7 +77,7 @@ ES|QL can access the following metadata fields:
Use the \`METADATA\` directive to enable metadata fields:
\`\`\`
FROM index [METADATA _index, _id]
FROM index METADATA _index, _id
\`\`\`
Metadata fields are only available if the source of the data is an index. Consequently, \`FROM\` is the only source commands that supports the \`METADATA\` directive.
@ -85,7 +85,7 @@ Metadata fields are only available if the source of the data is an index. Conseq
Once enabled, the fields are then available to subsequent processing commands, just like the other index fields:
\`\`\`
FROM ul_logs, apps [METADATA _index, _version]
FROM ul_logs, apps METADATA _index, _version
| WHERE id IN (13, 14) AND _version == 1
| EVAL key = CONCAT(_index, "_", TO_STR(id))
| SORT id, _index
@ -95,7 +95,7 @@ FROM ul_logs, apps [METADATA _index, _version]
Also, similar to the index fields, once an aggregation is performed, a metadata field will no longer be accessible to subsequent commands, unless used as grouping field:
\`\`\`
FROM employees [METADATA _index, _id]
FROM employees METADATA _index, _id
| STATS max = MAX(emp_no) BY _index
\`\`\`
`,
@ -114,7 +114,7 @@ FROM employees [METADATA _index, _id]
markdownContent={i18n.translate('languageDocumentation.documentationESQL.row.markdown', {
defaultMessage: `### ROW
The \`ROW\` source command produces a row with one or more columns with values that you specify. This can be useful for testing.
\`\`\`
ROW a = 1, b = "two", c = null
\`\`\`
@ -207,7 +207,7 @@ ROW a = "1953-01-23T12:15:00Z - some text - 127.0.0.1"
markdownContent={i18n.translate('languageDocumentation.documentationESQL.drop.markdown', {
defaultMessage: `### DROP
Use \`DROP\` to remove columns from a table:
\`\`\`
FROM employees
| DROP height
@ -347,7 +347,7 @@ ROW a = "12 15.5 15.6 true"
The \`KEEP\` command enables you to specify what columns are returned and the order in which they are returned.
To limit the columns that are returned, use a comma-separated list of column names. The columns are returned in the specified order:
\`\`\`
FROM employees
| KEEP first_name, last_name, height
@ -384,7 +384,7 @@ FROM employees
{
defaultMessage: `### LIMIT
The \`LIMIT\` processing command enables you to limit the number of rows:
\`\`\`
FROM employees
| LIMIT 5
@ -407,7 +407,7 @@ FROM employees
'languageDocumentation.documentationESQL.mvExpand.markdown',
{
defaultMessage: `### MV_EXPAND
The \`MV_EXPAND\` processing command expands multivalued fields into one row per value, duplicating other fields:
The \`MV_EXPAND\` processing command expands multivalued fields into one row per value, duplicating other fields:
\`\`\`
ROW a=[1,2,3], b="b", j=["a","b"]
| MV_EXPAND a
@ -607,7 +607,7 @@ FROM employees
{
defaultMessage: `### WHERE
Use \`WHERE\` to produce a table that contains all the rows from the input table for which the provided condition evaluates to \`true\`:
\`\`\`
FROM employees
| KEEP first_name, last_name, still_hired
@ -654,7 +654,7 @@ export const groupingFunctions = {
defaultMessage: `### BUCKET
Creates groups of values - buckets - out of a datetime or numeric input. The size of the buckets can either be provided directly, or chosen based on a recommended count and values range.
\`BUCKET\` works in two modes:
\`BUCKET\` works in two modes:
1. Where the size of the bucket is computed based on a buckets count recommendation (four parameters) and a range.
2. Where the bucket size is provided directly (two parameters).

View file

@ -148,7 +148,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await discover.selectTextBaseLang();
const testQuery = `from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500`;
const testQuery = `from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500`;
await monacoEditor.setCodeEditorValue(testQuery);
await testSubjects.click('querySubmitButton');
await header.waitUntilLoadingHasFinished();
@ -168,7 +168,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await unifiedFieldList.clickFieldListPlusFilter('bytes', '0');
const editorValue = await monacoEditor.getCodeEditorValue();
expect(editorValue).to.eql(
`from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`bytes\`==0`
`from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`bytes\`==0`
);
await unifiedFieldList.closeFieldPopover();
});
@ -188,7 +188,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await unifiedFieldList.clickFieldListPlusFilter('extension.raw', 'css');
const editorValue = await monacoEditor.getCodeEditorValue();
expect(editorValue).to.eql(
`from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`extension.raw\`=="css"`
`from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`extension.raw\`=="css"`
);
await unifiedFieldList.closeFieldPopover();
@ -209,7 +209,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await unifiedFieldList.clickFieldListPlusFilter('clientip', '216.126.255.31');
const editorValue = await monacoEditor.getCodeEditorValue();
expect(editorValue).to.eql(
`from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`clientip\`::string=="216.126.255.31"`
`from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`clientip\`::string=="216.126.255.31"`
);
await unifiedFieldList.closeFieldPopover();
@ -234,7 +234,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await unifiedFieldList.clickFieldListExistsFilter('@timestamp');
const editorValue = await monacoEditor.getCodeEditorValue();
expect(editorValue).to.eql(
`from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`@timestamp\` is not null`
`from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`@timestamp\` is not null`
);
await testSubjects.missingOrFail('dscFieldStats-statsFooter');
await unifiedFieldList.closeFieldPopover();
@ -269,7 +269,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await unifiedFieldList.clickFieldListPlusFilter('extension', 'css');
const editorValue = await monacoEditor.getCodeEditorValue();
expect(editorValue).to.eql(
`from logstash-* [METADATA _index, _id] | sort @timestamp desc | limit 500\n| WHERE \`extension\`=="css"`
`from logstash-* METADATA _index, _id | sort @timestamp desc | limit 500\n| WHERE \`extension\`=="css"`
);
await unifiedFieldList.closeFieldPopover();

View file

@ -22,11 +22,11 @@ const getQeryAst = (query: string) => {
describe('computeHasMetadataOperator', () => {
it('should be false if query does not have operator', () => {
expect(computeHasMetadataOperator(getQeryAst('from test*'))).toBe(false);
expect(computeHasMetadataOperator(getQeryAst('from test* [metadata]'))).toBe(false);
expect(computeHasMetadataOperator(getQeryAst('from test* [metadata id]'))).toBe(false);
expect(computeHasMetadataOperator(getQeryAst('from test* metadata'))).toBe(false);
expect(computeHasMetadataOperator(getQeryAst('from test* metadata id'))).toBe(false);
expect(computeHasMetadataOperator(getQeryAst('from metadata*'))).toBe(false);
expect(computeHasMetadataOperator(getQeryAst('from test* | keep metadata'))).toBe(false);
expect(computeHasMetadataOperator(getQeryAst('from test* | eval x="[metadata _id]"'))).toBe(
expect(computeHasMetadataOperator(getQeryAst('from test* | eval x="metadata _id"'))).toBe(
false
);
});
@ -48,19 +48,19 @@ describe('computeHasMetadataOperator', () => {
).toBe(true);
// still validates deprecated square bracket syntax
expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _id]'))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _id, _index]'))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* [metadata _index, _id]'))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* [ metadata _id ]'))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* [ metadata _id] '))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* [ metadata _id] | limit 10'))).toBe(
expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id'))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id, _index'))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* metadata _index, _id'))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id '))).toBe(true);
expect(computeHasMetadataOperator(getQeryAst('from test* metadata _id | limit 10'))).toBe(
true
);
expect(
computeHasMetadataOperator(
getQeryAst(`from packetbeat* [metadata
getQeryAst(`from packetbeat* metadata
_id ]
_id
| limit 100`)
)
).toBe(true);

View file

@ -154,7 +154,7 @@ export const parseEsqlQuery = (query: string) => {
return {
errors,
isEsqlQueryAggregating,
// non-aggregating query which does not have [metadata], is not a valid one
// non-aggregating query which does not have metadata, is not a valid one
isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(ast),
};
};