[ES|QL] Fixes autocomplete in case of comments with pipes (#219898)

## Summary

This PR is fixing the autocomplete in case of comments with pipes. 

<img width="990" alt="image"
src="https://github.com/user-attachments/assets/b0c4d39c-8da7-4957-89d3-1d9f534ec5b1"
/>

Also FORK suggests the _fork twice. I am fixing it here too.

### Checklist

- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/fork/fields_suggestions_after.ts

@@ -6,6 +6,7 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+import { uniqBy } from 'lodash';
 import { type ESQLAstCommand } from '@kbn/esql-ast';
 import type { ESQLRealField } from '../../../validation/types';
 
@@ -14,11 +15,14 @@ export const fieldsSuggestionsAfter = (
   previousCommandFields: ESQLRealField[],
   userDefinedColumns: ESQLRealField[]
 ) => {
-  return [
-    ...previousCommandFields,
-    {
-      name: '_fork',
-      type: 'keyword' as const,
-    },
-  ];
+  return uniqBy(
+    [
+      ...previousCommandFields,
+      {
+        name: '_fork',
+        type: 'keyword' as const,
+      },
+    ],
+    'name'
+  );
 };

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/shared/query_string_utils.test.ts

@@ -6,9 +6,78 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-import { removeLastPipe, processPipes, toSingleLine } from './query_string_utils';
+import { removeLastPipe, processPipes, toSingleLine, removeComments } from './query_string_utils';
 
 describe('query_string_utils', () => {
+  describe('removeComments', () => {
+    it('should remove single-line comments', () => {
+      const text = `
+        FROM users // This is a comment
+        | WHERE age > 18
+      `;
+      const expected = `FROM users 
+        | WHERE age > 18`;
+      expect(removeComments(text)).toBe(expected);
+    });
+
+    it('should remove multi-line comments', () => {
+      const text = `
+        FROM /* This is a
+        multi-line
+        comment */
+        products
+      `;
+      const expected = `FROM 
+        products`;
+      expect(removeComments(text)).toBe(expected);
+    });
+
+    it('should remove both single-line and multi-line comments', () => {
+      const text = `
+        FROM items // Get the name
+        /* The price of the
+        product */
+        | KEEP name, price
+      `;
+      const expected = `FROM items 
+        
+        | KEEP name, price`;
+      expect(removeComments(text)).toBe(expected);
+    });
+
+    it('should handle text with no comments', () => {
+      const text = `
+        FROM orders
+        | KEEP order_id, status
+        | WHERE status = 'pending';
+      `;
+      expect(removeComments(text)).toBe(text.trim());
+    });
+
+    it('should handle comments at the beginning and end of the text', () => {
+      const text = `
+        // Initial comment
+        FROM logs
+        /* Final
+        comment */
+      `;
+      const expected = `FROM logs`;
+      expect(removeComments(text)).toBe(expected);
+    });
+
+    it('should handle consecutive single-line comments', () => {
+      const text = `
+        // Comment line 1
+        // Comment line 2
+        FROM events | STATS COUNT(*)
+      `;
+      const expected = `
+        FROM events | STATS COUNT(*)
+      `;
+      expect(removeComments(text)).toBe(expected.trim());
+    });
+  });
+
   describe('removeLastPipe', () => {
     it('should remove the last pipe and any trailing whitespace', () => {
       expect(removeLastPipe('value1|value2|')).toBe('value1|value2');
@@ -16,14 +85,13 @@ describe('query_string_utils', () => {
     });
 
     it('should return the original string if there is no pipe', () => {
-      expect(removeLastPipe('value1value2')).toBe('value1value2');
-      expect(removeLastPipe('value1value2  ')).toBe('value1value2');
+      expect(removeLastPipe('FROM index')).toBe('FROM index');
+      expect(removeLastPipe('FROM index  ')).toBe('FROM index');
     });
 
     it('should handle strings with multiple pipes correctly', () => {
-      expect(removeLastPipe('a|b|c|d')).toBe('a|b|c');
-      expect(removeLastPipe('from index | stats count() | drop field1  ')).toBe(
-        'from index | stats count()'
+      expect(removeLastPipe('FROM index | STATS count() | DROP field1  ')).toBe(
+        'FROM index | STATS count()'
       );
     });
 
@@ -38,51 +106,65 @@ describe('query_string_utils', () => {
 
   describe('processPipes', () => {
     it('should return an array of strings, each progressively including parts separated by " | "', () => {
-      const input = 'value1|value2|value3';
-      const expected = ['value1', 'value1 | value2', 'value1 | value2 | value3'];
+      const input = 'FROM index|EVAL col = ABS(numeric) | KEEP col';
+      const expected = [
+        'FROM index',
+        'FROM index | EVAL col = ABS(numeric)',
+        'FROM index | EVAL col = ABS(numeric) | KEEP col',
+      ];
       expect(processPipes(input)).toEqual(expected);
     });
 
     it('should handle leading and trailing whitespace in parts', () => {
-      const input = ' valueA |  valueB  | valueC ';
-      const expected = ['valueA', 'valueA | valueB', 'valueA | valueB | valueC'];
+      const input = ' FROM index |  EVAL col = ABS(numeric)  | KEEP col ';
+      const expected = [
+        'FROM index',
+        'FROM index | EVAL col = ABS(numeric)',
+        'FROM index | EVAL col = ABS(numeric) | KEEP col',
+      ];
       expect(processPipes(input)).toEqual(expected);
     });
 
     it('should return an array with the trimmed input if there are no pipes', () => {
-      const input = 'from index';
-      const expected = ['from index'];
+      const input = 'FROM index';
+      const expected = ['FROM index'];
       expect(processPipes(input)).toEqual(expected);
 
-      const inputWithWhitespace = '  from index  ';
-      const expectedWithWhitespace = ['from index'];
+      const inputWithWhitespace = '  FROM index  ';
+      const expectedWithWhitespace = ['FROM index'];
       expect(processPipes(inputWithWhitespace)).toEqual(expectedWithWhitespace);
     });
+
+    it('should ignore comments', () => {
+      const input = '// This is an ES|QL query \n FROM index';
+      const expected = ['FROM index'];
+      expect(processPipes(input)).toEqual(expected);
+    });
   });
 
   describe('toSingleLine', () => {
     it('should convert a multi-line pipe-separated string to a single line with " | " as separator', () => {
-      const input = 'value1 \n|value2\n|value3';
-      const expected = 'value1 | value2 | value3';
+      const input = 'FROM index \n|EVAL col = ABS(numeric)\n|KEEP col';
+      const expected = 'FROM index | EVAL col = ABS(numeric) | KEEP col';
       expect(toSingleLine(input)).toBe(expected);
     });
 
     it('should trim whitespace from each part', () => {
-      const input = ' valueA |  valueB  | valueC ';
-      const expected = 'valueA | valueB | valueC';
+      const input = ' FROM index |  EVAL col = ABS(numeric)  | KEEP col ';
+      const expected = 'FROM index | EVAL col = ABS(numeric) | KEEP col';
       expect(toSingleLine(input)).toBe(expected);
     });
 
     it('should trim whitespace from each part for multi-line strings', () => {
-      const input = ' valueA \n|  valueB  \n| valueC ';
-      const expected = 'valueA | valueB | valueC';
+      const input = ' FROM index \n|  EVAL col = ABS(numeric)  \n| KEEP col ';
+      const expected = 'FROM index | EVAL col = ABS(numeric) | KEEP col';
       expect(toSingleLine(input)).toBe(expected);
     });
 
-    it('should handle parts with internal whitespace (which should be preserved)', () => {
-      const input = 'part with spaces|another part|yet another';
-      const expected = 'part with spaces | another part | yet another';
-      expect(toSingleLine(input)).toBe(expected);
+    it('should ignore comments', () => {
+      const input = '// This is an ES|QL query \n FROM index';
+      const expected = 'FROM index';
+      expect(toSingleLine(input)).toEqual(expected);
     });
   });
 });

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/shared/query_string_utils.ts

@@ -6,17 +6,26 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+export function removeComments(text: string): string {
+  // Remove single-line comments
+  const withoutSingleLineComments = text.replace(/\/\/.*$/gm, '');
+  // Remove multi-line comments
+  const withoutMultiLineComments = withoutSingleLineComments.replace(/\/\*[\s\S]*?\*\//g, '');
+  return withoutMultiLineComments.trim();
+}
 
 export function removeLastPipe(inputString: string): string {
-  const lastPipeIndex = inputString.lastIndexOf('|');
+  const queryNoComments = removeComments(inputString);
+  const lastPipeIndex = queryNoComments.lastIndexOf('|');
   if (lastPipeIndex !== -1) {
-    return inputString.substring(0, lastPipeIndex).trimEnd();
+    return queryNoComments.substring(0, lastPipeIndex).trimEnd();
   }
-  return inputString.trimEnd();
+  return queryNoComments.trimEnd();
 }
 
 export function processPipes(inputString: string) {
-  const parts = inputString.split('|');
+  const queryNoComments = removeComments(inputString);
+  const parts = queryNoComments.split('|');
   const results = [];
   let currentString = '';
 
@@ -33,7 +42,8 @@ export function processPipes(inputString: string) {
 }
 
 export function toSingleLine(inputString: string): string {
-  return inputString
+  const queryNoComments = removeComments(inputString);
+  return queryNoComments
     .split('|')
     .map((line) => line.trim())
     .filter((line) => line !== '')
@@ -41,9 +51,10 @@ export function toSingleLine(inputString: string): string {
 }
 
 export function getFirstPipeValue(inputString: string): string {
-  const parts = inputString.split('|');
+  const queryNoComments = removeComments(inputString);
+  const parts = queryNoComments.split('|');
   if (parts.length > 1) {
     return parts[0].trim();
   }
-  return inputString.trim();
+  return queryNoComments.trim();
 }