[ES|QL] Allows to retrieve empty columns (#218085)

## Summary

Closes https://github.com/elastic/kibana/issues/200039

Here we are asking ES to do a grouping between empty columns and
non-empty. The `dropNullColumns: true` in the request does the trick.

In case of true the response comes as:

```
// only columns with data
columns: [...]
// all columns
all_columns: [...]
```

When the query is empty the columns array comes empty but the
all_columns has all the columns information. The PR just takes the empty
columns scenario under consideration in order to serve the `all_columns`
instead. In that case the text based datasource has the info it needs to
serve a valid visualization state.


<img width="990" alt="image"
src="https://github.com/user-attachments/assets/7d0b2c58-eda2-4807-9203-36f7da48a6ff"
/>


<img width="814" alt="image"
src="https://github.com/user-attachments/assets/8b0ef3bf-14d5-4438-b8fd-a13d346da420"
/>


### Checklist

- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/platform/plugins/shared/lens/public/app_plugin/shared/edit_on_the_fly/helpers.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
-import { getESQLResults } from '@kbn/esql-utils';
+import { getESQLResults, formatESQLColumns } from '@kbn/esql-utils';
 import type { LensPluginStartDependencies } from '../../../plugin';
 import type { TypedLensSerializedState } from '../../../react_embeddable/types';
 import { createMockStartDependencies } from '../../../editor_frame_service/mocks';
@@ -16,70 +16,51 @@ import {
   mockAllSuggestions,
 } from '../../../mocks';
 import { suggestionsApi } from '../../../lens_suggestions_api';
-import { getSuggestions, injectESQLQueryIntoLensLayers } from './helpers';
+import { getSuggestions, injectESQLQueryIntoLensLayers, getGridAttrs } from './helpers';
 
 const mockSuggestionApi = suggestionsApi as jest.Mock;
 const mockFetchData = getESQLResults as jest.Mock;
+const mockformatESQLColumns = formatESQLColumns as jest.Mock;
 
 jest.mock('../../../lens_suggestions_api', () => ({
   suggestionsApi: jest.fn(() => mockAllSuggestions),
 }));
 
+const queryResponseColumns = [
+  {
+    name: '@timestamp',
+    id: '@timestamp',
+    meta: {
+      type: 'date',
+    },
+  },
+  {
+    name: 'bytes',
+    id: 'bytes',
+    meta: {
+      type: 'number',
+    },
+  },
+  {
+    name: 'memory',
+    id: 'memory',
+    meta: {
+      type: 'number',
+    },
+  },
+];
+
 jest.mock('@kbn/esql-utils', () => {
   return {
     getESQLResults: jest.fn().mockResolvedValue({
       response: {
-        columns: [
-          {
-            name: '@timestamp',
-            id: '@timestamp',
-            meta: {
-              type: 'date',
-            },
-          },
-          {
-            name: 'bytes',
-            id: 'bytes',
-            meta: {
-              type: 'number',
-            },
-          },
-          {
-            name: 'memory',
-            id: 'memory',
-            meta: {
-              type: 'number',
-            },
-          },
-        ],
+        columns: queryResponseColumns,
         values: [],
       },
     }),
     getIndexPatternFromESQLQuery: jest.fn().mockReturnValue('index1'),
     getESQLAdHocDataview: jest.fn().mockResolvedValue({}),
-    formatESQLColumns: jest.fn().mockReturnValue([
-      {
-        name: '@timestamp',
-        id: '@timestamp',
-        meta: {
-          type: 'date',
-        },
-      },
-      {
-        name: 'bytes',
-        id: 'bytes',
-        meta: {
-          type: 'number',
-        },
-      },
-      {
-        name: 'memory',
-        id: 'memory',
-        meta: {
-          type: 'number',
-        },
-      },
-    ]),
+    formatESQLColumns: jest.fn().mockReturnValue(queryResponseColumns),
   };
 });
 
@@ -226,4 +207,69 @@ describe('Lens inline editing helpers', () => {
       );
     });
   });
+
+  describe('getGridAttrs', () => {
+    const query = {
+      esql: 'from index1 | limit 10 | stats average = avg(bytes)',
+    };
+    const mockStartDependencies =
+      createMockStartDependencies() as unknown as LensPluginStartDependencies;
+    const dataViews = dataViewPluginMocks.createStartContract();
+    dataViews.create.mockResolvedValue(mockDataViewWithTimefield);
+    mockStartDependencies.data.dataViews = dataViews;
+    const dataviewSpecArr = [
+      {
+        id: 'd2588ae7-9ea0-4439-9f5b-f808754a3b97',
+        title: 'index1',
+        timeFieldName: '@timestamp',
+        sourceFilters: [],
+        fieldFormats: {},
+        runtimeFieldMap: {},
+        fieldAttrs: {},
+        allowNoIndex: false,
+        name: 'index1',
+      },
+    ];
+    const startDependencies = {
+      ...mockStartDependencies,
+      dataViews,
+    };
+
+    it('returns the columns if the array is not empty in the response', async () => {
+      mockFetchData.mockImplementation(() => {
+        return {
+          response: {
+            columns: queryResponseColumns,
+            values: [],
+          },
+        };
+      });
+      const gridAttributes = await getGridAttrs(query, dataviewSpecArr, startDependencies.data);
+      expect(gridAttributes.columns).toStrictEqual(queryResponseColumns);
+    });
+
+    it('returns all_columns if the columns array is empty in the response and all_columns exist', async () => {
+      const emptyColumns = [
+        {
+          name: 'bytes',
+          id: 'bytes',
+          meta: {
+            type: 'number',
+          },
+        },
+      ];
+      mockFetchData.mockImplementation(() => {
+        return {
+          response: {
+            columns: [],
+            values: [],
+            all_columns: emptyColumns,
+          },
+        };
+      });
+      mockformatESQLColumns.mockImplementation(() => emptyColumns);
+      const gridAttributes = await getGridAttrs(query, dataviewSpecArr, startDependencies.data);
+      expect(gridAttributes.columns).toStrictEqual(emptyColumns);
+    });
+  });
 });

x-pack/platform/plugins/shared/lens/public/app_plugin/shared/edit_on_the_fly/helpers.ts

@@ -78,7 +78,14 @@ export const getGridAttrs = async (
     variables: esqlVariables,
   });
 
-  const columns = formatESQLColumns(results.response.columns);
+  let queryColumns = results.response.columns;
+  // if the query columns are empty, we need to use the all_columns property
+  // which has all columns regardless if they have data or not
+  if (queryColumns.length === 0 && results.response.all_columns) {
+    queryColumns = results.response.all_columns;
+  }
+
+  const columns = formatESQLColumns(queryColumns);
 
   return {
     rows: results.response.values,