check data present for apm indices to consider it exists on seach strategy (#98940) (#99057)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/security_solution/server/search_strategy/index_fields/index.test.ts

@@ -7,9 +7,19 @@
 
 import { sortBy } from 'lodash/fp';
 
-import { formatIndexFields, formatFirstFields, formatSecondFields, createFieldItem } from './index';
+import {
+  formatIndexFields,
+  formatFirstFields,
+  formatSecondFields,
+  createFieldItem,
+  requestIndexFieldSearch,
+} from './index';
 import { mockAuditbeatIndexField, mockFilebeatIndexField, mockPacketbeatIndexField } from './mock';
 import { fieldsBeat as beatFields } from '../../utils/beat_schema/fields';
+import {
+  IndexPatternsFetcher,
+  SearchStrategyDependencies,
+} from '../../../../../../src/plugins/data/server';
 
 describe('Index Fields', () => {
   describe('formatIndexFields', () => {
@@ -780,3 +790,119 @@ describe('Index Fields', () => {
     });
   });
 });
+
+describe('Fields Provider', () => {
+  describe('search', () => {
+    const getFieldsForWildcardMock = jest.fn();
+    const esClientSearchMock = jest.fn();
+
+    const deps = ({
+      esClient: { asCurrentUser: { search: esClientSearchMock } },
+    } as unknown) as SearchStrategyDependencies;
+
+    beforeAll(() => {
+      getFieldsForWildcardMock.mockResolvedValue([]);
+      IndexPatternsFetcher.prototype.getFieldsForWildcard = getFieldsForWildcardMock;
+    });
+
+    beforeEach(() => {
+      getFieldsForWildcardMock.mockClear();
+      esClientSearchMock.mockClear();
+    });
+
+    afterAll(() => {
+      getFieldsForWildcardMock.mockRestore();
+    });
+
+    it('should check index exists', async () => {
+      const indices = ['some-index-pattern-*'];
+      const request = {
+        indices,
+        onlyCheckIfIndicesExist: true,
+      };
+
+      const response = await requestIndexFieldSearch(request, deps, beatFields);
+
+      expect(getFieldsForWildcardMock).toHaveBeenCalledWith({ pattern: indices[0] });
+
+      expect(response.indexFields).toHaveLength(0);
+      expect(response.indicesExist).toEqual(indices);
+    });
+
+    it('should search index fields', async () => {
+      const indices = ['some-index-pattern-*'];
+      const request = {
+        indices,
+        onlyCheckIfIndicesExist: false,
+      };
+
+      const response = await requestIndexFieldSearch(request, deps, beatFields);
+
+      expect(getFieldsForWildcardMock).toHaveBeenCalledWith({ pattern: indices[0] });
+
+      expect(response.indexFields).not.toHaveLength(0);
+      expect(response.indicesExist).toEqual(indices);
+    });
+
+    it('should search apm index fields', async () => {
+      const indices = ['apm-*-transaction*'];
+      const request = {
+        indices,
+        onlyCheckIfIndicesExist: false,
+      };
+
+      const response = await requestIndexFieldSearch(request, deps, beatFields);
+
+      expect(getFieldsForWildcardMock).toHaveBeenCalledWith({ pattern: indices[0] });
+      expect(esClientSearchMock).not.toHaveBeenCalled();
+
+      expect(response.indexFields).not.toHaveLength(0);
+      expect(response.indicesExist).toEqual(indices);
+    });
+
+    it('should check apm index exists with data', async () => {
+      const indices = ['apm-*-transaction*'];
+      const request = {
+        indices,
+        onlyCheckIfIndicesExist: true,
+      };
+
+      esClientSearchMock.mockResolvedValueOnce({
+        body: { hits: { total: { value: 1 } } },
+      });
+      const response = await requestIndexFieldSearch(request, deps, beatFields);
+
+      expect(esClientSearchMock).toHaveBeenCalledWith({
+        index: indices[0],
+        body: { query: { match_all: {} }, size: 0 },
+      });
+      expect(getFieldsForWildcardMock).not.toHaveBeenCalled();
+
+      expect(response.indexFields).toHaveLength(0);
+      expect(response.indicesExist).toEqual(indices);
+    });
+
+    it('should check apm index exists with no data', async () => {
+      const indices = ['apm-*-transaction*'];
+      const request = {
+        indices,
+        onlyCheckIfIndicesExist: true,
+      };
+
+      esClientSearchMock.mockResolvedValueOnce({
+        body: { hits: { total: { value: 0 } } },
+      });
+
+      const response = await requestIndexFieldSearch(request, deps, beatFields);
+
+      expect(esClientSearchMock).toHaveBeenCalledWith({
+        index: indices[0],
+        body: { query: { match_all: {} }, size: 0 },
+      });
+      expect(getFieldsForWildcardMock).not.toHaveBeenCalled();
+
+      expect(response.indexFields).toHaveLength(0);
+      expect(response.indicesExist).toEqual([]);
+    });
+  });
+});

x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts

@@ -7,7 +7,12 @@
 
 import { from } from 'rxjs';
 import isEmpty from 'lodash/isEmpty';
-import { IndexPatternsFetcher, ISearchStrategy } from '../../../../../../src/plugins/data/server';
+import get from 'lodash/get';
+import {
+  IndexPatternsFetcher,
+  ISearchStrategy,
+  SearchStrategyDependencies,
+} from '../../../../../../src/plugins/data/server';
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
 import { FieldDescriptor } from '../../../../../../src/plugins/data/server/index_patterns';
 import {
@@ -17,6 +22,8 @@ import {
   BeatFields,
 } from '../../../common/search_strategy/index_fields';
 
+const apmIndexPattern = 'apm-*-transaction*';
+
 export const securitySolutionIndexFieldsProvider = (): ISearchStrategy<
   IndexFieldsStrategyRequest,
   IndexFieldsStrategyResponse
@@ -27,60 +34,73 @@ export const securitySolutionIndexFieldsProvider = (): ISearchStrategy<
   const beatFields: BeatFields = require('../../utils/beat_schema/fields').fieldsBeat;
 
   return {
-    search: (request, options, { esClient }) =>
-      from(
-        new Promise<IndexFieldsStrategyResponse>(async (resolve) => {
-          const indexPatternsFetcher = new IndexPatternsFetcher(esClient.asCurrentUser);
-          const dedupeIndices = dedupeIndexName(request.indices);
+    search: (request, options, deps) => from(requestIndexFieldSearch(request, deps, beatFields)),
+  };
+};
 
-          const responsesIndexFields = await Promise.all(
-            dedupeIndices
-              .map((index) =>
-                indexPatternsFetcher.getFieldsForWildcard({
-                  pattern: index,
-                })
-              )
-              .map((p) => p.catch((e) => false))
-          );
-          let indexFields: IndexField[] = [];
+export const requestIndexFieldSearch = async (
+  request: IndexFieldsStrategyRequest,
+  { esClient }: SearchStrategyDependencies,
+  beatFields: BeatFields
+): Promise<IndexFieldsStrategyResponse> => {
+  const indexPatternsFetcher = new IndexPatternsFetcher(esClient.asCurrentUser);
+  const dedupeIndices = dedupeIndexName(request.indices);
 
-          if (!request.onlyCheckIfIndicesExist) {
-            indexFields = await formatIndexFields(
-              beatFields,
-              responsesIndexFields.filter((rif) => rif !== false) as FieldDescriptor[][],
-              dedupeIndices
-            );
-          }
-
-          return resolve({
-            indexFields,
-            indicesExist: dedupeIndices.filter((index, i) => responsesIndexFields[i] !== false),
-            rawResponse: {
-              timed_out: false,
-              took: -1,
-              _shards: {
-                total: -1,
-                successful: -1,
-                failed: -1,
-                skipped: -1,
-              },
-              hits: {
-                total: -1,
-                max_score: -1,
-                hits: [
-                  {
-                    _index: '',
-                    _type: '',
-                    _id: '',
-                    _score: -1,
-                    _source: null,
-                  },
-                ],
-              },
-            },
+  const responsesIndexFields = await Promise.all(
+    dedupeIndices
+      .map(async (index) => {
+        if (request.onlyCheckIfIndicesExist && index.includes(apmIndexPattern)) {
+          // for apm index pattern check also if there's data https://github.com/elastic/kibana/issues/90661
+          const searchResponse = await esClient.asCurrentUser.search({
+            index,
+            body: { query: { match_all: {} }, size: 0 },
           });
-        })
-      ),
+          return get(searchResponse, 'body.hits.total.value', 0) > 0;
+        } else {
+          return indexPatternsFetcher.getFieldsForWildcard({
+            pattern: index,
+          });
+        }
+      })
+      .map((p) => p.catch((e) => false))
+  );
+
+  let indexFields: IndexField[] = [];
+
+  if (!request.onlyCheckIfIndicesExist) {
+    indexFields = await formatIndexFields(
+      beatFields,
+      responsesIndexFields.filter((rif) => rif !== false) as FieldDescriptor[][],
+      dedupeIndices
+    );
+  }
+
+  return {
+    indexFields,
+    indicesExist: dedupeIndices.filter((index, i) => responsesIndexFields[i] !== false),
+    rawResponse: {
+      timed_out: false,
+      took: -1,
+      _shards: {
+        total: -1,
+        successful: -1,
+        failed: -1,
+        skipped: -1,
+      },
+      hits: {
+        total: -1,
+        max_score: -1,
+        hits: [
+          {
+            _index: '',
+            _type: '',
+            _id: '',
+            _score: -1,
+            _source: null,
+          },
+        ],
+      },
+    },
   };
 };