[7.x] [Lens] Adds using queries/filters for field existence en… (#59984)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
2025-04-24 09:48:58 -04:00 · 2020-03-12 15:00:31 +01:00 · 2020-03-12 15:00:31 +01:00 · e84ac45171
commit e84ac45171
parent ed0a0826c2
8 changed files with 157 additions and 81 deletions
--- a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
+++ b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx
@ -206,6 +206,9 @@ const initialState: IndexPatternPrivateState = {
    },
  },
 };
+
+const dslQuery = { bool: { must: [{ match_all: {} }], filter: [], should: [], must_not: [] } };
+
 describe('IndexPattern Data Panel', () => {
  let defaultProps: Parameters<typeof InnerIndexPatternDataPanel>[0];
  let core: ReturnType<typeof coreMock['createSetup']>;
@ -271,8 +274,8 @@ describe('IndexPattern Data Panel', () => {
  describe('loading existence data', () => {
    function testProps() {
      const setState = jest.fn();
-      core.http.get.mockImplementation(async ({ path }) => {
-        const parts = path.split('/');
+      core.http.post.mockImplementation(async path => {
+        const parts = ((path as unknown) as string).split('/');
        const indexPatternTitle = parts[parts.length - 1];
        return {
          indexPatternTitle: `${indexPatternTitle}_testtitle`,
@ -385,24 +388,24 @@ describe('IndexPattern Data Panel', () => {
      });

      expect(setState).toHaveBeenCalledTimes(2);
-      expect(core.http.get).toHaveBeenCalledTimes(2);
+      expect(core.http.post).toHaveBeenCalledTimes(2);

-      expect(core.http.get).toHaveBeenCalledWith({
-        path: '/api/lens/existing_fields/a',
-        query: {
+      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
+        body: JSON.stringify({
+          dslQuery,
          fromDate: '2019-01-01',
          toDate: '2020-01-01',
          timeFieldName: 'atime',
-        },
+        }),
      });

-      expect(core.http.get).toHaveBeenCalledWith({
-        path: '/api/lens/existing_fields/a',
-        query: {
+      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
+        body: JSON.stringify({
+          dslQuery,
          fromDate: '2019-01-01',
          toDate: '2020-01-02',
          timeFieldName: 'atime',
-        },
+        }),
      });

      const nextState = setState.mock.calls[1][0]({
@ -428,22 +431,22 @@ describe('IndexPattern Data Panel', () => {

      expect(setState).toHaveBeenCalledTimes(2);

-      expect(core.http.get).toHaveBeenCalledWith({
-        path: '/api/lens/existing_fields/a',
-        query: {
+      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/a', {
+        body: JSON.stringify({
+          dslQuery,
          fromDate: '2019-01-01',
          toDate: '2020-01-01',
          timeFieldName: 'atime',
-        },
+        }),
      });

-      expect(core.http.get).toHaveBeenCalledWith({
-        path: '/api/lens/existing_fields/b',
-        query: {
+      expect(core.http.post).toHaveBeenCalledWith('/api/lens/existing_fields/b', {
+        body: JSON.stringify({
+          dslQuery,
          fromDate: '2019-01-01',
          toDate: '2020-01-01',
          timeFieldName: 'btime',
-        },
+        }),
      });

      const nextState = setState.mock.calls[1][0]({
@ -476,13 +479,13 @@ describe('IndexPattern Data Panel', () => {
      let overlapCount = 0;
      const props = testProps();

-      core.http.get.mockImplementation(({ path }) => {
+      core.http.post.mockImplementation(path => {
        if (queryCount) {
          ++overlapCount;
        }
        ++queryCount;

-        const parts = path.split('/');
+        const parts = ((path as unknown) as string).split('/');
        const indexPatternTitle = parts[parts.length - 1];
        const result = Promise.resolve({
          indexPatternTitle,
@ -516,7 +519,7 @@ describe('IndexPattern Data Panel', () => {
        inst.update();
      });

-      expect(core.http.get).toHaveBeenCalledTimes(2);
+      expect(core.http.post).toHaveBeenCalledTimes(2);
      expect(overlapCount).toEqual(0);
    });
  });
--- a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx
+++ b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/datapanel.tsx
@ -40,6 +40,7 @@ import { trackUiEvent } from '../lens_ui_telemetry';
 import { syncExistingFields } from './loader';
 import { fieldExists } from './pure_helpers';
 import { Loader } from '../loader';
+import { esQuery, IIndexPattern } from '../../../../../../src/plugins/data/public';

 export type Props = DatasourceDataPanelProps<IndexPatternPrivateState> & {
  changeIndexPattern: (
@ -113,6 +114,13 @@ export function IndexPatternDataPanel({
      timeFieldName: indexPatterns[id].timeFieldName,
    }));

+  const dslQuery = esQuery.buildEsQuery(
+    indexPatterns[currentIndexPatternId] as IIndexPattern,
+    query,
+    filters,
+    esQuery.getEsQueryConfig(core.uiSettings)
+  );
+
  return (
    <>
      <Loader
@ -121,10 +129,13 @@ export function IndexPatternDataPanel({
            dateRange,
            setState,
            indexPatterns: indexPatternList,
-            fetchJson: core.http.get,
+            fetchJson: core.http.post,
+            dslQuery,
          })
        }
        loadDeps={[
+          query,
+          filters,
          dateRange.fromDate,
          dateRange.toDate,
          indexPatternList.map(x => `${x.title}:${x.timeFieldName}`).join(','),
--- a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts
+++ b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.test.ts
@ -535,9 +535,18 @@ describe('loader', () => {
  });

  describe('syncExistingFields', () => {
+    const dslQuery = {
+      bool: {
+        must: [],
+        filter: [{ match_all: {} }],
+        should: [],
+        must_not: [],
+      },
+    };
+
    it('should call once for each index pattern', async () => {
      const setState = jest.fn();
-      const fetchJson = jest.fn(({ path }: { path: string }) => {
+      const fetchJson = jest.fn((path: string) => {
        const indexPatternTitle = _.last(path.split('/'));
        return {
          indexPatternTitle,
@ -553,6 +562,7 @@ describe('loader', () => {
        fetchJson: fetchJson as any,
        indexPatterns: [{ id: 'a' }, { id: 'b' }, { id: 'c' }],
        setState,
+        dslQuery,
      });

      expect(fetchJson).toHaveBeenCalledTimes(3);
--- a/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts
+++ b/x-pack/legacy/plugins/lens/public/indexpattern_datasource/loader.ts
@ -215,26 +215,28 @@ export async function syncExistingFields({
  dateRange,
  fetchJson,
  setState,
+  dslQuery,
 }: {
  dateRange: DateRange;
  indexPatterns: Array<{ id: string; timeFieldName?: string | null }>;
-  fetchJson: HttpSetup['get'];
+  fetchJson: HttpSetup['post'];
  setState: SetState;
+  dslQuery: object;
 }) {
  const emptinessInfo = await Promise.all(
    indexPatterns.map(pattern => {
-      const query: Record<string, string> = {
+      const body: Record<string, string | object> = {
+        dslQuery,
        fromDate: dateRange.fromDate,
        toDate: dateRange.toDate,
      };

      if (pattern.timeFieldName) {
-        query.timeFieldName = pattern.timeFieldName;
+        body.timeFieldName = pattern.timeFieldName;
      }

-      return fetchJson({
-        path: `${BASE_API_URL}/existing_fields/${pattern.id}`,
-        query,
+      return fetchJson(`${BASE_API_URL}/existing_fields/${pattern.id}`, {
+        body: JSON.stringify(body),
      }) as Promise<ExistingFields>;
    })
  );
--- a/x-pack/plugins/lens/server/routes/existing_fields.ts
+++ b/x-pack/plugins/lens/server/routes/existing_fields.ts
@ -46,14 +46,16 @@ const metaFields = ['_source', '_id', '_type', '_index', '_score'];

 export async function existingFieldsRoute(setup: CoreSetup) {
  const router = setup.http.createRouter();
-  router.get(
+
+  router.post(
    {
      path: `${BASE_API_URL}/existing_fields/{indexPatternId}`,
      validate: {
        params: schema.object({
          indexPatternId: schema.string(),
        }),
-        query: schema.object({
+        body: schema.object({
+          dslQuery: schema.object({}, { allowUnknowns: true }),
          fromDate: schema.maybe(schema.string()),
          toDate: schema.maybe(schema.string()),
          timeFieldName: schema.maybe(schema.string()),
@ -64,8 +66,8 @@ export async function existingFieldsRoute(setup: CoreSetup) {
      try {
        return res.ok({
          body: await fetchFieldExistence({
-            ...req.query,
            ...req.params,
+            ...req.body,
            context,
          }),
        });
@ -91,12 +93,14 @@ export async function existingFieldsRoute(setup: CoreSetup) {
 async function fetchFieldExistence({
  context,
  indexPatternId,
+  dslQuery = { match_all: {} },
  fromDate,
  toDate,
  timeFieldName,
 }: {
  indexPatternId: string;
  context: RequestHandlerContext;
+  dslQuery: object;
  fromDate?: string;
  toDate?: string;
  timeFieldName?: string;
@ -109,10 +113,10 @@ async function fetchFieldExistence({
  } = await fetchIndexPatternDefinition(indexPatternId, context);

  const fields = buildFieldList(indexPattern, mappings, fieldDescriptors);
-
  const docs = await fetchIndexPatternStats({
    fromDate,
    toDate,
+    dslQuery,
    client: context.core.elasticsearch.dataClient,
    index: indexPatternTitle,
    timeFieldName: timeFieldName || indexPattern.attributes.timeFieldName,
@ -197,6 +201,7 @@ export function buildFieldList(
 async function fetchIndexPatternStats({
  client,
  index,
+  dslQuery,
  timeFieldName,
  fromDate,
  toDate,
@ -204,17 +209,15 @@ async function fetchIndexPatternStats({
 }: {
  client: IScopedClusterClient;
  index: string;
+  dslQuery: object;
  timeFieldName?: string;
  fromDate?: string;
  toDate?: string;
  fields: Field[];
 }) {
-  let query;
-
-  if (timeFieldName && fromDate && toDate) {
-    query = {
-      bool: {
-        filter: [
+  const filter =
+    timeFieldName && fromDate && toDate
+      ? [
          {
            range: {
              [timeFieldName]: {
@ -223,16 +226,17 @@ async function fetchIndexPatternStats({
              },
            },
          },
-        ],
-      },
-    };
-  } else {
-    query = {
-      match_all: {},
-    };
-  }
-  const scriptedFields = fields.filter(f => f.isScript);
+          dslQuery,
+        ]
+      : [dslQuery];

+  const query = {
+    bool: {
+      filter,
+    },
+  };
+
+  const scriptedFields = fields.filter(f => f.isScript);
  const result = await client.callAsCurrentUser('search', {
    index,
    body: {
@ -251,7 +255,6 @@ async function fetchIndexPatternStats({
      }, {} as Record<string, unknown>),
    },
  });
-
  return result.hits.hits;
 }

--- a/x-pack/test/api_integration/apis/lens/existing_fields.ts
+++ b/x-pack/test/api_integration/apis/lens/existing_fields.ts
@ -8,8 +8,8 @@ import expect from '@kbn/expect';

 import { FtrProviderContext } from '../../ftr_provider_context';

-const TEST_START_TIME = encodeURIComponent('2015-09-19T06:31:44.000');
-const TEST_END_TIME = encodeURIComponent('2015-09-23T18:31:44.000');
+const TEST_START_TIME = '2015-09-19T06:31:44.000';
+const TEST_END_TIME = '2015-09-23T18:31:44.000';
 const COMMON_HEADERS = {
  'kbn-xsrf': 'some-xsrf-token',
 };
@ -147,12 +147,17 @@ export default ({ getService }: FtrProviderContext) => {
    describe('existence', () => {
      it('should find which fields exist in the sample documents', async () => {
        const { body } = await supertest
-          .get(
-            `/api/lens/existing_fields/${encodeURIComponent(
-              'logstash-*'
-            )}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}`
-          )
+          .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`)
          .set(COMMON_HEADERS)
+          .send({
+            dslQuery: {
+              bool: {
+                filter: [{ match_all: {} }],
+              },
+            },
+            fromDate: TEST_START_TIME,
+            toDate: TEST_END_TIME,
+          })
          .expect(200);

        expect(body.indexPatternTitle).to.eql('logstash-*');
@ -161,25 +166,67 @@ export default ({ getService }: FtrProviderContext) => {

      it('should succeed for thousands of fields', async () => {
        const { body } = await supertest
-          .get(
-            `/api/lens/existing_fields/${encodeURIComponent(
-              'metricbeat-*'
-            )}?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}`
-          )
+          .post(`/api/lens/existing_fields/${encodeURIComponent('metricbeat-*')}`)
          .set(COMMON_HEADERS)
+          .send({
+            dslQuery: { match_all: {} },
+            fromDate: TEST_START_TIME,
+            toDate: TEST_END_TIME,
+          })
          .expect(200);

        expect(body.indexPatternTitle).to.eql('metricbeat-*');
        expect(body.existingFieldNames.sort()).to.eql(metricBeatData.sort());
      });

-      it('should throw a 404 for a non-existent index', async () => {
-        await supertest
-          .get(
-            `/api/lens/existing_fields/nadachance?fromDate=${TEST_START_TIME}&toDate=${TEST_END_TIME}`
-          )
+      it('should return fields filtered by query and filters', async () => {
+        const expectedFieldNames = [
+          '@message',
+          '@message.raw',
+          '@tags',
+          '@tags.raw',
+          '@timestamp',
+          'agent',
+          'agent.raw',
+          'bytes',
+          'clientip',
+          'extension',
+          'extension.raw',
+          'headings',
+          'headings.raw',
+          'host',
+          'host.raw',
+          'index',
+          'index.raw',
+          'referer',
+          'request',
+          'request.raw',
+          'response',
+          'response.raw',
+          'spaces',
+          'spaces.raw',
+          'type',
+          'url',
+          'url.raw',
+          'utc_time',
+          'xss',
+          'xss.raw',
+        ];
+
+        const { body } = await supertest
+          .post(`/api/lens/existing_fields/${encodeURIComponent('logstash-*')}`)
          .set(COMMON_HEADERS)
-          .expect(404);
+          .send({
+            dslQuery: {
+              bool: {
+                filter: [{ match: { referer: 'https://www.taylorswift.com/' } }],
+              },
+            },
+            fromDate: TEST_START_TIME,
+            toDate: TEST_END_TIME,
+          })
+          .expect(200);
+        expect(body.existingFieldNames.sort()).to.eql(expectedFieldNames.sort());
      });
    });
  });
--- a/x-pack/test/api_integration/apis/lens/field_stats.ts
+++ b/x-pack/test/api_integration/apis/lens/field_stats.ts
@ -62,7 +62,7 @@ export default ({ getService }: FtrProviderContext) => {
          })
          .expect(200);

-        expect(body).to.have.property('totalDocuments', 4633);
+        expect(body).to.have.property('totalDocuments', 4634);
      });

      it('should return an auto histogram for numbers and top values', async () => {
@ -82,9 +82,9 @@ export default ({ getService }: FtrProviderContext) => {
          .expect(200);

        expect(body).to.eql({
-          totalDocuments: 4633,
-          sampledDocuments: 4633,
-          sampledValues: 4633,
+          totalDocuments: 4634,
+          sampledDocuments: 4634,
+          sampledValues: 4634,
          histogram: {
            buckets: [
              {
@ -96,7 +96,7 @@ export default ({ getService }: FtrProviderContext) => {
                key: 1999,
              },
              {
-                count: 885,
+                count: 886,
                key: 3998,
              },
              {
@ -139,6 +139,10 @@ export default ({ getService }: FtrProviderContext) => {
                count: 5,
                key: 3954,
              },
+              {
+                count: 5,
+                key: 5846,
+              },
              {
                count: 5,
                key: 6497,
@ -159,10 +163,6 @@ export default ({ getService }: FtrProviderContext) => {
                count: 4,
                key: 4669,
              },
-              {
-                count: 4,
-                key: 5846,
-              },
              {
                count: 4,
                key: 5863,
@ -193,11 +193,11 @@ export default ({ getService }: FtrProviderContext) => {
          .expect(200);

        expect(body).to.eql({
-          totalDocuments: 4633,
+          totalDocuments: 4634,
          histogram: {
            buckets: [
              {
-                count: 1161,
+                count: 1162,
                key: 1442875680000,
              },
              {
@ -230,8 +230,8 @@ export default ({ getService }: FtrProviderContext) => {
          .expect(200);

        expect(body).to.eql({
-          totalDocuments: 4633,
-          sampledDocuments: 4633,
+          totalDocuments: 4634,
+          sampledDocuments: 4634,
          sampledValues: 4633,
          topValues: {
            buckets: [
--- a/x-pack/test/functional/es_archives/logstash_functional/data.json.gz
+++ b/x-pack/test/functional/es_archives/logstash_functional/data.json.gz