[Security Solution] Migrated Users search strategy to fields API (#130213)

* [Security Solution] Migrated Users search strategy to fields API * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * fixed tests * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * fixed tests * fixed tests * merge fix * fixed tests * fixed tests * fixed tests * fixed tests * fixed tests Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
2025-04-24 01:38:56 -04:00 · 2022-04-29 10:00:28 -07:00 · 2022-04-29 10:00:28 -07:00 · 7c8e144dd1
commit 7c8e144dd1
parent f476109d57
13 changed files with 236 additions and 1689 deletions
--- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/users/authentications/index.ts
+++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/users/authentications/index.ts
@ -59,7 +59,7 @@ export interface LastSourceHost {
 }

 export interface AuthenticationHit extends Hit {
-  _source: {
+  fields: {
    '@timestamp': string;
    lastSuccess?: LastSourceHost;
    lastFailure?: LastSourceHost;
--- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/users/common/index.ts
+++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/users/common/index.ts
@ -67,7 +67,7 @@ export interface AllUsersAggEsItem {
 export interface UsersDomainHitsItem {
  hits: {
    hits: Array<{
-      _source: { user: { domain: Maybe<string> } };
+      fields: { user: { domain: Maybe<string[]> } };
    }>;
  };
 }
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/mocks/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/mocks/index.ts
@ -13,12 +13,6 @@ import { UsersFields } from '../../../../../../../common/search_strategy/securit

 export const mockOptions: UsersRequestOptions = {
  defaultIndex: ['test_indices*'],
-  docValueFields: [
-    {
-      field: '@timestamp',
-      format: 'date_time',
-    },
-  ],
  factoryQueryType: UsersQueries.users,
  filterQuery:
    '{"bool":{"must":[],"filter":[{"match_all":{}},{"match_phrase":{"user.name":{"query":"test_user"}}}],"should":[],"must_not":[]}}',
@ -78,10 +72,12 @@ export const mockSearchStrategyResponse: IEsSearchResponse<unknown> = {
                    _index: 'endgame-00001',
                    _id: 'inT0934BjUd1_U2597Vf',
                    _score: null,
-                    _source: {
-                      user: {
-                        domain: 'ENDPOINT-W-8-03',
-                      },
+                    fields: {
+                      'user.name': ['jose52'],
+                      '@timestamp': ['2022-04-13T17:16:34.540Z'],
+                      'user.id': ['17'],
+                      'user.email': ['jose52@barrett.com'],
+                      'user.domain': ['ENDPOINT-W-8-03'],
                    },
                    sort: [1644837532000],
                  },
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/snapshots/index.test.ts.snap
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/snapshots/index.test.ts.snap
@ -3,7 +3,9 @@
 exports[`allHosts search strategy parse should parse data correctly 1`] = `
 Array [
  Object {
-    "domain": "ENDPOINT-W-8-03",
+    "domain": Array [
+      "ENDPOINT-W-8-03",
+    ],
    "lastSeen": "2022-02-14T11:18:52.000Z",
    "name": "vagrant",
  },
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/snapshots/query.all_users.dsl.test.ts.snap
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/snapshots/query.all_users.dsl.test.ts.snap
@ -4,6 +4,7 @@ exports[`buildUsersQuery build query from options correctly 1`] = `
 Object {
  "allow_no_indices": true,
  "body": Object {
+    "_source": false,
    "aggregations": Object {
      "user_count": Object {
        "cardinality": Object {
@ -14,11 +15,7 @@ Object {
        "aggs": Object {
          "domain": Object {
            "top_hits": Object {
-              "_source": Object {
-                "includes": Array [
-                  "user.domain",
-                ],
-              },
+              "_source": false,
              "size": 1,
              "sort": Array [
                Object {
@ -44,10 +41,12 @@ Object {
        },
      },
    },
-    "docvalue_fields": Array [
+    "fields": Array [
+      "user.name",
+      "user.domain",
      Object {
        "field": "@timestamp",
-        "format": "date_time",
+        "format": "strict_date_optional_time",
      },
    ],
    "query": Object {
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/index.ts
@ -50,7 +50,7 @@ export const allUsers: SecuritySolutionFactory<UsersQueries.users> = {
      (bucket: AllUsersAggEsItem) => ({
        name: bucket.key,
        lastSeen: getOr(null, `lastSeen.value_as_string`, bucket),
-        domain: getOr(null, `domain.hits.hits[0]._source.user.domain`, bucket),
+        domain: getOr(null, `domain.hits.hits[0].fields['user.domain']`, bucket),
      }),
      {}
    );
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/query.all_users.dsl.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/all/query.all_users.dsl.ts
@ -5,7 +5,6 @@
 * 2.0.
 */

-import { isEmpty } from 'lodash/fp';
 import type { ISearchRequestParams } from '@kbn/data-plugin/common';
 import { Direction } from '../../../../../../common/search_strategy';
 import { createQueryFilterClauses } from '../../../../../utils/build_query';
@ -18,7 +17,6 @@ import { assertUnreachable } from '../../../../../../common/utility_types';

 export const buildUsersQuery = ({
  defaultIndex,
-  docValueFields,
  filterQuery,
  pagination: { querySize },
  sort,
@ -43,7 +41,6 @@ export const buildUsersQuery = ({
    ignore_unavailable: true,
    track_total_hits: false,
    body: {
-      ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}),
      aggregations: {
        user_count: { cardinality: { field: 'user.name' } },
        user_data: {
@ -60,15 +57,22 @@ export const buildUsersQuery = ({
                    },
                  },
                ],
-                _source: {
-                  includes: ['user.domain'],
-                },
+                _source: false,
              },
            },
          },
        },
      },
      query: { bool: { filter } },
+      _source: false,
+      fields: [
+        'user.name',
+        'user.domain',
+        {
+          field: '@timestamp',
+          format: 'strict_date_optional_time',
+        },
+      ],
      size: 0,
    },
  };
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/mocks/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/mocks/index.ts
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/dsl/query.dsl.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/dsl/query.dsl.ts
@ -5,26 +5,10 @@
 * 2.0.
 */

-import { isEmpty } from 'lodash/fp';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { UserAuthenticationsRequestOptions } from '../../../../../../../common/search_strategy/security_solution/users/authentications';
-import { sourceFieldsMap, hostFieldsMap } from '../../../../../../../common/ecs/ecs_fields';
-
 import { createQueryFilterClauses } from '../../../../../../utils/build_query';
-import { reduceFields } from '../../../../../../utils/build_query/reduce_fields';
-
 import { authenticationsFields } from '../helpers';
-import { extendMap } from '../../../../../../../common/ecs/ecs_fields/extend_map';
-
-export const auditdFieldsMap: Readonly<Record<string, string>> = {
-  latest: '@timestamp',
-  'lastSuccess.timestamp': 'lastSuccess.@timestamp',
-  'lastFailure.timestamp': 'lastFailure.@timestamp',
-  ...{ ...extendMap('lastSuccess', sourceFieldsMap) },
-  ...{ ...extendMap('lastSuccess', hostFieldsMap) },
-  ...{ ...extendMap('lastFailure', sourceFieldsMap) },
-  ...{ ...extendMap('lastFailure', hostFieldsMap) },
-};

 export const buildQuery = ({
  filterQuery,
@ -32,13 +16,7 @@ export const buildQuery = ({
  timerange: { from, to },
  pagination: { querySize },
  defaultIndex,
-  docValueFields,
 }: UserAuthenticationsRequestOptions) => {
-  const esFields = reduceFields(authenticationsFields, {
-    ...hostFieldsMap,
-    ...sourceFieldsMap,
-  }) as string[];
-
  const filter = [
    ...createQueryFilterClauses(filterQuery),
    { term: { 'event.category': 'authentication' } },
@ -52,13 +30,13 @@ export const buildQuery = ({
      },
    },
  ];
+  const queryFields = authenticationsFields.filter((field) => field !== 'timestamp');

  const dslQuery = {
    allow_no_indices: true,
    index: defaultIndex,
    ignore_unavailable: true,
    body: {
-      ...(!isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}),
      aggregations: {
        stack_by_count: {
          cardinality: {
@ -85,7 +63,7 @@ export const buildQuery = ({
                lastFailure: {
                  top_hits: {
                    size: 1,
-                    _source: esFields,
+                    _source: false,
                    sort: [{ '@timestamp': { order: 'desc' as const } }],
                  },
                },
@ -101,7 +79,7 @@ export const buildQuery = ({
                lastSuccess: {
                  top_hits: {
                    size: 1,
-                    _source: esFields,
+                    _source: false,
                    sort: [{ '@timestamp': { order: 'desc' as const } }],
                  },
                },
@ -116,6 +94,14 @@ export const buildQuery = ({
        },
      },
      size: 0,
+      _source: false,
+      fields: [
+        ...queryFields,
+        {
+          field: '@timestamp',
+          format: 'strict_date_optional_time',
+        },
+      ],
    },
    track_total_hits: false,
  };
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.test.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.test.ts
@ -6,15 +6,12 @@
 */

 import { AuthenticationsEdges } from '../../../../../../common/search_strategy';
-import { auditdFieldsMap } from './dsl/query.dsl';
-
 import { formatAuthenticationData } from './helpers';
 import { mockHit } from './__mocks__';

 describe('#formatAuthenticationsData', () => {
  test('it formats a authentication with an empty set', () => {
-    const fields: readonly string[] = [''];
-    const data = formatAuthenticationData(fields, mockHit, auditdFieldsMap);
+    const data = formatAuthenticationData(mockHit);
    const expected: AuthenticationsEdges = {
      cursor: {
        tiebreaker: null,
@ -32,8 +29,7 @@ describe('#formatAuthenticationsData', () => {
  });

  test('it formats a authentications with a source ip correctly', () => {
-    const fields: readonly string[] = ['lastSuccess.source.ip'];
-    const data = formatAuthenticationData(fields, mockHit, auditdFieldsMap);
+    const data = formatAuthenticationData(mockHit);
    const expected: AuthenticationsEdges = {
      cursor: {
        tiebreaker: null,
@ -51,8 +47,7 @@ describe('#formatAuthenticationsData', () => {
  });

  test('it formats a authentications with a host name only', () => {
-    const fields: readonly string[] = ['lastSuccess.host.name'];
-    const data = formatAuthenticationData(fields, mockHit, auditdFieldsMap);
+    const data = formatAuthenticationData(mockHit);
    const expected: AuthenticationsEdges = {
      cursor: {
        tiebreaker: null,
@ -70,8 +65,7 @@ describe('#formatAuthenticationsData', () => {
  });

  test('it formats a authentications with a host id only', () => {
-    const fields: readonly string[] = ['lastSuccess.host.id'];
-    const data = formatAuthenticationData(fields, mockHit, auditdFieldsMap);
+    const data = formatAuthenticationData(mockHit);
    const expected: AuthenticationsEdges = {
      cursor: {
        tiebreaker: null,
@ -89,8 +83,7 @@ describe('#formatAuthenticationsData', () => {
  });

  test('it formats a authentications with a host name and id correctly', () => {
-    const fields: readonly string[] = ['lastSuccess.host.name', 'lastSuccess.host.id'];
-    const data = formatAuthenticationData(fields, mockHit, auditdFieldsMap);
+    const data = formatAuthenticationData(mockHit);
    const expected: AuthenticationsEdges = {
      cursor: {
        tiebreaker: null,
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts
@ -7,8 +7,8 @@

 import { get, getOr, isEmpty } from 'lodash/fp';
 import { set } from '@elastic/safer-lodash-set/fp';
-import { mergeFieldsWithHit } from '../../../../../utils/build_query';
 import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array';
+import { sourceFieldsMap, hostFieldsMap } from '../../../../../../common/ecs/ecs_fields';
 import {
  AuthenticationsEdges,
  AuthenticationHit,
@ -17,78 +17,79 @@ import {
  StrategyResponseType,
 } from '../../../../../../common/search_strategy/security_solution';

-export const authenticationsFields = [
-  '_id',
-  'failures',
-  'successes',
-  'stackedValue',
-  'lastSuccess.timestamp',
-  'lastSuccess.source.ip',
-  'lastSuccess.host.id',
-  'lastSuccess.host.name',
-  'lastFailure.timestamp',
-  'lastFailure.source.ip',
-  'lastFailure.host.id',
-  'lastFailure.host.name',
-];
+export const authenticationsFields = ['timestamp', 'source.ip', 'host.id', 'host.name'];
+export const authenticationsFieldsMap: Readonly<Record<string, unknown>> = {
+  latest: '@timestamp',
+  lastSuccess: {
+    timestamp: '@timestamp',
+    ...sourceFieldsMap,
+    ...hostFieldsMap,
+  },
+  lastFailure: {
+    timestamp: '@timestamp',
+    ...sourceFieldsMap,
+    ...hostFieldsMap,
+  },
+};

-export const formatAuthenticationData = (
-  fields: readonly string[] = authenticationsFields,
-  hit: AuthenticationHit,
-  fieldMap: Readonly<Record<string, string>>
-): AuthenticationsEdges =>
-  fields.reduce<AuthenticationsEdges>(
-    (flattenedFields, fieldName) => {
-      if (hit.cursor) {
-        flattenedFields.cursor.value = hit.cursor;
-      }
-      flattenedFields.node = {
-        ...flattenedFields.node,
-        ...{
-          _id: hit._id,
-          stackedValue: [hit.stackedValue],
-          failures: hit.failures,
-          successes: hit.successes,
-        },
-      };
-      const mergedResult = mergeFieldsWithHit(fieldName, flattenedFields, fieldMap, hit);
-      const fieldPath = `node.${fieldName}`;
-      const fieldValue = get(fieldPath, mergedResult);
+export const formatAuthenticationData = (hit: AuthenticationHit): AuthenticationsEdges => {
+  let flattenedFields = {
+    node: {
+      _id: hit._id,
+      stackedValue: [hit.stackedValue],
+      failures: hit.failures,
+      successes: hit.successes,
+    },
+    cursor: {
+      value: hit.cursor,
+      tiebreaker: null,
+    },
+  };
+
+  const lastSuccessFields = getAuthenticationFields(authenticationsFields, hit, 'lastSuccess');
+  if (Object.keys(lastSuccessFields).length > 0) {
+    flattenedFields = set('node.lastSuccess', lastSuccessFields, flattenedFields);
+  }
+
+  const lastFailureFields = getAuthenticationFields(authenticationsFields, hit, 'lastFailure');
+  if (Object.keys(lastFailureFields).length > 0) {
+    flattenedFields = set('node.lastFailure', lastFailureFields, flattenedFields);
+  }
+
+  return flattenedFields;
+};
+
+const getAuthenticationFields = (fields: string[], hit: AuthenticationHit, parentField: string) => {
+  return fields.reduce((flattenedFields, fieldName) => {
+    const fieldPath = `${fieldName}`;
+    const esField = get(`${parentField}['${fieldName}']`, authenticationsFieldsMap);
+
+    if (!isEmpty(esField)) {
+      const fieldValue = get(`${parentField}['${esField}']`, hit.fields);
      if (!isEmpty(fieldValue)) {
        return set(
          fieldPath,
          toObjectArrayOfStrings(fieldValue).map(({ str }) => str),
-          mergedResult
+          flattenedFields
        );
-      } else {
-        return mergedResult;
      }
-    },
-    {
-      node: {
-        failures: 0,
-        successes: 0,
-        _id: '',
-        stackedValue: [''],
-      },
-      cursor: {
-        value: '',
-        tiebreaker: null,
-      },
    }
-  );
+
+    return flattenedFields;
+  }, {});
+};

 export const getHits = <T extends FactoryQueryTypes>(response: StrategyResponseType<T>) =>
  getOr([], 'aggregations.stack_by.buckets', response.rawResponse).map(
    (bucket: AuthenticationBucket) => ({
      _id: getOr(
        `${bucket.key}+${bucket.doc_count}`,
-        'failures.lastFailure.hits.hits[0].id',
+        'failures.lastFailure.hits.hits[0]._id',
        bucket
      ),
-      _source: {
-        lastSuccess: getOr(null, 'successes.lastSuccess.hits.hits[0]._source', bucket),
-        lastFailure: getOr(null, 'failures.lastFailure.hits.hits[0]._source', bucket),
+      fields: {
+        lastSuccess: getOr(null, 'successes.lastSuccess.hits.hits[0].fields', bucket),
+        lastFailure: getOr(null, 'failures.lastFailure.hits.hits[0].fields', bucket),
      },
      stackedValue: bucket.key,
      failures: bucket.failures.doc_count,
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/index.tsx
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/index.tsx
@ -20,9 +20,9 @@ import { UsersQueries } from '../../../../../../common/search_strategy/security_

 import { inspectStringifyObject } from '../../../../../utils/build_query';
 import { SecuritySolutionFactory } from '../../types';
-import { auditdFieldsMap, buildQuery as buildAuthenticationQuery } from './dsl/query.dsl';
+import { buildQuery as buildAuthenticationQuery } from './dsl/query.dsl';

-import { authenticationsFields, formatAuthenticationData, getHits } from './helpers';
+import { formatAuthenticationData, getHits } from './helpers';

 export const authentications: SecuritySolutionFactory<UsersQueries.authentications> = {
  buildDsl: (options: UserAuthenticationsRequestOptions) => {
@ -42,7 +42,7 @@ export const authentications: SecuritySolutionFactory<UsersQueries.authenticatio
    const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount;
    const hits: AuthenticationHit[] = getHits(response);
    const authenticationEdges: AuthenticationsEdges[] = hits.map((hit) =>
-      formatAuthenticationData(authenticationsFields, hit, auditdFieldsMap)
+      formatAuthenticationData(hit)
    );

    const edges = authenticationEdges.splice(cursorStart, querySize - cursorStart);
--- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/details/mocks/index.ts
+++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/details/mocks/index.ts
@ -12,12 +12,6 @@ import { UserDetailsRequestOptions } from '../../../../../../../common/search_st

 export const mockOptions: UserDetailsRequestOptions = {
  defaultIndex: ['test_indices*'],
-  docValueFields: [
-    {
-      field: '@timestamp',
-      format: 'date_time',
-    },
-  ],
  factoryQueryType: UsersQueries.details,
  filterQuery:
    '{"bool":{"must":[],"filter":[{"match_all":{}},{"match_phrase":{"user.name":{"query":"test_user"}}}],"should":[],"must_not":[]}}',