[logging] Upgrade Kibana logs to ECS 8.4 (#136362)

* 8.1.0

* 8.2.0

* 8.3.0

* Upgrade ECS to 8.4.0

* Simplify EcsFields type in rule_registry.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Xavier Mouligneau <xavier.mouligneau@elastic.co>

packages/core/logging/core-logging-server-internal/src/__snapshots__/logging_system.test.ts.snap

@@ -16,7 +16,7 @@ exports[`asLoggerFactory() only allows to create new loggers. 1`] = `
 Object {
   "@timestamp": "2012-01-30T22:33:22.011-05:00",
   "ecs": Object {
-    "version": "8.0.0",
+    "version": "8.4.0",
   },
   "log": Object {
     "level": "TRACE",
@@ -33,7 +33,7 @@ exports[`asLoggerFactory() only allows to create new loggers. 2`] = `
 Object {
   "@timestamp": "2012-01-30T17:33:22.011-05:00",
   "ecs": Object {
-    "version": "8.0.0",
+    "version": "8.4.0",
   },
   "log": Object {
     "level": "INFO",
@@ -51,7 +51,7 @@ exports[`asLoggerFactory() only allows to create new loggers. 3`] = `
 Object {
   "@timestamp": "2012-01-30T12:33:22.011-05:00",
   "ecs": Object {
-    "version": "8.0.0",
+    "version": "8.4.0",
   },
   "log": Object {
     "level": "FATAL",
@@ -68,7 +68,7 @@ exports[`flushes memory buffer logger and switches to real logger once config is
 Object {
   "@timestamp": "2012-02-01T09:33:22.011-05:00",
   "ecs": Object {
-    "version": "8.0.0",
+    "version": "8.4.0",
   },
   "log": Object {
     "level": "INFO",
@@ -86,7 +86,7 @@ exports[`flushes memory buffer logger and switches to real logger once config is
 Object {
   "@timestamp": "2012-01-31T23:33:22.011-05:00",
   "ecs": Object {
-    "version": "8.0.0",
+    "version": "8.4.0",
   },
   "log": Object {
     "level": "INFO",

packages/core/logging/core-logging-server-internal/src/layouts/json_layout.test.ts

@@ -81,7 +81,7 @@ test('`format()` correctly formats record and includes correct ECS version.', ()
 
   for (const record of records) {
     const { ecs, ...restOfRecord } = JSON.parse(layout.format(record));
-    expect(ecs).toStrictEqual({ version: '8.0.0' });
+    expect(ecs).toStrictEqual({ version: '8.4.0' });
     expect(restOfRecord).toMatchSnapshot();
   }
 });

packages/core/logging/core-logging-server-internal/src/layouts/json_layout.ts

@@ -42,7 +42,7 @@ export class JsonLayout implements Layout {
     const transactionId = record.meta?.transaction?.id ?? record.transactionId;
 
     const log: Ecs = {
-      ecs: { version: '8.0.0' },
+      ecs: { version: '8.4.0' },
       '@timestamp': moment(record.timestamp).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
       message: record.message,
       error: JsonLayout.errorToSerializableObject(record.error),

packages/core/node/core-node-server-internal/src/node_service.ts

@@ -49,7 +49,6 @@ export class NodeService {
 
   public async preboot({ loggingSystem }: PrebootDeps): Promise<InternalNodeServicePreboot> {
     const roles = await this.getNodeRoles();
-    // @ts-expect-error Custom ECS field
     loggingSystem.setGlobalContext({ service: { node: { roles } } });
     this.log.info(`Kibana process configured with roles: [${roles.join(', ')}]`);
 

packages/kbn-logging/src/ecs/base.ts

@@ -13,7 +13,7 @@
  */
 export interface EcsBase {
   ['@timestamp']: string;
-  labels?: Record<string, unknown>;
+  labels?: Record<string, string>;
   message?: string;
   tags?: string[];
 }

packages/kbn-logging/src/ecs/container.ts

@@ -15,9 +15,11 @@ export interface EcsContainer {
   cpu?: { usage?: number };
   disk?: Disk;
   id?: string;
-  image?: { name?: string; tag?: string[] };
-  labels?: Record<string, unknown>;
+  image?: Image;
+  labels?: Record<string, string>;
+  memory?: { usage?: number };
   name?: string;
+  network?: Network;
   runtime?: string;
 }
 
@@ -25,3 +27,14 @@ interface Disk {
   read?: { bytes?: number };
   write?: { bytes?: number };
 }
+
+interface Image {
+  hash?: { all: string[] };
+  name?: string;
+  tag?: string[];
+}
+
+interface Network {
+  egress?: { bytes?: number };
+  ingress?: { bytes?: number };
+}

packages/kbn-logging/src/ecs/dll.ts

@@ -8,12 +8,10 @@
 
 import { EcsCodeSignature } from './code_signature';
 import { EcsHash } from './hash';
-import { EcsPe } from './pe';
 
 interface NestedFields {
   code_signature?: EcsCodeSignature;
   hash?: EcsHash;
-  pe?: EcsPe;
 }
 
 /**

packages/kbn-logging/src/ecs/dns.ts

@@ -13,7 +13,7 @@
  */
 export interface EcsDns {
   answers?: Answer[];
-  header_flags?: string[];
+  header_flags?: HeaderFlags[];
   id?: number;
   op_code?: string;
   question?: Question;
@@ -22,6 +22,8 @@ export interface EcsDns {
   type?: string;
 }
 
+type HeaderFlags = 'AA' | 'TC' | 'RD' | 'RA' | 'AD' | 'CD' | 'DO';
+
 interface Answer {
   data: string;
   class?: string;

packages/kbn-logging/src/ecs/event.ts

@@ -48,6 +48,7 @@ export type EcsEventCategory =
   | 'configuration'
   | 'database'
   | 'driver'
+  | 'email'
   | 'file'
   | 'host'
   | 'iam'

packages/kbn-logging/src/ecs/faas.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * https://www.elastic.co/guide/en/ecs/master/ecs-faas.html
+ *
+ * @internal
+ */
+export interface EcsFaas {
+  coldstart?: boolean;
+  execution?: string;
+  id?: string;
+  name?: string;
+  trigger?: Trigger;
+  version?: string;
+}
+
+interface Trigger {
+  request_id?: string;
+  type?: 'http' | 'pubsub' | 'datasource' | 'timer' | 'other';
+}

packages/kbn-logging/src/ecs/file.ts

@@ -9,14 +9,12 @@
 import { EcsCodeSignature } from './code_signature';
 import { EcsElf } from './elf';
 import { EcsHash } from './hash';
-import { EcsPe } from './pe';
 import { EcsX509 } from './x509';
 
 interface NestedFields {
   code_signature?: EcsCodeSignature;
   elf?: EcsElf;
   hash?: EcsHash;
-  pe?: EcsPe;
   x509?: EcsX509;
 }
 

packages/kbn-logging/src/ecs/hash.ts

@@ -15,6 +15,8 @@ export interface EcsHash {
   md5?: string;
   sha1?: string;
   sha256?: string;
+  sha384?: string;
   sha512?: string;
   ssdeep?: string;
+  tlsh?: string;
 }

packages/kbn-logging/src/ecs/host.ts

@@ -21,6 +21,7 @@ interface NestedFields {
  */
 export interface EcsHost extends NestedFields {
   architecture?: string;
+  boot?: { id: string };
   cpu?: { usage: number };
   disk?: Disk;
   domain?: string;
@@ -30,6 +31,7 @@ export interface EcsHost extends NestedFields {
   mac?: string[];
   name?: string;
   network?: Network;
+  pid_ns_ino?: string;
   type?: string;
   uptime?: number;
 }

packages/kbn-logging/src/ecs/index.ts

@@ -19,6 +19,7 @@ import { EcsDns } from './dns';
 import { EcsEmail } from './email';
 import { EcsError } from './error';
 import { EcsEvent } from './event';
+import { EcsFaas } from './faas';
 import { EcsFile } from './file';
 import { EcsGroup } from './group';
 import { EcsHost } from './host';
@@ -48,13 +49,13 @@ export type { EcsEventCategory, EcsEventKind, EcsEventOutcome, EcsEventType } fr
 
 interface EcsField {
   /**
-   * These typings were written as of ECS 8.0.0.
+   * These typings were written as of ECS 8.4.0.
    * Don't change this value without checking the rest
    * of the types to conform to that ECS version.
    *
    * https://www.elastic.co/guide/en/ecs/master/index.html
    */
-  version: '8.0.0';
+  version: '8.4.0';
 }
 
 /**
@@ -77,6 +78,7 @@ export type Ecs = EcsBase &
     email?: EcsEmail;
     error?: EcsError;
     event?: EcsEvent;
+    faas?: EcsFaas;
     file?: EcsFile;
     group?: EcsGroup;
     host?: EcsHost;

packages/kbn-logging/src/ecs/log.ts

@@ -25,7 +25,13 @@ interface Origin {
 }
 
 interface Syslog {
+  appname?: string;
   facility?: { code?: number; name?: string };
+  hostname?: string;
+  msgid?: string;
   priority?: number;
+  procid?: string;
   severity?: { code?: number; name?: string };
+  structured_data?: Record<string, string>;
+  version?: string;
 }

packages/kbn-logging/src/ecs/network.ts

@@ -22,7 +22,7 @@ export interface EcsNetwork extends NestedFields {
   application?: string;
   bytes?: number;
   community_id?: string;
-  direction?: string;
+  direction?: Direction;
   forwarded_ip?: string;
   iana_number?: string;
   name?: string;
@@ -31,3 +31,12 @@ export interface EcsNetwork extends NestedFields {
   transport?: string;
   type?: string;
 }
+
+type Direction =
+  | 'ingress'
+  | 'egress'
+  | 'inbound'
+  | 'outbound'
+  | 'internal'
+  | 'external'
+  | 'unknown';

packages/kbn-logging/src/ecs/orchestrator.ts

@@ -21,12 +21,16 @@ export interface EcsOrchestrator {
 }
 
 interface Cluster {
+  id?: string;
   name?: string;
   url?: string;
   version?: string;
 }
 
 interface Resource {
+  id?: string;
+  ip?: string[];
   name?: string;
+  parent?: { type: string };
   type?: string;
 }

packages/kbn-logging/src/ecs/os.ts

@@ -17,6 +17,6 @@ export interface EcsOs {
   kernel?: string;
   name?: string;
   platform?: string;
-  type?: string;
+  type?: 'linux' | 'macos' | 'unix' | 'windows';
   version?: string;
 }

packages/kbn-logging/src/ecs/pe.ts

@@ -18,5 +18,6 @@ export interface EcsPe {
   file_version?: string;
   imphash?: string;
   original_file_name?: string;
+  pehash?: string;
   product?: string;
 }

packages/kbn-logging/src/ecs/process.ts

@@ -8,16 +8,32 @@
 
 import { EcsCodeSignature } from './code_signature';
 import { EcsElf } from './elf';
+import { EcsGroup } from './group';
 import { EcsHash } from './hash';
-import { EcsPe } from './pe';
+import { EcsSource } from './source';
+import { EcsUser } from './user';
 
 interface NestedFields {
   code_signature?: EcsCodeSignature;
   elf?: EcsElf;
+  entry_leader?: EcsProcess;
+  group?: EcsGroup;
+  group_leader?: EcsProcess;
   hash?: EcsHash;
   parent?: EcsProcess;
-  pe?: EcsPe;
-  target?: EcsProcess;
+  previous?: EcsProcess;
+  real_group?: EcsGroup;
+  real_user?: EcsUser;
+  saved_group?: EcsGroup;
+  saved_user?: EcsUser;
+  session_leader?: EcsProcess & { entry_meta?: EntryMeta };
+  supplemental_groups?: EcsGroup;
+  user?: EcsUser;
+}
+
+interface EntryMeta {
+  type?: string;
+  source?: EcsSource;
 }
 
 /**
@@ -31,6 +47,7 @@ export interface EcsProcess extends NestedFields {
   command_line?: string;
   end?: string;
   entity_id?: string;
+  env_vars?: Record<string, string>;
   executable?: string;
   exit_code?: number;
   name?: string;

packages/kbn-logging/src/ecs/service.ts

@@ -17,8 +17,13 @@ export interface EcsService {
   ephemeral_id?: string;
   id?: string;
   name?: string;
-  node?: { name: string };
+  node?: Node;
   state?: string;
   type?: string;
   version?: string;
 }
+
+interface Node {
+  name?: string;
+  roles?: string[];
+}

packages/kbn-logging/src/ecs/threat.ts

@@ -30,6 +30,7 @@ interface IndicatorNestedFields {
 export interface EcsThreat {
   enrichments?: Enrichment[];
   indicator?: Indicator;
+  feed?: Feed;
   framework?: string;
   group?: Group;
   software?: Software;
@@ -43,20 +44,31 @@ interface Enrichment {
 }
 
 interface Indicator extends IndicatorNestedFields {
-  confidence?: string;
+  confidence?: 'Not Specified' | 'None' | 'Low' | 'Medium' | 'High';
   description?: string;
   email?: { address?: string };
   first_seen?: string;
   ip?: string;
   last_seen?: string;
-  marking?: { tlp?: string };
+  marking?: Marking;
   modified_at?: string;
   port?: number;
   provider?: string;
   reference?: string;
   scanner_stats?: number;
   sightings?: number;
-  type?: string;
+  type?: IndicatorType;
+}
+
+interface Feed {
+  dashboard_id?: string;
+  description?: string;
+  name?: string;
+  reference?: string;
+}
+
+interface Marking {
+  tlp?: 'WHITE' | 'GREEN' | 'AMBER' | 'RED';
 }
 
 interface Matched {
@@ -77,11 +89,23 @@ interface Group {
 interface Software {
   id?: string;
   name?: string;
-  platforms?: string[];
+  platforms?: SoftwarePlatforms[];
   reference?: string;
-  type?: string;
+  type?: 'Malware' | 'Tool';
 }
 
+type SoftwarePlatforms =
+  | 'AWS'
+  | 'Azure'
+  | 'Azure AD'
+  | 'GCP'
+  | 'Linux'
+  | 'macOS'
+  | 'Network'
+  | 'Office 365'
+  | 'SaaS'
+  | 'Windows';
+
 interface Tactic {
   id?: string[];
   name?: string[];
@@ -94,3 +118,22 @@ interface Technique {
   reference?: string[];
   subtechnique?: Technique;
 }
+
+type IndicatorType =
+  | 'autonomous-system'
+  | 'artifact'
+  | 'directory'
+  | 'domain-name'
+  | 'email-addr'
+  | 'file'
+  | 'ipv4-addr'
+  | 'ipv6-addr'
+  | 'mac-addr'
+  | 'mutex'
+  | 'port'
+  | 'process'
+  | 'software'
+  | 'url'
+  | 'user-account'
+  | 'windows-registry-key'
+  | 'x509-certificate';

x-pack/plugins/observability/public/pages/alerts/containers/alerts_table_t_grid/get_row_actions.tsx

@@ -12,7 +12,7 @@ import type { ObservabilityActionsProps } from './alerts_table_t_grid';
 
 const buildData = (alerts: EcsFieldsResponse): ObservabilityActionsProps['data'] => {
   return Object.entries(alerts).reduce<ObservabilityActionsProps['data']>(
-    (acc, [field, value]) => [...acc, { field, value }],
+    (acc, [field, value]) => [...acc, { field, value: value as string[] }],
     []
   );
 };

x-pack/plugins/rule_registry/common/search_strategy/index.ts

@@ -5,8 +5,6 @@
  * 2.0.
  */
 import { TechnicalRuleDataFieldName, ValidFeatureId } from '@kbn/rule-data-utils';
-// eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import { Ecs } from '@kbn/core/server';
 import { IEsSearchRequest, IEsSearchResponse } from '@kbn/data-plugin/common';
 import type {
   QueryDslFieldAndFormat,
@@ -27,54 +25,13 @@ export interface RuleRegistrySearchRequestPagination {
   pageSize: number;
 }
 
-type Prev = [
-  never,
-  0,
-  1,
-  2,
-  3,
-  4,
-  5,
-  6,
-  7,
-  8,
-  9,
-  10,
-  11,
-  12,
-  13,
-  14,
-  15,
-  16,
-  17,
-  18,
-  19,
-  20,
-  ...Array<0>
-];
-
-type Join<K, P> = K extends string | number
-  ? P extends string | number
-    ? `${K}${'' extends P ? '' : '.'}${P}`
-    : never
-  : string;
-
-type DotNestedKeys<T, D extends number = 10> = [D] extends [never]
-  ? never
-  : T extends object
-  ? { [K in keyof T]-?: Join<K, DotNestedKeys<T[K], Prev[D]>> }[keyof T]
-  : never;
-
-export type EcsFields = DotNestedKeys<Omit<Ecs, 'ecs'>>;
-
 export interface BasicFields {
   _id: string;
   _index: string;
 }
-
-export type EcsFieldsResponse = {
-  [Property in EcsFields]: string[];
-} & BasicFields & {
-    [Property in TechnicalRuleDataFieldName]?: string[];
-  };
+export type EcsFieldsResponse = BasicFields & {
+  [Property in TechnicalRuleDataFieldName]?: string[];
+} & {
+  [x: string]: unknown[];
+};
 export type RuleRegistrySearchResponse = IEsSearchResponse<EcsFieldsResponse>;

x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_flyout/alerts_flyout.test.tsx

@@ -6,6 +6,7 @@
  */
 import React from 'react';
 import { mountWithIntl, nextTick } from '@kbn/test-jest-helpers';
+import { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common/search_strategy';
 import { act } from 'react-dom/test-utils';
 import { AlertsFlyout } from './alerts_flyout';
 import { AlertsField } from '../../../../types';
@@ -18,7 +19,7 @@ const props = {
     [AlertsField.reason]: ['two'],
     _id: '0123456789',
     _index: '.alerts-default',
-  },
+  } as unknown as EcsFieldsResponse,
   alertsTableConfiguration: {
     id: 'test',
     columns: [

x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.tsx

@@ -220,7 +220,7 @@ const AlertsTable: React.FunctionComponent<AlertsTableProps> = (props: AlertsTab
       const alert = alerts[_props.rowIndex - pagination.pageSize * pagination.pageIndex];
       const data: Array<{ field: string; value: string[] }> = [];
       Object.entries(alert ?? {}).forEach(([key, value]) => {
-        data.push({ field: key, value });
+        data.push({ field: key, value: value as string[] });
       });
       return renderCellValue({
         ..._props,