[Fields Metadata] Add metadata fields static source (#188453)

## 📓 Summary

Closes #188443 

Adding a static source repository for [metadata
fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-fields.html#_indexing_metadata_fields)
in the resolution chain, so that it's now possible to retrieve metadata
info for them too.

**GET /internal/fields_metadata?fieldNames=_index,_source**
```json
{
  "fields": {
    "_index": {
      "dashed_name": "index",
      "description": "The index to which the document belongs. This metadata field specifies the exact index name in which the document is stored.",
      "example": "index_1",
      "flat_name": "_index",
      "name": "_index",
      "short": "The index to which the document belongs.",
      "type": "keyword",
      "documentation_url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index-field.html",
      "source": "metadata",
      "normalize": []
    },
    "_source": {
      "dashed_name": "source",
      "description": "The original JSON representing the body of the document. This field contains all the source data that was provided at the time of indexing.",
      "example": "{\"user\": \"John Doe\", \"message\": \"Hello\"}",
      "flat_name": "_source",
      "name": "_source",
      "short": "The original JSON representing the body of the document.",
      "documentation_url": "https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html",
      "source": "metadata",
      "normalize": [],
      "type": "unknown"
    }
  }
}
```

---------

Co-authored-by: Marco Antonio Ghiani <marcoantonio.ghiani@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

docs/developer/plugin-list.asciidoc

@@ -588,7 +588,7 @@ activities.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/fields_metadata/README.md[fieldsMetadata]
-|The @kbn/fields-metadata-plugin is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future.
+|The @kbn/fields-metadata-plugin is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS/Metadata definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future.
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload]

x-pack/plugins/fields_metadata/README.md

@@ -1,6 +1,6 @@
 # Fields Metadata Plugin
 
-The `@kbn/fields-metadata-plugin` is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future.
+The `@kbn/fields-metadata-plugin` is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS/Metadata definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future.
 
 ## Components and Mechanisms
 

x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts

@@ -14,6 +14,10 @@ export type FieldsMetadataMap = Record<string, FieldMetadata>;
 export class FieldsMetadataDictionary {
   private constructor(private readonly fields: FieldsMetadataMap) {}
 
+  getFields() {
+    return this.fields;
+  }
+
   pick(attributes: FieldAttribute[]): Record<string, PartialFieldMetadataPlain> {
     return mapValues(this.fields, (field) => field.pick(attributes));
   }

x-pack/plugins/fields_metadata/common/fields_metadata/types.ts

@@ -7,10 +7,12 @@
 
 import { EcsFlat } from '@elastic/ecs';
 import * as rt from 'io-ts';
+import { MetadataFields } from '../metadata_fields';
 
 export const fieldSourceRT = rt.keyof({
   ecs: null,
   integration: null,
+  metadata: null,
   unknown: null,
 });
 
@@ -63,6 +65,7 @@ const optionalMetadataPlainRT = rt.partial({
   short: rt.string,
   source: fieldSourceRT,
   type: rt.string,
+  documentation_url: rt.string,
 });
 
 export const partialFieldMetadataPlainRT = rt.intersection([
@@ -80,11 +83,14 @@ export const fieldAttributeRT = rt.union([
   rt.keyof(optionalMetadataPlainRT.props),
 ]);
 
+export type AnyFieldName = string & {};
+export type TMetadataFields = typeof MetadataFields;
+export type MetadataFieldName = keyof TMetadataFields;
 export type TEcsFields = typeof EcsFlat;
 export type EcsFieldName = keyof TEcsFields;
-export type IntegrationFieldName = string;
+export type IntegrationFieldName = AnyFieldName;
 
-export type FieldName = EcsFieldName | (IntegrationFieldName & {});
+export type FieldName = MetadataFieldName | EcsFieldName | IntegrationFieldName;
 export type FieldMetadataPlain = rt.TypeOf<typeof fieldMetadataPlainRT>;
 export type PartialFieldMetadataPlain = rt.TypeOf<typeof partialFieldMetadataPlainRT>;
 

x-pack/plugins/fields_metadata/common/index.ts

@@ -7,6 +7,7 @@
 
 export { fieldMetadataPlainRT } from './fields_metadata/types';
 export type {
+  AnyFieldName,
   EcsFieldName,
   FieldAttribute,
   FieldMetadataPlain,
@@ -14,6 +15,7 @@ export type {
   IntegrationFieldName,
   PartialFieldMetadataPlain,
   TEcsFields,
+  TMetadataFields,
 } from './fields_metadata/types';
 
 export { FieldMetadata } from './fields_metadata/models/field_metadata';

x-pack/plugins/fields_metadata/common/metadata_fields.ts

@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const MetadataFields = {
+  _index: {
+    dashed_name: 'index',
+    description:
+      'The index to which the document belongs. This metadata field specifies the exact index name in which the document is stored.',
+    example: 'index_1',
+    flat_name: '_index',
+    name: '_index',
+    short: 'The index to which the document belongs.',
+    type: 'keyword',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index-field.html',
+  },
+  _id: {
+    dashed_name: 'id',
+    description:
+      'The document’s ID. This unique identifier is used to fetch, update, or delete a document within an index.',
+    example: '1',
+    flat_name: '_id',
+    name: '_id',
+    short: 'The document’s ID.',
+    type: 'keyword',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-id-field.html',
+  },
+  _source: {
+    dashed_name: 'source',
+    description:
+      'The original JSON representing the body of the document. This field contains all the source data that was provided at the time of indexing.',
+    example: '{"user": "John Doe", "message": "Hello"}',
+    flat_name: '_source',
+    name: '_source',
+    short: 'The original JSON representing the body of the document.',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html',
+  },
+  _size: {
+    dashed_name: 'size',
+    description:
+      'The size of the _source field in bytes. Provided by the mapper-size plugin, this metadata field helps in understanding the storage impact of the document.',
+    example: '150',
+    flat_name: '_size',
+    name: '_size',
+    short: 'The size of the _source field in bytes, provided by the mapper-size plugin.',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/plugins/current/mapper-size.html',
+  },
+  _doc_count: {
+    dashed_name: 'doc_count',
+    description:
+      'A custom field used for storing document counts when a document represents pre-aggregated data. It helps in scenarios involving pre-computed data aggregation.',
+    example: '42',
+    flat_name: '_doc_count',
+    name: '_doc_count',
+    short:
+      'A custom field used for storing doc counts when a document represents pre-aggregated data.',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-doc-count-field.html',
+  },
+  _field_names: {
+    dashed_name: 'field_names',
+    description:
+      'All fields in the document which contain non-null values. This metadata field lists the field names that have valid data.',
+    example: '["user", "message"]',
+    flat_name: '_field_names',
+    name: '_field_names',
+    short: 'Fields with non-null values.',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-field-names-field.html',
+  },
+  _ignored: {
+    dashed_name: 'ignored',
+    description:
+      'All fields in the document that have been ignored at index time because of ignore_malformed. It indicates fields that were not indexed due to malformation.',
+    example: '["malformed_field"]',
+    flat_name: '_ignored',
+    name: '_ignored',
+    short: 'Fields ignored during indexing.',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-ignored-field.html',
+  },
+  _routing: {
+    dashed_name: 'routing',
+    description:
+      'A custom routing value which routes a document to a particular shard. This field is used to control the shard placement of a document.',
+    example: 'user_routing_value',
+    flat_name: '_routing',
+    name: '_routing',
+    short: 'Custom shard routing value.',
+    type: 'keyword',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-routing-field.html',
+  },
+  _meta: {
+    dashed_name: 'meta',
+    description:
+      'Application specific metadata. This field can store any custom metadata relevant to the application using Elasticsearch.',
+    example: '{"app": "my_app"}',
+    flat_name: '_meta',
+    name: '_meta',
+    short: 'Custom application metadata.',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-meta-field.html',
+  },
+  _tier: {
+    dashed_name: 'tier',
+    description:
+      'The current data tier preference of the index to which the document belongs. It helps in managing the index’s storage tier.',
+    example: 'hot',
+    flat_name: '_tier',
+    name: '_tier',
+    short: 'Index data tier preference.',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-tier-field.html',
+  },
+};

x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts

@@ -4,11 +4,12 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { FieldMetadata, TEcsFields } from '../../../common';
+import { FieldMetadata, TEcsFields, TMetadataFields } from '../../../common';
 import { loggerMock } from '@kbn/logging-mocks';
 import { FieldsMetadataClient } from './fields_metadata_client';
 import { EcsFieldsRepository } from './repositories/ecs_fields_repository';
 import { IntegrationFieldsRepository } from './repositories/integration_fields_repository';
+import { MetadataFieldsRepository } from './repositories/metadata_fields_repository';
 
 const ecsFields = {
   '@timestamp': {
@@ -26,6 +27,21 @@ const ecsFields = {
   },
 } as TEcsFields;
 
+const metadataFields = {
+  _index: {
+    dashed_name: 'index',
+    description:
+      'The index to which the document belongs. This metadata field specifies the exact index name in which the document is stored.',
+    example: 'index_1',
+    flat_name: '_index',
+    name: '_index',
+    short: 'The index to which the document belongs.',
+    type: 'keyword',
+    documentation_url:
+      'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index-field.html',
+  },
+} as TMetadataFields;
+
 const integrationFields = {
   '1password.item_usages': {
     'onepassword.client.platform_version': {
@@ -46,6 +62,7 @@ const integrationFields = {
 describe('FieldsMetadataClient class', () => {
   const logger = loggerMock.create();
   const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields });
+  const metadataFieldsRepository = MetadataFieldsRepository.create({ metadataFields });
   const integrationFieldsExtractor = jest.fn();
   integrationFieldsExtractor.mockImplementation(() => Promise.resolve(integrationFields));
 
@@ -60,12 +77,13 @@ describe('FieldsMetadataClient class', () => {
     fieldsMetadataClient = FieldsMetadataClient.create({
       ecsFieldsRepository,
       integrationFieldsRepository,
+      metadataFieldsRepository,
       logger,
     });
   });
 
   describe('#getByName', () => {
-    it('should resolve a single ECS FieldMetadata instance by default', async () => {
+    it('should resolve a single ECS/Metadata FieldMetadata instance by default', async () => {
       const timestampFieldInstance = await fieldsMetadataClient.getByName('@timestamp');
 
       expect(integrationFieldsExtractor).not.toHaveBeenCalled();
@@ -87,7 +105,7 @@ describe('FieldsMetadataClient class', () => {
       expect(timestampField.hasOwnProperty('type')).toBeTruthy();
     });
 
-    it('should attempt resolving the field from an integration if it does not exist in ECS  and the integration and dataset params are provided', async () => {
+    it('should attempt resolving the field from an integration if it does not exist in ECS/Metadata and the integration and dataset params are provided', async () => {
       const onePasswordFieldInstance = await fieldsMetadataClient.getByName(
         'onepassword.client.platform_version',
         { integration: '1password', dataset: '1password.item_usages' }

x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts

@@ -9,12 +9,14 @@ import { Logger } from '@kbn/core/server';
 import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common';
 import { EcsFieldsRepository } from './repositories/ecs_fields_repository';
 import { IntegrationFieldsRepository } from './repositories/integration_fields_repository';
+import { MetadataFieldsRepository } from './repositories/metadata_fields_repository';
 import { IntegrationFieldsSearchParams } from './repositories/types';
 import { FindFieldsMetadataOptions, IFieldsMetadataClient } from './types';
 
 interface FieldsMetadataClientDeps {
   logger: Logger;
   ecsFieldsRepository: EcsFieldsRepository;
+  metadataFieldsRepository: MetadataFieldsRepository;
   integrationFieldsRepository: IntegrationFieldsRepository;
 }
 
@@ -22,6 +24,7 @@ export class FieldsMetadataClient implements IFieldsMetadataClient {
   private constructor(
     private readonly logger: Logger,
     private readonly ecsFieldsRepository: EcsFieldsRepository,
+    private readonly metadataFieldsRepository: MetadataFieldsRepository,
     private readonly integrationFieldsRepository: IntegrationFieldsRepository
   ) {}
 
@@ -31,8 +34,13 @@ export class FieldsMetadataClient implements IFieldsMetadataClient {
   ): Promise<FieldMetadata | undefined> {
     this.logger.debug(`Retrieving field metadata for: ${fieldName}`);
 
-    // 1. Try resolving from ecs static metadata
-    let field = this.ecsFieldsRepository.getByName(fieldName);
+    // 1. Try resolving from metadata-fields static metadata
+    let field = this.metadataFieldsRepository.getByName(fieldName);
+
+    // 2. Try resolving from ecs static metadata
+    if (!field) {
+      field = this.ecsFieldsRepository.getByName(fieldName);
+    }
 
     // 2. Try searching for the fiels in the Elastic Package Registry
     if (!field && integration) {
@@ -48,7 +56,10 @@ export class FieldsMetadataClient implements IFieldsMetadataClient {
     dataset,
   }: FindFieldsMetadataOptions = {}): Promise<FieldsMetadataDictionary> {
     if (!fieldNames) {
-      return this.ecsFieldsRepository.find();
+      return FieldsMetadataDictionary.create({
+        ...this.metadataFieldsRepository.find().getFields(),
+        ...this.ecsFieldsRepository.find().getFields(),
+      });
     }
 
     const fields: Record<string, FieldMetadata> = {};
@@ -66,8 +77,14 @@ export class FieldsMetadataClient implements IFieldsMetadataClient {
   public static create({
     logger,
     ecsFieldsRepository,
+    metadataFieldsRepository,
     integrationFieldsRepository,
   }: FieldsMetadataClientDeps) {
-    return new FieldsMetadataClient(logger, ecsFieldsRepository, integrationFieldsRepository);
+    return new FieldsMetadataClient(
+      logger,
+      ecsFieldsRepository,
+      metadataFieldsRepository,
+      integrationFieldsRepository
+    );
   }
 }

x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts

@@ -10,8 +10,10 @@ import { Logger } from '@kbn/core/server';
 import { FieldsMetadataClient } from './fields_metadata_client';
 import { EcsFieldsRepository } from './repositories/ecs_fields_repository';
 import { IntegrationFieldsRepository } from './repositories/integration_fields_repository';
+import { MetadataFieldsRepository } from './repositories/metadata_fields_repository';
 import { IntegrationFieldsExtractor } from './repositories/types';
 import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types';
+import { MetadataFields as metadataFields } from '../../../common/metadata_fields';
 
 export class FieldsMetadataService {
   private integrationFieldsExtractor: IntegrationFieldsExtractor = () => Promise.resolve({});
@@ -30,6 +32,7 @@ export class FieldsMetadataService {
     const { logger, integrationFieldsExtractor } = this;
 
     const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields });
+    const metadataFieldsRepository = MetadataFieldsRepository.create({ metadataFields });
     const integrationFieldsRepository = IntegrationFieldsRepository.create({
       integrationFieldsExtractor,
     });
@@ -39,6 +42,7 @@ export class FieldsMetadataService {
         return FieldsMetadataClient.create({
           logger,
           ecsFieldsRepository,
+          metadataFieldsRepository,
           integrationFieldsRepository,
         });
       },

x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts

@@ -7,18 +7,18 @@
 
 import mapValues from 'lodash/mapValues';
 import { FieldsMetadataDictionary } from '../../../../common/fields_metadata/models/fields_metadata_dictionary';
-import { FieldMetadata, FieldName, TEcsFields } from '../../../../common';
+import { AnyFieldName, EcsFieldName, FieldMetadata, TEcsFields } from '../../../../common';
 
 interface EcsFieldsRepositoryDeps {
   ecsFields: TEcsFields;
 }
 
 interface FindOptions {
-  fieldNames?: FieldName[];
+  fieldNames?: EcsFieldName[];
 }
 
 export class EcsFieldsRepository {
-  private readonly ecsFields: Record<FieldName, FieldMetadata>;
+  private readonly ecsFields: Record<EcsFieldName, FieldMetadata>;
 
   private constructor(ecsFields: TEcsFields) {
     this.ecsFields = mapValues(ecsFields, (field) =>
@@ -26,8 +26,8 @@ export class EcsFieldsRepository {
     );
   }
 
-  getByName(fieldName: FieldName): FieldMetadata | undefined {
-    return this.ecsFields[fieldName];
+  getByName(fieldName: EcsFieldName | AnyFieldName): FieldMetadata | undefined {
+    return this.ecsFields[fieldName as EcsFieldName];
   }
 
   find({ fieldNames }: FindOptions = {}): FieldsMetadataDictionary {
@@ -43,7 +43,7 @@ export class EcsFieldsRepository {
       }
 
       return fieldsMetadata;
-    }, {} as Record<FieldName, FieldMetadata>);
+    }, {} as Record<EcsFieldName, FieldMetadata>);
 
     return FieldsMetadataDictionary.create(fields);
   }

x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/metadata_fields_repository.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import mapValues from 'lodash/mapValues';
+import { MetadataFieldName } from '../../../../common/fields_metadata';
+import { FieldsMetadataDictionary } from '../../../../common/fields_metadata/models/fields_metadata_dictionary';
+import { AnyFieldName, FieldMetadata, TMetadataFields } from '../../../../common';
+
+interface MetadataFieldsRepositoryDeps {
+  metadataFields: TMetadataFields;
+}
+
+interface FindOptions {
+  fieldNames?: MetadataFieldName[];
+}
+
+export class MetadataFieldsRepository {
+  private readonly metadataFields: Record<MetadataFieldName, FieldMetadata>;
+
+  private constructor(metadataFields: TMetadataFields) {
+    this.metadataFields = mapValues(metadataFields, (field) =>
+      FieldMetadata.create({ ...field, source: 'metadata' })
+    );
+  }
+
+  getByName(fieldName: MetadataFieldName | AnyFieldName): FieldMetadata | undefined {
+    return this.metadataFields[fieldName as MetadataFieldName];
+  }
+
+  find({ fieldNames }: FindOptions = {}): FieldsMetadataDictionary {
+    if (!fieldNames) {
+      return FieldsMetadataDictionary.create(this.metadataFields);
+    }
+
+    const fields = fieldNames.reduce((fieldsMetadata, fieldName) => {
+      const field = this.getByName(fieldName);
+
+      if (field) {
+        fieldsMetadata[fieldName] = field;
+      }
+
+      return fieldsMetadata;
+    }, {} as Record<MetadataFieldName, FieldMetadata>);
+
+    return FieldsMetadataDictionary.create(fields);
+  }
+
+  public static create({ metadataFields }: MetadataFieldsRepositoryDeps) {
+    return new MetadataFieldsRepository(metadataFields);
+  }
+}