[EEM] Add index and alias pattern helpers (#190142)

This PR adds two helpers `entitiesIndexPattern` and
`entitiesAliasPattern` to resolve patterns for entities data.

The first one is meant to be used when **creating** indices while the
second is meant to be used when **querying** entities based on current
needs.

I've on purpose excluded the ability to provide wildcards or multiple
options to these functions to keep things simple and in line with our
_current_ needs.
I feel this also makes it easier to consume by being in a package
compared to the data access plugin.

This gives us a clear place to:
1. Encapsulate knowledge about how to access data from the Elastic
Entity Model
2. Gives us a clear place to expand when/if our needs change

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/packages/kbn-entities-schema/index.ts

@@ -8,6 +8,7 @@
 export * from './src/schema/entity_definition';
 export * from './src/schema/entity';
 export * from './src/schema/common';
+export * from './src/schema/patterns';
 export * from './src/rest_spec/delete';
 export * from './src/rest_spec/reset';
 export * from './src/rest_spec/get';

x-pack/packages/kbn-entities-schema/src/schema/patterns.test.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { entitiesIndexPattern, entitiesAliasPattern } from './patterns';
+
+describe('index/alias pattern helpers', () => {
+  describe('entitiesIndexPattern', () => {
+    it('generates a index pattern', () => {
+      expect(
+        entitiesIndexPattern({
+          definitionId: 'my-definition',
+          schemaVersion: 'v1',
+          dataset: 'latest',
+        })
+      ).toEqual('.entities.v1.latest.my-definition');
+    });
+  });
+
+  describe('entitiesAliasPattern', () => {
+    it('generates a alias pattern', () => {
+      expect(
+        entitiesAliasPattern({
+          type: 'service',
+          dataset: 'latest',
+        })
+      ).toEqual('entities-service-latest');
+    });
+  });
+});

x-pack/packages/kbn-entities-schema/src/schema/patterns.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const ENTITY_BASE_PREFIX = 'entities';
+export const ENTITY_HISTORY = 'history' as const;
+export const ENTITY_LATEST = 'latest' as const;
+
+export const ENTITY_SCHEMA_VERSION_V1 = 'v1';
+
+type SchemaVersion = `v${number}`;
+type Dataset = typeof ENTITY_LATEST | typeof ENTITY_HISTORY;
+
+interface IndexPatternOptions<TDataset extends Dataset> {
+  dataset: TDataset;
+  schemaVersion: SchemaVersion;
+  definitionId: string;
+}
+
+interface AliasPatternOptions<TDataset extends Dataset> {
+  dataset: TDataset;
+  type: string;
+}
+
+export function entitiesIndexPattern<TDataset extends Dataset>({
+  schemaVersion,
+  dataset,
+  definitionId,
+}: IndexPatternOptions<TDataset>) {
+  return `.${ENTITY_BASE_PREFIX}.${schemaVersion}.${dataset}.${definitionId}` as const;
+}
+
+export function entitiesAliasPattern<TDataset extends Dataset>({
+  type,
+  dataset,
+}: AliasPatternOptions<TDataset>) {
+  return `${ENTITY_BASE_PREFIX}-${type}-${dataset}` as const;
+}

x-pack/plugins/observability_solution/apm/server/lib/helpers/create_es_client/create_assets_es_client/create_assets_es_clients.ts

@@ -4,9 +4,11 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import type { KibanaRequest } from '@kbn/core/server';
 import { ElasticsearchClient } from '@kbn/core/server';
+import { entitiesAliasPattern, ENTITY_LATEST, ENTITY_HISTORY } from '@kbn/entities-schema';
 import { unwrapEsResponse } from '@kbn/observability-plugin/common/utils/unwrap_es_response';
 import {
   MsearchMultisearchBody,
@@ -15,8 +17,14 @@ import {
 import { withApmSpan } from '../../../../utils/with_apm_span';
 import { EntityType } from '../../../../routes/entities/types';
 
-const ENTITIES_LATEST_INDEX_NAME = `entities-${EntityType.SERVICE}-latest`;
-const ENTITIES_HISTORY_INDEX_NAME = `entities-${EntityType.SERVICE}-history`;
+const SERVICE_ENTITIES_LATEST_ALIAS = entitiesAliasPattern({
+  type: EntityType.SERVICE,
+  dataset: ENTITY_LATEST,
+});
+const SERVICE_ENTITIES_HISTORY_ALIAS = entitiesAliasPattern({
+  type: EntityType.SERVICE,
+  dataset: ENTITY_HISTORY,
+});
 
 export function cancelEsRequestOnAbort<T extends Promise<any>>(
   promise: T,
@@ -82,14 +90,14 @@ export async function createEntitiesESClient({
       operationName: string,
       searchRequest: TSearchRequest
     ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>> {
-      return search(ENTITIES_LATEST_INDEX_NAME, operationName, searchRequest);
+      return search(SERVICE_ENTITIES_LATEST_ALIAS, operationName, searchRequest);
     },
 
     searchHistory<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
       operationName: string,
       searchRequest: TSearchRequest
     ): Promise<InferSearchResponseOf<TDocument, TSearchRequest>> {
-      return search(ENTITIES_HISTORY_INDEX_NAME, operationName, searchRequest);
+      return search(SERVICE_ENTITIES_HISTORY_ALIAS, operationName, searchRequest);
     },
 
     async msearch<TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest>(
@@ -99,7 +107,7 @@ export async function createEntitiesESClient({
         .map((params) => {
           const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [
             {
-              index: [ENTITIES_LATEST_INDEX_NAME],
+              index: [SERVICE_ENTITIES_LATEST_ALIAS],
               ignore_unavailable: true,
             },
             {

x-pack/plugins/observability_solution/apm/tsconfig.json

@@ -126,7 +126,8 @@
     "@kbn/react-hooks",
     "@kbn/server-route-repository-utils",
     "@kbn/core-analytics-browser",
-    "@kbn/apm-types"
+    "@kbn/apm-types",
+    "@kbn/entities-schema"
   ],
   "exclude": [
     "target/**/*"

x-pack/plugins/observability_solution/entity_manager/common/constants_entities.ts

@@ -5,11 +5,15 @@
  * 2.0.
  */
 
+import {
+  ENTITY_BASE_PREFIX,
+  ENTITY_SCHEMA_VERSION_V1,
+  ENTITY_HISTORY,
+  ENTITY_LATEST,
+} from '@kbn/entities-schema';
+
 // Base constants
-export const ENTITY_BASE_PREFIX = 'entities';
-export const ENTITY_SCHEMA_VERSION_V1 = 'v1';
-export const ENTITY_INDEX_PREFIX = `.${ENTITY_BASE_PREFIX}` as const;
-export const ENTITY_INDICES_PATTERN = `${ENTITY_INDEX_PREFIX}*` as const;
+export const ENTITY_INTERNAL_INDICES_PATTERN = `.${ENTITY_BASE_PREFIX}*` as const;
 
 export const ENTITY_ENTITY_COMPONENT_TEMPLATE_V1 =
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_entity` as const;
@@ -17,24 +21,16 @@ export const ENTITY_EVENT_COMPONENT_TEMPLATE_V1 =
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_event` as const;
 
 // History constants
-export const ENTITY_HISTORY = 'history' as const;
 export const ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1 =
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_base` as const;
 export const ENTITY_HISTORY_PREFIX_V1 =
   `${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_HISTORY}` as const;
-export const ENTITY_HISTORY_BACKFILL_PREFIX_V1 =
-  `${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_HISTORY}-backfill` as const;
-export const ENTITY_HISTORY_INDEX_PREFIX_V1 =
-  `${ENTITY_INDEX_PREFIX}.${ENTITY_SCHEMA_VERSION_V1}.${ENTITY_HISTORY}` as const;
 
 // Latest constants
-export const ENTITY_LATEST = 'latest' as const;
 export const ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1 =
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_base` as const;
 export const ENTITY_LATEST_PREFIX_V1 =
   `${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_LATEST}` as const;
-export const ENTITY_LATEST_INDEX_PREFIX_V1 =
-  `${ENTITY_INDEX_PREFIX}.${ENTITY_SCHEMA_VERSION_V1}.${ENTITY_LATEST}` as const;
 
 // Transform constants
 export const ENTITY_DEFAULT_HISTORY_FREQUENCY = '1m';

x-pack/plugins/observability_solution/entity_manager/common/helpers.ts

@@ -7,10 +7,10 @@
 
 import {
   ENTITY_BASE_PREFIX,
+  ENTITY_SCHEMA_VERSION_V1,
   ENTITY_HISTORY,
   ENTITY_LATEST,
-  ENTITY_SCHEMA_VERSION_V1,
-} from './constants_entities';
+} from '@kbn/entities-schema';
 
 export const getEntityHistoryIndexTemplateV1 = (definitionId: string) =>
   `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_${definitionId}_index_template` as const;

x-pack/plugins/observability_solution/entity_manager/server/lib/auth/privileges.ts

@@ -6,7 +6,7 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
-import { ENTITY_INDICES_PATTERN } from '../../../common/constants_entities';
+import { ENTITY_INTERNAL_INDICES_PATTERN } from '../../../common/constants_entities';
 import { SO_ENTITY_DEFINITION_TYPE, SO_ENTITY_DISCOVERY_API_KEY_TYPE } from '../../saved_objects';
 import { BUILT_IN_ALLOWED_INDICES } from '../entities/built_in/constants';
 
@@ -58,11 +58,11 @@ export const entityDefinitionRuntimePrivileges = {
   cluster: ['manage_transform', 'manage_ingest_pipelines', 'manage_index_templates'],
   index: [
     {
-      names: [ENTITY_INDICES_PATTERN],
+      names: [ENTITY_INTERNAL_INDICES_PATTERN],
       privileges: ['create_index', 'index', 'create_doc', 'auto_configure', 'read'],
     },
     {
-      names: [...BUILT_IN_ALLOWED_INDICES, ENTITY_INDICES_PATTERN],
+      names: [...BUILT_IN_ALLOWED_INDICES, ENTITY_INTERNAL_INDICES_PATTERN],
       privileges: ['read', 'view_index_metadata'],
     },
   ],
@@ -79,7 +79,7 @@ export const entityDefinitionDeletionPrivileges = {
   cluster: ['manage_transform', 'manage_ingest_pipelines', 'manage_index_templates'],
   index: [
     {
-      names: [ENTITY_INDICES_PATTERN],
+      names: [ENTITY_INTERNAL_INDICES_PATTERN],
       privileges: ['delete_index'],
     },
   ],

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/generate_component_id.ts

@@ -5,40 +5,51 @@
  * 2.0.
  */
 
-import { EntityDefinition } from '@kbn/entities-schema';
 import {
-  ENTITY_HISTORY_BACKFILL_PREFIX_V1,
-  ENTITY_HISTORY_INDEX_PREFIX_V1,
+  ENTITY_HISTORY,
+  ENTITY_LATEST,
+  ENTITY_SCHEMA_VERSION_V1,
+  EntityDefinition,
+  entitiesIndexPattern,
+} from '@kbn/entities-schema';
+import {
   ENTITY_HISTORY_PREFIX_V1,
-  ENTITY_LATEST_INDEX_PREFIX_V1,
   ENTITY_LATEST_PREFIX_V1,
 } from '../../../../common/constants_entities';
 
 // History
 function generateHistoryId(definition: EntityDefinition) {
-  return `${ENTITY_HISTORY_PREFIX_V1}-${definition.id}`;
+  return `${ENTITY_HISTORY_PREFIX_V1}-${definition.id}` as const;
 }
 
 // History Backfill
 export function generateHistoryBackfillTransformId(definition: EntityDefinition) {
-  return `${ENTITY_HISTORY_BACKFILL_PREFIX_V1}-${definition.id}`;
+  return `${ENTITY_HISTORY_PREFIX_V1}-backfill-${definition.id}` as const;
 }
 
 export const generateHistoryTransformId = generateHistoryId;
 export const generateHistoryIngestPipelineId = generateHistoryId;
 
 export function generateHistoryIndexName(definition: EntityDefinition) {
-  return `${ENTITY_HISTORY_INDEX_PREFIX_V1}.${definition.id}`;
+  return entitiesIndexPattern({
+    schemaVersion: ENTITY_SCHEMA_VERSION_V1,
+    dataset: ENTITY_HISTORY,
+    definitionId: definition.id,
+  });
 }
 
 // Latest
 function generateLatestId(definition: EntityDefinition) {
-  return `${ENTITY_LATEST_PREFIX_V1}-${definition.id}`;
+  return `${ENTITY_LATEST_PREFIX_V1}-${definition.id}` as const;
 }
 
 export const generateLatestTransformId = generateLatestId;
 export const generateLatestIngestPipelineId = generateLatestId;
 
 export function generateLatestIndexName(definition: EntityDefinition) {
-  return `${ENTITY_LATEST_INDEX_PREFIX_V1}.${definition.id}`;
+  return entitiesIndexPattern({
+    schemaVersion: ENTITY_SCHEMA_VERSION_V1,
+    dataset: ENTITY_LATEST,
+    definitionId: definition.id,
+  });
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts

@@ -5,8 +5,7 @@
  * 2.0.
  */
 
-import { EntityDefinition } from '@kbn/entities-schema';
-import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities';
+import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1 } from '@kbn/entities-schema';
 import {
   initializePathScript,
   cleanScript,

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts

@@ -5,8 +5,7 @@
  * 2.0.
  */
 
-import { EntityDefinition } from '@kbn/entities-schema';
-import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities';
+import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1 } from '@kbn/entities-schema';
 import {
   initializePathScript,
   cleanScript,

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.ts

@@ -6,15 +6,18 @@
  */
 
 import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
-import { EntityDefinition } from '@kbn/entities-schema';
+import {
+  ENTITY_HISTORY,
+  EntityDefinition,
+  entitiesIndexPattern,
+  entitiesAliasPattern,
+  ENTITY_SCHEMA_VERSION_V1,
+} from '@kbn/entities-schema';
 import { getEntityHistoryIndexTemplateV1 } from '../../../../common/helpers';
 import {
-  ENTITY_BASE_PREFIX,
   ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
   ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
-  ENTITY_HISTORY,
   ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1,
-  ENTITY_HISTORY_INDEX_PREFIX_V1,
 } from '../../../../common/constants_entities';
 import { getCustomHistoryTemplateComponents } from '../../../templates/components/helpers';
 
@@ -36,11 +39,17 @@ export const getEntitiesHistoryIndexTemplateConfig = (
     ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
     ...getCustomHistoryTemplateComponents(definition.id),
   ],
-  index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.${definition.id}.*`],
+  index_patterns: [
+    `${entitiesIndexPattern({
+      schemaVersion: ENTITY_SCHEMA_VERSION_V1,
+      dataset: ENTITY_HISTORY,
+      definitionId: definition.id,
+    })}.*`,
+  ],
   priority: 200,
   template: {
     aliases: {
-      [`${ENTITY_BASE_PREFIX}-${definition.type}-${ENTITY_HISTORY}`]: {},
+      [entitiesAliasPattern({ type: definition.type, dataset: ENTITY_HISTORY })]: {},
     },
     mappings: {
       _meta: {

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.ts

@@ -6,15 +6,18 @@
  */
 
 import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
-import { EntityDefinition } from '@kbn/entities-schema';
+import {
+  ENTITY_LATEST,
+  ENTITY_SCHEMA_VERSION_V1,
+  EntityDefinition,
+  entitiesIndexPattern,
+  entitiesAliasPattern,
+} from '@kbn/entities-schema';
 import { getEntityLatestIndexTemplateV1 } from '../../../../common/helpers';
 import {
-  ENTITY_BASE_PREFIX,
   ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
   ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
-  ENTITY_LATEST,
   ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1,
-  ENTITY_LATEST_INDEX_PREFIX_V1,
 } from '../../../../common/constants_entities';
 import { getCustomLatestTemplateComponents } from '../../../templates/components/helpers';
 
@@ -36,11 +39,17 @@ export const getEntitiesLatestIndexTemplateConfig = (
     ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
     ...getCustomLatestTemplateComponents(definition.id),
   ],
-  index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.${definition.id}`],
+  index_patterns: [
+    entitiesIndexPattern({
+      schemaVersion: ENTITY_SCHEMA_VERSION_V1,
+      dataset: ENTITY_LATEST,
+      definitionId: definition.id,
+    }),
+  ],
   priority: 200,
   template: {
     aliases: {
-      [`${ENTITY_BASE_PREFIX}-${definition.type}-${ENTITY_LATEST}`]: {},
+      [entitiesAliasPattern({ type: definition.type, dataset: ENTITY_LATEST })]: {},
     },
     mappings: {
       _meta: {