github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[eem] remove history transforms (#193999)

Browse source 
### Summary

Remove history and backfill transforms, leaving latest transform in
place.

Notable changes to latest transform:
- it does not read from history output anymore but source indices
defined on the definition
- it defines a `latest.lookbackPeriod` to limit the amount of data
ingested, which defaults to 24h
- each metadata aggregation now accepts a
`metadata.aggregation.lookbackPeriod` which defaults to the
`latest.lookbackPeriod`
- `entity.firstSeenTimestamp` is removed. this should be temporary until
we have a solution for
https://github.com/elastic/elastic-entity-model/issues/174
- latest metrics used to get the latest pre-computed value from history
data, but is it now aggregating over the `lookbackPeriod` in the source
indices (which can be filtered down with `metrics.filter`)
- `latest` block on the entity definition is now mandatory

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Mark Hopkin <mark.hopkin@elastic.co>
This commit is contained in:
Kevin Lacabane 2024-10-09 23:15:33 +02:00 • committed by GitHub
parent 742cd1336e
commit 8f8e9883e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
59 changed files with 742 additions and 2425 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
View file

@ -91,7 +91,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e",
"endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
"enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
"entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9",
"entity-definition": "e3811fd5fbb878d170067c0d6897a2e63010af36",
"entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
"entity-engine-status": "0738aa1a06d3361911740f8f166071ea43a00927",
"epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",

12
x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap generated
View file

@ -78,7 +78,8 @@ exports[`schemas metadataSchema should parse successfully with a source and desi
Object {
"data": Object {
"aggregation": Object {
"limit": 1000,
"limit": 10,
"lookbackPeriod": undefined,
"type": "terms",
},
"destination": "hostName",
@ -92,7 +93,8 @@ exports[`schemas metadataSchema should parse successfully with an valid string 1
Object {
"data": Object {
"aggregation": Object {
"limit": 1000,
"limit": 10,
"lookbackPeriod": undefined,
"type": "terms",
},
"destination": "host.name",
@ -106,7 +108,8 @@ exports[`schemas metadataSchema should parse successfully with just a source 1`]
Object {
"data": Object {
"aggregation": Object {
"limit": 1000,
"limit": 10,
"lookbackPeriod": undefined,
"type": "terms",
},
"destination": "host.name",
@ -120,7 +123,8 @@ exports[`schemas metadataSchema should parse successfully with valid object 1`]
Object {
"data": Object {
"aggregation": Object {
"limit": 1000,
"limit": 10,
"lookbackPeriod": undefined,
"type": "terms",
},
"destination": "hostName",

30
x-pack/packages/kbn-entities-schema/src/schema/common.test.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { durationSchema, metadataSchema, semVerSchema, historySettingsSchema } from './common';
import { durationSchema, metadataSchema, semVerSchema } from './common';
describe('schemas', () => {
describe('metadataSchema', () => {
@ -66,7 +66,7 @@ describe('schemas', () => {
expect(result.data).toEqual({
source: 'host.name',
destination: 'hostName',
aggregation: { type: 'terms', limit: 1000 },
aggregation: { type: 'terms', limit: 10, lookbackPeriod: undefined },
});
});
@ -139,30 +139,4 @@ describe('schemas', () => {
expect(result).toMatchSnapshot();
});
});
describe('historySettingsSchema', () => {
it('should return default values when not defined', () => {
let result = historySettingsSchema.safeParse(undefined);
expect(result.success).toBeTruthy();
expect(result.data).toEqual({ lookbackPeriod: '1h' });
result = historySettingsSchema.safeParse({ syncDelay: '1m' });
expect(result.success).toBeTruthy();
expect(result.data).toEqual({ syncDelay: '1m', lookbackPeriod: '1h' });
});
it('should return user defined values when defined', () => {
const result = historySettingsSchema.safeParse({
lookbackPeriod: '30m',
syncField: 'event.ingested',
syncDelay: '5m',
});
expect(result.success).toBeTruthy();
expect(result.data).toEqual({
lookbackPeriod: '30m',
syncField: 'event.ingested',
syncDelay: '5m',
});
});
});
});

10
x-pack/packages/kbn-entities-schema/src/schema/common.ts
View file

@ -85,7 +85,11 @@ export const keyMetricSchema = z.object({
export type KeyMetric = z.infer<typeof keyMetricSchema>;
export const metadataAggregation = z.union([
z.object({ type: z.literal('terms'), limit: z.number().default(1000) }),
z.object({
type: z.literal('terms'),
limit: z.number().default(10),
lookbackPeriod: z.optional(durationSchema),
}),
z.object({
type: z.literal('top_value'),
sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])),
@ -99,13 +103,13 @@ export const metadataSchema = z
destination: z.optional(z.string()),
aggregation: z
.optional(metadataAggregation)
.default({ type: z.literal('terms').value, limit: 1000 }),
.default({ type: z.literal('terms').value, limit: 10, lookbackPeriod: undefined }),
})
.or(
z.string().transform((value) => ({
source: value,
destination: value,
aggregation: { type: z.literal('terms').value, limit: 1000 },
aggregation: { type: z.literal('terms').value, limit: 10, lookbackPeriod: undefined },
}))
)
.transform((metadata) => ({

1
x-pack/packages/kbn-entities-schema/src/schema/entity.ts
View file

@ -35,7 +35,6 @@ export const entityLatestSchema = z
entity: entityBaseSchema.merge(
z.object({
lastSeenTimestamp: z.string(),
firstSeenTimestamp: z.string(),
})
),
})

39
x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts
View file

@ -14,8 +14,6 @@ import {
durationSchema,
identityFieldsSchema,
semVerSchema,
historySettingsSchema,
durationSchemaWithMinimum,
} from './common';
export const entityDefinitionSchema = z.object({
@ -32,22 +30,17 @@ export const entityDefinitionSchema = z.object({
metrics: z.optional(z.array(keyMetricSchema)),
staticFields: z.optional(z.record(z.string(), z.string())),
managed: z.optional(z.boolean()).default(false),
history: z.object({
latest: z.object({
timestampField: z.string(),
interval: durationSchemaWithMinimum(1),
settings: historySettingsSchema,
lookbackPeriod: z.optional(durationSchema).default('24h'),
settings: z.optional(
z.object({
syncField: z.optional(z.string()),
syncDelay: z.optional(durationSchema),
frequency: z.optional(durationSchema),
})
),
}),
latest: z.optional(
z.object({
settings: z.optional(
z.object({
syncField: z.optional(z.string()),
syncDelay: z.optional(durationSchema),
frequency: z.optional(durationSchema),
})
),
})
),
installStatus: z.optional(
z.union([
z.literal('installing'),
@ -57,6 +50,18 @@ export const entityDefinitionSchema = z.object({
])
),
installStartedAt: z.optional(z.string()),
installedComponents: z.optional(
z.array(
z.object({
type: z.union([
z.literal('transform'),
z.literal('ingest_pipeline'),
z.literal('template'),
]),
id: z.string(),
})
)
),
});
export const entityDefinitionUpdateSchema = entityDefinitionSchema
@ -69,7 +74,7 @@ export const entityDefinitionUpdateSchema = entityDefinitionSchema
.partial()
.merge(
z.object({
history: z.optional(entityDefinitionSchema.shape.history.partial()),
latest: z.optional(entityDefinitionSchema.shape.latest.partial()),
version: semVerSchema,
})
);

8
x-pack/plugins/entity_manager/common/constants_entities.ts
View file

@ -33,8 +33,6 @@ export const ENTITY_LATEST_PREFIX_V1 =
`${ENTITY_BASE_PREFIX}-${ENTITY_SCHEMA_VERSION_V1}-${ENTITY_LATEST}` as const;
// Transform constants
export const ENTITY_DEFAULT_HISTORY_FREQUENCY = '1m';
export const ENTITY_DEFAULT_HISTORY_SYNC_DELAY = '60s';
export const ENTITY_DEFAULT_LATEST_FREQUENCY = '30s';
export const ENTITY_DEFAULT_LATEST_SYNC_DELAY = '1s';
export const ENTITY_DEFAULT_METADATA_LIMIT = 1000;
export const ENTITY_DEFAULT_LATEST_FREQUENCY = '1m';
export const ENTITY_DEFAULT_LATEST_SYNC_DELAY = '60s';
export const ENTITY_DEFAULT_METADATA_LIMIT = 10;

4
x-pack/plugins/entity_manager/server/lib/entities/built_in/containers_from_ecs_data.ts
View file

@ -20,9 +20,9 @@ export const builtInContainersFromEcsEntityDefinition: EntityDefinition =
indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'],
identityFields: ['container.id'],
displayNameTemplate: '{{container.id}}',
history: {
latest: {
timestampField: '@timestamp',
interval: '5m',
lookbackPeriod: '10m',
settings: {
frequency: '5m',
},

4
x-pack/plugins/entity_manager/server/lib/entities/built_in/hosts_from_ecs_data.ts
View file

@ -19,9 +19,9 @@ export const builtInHostsFromEcsEntityDefinition: EntityDefinition = entityDefin
indexPatterns: ['filebeat-*', 'logs-*', 'metrics-*', 'metricbeat-*'],
identityFields: ['host.name'],
displayNameTemplate: '{{host.name}}',
history: {
latest: {
timestampField: '@timestamp',
interval: '5m',
lookbackPeriod: '10m',
settings: {
frequency: '5m',
},

5
x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts
View file

@ -18,11 +18,10 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
type: 'service',
managed: true,
indexPatterns: ['logs-*', 'filebeat*', 'traces-apm*'],
history: {
latest: {
timestampField: '@timestamp',
interval: '1m',
lookbackPeriod: '10m',
settings: {
lookbackPeriod: '10m',
frequency: '2m',
syncDelay: '2m',
},

40
x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts
View file

@ -7,46 +7,15 @@
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import {
generateHistoryIngestPipelineId,
generateLatestIngestPipelineId,
} from './helpers/generate_component_id';
import { generateLatestIngestPipelineId } from './helpers/generate_component_id';
import { retryTransientEsErrors } from './helpers/retry';
import { generateHistoryProcessors } from './ingest_pipeline/generate_history_processors';
import { generateLatestProcessors } from './ingest_pipeline/generate_latest_processors';
export async function createAndInstallHistoryIngestPipeline(
export async function createAndInstallIngestPipelines(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
try {
const historyProcessors = generateHistoryProcessors(definition);
const historyId = generateHistoryIngestPipelineId(definition);
await retryTransientEsErrors(
() =>
esClient.ingest.putPipeline({
id: historyId,
processors: historyProcessors,
_meta: {
definitionVersion: definition.version,
managed: definition.managed,
},
}),
{ logger }
);
} catch (e) {
logger.error(
`Cannot create entity history ingest pipelines for [${definition.id}] entity defintion`
);
throw e;
}
}
export async function createAndInstallLatestIngestPipeline(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
): Promise<Array<{ type: 'ingest_pipeline'; id: string }>> {
try {
const latestProcessors = generateLatestProcessors(definition);
const latestId = generateLatestIngestPipelineId(definition);
@ -62,9 +31,10 @@ export async function createAndInstallLatestIngestPipeline(
}),
{ logger }
);
return [{ type: 'ingest_pipeline', id: latestId }];
} catch (e) {
logger.error(
`Cannot create entity latest ingest pipelines for [${definition.id}] entity defintion`
`Cannot create entity latest ingest pipelines for [${definition.id}] entity definition`
);
throw e;
}

45
x-pack/plugins/entity_manager/server/lib/entities/create_and_install_transform.ts
View file

@ -9,57 +9,20 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import { retryTransientEsErrors } from './helpers/retry';
import { generateLatestTransform } from './transform/generate_latest_transform';
import {
generateBackfillHistoryTransform,
generateHistoryTransform,
} from './transform/generate_history_transform';
export async function createAndInstallHistoryTransform(
export async function createAndInstallTransforms(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
try {
const historyTransform = generateHistoryTransform(definition);
await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), {
logger,
});
} catch (e) {
logger.error(`Cannot create entity history transform for [${definition.id}] entity definition`);
throw e;
}
}
export async function createAndInstallHistoryBackfillTransform(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
try {
const historyTransform = generateBackfillHistoryTransform(definition);
await retryTransientEsErrors(() => esClient.transform.putTransform(historyTransform), {
logger,
});
} catch (e) {
logger.error(
`Cannot create entity history backfill transform for [${definition.id}] entity definition`
);
throw e;
}
}
export async function createAndInstallLatestTransform(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
): Promise<Array<{ type: 'transform'; id: string }>> {
try {
const latestTransform = generateLatestTransform(definition);
await retryTransientEsErrors(() => esClient.transform.putTransform(latestTransform), {
logger,
});
return [{ type: 'transform', id: latestTransform.transform_id }];
} catch (e) {
logger.error(`Cannot create entity latest transform for [${definition.id}] entity definition`);
logger.error(`Cannot create entity history transform for [${definition.id}] entity definition`);
throw e;
}
}

24
x-pack/plugins/entity_manager/server/lib/entities/delete_ingest_pipeline.ts
View file

@ -7,24 +7,24 @@
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import {
generateHistoryIngestPipelineId,
generateLatestIngestPipelineId,
} from './helpers/generate_component_id';
import { retryTransientEsErrors } from './helpers/retry';
import { generateLatestIngestPipelineId } from './helpers/generate_component_id';
export async function deleteHistoryIngestPipeline(
export async function deleteIngestPipelines(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
try {
const historyPipelineId = generateHistoryIngestPipelineId(definition);
await retryTransientEsErrors(() =>
esClient.ingest.deletePipeline({ id: historyPipelineId }, { ignore: [404] })
await Promise.all(
(definition.installedComponents ?? [])
.filter(({ type }) => type === 'ingest_pipeline')
.map(({ id }) =>
retryTransientEsErrors(() => esClient.ingest.deletePipeline({ id }, { ignore: [404] }))
)
);
} catch (e) {
logger.error(`Unable to delete history ingest pipeline [${definition.id}]: ${e}`);
logger.error(`Unable to delete ingest pipelines for definition [${definition.id}]: ${e}`);
throw e;
}
}
@ -35,9 +35,11 @@ export async function deleteLatestIngestPipeline(
logger: Logger
) {
try {
const latestPipelineId = generateLatestIngestPipelineId(definition);
await retryTransientEsErrors(() =>
esClient.ingest.deletePipeline({ id: latestPipelineId }, { ignore: [404] })
esClient.ingest.deletePipeline(
{ id: generateLatestIngestPipelineId(definition) },
{ ignore: [404] }
)
);
} catch (e) {
logger.error(`Unable to delete latest ingest pipeline [${definition.id}]: ${e}`);

57
x-pack/plugins/entity_manager/server/lib/entities/delete_transforms.ts
View file

@ -7,14 +7,8 @@
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import {
generateHistoryTransformId,
generateHistoryBackfillTransformId,
generateLatestTransformId,
} from './helpers/generate_component_id';
import { retryTransientEsErrors } from './helpers/retry';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
import { generateLatestTransformId } from './helpers/generate_component_id';
export async function deleteTransforms(
esClient: ElasticsearchClient,
@ -22,37 +16,42 @@ export async function deleteTransforms(
logger: Logger
) {
try {
const historyTransformId = generateHistoryTransformId(definition);
const latestTransformId = generateLatestTransformId(definition);
await retryTransientEsErrors(
() =>
esClient.transform.deleteTransform(
{ transform_id: historyTransformId, force: true },
{ ignore: [404] }
),
{ logger }
await Promise.all(
(definition.installedComponents ?? [])
.filter(({ type }) => type === 'transform')
.map(({ id }) =>
retryTransientEsErrors(
() =>
esClient.transform.deleteTransform(
{ transform_id: id, force: true },
{ ignore: [404] }
),
{ logger }
)
)
);
if (isBackfillEnabled(definition)) {
const historyBackfillTransformId = generateHistoryBackfillTransformId(definition);
await retryTransientEsErrors(
() =>
esClient.transform.deleteTransform(
{ transform_id: historyBackfillTransformId, force: true },
{ ignore: [404] }
),
{ logger }
);
}
} catch (e) {
logger.error(`Cannot delete transforms for definition [${definition.id}]: ${e}`);
throw e;
}
}
export async function deleteLatestTransform(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
try {
await retryTransientEsErrors(
() =>
esClient.transform.deleteTransform(
{ transform_id: latestTransformId, force: true },
{ transform_id: generateLatestTransformId(definition), force: true },
{ ignore: [404] }
),
{ logger }
);
} catch (e) {
logger.error(`Cannot delete history transform [${definition.id}]: ${e}`);
logger.error(`Cannot delete latest transform for definition [${definition.id}]: ${e}`);
throw e;
}
}

33
x-pack/plugins/entity_manager/server/lib/entities/find_entity_definition.ts
View file

@ -10,18 +10,8 @@ import { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/serve
import { EntityDefinition } from '@kbn/entities-schema';
import { NodesIngestTotal } from '@elastic/elasticsearch/lib/api/types';
import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
import {
generateHistoryTransformId,
generateHistoryBackfillTransformId,
generateHistoryIngestPipelineId,
generateHistoryIndexTemplateId,
generateLatestTransformId,
generateLatestIngestPipelineId,
generateLatestIndexTemplateId,
} from './helpers/generate_component_id';
import { BUILT_IN_ID_PREFIX } from './built_in';
import { EntityDefinitionState, EntityDefinitionWithState } from './types';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
export async function findEntityDefinitions({
soClient,
@ -120,11 +110,9 @@ async function getTransformState({
definition: EntityDefinition;
esClient: ElasticsearchClient;
}) {
const transformIds = [
generateHistoryTransformId(definition),
generateLatestTransformId(definition),
...(isBackfillEnabled(definition) ? [generateHistoryBackfillTransformId(definition)] : []),
];
const transformIds = (definition.installedComponents ?? [])
.filter(({ type }) => type === 'transform')
.map(({ id }) => id);
const transformStats = await Promise.all(
transformIds.map((id) => esClient.transform.getTransformStats({ transform_id: id }))
@ -152,10 +140,10 @@ async function getIngestPipelineState({
definition: EntityDefinition;
esClient: ElasticsearchClient;
}) {
const ingestPipelineIds = [
generateHistoryIngestPipelineId(definition),
generateLatestIngestPipelineId(definition),
];
const ingestPipelineIds = (definition.installedComponents ?? [])
.filter(({ type }) => type === 'ingest_pipeline')
.map(({ id }) => id);
const [ingestPipelines, ingestPipelinesStats] = await Promise.all([
esClient.ingest.getPipeline({ id: ingestPipelineIds.join(',') }, { ignore: [404] }),
esClient.nodes.stats({
@ -193,10 +181,9 @@ async function getIndexTemplatesState({
definition: EntityDefinition;
esClient: ElasticsearchClient;
}) {
const indexTemplatesIds = [
generateLatestIndexTemplateId(definition),
generateHistoryIndexTemplateId(definition),
];
const indexTemplatesIds = (definition.installedComponents ?? [])
.filter(({ type }) => type === 'template')
.map(({ id }) => id);
const templates = await Promise.all(
indexTemplatesIds.map((id) =>
esClient.indices

34
x-pack/plugins/entity_manager/server/lib/entities/helpers/calculate_offset.ts
View file

@ -1,34 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
import moment from 'moment';
import {
ENTITY_DEFAULT_HISTORY_FREQUENCY,
ENTITY_DEFAULT_HISTORY_SYNC_DELAY,
} from '../../../../common/constants_entities';
const durationToSeconds = (dateMath: string) => {
const parts = dateMath.match(/(\d+)([m|s|h|d])/);
if (!parts) {
throw new Error(`Invalid date math supplied: ${dateMath}`);
}
const value = parseInt(parts[1], 10);
const unit = parts[2] as 'm' | 's' | 'h' | 'd';
return moment.duration(value, unit).asSeconds();
};
export function calculateOffset(definition: EntityDefinition) {
const syncDelay = durationToSeconds(
definition.history.settings.syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY
);
const frequency =
durationToSeconds(definition.history.settings.frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY) *
2;
return syncDelay + frequency;
}

3
x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts
View file

@ -13,9 +13,8 @@ export const builtInEntityDefinition = entityDefinitionSchema.parse({
type: 'service',
indexPatterns: ['kbn-data-forge-fake_stack.*'],
managed: true,
history: {
latest: {
timestampField: '@timestamp',
interval: '1m',
},
identityFields: ['log.logger', { field: 'event.category', optional: true }],
displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}',

9
x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts
View file

@ -12,13 +12,12 @@ export const rawEntityDefinition = {
name: 'Services for Admin Console',
type: 'service',
indexPatterns: ['kbn-data-forge-fake_stack.*'],
history: {
latest: {
timestampField: '@timestamp',
interval: '1m',
lookbackPeriod: '10m',
settings: {
lookbackPeriod: '10m',
frequency: '2m',
syncDelay: '2m',
frequency: '30s',
syncDelay: '10s',
},
},
identityFields: ['log.logger', { field: 'event.category', optional: true }],

51
x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts
View file

@ -1,51 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { entityDefinitionSchema } from '@kbn/entities-schema';
export const entityDefinitionWithBackfill = entityDefinitionSchema.parse({
id: 'admin-console-services-backfill',
version: '999.999.999',
name: 'Services for Admin Console',
type: 'service',
indexPatterns: ['kbn-data-forge-fake_stack.*'],
history: {
timestampField: '@timestamp',
interval: '1m',
settings: {
backfillSyncDelay: '15m',
backfillLookbackPeriod: '72h',
backfillFrequency: '5m',
},
},
identityFields: ['log.logger', { field: 'event.category', optional: true }],
displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}',
metadata: ['tags', 'host.name', 'host.os.name', { source: '_index', destination: 'sourceIndex' }],
metrics: [
{
name: 'logRate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'log.level: *',
},
],
},
{
name: 'errorRate',
equation: 'A',
metrics: [
{
name: 'A',
aggregation: 'doc_count',
filter: 'log.level: "ERROR"',
},
],
},
],
});

1
x-pack/plugins/entity_manager/server/lib/entities/helpers/fixtures/index.ts
View file

@ -6,5 +6,4 @@
*/
export { entityDefinition } from './entity_definition';
export { entityDefinitionWithBackfill } from './entity_definition_with_backfill';
export { builtInEntityDefinition } from './builtin_entity_definition';

12
x-pack/plugins/entity_manager/server/lib/entities/helpers/is_backfill_enabled.ts
View file

@ -1,12 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
export function isBackfillEnabled(definition: EntityDefinition) {
return definition.history.settings.backfillSyncDelay != null;
}

327
x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap generated
View file

@ -1,327 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`generateHistoryProcessors(definition) should generate a valid pipeline for builtin definition 1`] = `
Array [
Object {
"set": Object {
"field": "event.ingested",
"value": "{{{_ingest.timestamp}}}",
},
},
Object {
"set": Object {
"field": "entity.type",
"value": "service",
},
},
Object {
"set": Object {
"field": "entity.definitionId",
"value": "builtin_mock_entity_definition",
},
},
Object {
"set": Object {
"field": "entity.definitionVersion",
"value": "1.0.0",
},
},
Object {
"set": Object {
"field": "entity.schemaVersion",
"value": "v1",
},
},
Object {
"set": Object {
"field": "entity.identityFields",
"value": Array [
"log.logger",
"event.category",
],
},
},
Object {
"script": Object {
"description": "Generated the entity.id field",
"source": "// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx[\\"entity\\"][\\"identity\\"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(\\":\\");
}
// Assign the entity.id
ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\";
}",
},
},
Object {
"fingerprint": Object {
"fields": Array [
"entity.id",
],
"method": "MurmurHash3",
"target_field": "entity.id",
},
},
Object {
"script": Object {
"source": "if (ctx.entity?.metadata?.tags != null) {
ctx.tags = ctx.entity.metadata.tags.keySet();
}
if (ctx.entity?.metadata?.host?.name != null) {
if (ctx.host == null) {
ctx.host = new HashMap();
}
ctx.host.name = ctx.entity.metadata.host.name.keySet();
}
if (ctx.entity?.metadata?.host?.os?.name != null) {
if (ctx.host == null) {
ctx.host = new HashMap();
}
if (ctx.host.os == null) {
ctx.host.os = new HashMap();
}
ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet();
}
if (ctx.entity?.metadata?.sourceIndex != null) {
ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet();
}",
},
},
Object {
"remove": Object {
"field": "entity.metadata",
"ignore_missing": true,
},
},
Object {
"set": Object {
"field": "log.logger",
"if": "ctx.entity?.identity?.log?.logger != null",
"value": "{{entity.identity.log.logger}}",
},
},
Object {
"set": Object {
"field": "event.category",
"if": "ctx.entity?.identity?.event?.category != null",
"value": "{{entity.identity.event.category}}",
},
},
Object {
"remove": Object {
"field": "entity.identity",
"ignore_missing": true,
},
},
Object {
"date_index_name": Object {
"date_formats": Array [
"UNIX_MS",
"ISO8601",
"yyyy-MM-dd'T'HH:mm:ss.SSSXX",
],
"date_rounding": "M",
"field": "@timestamp",
"index_name_prefix": ".entities.v1.history.builtin_mock_entity_definition.",
},
},
]
`;
exports[`generateHistoryProcessors(definition) should generate a valid pipeline for custom definition 1`] = `
Array [
Object {
"set": Object {
"field": "event.ingested",
"value": "{{{_ingest.timestamp}}}",
},
},
Object {
"set": Object {
"field": "entity.type",
"value": "service",
},
},
Object {
"set": Object {
"field": "entity.definitionId",
"value": "admin-console-services",
},
},
Object {
"set": Object {
"field": "entity.definitionVersion",
"value": "1.0.0",
},
},
Object {
"set": Object {
"field": "entity.schemaVersion",
"value": "v1",
},
},
Object {
"set": Object {
"field": "entity.identityFields",
"value": Array [
"log.logger",
"event.category",
],
},
},
Object {
"script": Object {
"description": "Generated the entity.id field",
"source": "// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx[\\"entity\\"][\\"identity\\"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(\\":\\");
}
// Assign the entity.id
ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\";
}",
},
},
Object {
"fingerprint": Object {
"fields": Array [
"entity.id",
],
"method": "MurmurHash3",
"target_field": "entity.id",
},
},
Object {
"script": Object {
"source": "if (ctx.entity?.metadata?.tags != null) {
ctx.tags = ctx.entity.metadata.tags.keySet();
}
if (ctx.entity?.metadata?.host?.name != null) {
if (ctx.host == null) {
ctx.host = new HashMap();
}
ctx.host.name = ctx.entity.metadata.host.name.keySet();
}
if (ctx.entity?.metadata?.host?.os?.name != null) {
if (ctx.host == null) {
ctx.host = new HashMap();
}
if (ctx.host.os == null) {
ctx.host.os = new HashMap();
}
ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet();
}
if (ctx.entity?.metadata?.sourceIndex != null) {
ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet();
}",
},
},
Object {
"remove": Object {
"field": "entity.metadata",
"ignore_missing": true,
},
},
Object {
"set": Object {
"field": "log.logger",
"if": "ctx.entity?.identity?.log?.logger != null",
"value": "{{entity.identity.log.logger}}",
},
},
Object {
"set": Object {
"field": "event.category",
"if": "ctx.entity?.identity?.event?.category != null",
"value": "{{entity.identity.event.category}}",
},
},
Object {
"remove": Object {
"field": "entity.identity",
"ignore_missing": true,
},
},
Object {
"date_index_name": Object {
"date_formats": Array [
"UNIX_MS",
"ISO8601",
"yyyy-MM-dd'T'HH:mm:ss.SSSXX",
],
"date_rounding": "M",
"field": "@timestamp",
"index_name_prefix": ".entities.v1.history.admin-console-services.",
},
},
Object {
"pipeline": Object {
"ignore_missing_pipeline": true,
"name": "admin-console-services@platform",
},
},
Object {
"pipeline": Object {
"ignore_missing_pipeline": true,
"name": "admin-console-services-history@platform",
},
},
Object {
"pipeline": Object {
"ignore_missing_pipeline": true,
"name": "admin-console-services@custom",
},
},
Object {
"pipeline": Object {
"ignore_missing_pipeline": true,
"name": "admin-console-services-history@custom",
},
},
]
`;

140
x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap generated
View file

@ -43,16 +43,60 @@ Array [
},
Object {
"script": Object {
"source": "if (ctx.entity?.metadata?.tags.data != null) {
"description": "Generated the entity.id field",
"source": "// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx[\\"entity\\"][\\"identity\\"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(\\":\\");
}
// Assign the entity.id
ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\";
}",
},
},
Object {
"fingerprint": Object {
"fields": Array [
"entity.id",
],
"method": "MurmurHash3",
"target_field": "entity.id",
},
},
Object {
"script": Object {
"source": "if (ctx.entity?.metadata?.tags?.data != null) {
ctx.tags = ctx.entity.metadata.tags.data.keySet();
}
if (ctx.entity?.metadata?.host?.name.data != null) {
if (ctx.entity?.metadata?.host?.name?.data != null) {
if (ctx.host == null) {
ctx.host = new HashMap();
}
ctx.host.name = ctx.entity.metadata.host.name.data.keySet();
}
if (ctx.entity?.metadata?.host?.os?.name.data != null) {
if (ctx.entity?.metadata?.host?.os?.name?.data != null) {
if (ctx.host == null) {
ctx.host = new HashMap();
}
@ -61,7 +105,7 @@ if (ctx.entity?.metadata?.host?.os?.name.data != null) {
}
ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet();
}
if (ctx.entity?.metadata?.sourceIndex.data != null) {
if (ctx.entity?.metadata?.sourceIndex?.data != null) {
ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet();
}",
},
@ -72,28 +116,18 @@ if (ctx.entity?.metadata?.sourceIndex.data != null) {
"ignore_missing": true,
},
},
Object {
"dot_expander": Object {
"field": "log.logger",
"path": "entity.identity.log.logger.top_metric",
},
},
Object {
"set": Object {
"field": "log.logger",
"value": "{{entity.identity.log.logger.top_metric.log.logger}}",
},
},
Object {
"dot_expander": Object {
"field": "event.category",
"path": "entity.identity.event.category.top_metric",
"if": "ctx.entity?.identity?.log?.logger != null",
"value": "{{entity.identity.log.logger}}",
},
},
Object {
"set": Object {
"field": "event.category",
"value": "{{entity.identity.event.category.top_metric.event.category}}",
"if": "ctx.entity?.identity?.event?.category != null",
"value": "{{entity.identity.event.category}}",
},
},
Object {
@ -160,16 +194,60 @@ Array [
},
Object {
"script": Object {
"source": "if (ctx.entity?.metadata?.tags.data != null) {
"description": "Generated the entity.id field",
"source": "// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx[\\"entity\\"][\\"identity\\"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(\\":\\");
}
// Assign the entity.id
ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\";
}",
},
},
Object {
"fingerprint": Object {
"fields": Array [
"entity.id",
],
"method": "MurmurHash3",
"target_field": "entity.id",
},
},
Object {
"script": Object {
"source": "if (ctx.entity?.metadata?.tags?.data != null) {
ctx.tags = ctx.entity.metadata.tags.data.keySet();
}
if (ctx.entity?.metadata?.host?.name.data != null) {
if (ctx.entity?.metadata?.host?.name?.data != null) {
if (ctx.host == null) {
ctx.host = new HashMap();
}
ctx.host.name = ctx.entity.metadata.host.name.data.keySet();
}
if (ctx.entity?.metadata?.host?.os?.name.data != null) {
if (ctx.entity?.metadata?.host?.os?.name?.data != null) {
if (ctx.host == null) {
ctx.host = new HashMap();
}
@ -178,7 +256,7 @@ if (ctx.entity?.metadata?.host?.os?.name.data != null) {
}
ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet();
}
if (ctx.entity?.metadata?.sourceIndex.data != null) {
if (ctx.entity?.metadata?.sourceIndex?.data != null) {
ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet();
}",
},
@ -189,28 +267,18 @@ if (ctx.entity?.metadata?.sourceIndex.data != null) {
"ignore_missing": true,
},
},
Object {
"dot_expander": Object {
"field": "log.logger",
"path": "entity.identity.log.logger.top_metric",
},
},
Object {
"set": Object {
"field": "log.logger",
"value": "{{entity.identity.log.logger.top_metric.log.logger}}",
},
},
Object {
"dot_expander": Object {
"field": "event.category",
"path": "entity.identity.event.category.top_metric",
"if": "ctx.entity?.identity?.log?.logger != null",
"value": "{{entity.identity.log.logger}}",
},
},
Object {
"set": Object {
"field": "event.category",
"value": "{{entity.identity.event.category.top_metric.event.category}}",
"if": "ctx.entity?.identity?.event?.category != null",
"value": "{{entity.identity.event.category}}",
},
},
Object {

21
x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.test.ts
View file

@ -1,21 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { entityDefinition, builtInEntityDefinition } from '../helpers/fixtures';
import { generateHistoryProcessors } from './generate_history_processors';
describe('generateHistoryProcessors(definition)', () => {
it('should generate a valid pipeline for custom definition', () => {
const processors = generateHistoryProcessors(entityDefinition);
expect(processors).toMatchSnapshot();
});
it('should generate a valid pipeline for builtin definition', () => {
const processors = generateHistoryProcessors(builtInEntityDefinition);
expect(processors).toMatchSnapshot();
});
});

222
x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts
View file

@ -1,222 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1, MetadataField } from '@kbn/entities-schema';
import {
initializePathScript,
cleanScript,
} from '../helpers/ingest_pipeline_script_processor_helpers';
import { generateHistoryIndexName } from '../helpers/generate_component_id';
import { isBuiltinDefinition } from '../helpers/is_builtin_definition';
function getMetadataSourceField({ aggregation, destination, source }: MetadataField) {
if (aggregation.type === 'terms') {
return `ctx.entity.metadata.${destination}.keySet()`;
} else if (aggregation.type === 'top_value') {
return `ctx.entity.metadata.${destination}.top_value["${source}"]`;
}
}
function mapDestinationToPainless(metadata: MetadataField) {
const field = metadata.destination;
return `
${initializePathScript(field)}
ctx.${field} = ${getMetadataSourceField(metadata)};
`;
}
function createMetadataPainlessScript(definition: EntityDefinition) {
if (!definition.metadata) {
return '';
}
return definition.metadata.reduce((acc, metadata) => {
const { destination, source } = metadata;
const optionalFieldPath = destination.replaceAll('.', '?.');
if (metadata.aggregation.type === 'terms') {
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath} != null) {
${mapDestinationToPainless(metadata)}
}
`;
return `${acc}\n${next}`;
} else if (metadata.aggregation.type === 'top_value') {
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) {
${mapDestinationToPainless(metadata)}
}
`;
return `${acc}\n${next}`;
}
return acc;
}, '');
}
function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) {
return definition.identityFields.map((key) => ({
set: {
if: `ctx.entity?.identity?.${key.field.replaceAll('.', '?.')} != null`,
field: key.field,
value: `{{entity.identity.${key.field}}}`,
},
}));
}
function getCustomIngestPipelines(definition: EntityDefinition) {
if (isBuiltinDefinition(definition)) {
return [];
}
return [
{
pipeline: {
ignore_missing_pipeline: true,
name: `${definition.id}@platform`,
},
},
{
pipeline: {
ignore_missing_pipeline: true,
name: `${definition.id}-history@platform`,
},
},
{
pipeline: {
ignore_missing_pipeline: true,
name: `${definition.id}@custom`,
},
},
{
pipeline: {
ignore_missing_pipeline: true,
name: `${definition.id}-history@custom`,
},
},
];
}
export function generateHistoryProcessors(definition: EntityDefinition) {
return [
{
set: {
field: 'event.ingested',
value: '{{{_ingest.timestamp}}}',
},
},
{
set: {
field: 'entity.type',
value: definition.type,
},
},
{
set: {
field: 'entity.definitionId',
value: definition.id,
},
},
{
set: {
field: 'entity.definitionVersion',
value: definition.version,
},
},
{
set: {
field: 'entity.schemaVersion',
value: ENTITY_SCHEMA_VERSION_V1,
},
},
{
set: {
field: 'entity.identityFields',
value: definition.identityFields.map((identityField) => identityField.field),
},
},
{
script: {
description: 'Generated the entity.id field',
source: cleanScript(`
// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx["entity"]["identity"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx["entity"]["identity"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(":");
}
// Assign the entity.id
ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown";
}
`),
},
},
{
fingerprint: {
fields: ['entity.id'],
target_field: 'entity.id',
method: 'MurmurHash3',
},
},
...(definition.staticFields != null
? Object.keys(definition.staticFields).map((field) => ({
set: { field, value: definition.staticFields![field] },
}))
: []),
...(definition.metadata != null
? [{ script: { source: cleanScript(createMetadataPainlessScript(definition)) } }]
: []),
{
remove: {
field: 'entity.metadata',
ignore_missing: true,
},
},
...liftIdentityFieldsToDocumentRoot(definition),
{
remove: {
field: 'entity.identity',
ignore_missing: true,
},
},
{
date_index_name: {
field: '@timestamp',
index_name_prefix: `${generateHistoryIndexName(definition)}.`,
date_rounding: 'M',
date_formats: ['UNIX_MS', 'ISO8601', "yyyy-MM-dd'T'HH:mm:ss.SSSXX"],
},
},
...getCustomIngestPipelines(definition),
];
}

90
x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts
View file

@ -17,7 +17,7 @@ function getMetadataSourceField({ aggregation, destination, source }: MetadataFi
if (aggregation.type === 'terms') {
return `ctx.entity.metadata.${destination}.data.keySet()`;
} else if (aggregation.type === 'top_value') {
return `ctx.entity.metadata.${destination}.top_value["${destination}"]`;
return `ctx.entity.metadata.${destination}.top_value["${source}"]`;
}
}
@ -35,19 +35,19 @@ function createMetadataPainlessScript(definition: EntityDefinition) {
}
return definition.metadata.reduce((acc, metadata) => {
const destination = metadata.destination;
const { destination, source } = metadata;
const optionalFieldPath = destination.replaceAll('.', '?.');
if (metadata.aggregation.type === 'terms') {
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) {
if (ctx.entity?.metadata?.${optionalFieldPath}?.data != null) {
${mapDestinationToPainless(metadata)}
}
`;
return `${acc}\n${next}`;
} else if (metadata.aggregation.type === 'top_value') {
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${destination}"] != null) {
if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) {
${mapDestinationToPainless(metadata)}
}
`;
@ -59,30 +59,13 @@ function createMetadataPainlessScript(definition: EntityDefinition) {
}
function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) {
return definition.identityFields
.map((identityField) => {
const setProcessor = {
set: {
field: identityField.field,
value: `{{entity.identity.${identityField.field}.top_metric.${identityField.field}}}`,
},
};
if (!identityField.field.includes('.')) {
return [setProcessor];
}
return [
{
dot_expander: {
field: identityField.field,
path: `entity.identity.${identityField.field}.top_metric`,
},
},
setProcessor,
];
})
.flat();
return definition.identityFields.map((key) => ({
set: {
if: `ctx.entity?.identity?.${key.field.replaceAll('.', '?.')} != null`,
field: key.field,
value: `{{entity.identity.${key.field}}}`,
},
}));
}
function getCustomIngestPipelines(definition: EntityDefinition) {
@ -156,6 +139,55 @@ export function generateLatestProcessors(definition: EntityDefinition) {
value: definition.identityFields.map((identityField) => identityField.field),
},
},
{
script: {
description: 'Generated the entity.id field',
source: cleanScript(`
// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx["entity"]["identity"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx["entity"]["identity"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(":");
}
// Assign the entity.id
ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown";
}
`),
},
},
{
fingerprint: {
fields: ['entity.id'],
target_field: 'entity.id',
method: 'MurmurHash3',
},
},
...(definition.staticFields != null
? Object.keys(definition.staticFields).map((field) => ({
set: { field, value: definition.staticFields![field] },
@ -177,8 +209,8 @@ export function generateLatestProcessors(definition: EntityDefinition) {
ignore_missing: true,
},
},
// This must happen AFTER we lift the identity fields into the root of the document
{
// This must happen AFTER we lift the identity fields into the root of the document
set: {
field: 'entity.displayName',
value: definition.displayNameTemplate,

143
x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts
View file

@ -19,19 +19,23 @@ import {
} from './install_entity_definition';
import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
import {
generateHistoryIndexTemplateId,
generateHistoryIngestPipelineId,
generateHistoryTransformId,
generateLatestIndexTemplateId,
generateLatestIngestPipelineId,
generateLatestTransformId,
} from './helpers/generate_component_id';
import { generateHistoryTransform } from './transform/generate_history_transform';
import { generateLatestTransform } from './transform/generate_latest_transform';
import { entityDefinition as mockEntityDefinition } from './helpers/fixtures/entity_definition';
import { EntityDefinitionIdInvalid } from './errors/entity_definition_id_invalid';
import { EntityIdConflict } from './errors/entity_id_conflict_error';
const getExpectedInstalledComponents = (definition: EntityDefinition) => {
return [
{ type: 'template', id: generateLatestIndexTemplateId(definition) },
{ type: 'ingest_pipeline', id: generateLatestIngestPipelineId(definition) },
{ type: 'transform', id: generateLatestTransformId(definition) },
];
};
const assertHasCreatedDefinition = (
definition: EntityDefinition,
soClient: SavedObjectsClientContract,
@ -44,6 +48,7 @@ const assertHasCreatedDefinition = (
...definition,
installStatus: 'installing',
installStartedAt: expect.any(String),
installedComponents: [],
},
{
id: definition.id,
@ -54,29 +59,17 @@ const assertHasCreatedDefinition = (
expect(soClient.update).toBeCalledTimes(1);
expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, {
installStatus: 'installed',
installedComponents: getExpectedInstalledComponents(definition),
});
expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2);
expect(esClient.indices.putIndexTemplate).toBeCalledWith(
expect.objectContaining({
name: `entities_v1_history_${definition.id}_index_template`,
})
);
expect(esClient.indices.putIndexTemplate).toBeCalledTimes(1);
expect(esClient.indices.putIndexTemplate).toBeCalledWith(
expect.objectContaining({
name: `entities_v1_latest_${definition.id}_index_template`,
})
);
expect(esClient.ingest.putPipeline).toBeCalledTimes(2);
expect(esClient.ingest.putPipeline).toBeCalledWith({
id: generateHistoryIngestPipelineId(definition),
processors: expect.anything(),
_meta: {
definitionVersion: definition.version,
managed: definition.managed,
},
});
expect(esClient.ingest.putPipeline).toBeCalledTimes(1);
expect(esClient.ingest.putPipeline).toBeCalledWith({
id: generateLatestIngestPipelineId(definition),
processors: expect.anything(),
@ -86,8 +79,7 @@ const assertHasCreatedDefinition = (
},
});
expect(esClient.transform.putTransform).toBeCalledTimes(2);
expect(esClient.transform.putTransform).toBeCalledWith(generateHistoryTransform(definition));
expect(esClient.transform.putTransform).toBeCalledTimes(1);
expect(esClient.transform.putTransform).toBeCalledWith(generateLatestTransform(definition));
};
@ -101,32 +93,21 @@ const assertHasUpgradedDefinition = (
...definition,
installStatus: 'upgrading',
installStartedAt: expect.any(String),
installedComponents: getExpectedInstalledComponents(definition),
});
expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, {
installStatus: 'installed',
installedComponents: getExpectedInstalledComponents(definition),
});
expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2);
expect(esClient.indices.putIndexTemplate).toBeCalledWith(
expect.objectContaining({
name: `entities_v1_history_${definition.id}_index_template`,
})
);
expect(esClient.indices.putIndexTemplate).toBeCalledTimes(1);
expect(esClient.indices.putIndexTemplate).toBeCalledWith(
expect.objectContaining({
name: `entities_v1_latest_${definition.id}_index_template`,
})
);
expect(esClient.ingest.putPipeline).toBeCalledTimes(2);
expect(esClient.ingest.putPipeline).toBeCalledWith({
id: generateHistoryIngestPipelineId(definition),
processors: expect.anything(),
_meta: {
definitionVersion: definition.version,
managed: definition.managed,
},
});
expect(esClient.ingest.putPipeline).toBeCalledTimes(1);
expect(esClient.ingest.putPipeline).toBeCalledWith({
id: generateLatestIngestPipelineId(definition),
processors: expect.anything(),
@ -136,8 +117,7 @@ const assertHasUpgradedDefinition = (
},
});
expect(esClient.transform.putTransform).toBeCalledTimes(2);
expect(esClient.transform.putTransform).toBeCalledWith(generateHistoryTransform(definition));
expect(esClient.transform.putTransform).toBeCalledTimes(1);
expect(esClient.transform.putTransform).toBeCalledWith(generateLatestTransform(definition));
};
@ -148,13 +128,7 @@ const assertHasDeletedDefinition = (
) => {
assertHasDeletedTransforms(definition, esClient);
expect(esClient.ingest.deletePipeline).toBeCalledTimes(2);
expect(esClient.ingest.deletePipeline).toBeCalledWith(
{
id: generateHistoryIngestPipelineId(definition),
},
{ ignore: [404] }
);
expect(esClient.ingest.deletePipeline).toBeCalledTimes(1);
expect(esClient.ingest.deletePipeline).toBeCalledWith(
{
id: generateLatestIngestPipelineId(definition),
@ -162,13 +136,7 @@ const assertHasDeletedDefinition = (
{ ignore: [404] }
);
expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2);
expect(esClient.indices.deleteIndexTemplate).toBeCalledWith(
{
name: generateHistoryIndexTemplateId(definition),
},
{ ignore: [404] }
);
expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(1);
expect(esClient.indices.deleteIndexTemplate).toBeCalledWith(
{
name: generateLatestIndexTemplateId(definition),
@ -184,33 +152,21 @@ const assertHasDeletedTransforms = (
definition: EntityDefinition,
esClient: ElasticsearchClient
) => {
expect(esClient.transform.stopTransform).toBeCalledTimes(2);
expect(esClient.transform.stopTransform).toBeCalledTimes(1);
expect(esClient.transform.stopTransform).toBeCalledWith(
expect.objectContaining({
transform_id: generateHistoryTransformId(definition),
}),
expect.anything()
);
expect(esClient.transform.deleteTransform).toBeCalledWith(
expect.objectContaining({
transform_id: generateHistoryTransformId(definition),
}),
expect.anything()
);
expect(esClient.transform.stopTransform).toBeCalledWith(
expect.objectContaining({
transform_id: generateLatestTransformId(definition),
}),
expect.anything()
);
expect(esClient.transform.deleteTransform).toBeCalledWith(
expect.objectContaining({
transform_id: generateLatestTransformId(definition),
}),
expect.anything()
);
expect(esClient.transform.deleteTransform).toBeCalledTimes(2);
expect(esClient.transform.deleteTransform).toBeCalledTimes(1);
expect(esClient.transform.deleteTransform).toBeCalledWith(
expect.objectContaining({
transform_id: generateLatestTransformId(definition),
}),
expect.anything()
);
};
describe('install_entity_definition', () => {
@ -223,7 +179,7 @@ describe('install_entity_definition', () => {
installEntityDefinition({
esClient,
soClient,
definition: { id: 'a'.repeat(40) } as EntityDefinition,
definition: { id: 'a'.repeat(50) } as EntityDefinition,
logger: loggerMock.create(),
})
).rejects.toThrow(EntityDefinitionIdInvalid);
@ -242,6 +198,7 @@ describe('install_entity_definition', () => {
attributes: {
...mockEntityDefinition,
installStatus: 'installed',
installedComponents: [],
},
},
],
@ -264,6 +221,12 @@ describe('install_entity_definition', () => {
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 });
soClient.update.mockResolvedValue({
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
attributes: {},
});
await installEntityDefinition({
esClient,
@ -300,6 +263,12 @@ describe('install_entity_definition', () => {
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 });
soClient.update.mockResolvedValue({
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
attributes: {},
});
await installBuiltInEntityDefinitions({
esClient,
@ -329,6 +298,7 @@ describe('install_entity_definition', () => {
attributes: {
...mockEntityDefinition,
installStatus: 'installed',
installedComponents: getExpectedInstalledComponents(mockEntityDefinition),
},
},
],
@ -336,6 +306,12 @@ describe('install_entity_definition', () => {
page: 1,
per_page: 10,
});
soClient.update.mockResolvedValue({
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
attributes: {},
});
await installBuiltInEntityDefinitions({
esClient,
@ -367,6 +343,7 @@ describe('install_entity_definition', () => {
attributes: {
...mockEntityDefinition,
installStatus: 'installed',
installedComponents: getExpectedInstalledComponents(mockEntityDefinition),
},
},
],
@ -374,6 +351,12 @@ describe('install_entity_definition', () => {
page: 1,
per_page: 10,
});
soClient.update.mockResolvedValue({
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
attributes: {},
});
await installBuiltInEntityDefinitions({
esClient,
@ -407,6 +390,7 @@ describe('install_entity_definition', () => {
// upgrading for 1h
installStatus: 'upgrading',
installStartedAt: moment().subtract(1, 'hour').toISOString(),
installedComponents: getExpectedInstalledComponents(mockEntityDefinition),
},
},
],
@ -414,6 +398,12 @@ describe('install_entity_definition', () => {
page: 1,
per_page: 10,
});
soClient.update.mockResolvedValue({
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
attributes: {},
});
await installBuiltInEntityDefinitions({
esClient,
@ -442,6 +432,7 @@ describe('install_entity_definition', () => {
...mockEntityDefinition,
installStatus: 'failed',
installStartedAt: new Date().toISOString(),
installedComponents: getExpectedInstalledComponents(mockEntityDefinition),
},
},
],
@ -449,6 +440,12 @@ describe('install_entity_definition', () => {
page: 1,
per_page: 10,
});
soClient.update.mockResolvedValue({
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
attributes: {},
});
await installBuiltInEntityDefinitions({
esClient,

112
x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts
View file

@ -10,39 +10,25 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
import { Logger } from '@kbn/logging';
import {
generateHistoryIndexTemplateId,
generateLatestIndexTemplateId,
} from './helpers/generate_component_id';
import {
createAndInstallHistoryIngestPipeline,
createAndInstallLatestIngestPipeline,
} from './create_and_install_ingest_pipeline';
import {
createAndInstallHistoryBackfillTransform,
createAndInstallHistoryTransform,
createAndInstallLatestTransform,
} from './create_and_install_transform';
import { generateLatestIndexTemplateId } from './helpers/generate_component_id';
import { createAndInstallIngestPipelines } from './create_and_install_ingest_pipeline';
import { createAndInstallTransforms } from './create_and_install_transform';
import { validateDefinitionCanCreateValidTransformIds } from './transform/validate_transform_ids';
import { deleteEntityDefinition } from './delete_entity_definition';
import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline';
import { deleteLatestIngestPipeline } from './delete_ingest_pipeline';
import { findEntityDefinitionById } from './find_entity_definition';
import {
entityDefinitionExists,
saveEntityDefinition,
updateEntityDefinition,
} from './save_entity_definition';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
import { deleteTemplate, upsertTemplate } from '../manage_index_templates';
import { generateEntitiesLatestIndexTemplateConfig } from './templates/entities_latest_template';
import { generateEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template';
import { createAndInstallTemplates, deleteTemplate } from '../manage_index_templates';
import { EntityIdConflict } from './errors/entity_id_conflict_error';
import { EntityDefinitionNotFound } from './errors/entity_not_found';
import { mergeEntityDefinitionUpdate } from './helpers/merge_definition_update';
import { EntityDefinitionWithState } from './types';
import { stopTransforms } from './stop_transforms';
import { deleteTransforms } from './delete_transforms';
import { stopLatestTransform, stopTransforms } from './stop_transforms';
import { deleteLatestTransform, deleteTransforms } from './delete_transforms';
export interface InstallDefinitionParams {
esClient: ElasticsearchClient;
@ -51,16 +37,6 @@ export interface InstallDefinitionParams {
logger: Logger;
}
const throwIfRejected = (values: Array<PromiseFulfilledResult<any> | PromiseRejectedResult>) => {
const rejectedPromise = values.find(
(value) => value.status === 'rejected'
) as PromiseRejectedResult;
if (rejectedPromise) {
throw new Error(rejectedPromise.reason);
}
return values;
};
// install an entity definition from scratch with all its required components
// after verifying that the definition id is valid and available.
// attempt to remove all installed components if the installation fails.
@ -72,42 +48,35 @@ export async function installEntityDefinition({
}: InstallDefinitionParams): Promise<EntityDefinition> {
validateDefinitionCanCreateValidTransformIds(definition);
try {
if (await entityDefinitionExists(soClient, definition.id)) {
throw new EntityIdConflict(
`Entity definition with [${definition.id}] already exists.`,
definition
);
}
if (await entityDefinitionExists(soClient, definition.id)) {
throw new EntityIdConflict(
`Entity definition with [${definition.id}] already exists.`,
definition
);
}
try {
const entityDefinition = await saveEntityDefinition(soClient, {
...definition,
installStatus: 'installing',
installStartedAt: new Date().toISOString(),
installedComponents: [],
});
return await install({ esClient, soClient, logger, definition: entityDefinition });
} catch (e) {
logger.error(`Failed to install entity definition ${definition.id}: ${e}`);
await stopAndDeleteTransforms(esClient, definition, logger);
await Promise.all([
deleteHistoryIngestPipeline(esClient, definition, logger),
deleteLatestIngestPipeline(esClient, definition, logger),
]);
await stopLatestTransform(esClient, definition, logger);
await deleteLatestTransform(esClient, definition, logger);
await Promise.all([
deleteTemplate({
esClient,
logger,
name: generateHistoryIndexTemplateId(definition),
}),
deleteTemplate({
esClient,
logger,
name: generateLatestIndexTemplateId(definition),
}),
]);
await deleteLatestIngestPipeline(esClient, definition, logger);
await deleteTemplate({
esClient,
logger,
name: generateLatestIndexTemplateId(definition),
});
await deleteEntityDefinition(soClient, definition).catch((err) => {
if (err instanceof EntityDefinitionNotFound) {
@ -191,36 +160,19 @@ async function install({
);
logger.debug(`Installing index templates for definition ${definition.id}`);
await Promise.allSettled([
upsertTemplate({
esClient,
logger,
template: generateEntitiesHistoryIndexTemplateConfig(definition),
}),
upsertTemplate({
esClient,
logger,
template: generateEntitiesLatestIndexTemplateConfig(definition),
}),
]).then(throwIfRejected);
const templates = await createAndInstallTemplates(esClient, definition, logger);
logger.debug(`Installing ingest pipelines for definition ${definition.id}`);
await Promise.allSettled([
createAndInstallHistoryIngestPipeline(esClient, definition, logger),
createAndInstallLatestIngestPipeline(esClient, definition, logger),
]).then(throwIfRejected);
const pipelines = await createAndInstallIngestPipelines(esClient, definition, logger);
logger.debug(`Installing transforms for definition ${definition.id}`);
await Promise.allSettled([
createAndInstallHistoryTransform(esClient, definition, logger),
isBackfillEnabled(definition)
? createAndInstallHistoryBackfillTransform(esClient, definition, logger)
: Promise.resolve(),
createAndInstallLatestTransform(esClient, definition, logger),
]).then(throwIfRejected);
const transforms = await createAndInstallTransforms(esClient, definition, logger);
await updateEntityDefinition(soClient, definition.id, { installStatus: 'installed' });
return { ...definition, installStatus: 'installed' };
const updatedProps = await updateEntityDefinition(soClient, definition.id, {
installStatus: 'installed',
installedComponents: [...templates, ...pipelines, ...transforms],
});
return { ...definition, ...updatedProps.attributes };
}
// stop and delete the current transforms and reinstall all the components

2
x-pack/plugins/entity_manager/server/lib/entities/save_entity_definition.ts
View file

@ -41,5 +41,5 @@ export async function updateEntityDefinition(
id: string,
definition: Partial<EntityDefinition>
) {
await soClient.update<EntityDefinition>(SO_ENTITY_DEFINITION_TYPE, id, definition);
return await soClient.update<EntityDefinition>(SO_ENTITY_DEFINITION_TYPE, id, definition);
}

37
x-pack/plugins/entity_manager/server/lib/entities/start_transforms.ts
View file

@ -7,13 +7,7 @@
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import {
generateHistoryBackfillTransformId,
generateHistoryTransformId,
generateLatestTransformId,
} from './helpers/generate_component_id';
import { retryTransientEsErrors } from './helpers/retry';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
export async function startTransforms(
esClient: ElasticsearchClient,
@ -21,28 +15,15 @@ export async function startTransforms(
logger: Logger
) {
try {
const historyTransformId = generateHistoryTransformId(definition);
const latestTransformId = generateLatestTransformId(definition);
await retryTransientEsErrors(
() =>
esClient.transform.startTransform({ transform_id: historyTransformId }, { ignore: [409] }),
{ logger }
);
if (isBackfillEnabled(definition)) {
const historyBackfillTransformId = generateHistoryBackfillTransformId(definition);
await retryTransientEsErrors(
() =>
esClient.transform.startTransform(
{ transform_id: historyBackfillTransformId },
{ ignore: [409] }
),
{ logger }
);
}
await retryTransientEsErrors(
() =>
esClient.transform.startTransform({ transform_id: latestTransformId }, { ignore: [409] }),
{ logger }
await Promise.all(
(definition.installedComponents ?? [])
.filter(({ type }) => type === 'transform')
.map(({ id }) =>
retryTransientEsErrors(
() => esClient.transform.startTransform({ transform_id: id }, { ignore: [409] }),
{ logger }
)
)
);
} catch (err) {
logger.error(`Cannot start entity transforms [${definition.id}]: ${err}`);

65
x-pack/plugins/entity_manager/server/lib/entities/stop_transforms.ts
View file

@ -8,14 +8,8 @@
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import {
generateHistoryTransformId,
generateHistoryBackfillTransformId,
generateLatestTransformId,
} from './helpers/generate_component_id';
import { retryTransientEsErrors } from './helpers/retry';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
import { generateLatestTransformId } from './helpers/generate_component_id';
export async function stopTransforms(
esClient: ElasticsearchClient,
@ -23,43 +17,46 @@ export async function stopTransforms(
logger: Logger
) {
try {
const historyTransformId = generateHistoryTransformId(definition);
const latestTransformId = generateLatestTransformId(definition);
await retryTransientEsErrors(
() =>
esClient.transform.stopTransform(
{ transform_id: historyTransformId, wait_for_completion: true, force: true },
{ ignore: [409, 404] }
),
{ logger }
await Promise.all(
(definition.installedComponents ?? [])
.filter(({ type }) => type === 'transform')
.map(({ id }) =>
retryTransientEsErrors(
() =>
esClient.transform.stopTransform(
{ transform_id: id, wait_for_completion: true, force: true },
{ ignore: [409, 404] }
),
{ logger }
)
)
);
} catch (e) {
logger.error(`Cannot stop transforms for definition [${definition.id}]: ${e}`);
throw e;
}
}
if (isBackfillEnabled(definition)) {
const historyBackfillTransformId = generateHistoryBackfillTransformId(definition);
await retryTransientEsErrors(
() =>
esClient.transform.stopTransform(
{
transform_id: historyBackfillTransformId,
wait_for_completion: true,
force: true,
},
{ ignore: [409, 404] }
),
{ logger }
);
}
export async function stopLatestTransform(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
try {
await retryTransientEsErrors(
() =>
esClient.transform.stopTransform(
{ transform_id: latestTransformId, wait_for_completion: true, force: true },
{
transform_id: generateLatestTransformId(definition),
wait_for_completion: true,
force: true,
},
{ ignore: [409, 404] }
),
{ logger }
);
} catch (e) {
logger.error(`Cannot stop entity transforms [${definition.id}]: ${e}`);
logger.error(`Cannot stop latest transform for definition [${definition.id}]: ${e}`);
throw e;
}
}

152
x-pack/plugins/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap generated
View file

@ -1,152 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`generateEntitiesHistoryIndexTemplateConfig(definition) should generate a valid index template for builtin definition 1`] = `
Object {
"_meta": Object {
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset",
"ecs_version": "8.0.0",
"managed": true,
"managed_by": "elastic_entity_model",
},
"composed_of": Array [
"entities_v1_history_base",
"entities_v1_entity",
"entities_v1_event",
],
"ignore_missing_component_templates": Array [],
"index_patterns": Array [
".entities.v1.history.builtin_mock_entity_definition.*",
],
"name": "entities_v1_history_builtin_mock_entity_definition_index_template",
"priority": 200,
"template": Object {
"aliases": Object {
"entities-service-history": Object {},
},
"mappings": Object {
"_meta": Object {
"version": "1.6.0",
},
"date_detection": false,
"dynamic_templates": Array [
Object {
"strings_as_keyword": Object {
"mapping": Object {
"fields": Object {
"text": Object {
"type": "text",
},
},
"ignore_above": 1024,
"type": "keyword",
},
"match_mapping_type": "string",
},
},
Object {
"entity_metrics": Object {
"mapping": Object {
"type": "{dynamic_type}",
},
"match_mapping_type": Array [
"long",
"double",
],
"path_match": "entity.metrics.*",
},
},
],
},
"settings": Object {
"index": Object {
"codec": "best_compression",
"mapping": Object {
"total_fields": Object {
"limit": 2000,
},
},
},
},
},
}
`;
exports[`generateEntitiesHistoryIndexTemplateConfig(definition) should generate a valid index template for custom definition 1`] = `
Object {
"_meta": Object {
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset",
"ecs_version": "8.0.0",
"managed": true,
"managed_by": "elastic_entity_model",
},
"composed_of": Array [
"entities_v1_history_base",
"entities_v1_entity",
"entities_v1_event",
"admin-console-services@platform",
"admin-console-services-history@platform",
"admin-console-services@custom",
"admin-console-services-history@custom",
],
"ignore_missing_component_templates": Array [
"admin-console-services@platform",
"admin-console-services-history@platform",
"admin-console-services@custom",
"admin-console-services-history@custom",
],
"index_patterns": Array [
".entities.v1.history.admin-console-services.*",
],
"name": "entities_v1_history_admin-console-services_index_template",
"priority": 200,
"template": Object {
"aliases": Object {
"entities-service-history": Object {},
},
"mappings": Object {
"_meta": Object {
"version": "1.6.0",
},
"date_detection": false,
"dynamic_templates": Array [
Object {
"strings_as_keyword": Object {
"mapping": Object {
"fields": Object {
"text": Object {
"type": "text",
},
},
"ignore_above": 1024,
"type": "keyword",
},
"match_mapping_type": "string",
},
},
Object {
"entity_metrics": Object {
"mapping": Object {
"type": "{dynamic_type}",
},
"match_mapping_type": Array [
"long",
"double",
],
"path_match": "entity.metrics.*",
},
},
],
},
"settings": Object {
"index": Object {
"codec": "best_compression",
"mapping": Object {
"total_fields": Object {
"limit": 2000,
},
},
},
},
},
}
`;

21
x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.test.ts
View file

@ -1,21 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { entityDefinition, builtInEntityDefinition } from '../helpers/fixtures';
import { generateEntitiesHistoryIndexTemplateConfig } from './entities_history_template';
describe('generateEntitiesHistoryIndexTemplateConfig(definition)', () => {
it('should generate a valid index template for custom definition', () => {
const template = generateEntitiesHistoryIndexTemplateConfig(entityDefinition);
expect(template).toMatchSnapshot();
});
it('should generate a valid index template for builtin definition', () => {
const template = generateEntitiesHistoryIndexTemplateConfig(builtInEntityDefinition);
expect(template).toMatchSnapshot();
});
});

96
x-pack/plugins/entity_manager/server/lib/entities/templates/entities_history_template.ts
View file

@ -1,96 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
import {
ENTITY_HISTORY,
EntityDefinition,
entitiesIndexPattern,
entitiesAliasPattern,
ENTITY_SCHEMA_VERSION_V1,
} from '@kbn/entities-schema';
import { generateHistoryIndexTemplateId } from '../helpers/generate_component_id';
import {
ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1,
} from '../../../../common/constants_entities';
import { getCustomHistoryTemplateComponents } from '../../../templates/components/helpers';
export const generateEntitiesHistoryIndexTemplateConfig = (
definition: EntityDefinition
): IndicesPutIndexTemplateRequest => ({
name: generateHistoryIndexTemplateId(definition),
_meta: {
description:
"Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset",
ecs_version: '8.0.0',
managed: true,
managed_by: 'elastic_entity_model',
},
ignore_missing_component_templates: getCustomHistoryTemplateComponents(definition),
composed_of: [
ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1,
ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
...getCustomHistoryTemplateComponents(definition),
],
index_patterns: [
`${entitiesIndexPattern({
schemaVersion: ENTITY_SCHEMA_VERSION_V1,
dataset: ENTITY_HISTORY,
definitionId: definition.id,
})}.*`,
],
priority: 200,
template: {
aliases: {
[entitiesAliasPattern({ type: definition.type, dataset: ENTITY_HISTORY })]: {},
},
mappings: {
_meta: {
version: '1.6.0',
},
date_detection: false,
dynamic_templates: [
{
strings_as_keyword: {
mapping: {
ignore_above: 1024,
type: 'keyword',
fields: {
text: {
type: 'text',
},
},
},
match_mapping_type: 'string',
},
},
{
entity_metrics: {
mapping: {
type: '{dynamic_type}',
},
match_mapping_type: ['long', 'double'],
path_match: 'entity.metrics.*',
},
},
],
},
settings: {
index: {
codec: 'best_compression',
mapping: {
total_fields: {
limit: 2000,
},
},
},
},
},
});

15
x-pack/plugins/entity_manager/server/lib/entities/templates/entities_latest_template.ts
View file

@ -19,7 +19,7 @@ import {
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1,
} from '../../../../common/constants_entities';
import { getCustomLatestTemplateComponents } from '../../../templates/components/helpers';
import { isBuiltinDefinition } from '../helpers/is_builtin_definition';
export const generateEntitiesLatestIndexTemplateConfig = (
definition: EntityDefinition
@ -94,3 +94,16 @@ export const generateEntitiesLatestIndexTemplateConfig = (
},
},
});
function getCustomLatestTemplateComponents(definition: EntityDefinition) {
if (isBuiltinDefinition(definition)) {
return [];
}
return [
`${definition.id}@platform`, // @platform goes before so it can be overwritten by custom
`${definition.id}-latest@platform`,
`${definition.id}@custom`,
`${definition.id}-latest@custom`,
];
}

305
x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap generated
View file

@ -1,305 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`generateHistoryTransform(definition) should generate a valid history backfill transform 1`] = `
Object {
"_meta": Object {
"definitionVersion": "999.999.999",
"managed": false,
},
"defer_validation": true,
"dest": Object {
"index": ".entities.v1.history.noop",
"pipeline": "entities-v1-history-admin-console-services-backfill",
},
"frequency": "5m",
"pivot": Object {
"aggs": Object {
"_errorRate_A": Object {
"filter": Object {
"bool": Object {
"minimum_should_match": 1,
"should": Array [
Object {
"match_phrase": Object {
"log.level": "ERROR",
},
},
],
},
},
},
"_logRate_A": Object {
"filter": Object {
"bool": Object {
"minimum_should_match": 1,
"should": Array [
Object {
"exists": Object {
"field": "log.level",
},
},
],
},
},
},
"entity.lastSeenTimestamp": Object {
"max": Object {
"field": "@timestamp",
},
},
"entity.metadata.host.name": Object {
"terms": Object {
"field": "host.name",
"size": 1000,
},
},
"entity.metadata.host.os.name": Object {
"terms": Object {
"field": "host.os.name",
"size": 1000,
},
},
"entity.metadata.sourceIndex": Object {
"terms": Object {
"field": "_index",
"size": 1000,
},
},
"entity.metadata.tags": Object {
"terms": Object {
"field": "tags",
"size": 1000,
},
},
"entity.metrics.errorRate": Object {
"bucket_script": Object {
"buckets_path": Object {
"A": "_errorRate_A>_count",
},
"script": Object {
"lang": "painless",
"source": "params.A",
},
},
},
"entity.metrics.logRate": Object {
"bucket_script": Object {
"buckets_path": Object {
"A": "_logRate_A>_count",
},
"script": Object {
"lang": "painless",
"source": "params.A",
},
},
},
},
"group_by": Object {
"@timestamp": Object {
"date_histogram": Object {
"field": "@timestamp",
"fixed_interval": "1m",
},
},
"entity.identity.event.category": Object {
"terms": Object {
"field": "event.category",
"missing_bucket": true,
},
},
"entity.identity.log.logger": Object {
"terms": Object {
"field": "log.logger",
"missing_bucket": false,
},
},
},
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": Array [
"kbn-data-forge-fake_stack.*",
],
"query": Object {
"bool": Object {
"filter": Array [
Object {
"range": Object {
"@timestamp": Object {
"gte": "now-72h",
},
},
},
Object {
"exists": Object {
"field": "log.logger",
},
},
],
},
},
},
"sync": Object {
"time": Object {
"delay": "15m",
"field": "@timestamp",
},
},
"transform_id": "entities-v1-history-backfill-admin-console-services-backfill",
}
`;
exports[`generateHistoryTransform(definition) should generate a valid history transform 1`] = `
Object {
"_meta": Object {
"definitionVersion": "1.0.0",
"managed": false,
},
"defer_validation": true,
"dest": Object {
"index": ".entities.v1.history.noop",
"pipeline": "entities-v1-history-admin-console-services",
},
"frequency": "2m",
"pivot": Object {
"aggs": Object {
"_errorRate_A": Object {
"filter": Object {
"bool": Object {
"minimum_should_match": 1,
"should": Array [
Object {
"match_phrase": Object {
"log.level": "ERROR",
},
},
],
},
},
},
"_logRate_A": Object {
"filter": Object {
"bool": Object {
"minimum_should_match": 1,
"should": Array [
Object {
"exists": Object {
"field": "log.level",
},
},
],
},
},
},
"entity.lastSeenTimestamp": Object {
"max": Object {
"field": "@timestamp",
},
},
"entity.metadata.host.name": Object {
"terms": Object {
"field": "host.name",
"size": 1000,
},
},
"entity.metadata.host.os.name": Object {
"terms": Object {
"field": "host.os.name",
"size": 1000,
},
},
"entity.metadata.sourceIndex": Object {
"terms": Object {
"field": "_index",
"size": 1000,
},
},
"entity.metadata.tags": Object {
"terms": Object {
"field": "tags",
"size": 1000,
},
},
"entity.metrics.errorRate": Object {
"bucket_script": Object {
"buckets_path": Object {
"A": "_errorRate_A>_count",
},
"script": Object {
"lang": "painless",
"source": "params.A",
},
},
},
"entity.metrics.logRate": Object {
"bucket_script": Object {
"buckets_path": Object {
"A": "_logRate_A>_count",
},
"script": Object {
"lang": "painless",
"source": "params.A",
},
},
},
},
"group_by": Object {
"@timestamp": Object {
"date_histogram": Object {
"field": "@timestamp",
"fixed_interval": "1m",
},
},
"entity.identity.event.category": Object {
"terms": Object {
"field": "event.category",
"missing_bucket": true,
},
},
"entity.identity.log.logger": Object {
"terms": Object {
"field": "log.logger",
"missing_bucket": false,
},
},
},
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": Array [
"kbn-data-forge-fake_stack.*",
],
"query": Object {
"bool": Object {
"filter": Array [
Object {
"exists": Object {
"field": "log.logger",
},
},
Object {
"range": Object {
"@timestamp": Object {
"gte": "now-10m",
},
},
},
],
},
},
},
"sync": Object {
"time": Object {
"delay": "2m",
"field": "@timestamp",
},
},
"transform_id": "entities-v1-history-admin-console-services",
}
`;

150
x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap generated
View file

@ -14,76 +14,37 @@ Object {
"frequency": "30s",
"pivot": Object {
"aggs": Object {
"_errorRate": Object {
"top_metrics": Object {
"metrics": Array [
Object {
"field": "entity.metrics.errorRate",
},
],
"sort": Array [
Object {
"@timestamp": "desc",
},
],
},
},
"_logRate": Object {
"top_metrics": Object {
"metrics": Array [
Object {
"field": "entity.metrics.logRate",
},
],
"sort": Array [
Object {
"@timestamp": "desc",
},
],
},
},
"entity.firstSeenTimestamp": Object {
"min": Object {
"field": "@timestamp",
},
},
"entity.identity.event.category": Object {
"aggs": Object {
"top_metric": Object {
"top_metrics": Object {
"metrics": Object {
"field": "event.category",
},
"sort": "_score",
},
},
},
"_errorRate_A": Object {
"filter": Object {
"exists": Object {
"field": "event.category",
"bool": Object {
"minimum_should_match": 1,
"should": Array [
Object {
"match_phrase": Object {
"log.level": "ERROR",
},
},
],
},
},
},
"entity.identity.log.logger": Object {
"aggs": Object {
"top_metric": Object {
"top_metrics": Object {
"metrics": Object {
"field": "log.logger",
},
"sort": "_score",
},
},
},
"_logRate_A": Object {
"filter": Object {
"exists": Object {
"field": "log.logger",
"bool": Object {
"minimum_should_match": 1,
"should": Array [
Object {
"exists": Object {
"field": "log.level",
},
},
],
},
},
},
"entity.lastSeenTimestamp": Object {
"max": Object {
"field": "entity.lastSeenTimestamp",
"field": "@timestamp",
},
},
"entity.metadata.host.name": Object {
@ -91,14 +52,14 @@ Object {
"data": Object {
"terms": Object {
"field": "host.name",
"size": 1000,
"size": 10,
},
},
},
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "now-360s",
"gte": "now-10m",
},
},
},
@ -108,14 +69,14 @@ Object {
"data": Object {
"terms": Object {
"field": "host.os.name",
"size": 1000,
"size": 10,
},
},
},
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "now-360s",
"gte": "now-10m",
},
},
},
@ -124,15 +85,15 @@ Object {
"aggs": Object {
"data": Object {
"terms": Object {
"field": "sourceIndex",
"size": 1000,
"field": "_index",
"size": 10,
},
},
},
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "now-360s",
"gte": "now-10m",
},
},
},
@ -142,14 +103,14 @@ Object {
"data": Object {
"terms": Object {
"field": "tags",
"size": 1000,
"size": 10,
},
},
},
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "now-360s",
"gte": "now-10m",
},
},
},
@ -157,24 +118,37 @@ Object {
"entity.metrics.errorRate": Object {
"bucket_script": Object {
"buckets_path": Object {
"value": "_errorRate[entity.metrics.errorRate]",
"A": "_errorRate_A>_count",
},
"script": Object {
"lang": "painless",
"source": "params.A",
},
"script": "params.value",
},
},
"entity.metrics.logRate": Object {
"bucket_script": Object {
"buckets_path": Object {
"value": "_logRate[entity.metrics.logRate]",
"A": "_logRate_A>_count",
},
"script": Object {
"lang": "painless",
"source": "params.A",
},
"script": "params.value",
},
},
},
"group_by": Object {
"entity.id": Object {
"entity.identity.event.category": Object {
"terms": Object {
"field": "entity.id",
"field": "event.category",
"missing_bucket": true,
},
},
"entity.identity.log.logger": Object {
"terms": Object {
"field": "log.logger",
"missing_bucket": false,
},
},
},
@ -184,12 +158,32 @@ Object {
"unattended": true,
},
"source": Object {
"index": ".entities.v1.history.admin-console-services.*",
"index": Array [
"kbn-data-forge-fake_stack.*",
],
"query": Object {
"bool": Object {
"filter": Array [
Object {
"exists": Object {
"field": "log.logger",
},
},
Object {
"range": Object {
"@timestamp": Object {
"gte": "now-10m",
},
},
},
],
},
},
},
"sync": Object {
"time": Object {
"delay": "1s",
"field": "event.ingested",
"delay": "10s",
"field": "@timestamp",
},
},
"transform_id": "entities-v1-latest-admin-console-services",

24
x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.test.ts
View file

@ -1,24 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { entityDefinition } from '../helpers/fixtures/entity_definition';
import { entityDefinitionWithBackfill } from '../helpers/fixtures/entity_definition_with_backfill';
import {
generateBackfillHistoryTransform,
generateHistoryTransform,
} from './generate_history_transform';
describe('generateHistoryTransform(definition)', () => {
it('should generate a valid history transform', () => {
const transform = generateHistoryTransform(entityDefinition);
expect(transform).toMatchSnapshot();
});
it('should generate a valid history backfill transform', () => {
const transform = generateBackfillHistoryTransform(entityDefinitionWithBackfill);
expect(transform).toMatchSnapshot();
});
});

178
x-pack/plugins/entity_manager/server/lib/entities/transform/generate_history_transform.ts
View file

@ -1,178 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
import {
QueryDslQueryContainer,
TransformPutTransformRequest,
} from '@elastic/elasticsearch/lib/api/types';
import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw';
import { generateHistoryMetricAggregations } from './generate_metric_aggregations';
import {
ENTITY_DEFAULT_HISTORY_FREQUENCY,
ENTITY_DEFAULT_HISTORY_SYNC_DELAY,
} from '../../../../common/constants_entities';
import { generateHistoryMetadataAggregations } from './generate_metadata_aggregations';
import {
generateHistoryTransformId,
generateHistoryIngestPipelineId,
generateHistoryIndexName,
generateHistoryBackfillTransformId,
} from '../helpers/generate_component_id';
import { isBackfillEnabled } from '../helpers/is_backfill_enabled';
export function generateHistoryTransform(
definition: EntityDefinition
): TransformPutTransformRequest {
const filter: QueryDslQueryContainer[] = [];
if (definition.filter) {
filter.push(getElasticsearchQueryOrThrow(definition.filter));
}
if (definition.identityFields.some(({ optional }) => !optional)) {
definition.identityFields
.filter(({ optional }) => !optional)
.forEach(({ field }) => {
filter.push({ exists: { field } });
});
}
filter.push({
range: {
[definition.history.timestampField]: {
gte: `now-${definition.history.settings.lookbackPeriod}`,
},
},
});
return generateTransformPutRequest({
definition,
filter,
transformId: generateHistoryTransformId(definition),
frequency: definition.history.settings.frequency,
syncDelay: definition.history.settings.syncDelay,
});
}
export function generateBackfillHistoryTransform(
definition: EntityDefinition
): TransformPutTransformRequest {
if (!isBackfillEnabled(definition)) {
throw new Error(
'generateBackfillHistoryTransform called without history.settings.backfillSyncDelay set'
);
}
const filter: QueryDslQueryContainer[] = [];
if (definition.filter) {
filter.push(getElasticsearchQueryOrThrow(definition.filter));
}
if (definition.history.settings.backfillLookbackPeriod) {
filter.push({
range: {
[definition.history.timestampField]: {
gte: `now-${definition.history.settings.backfillLookbackPeriod}`,
},
},
});
}
if (definition.identityFields.some(({ optional }) => !optional)) {
definition.identityFields
.filter(({ optional }) => !optional)
.forEach(({ field }) => {
filter.push({ exists: { field } });
});
}
return generateTransformPutRequest({
definition,
filter,
transformId: generateHistoryBackfillTransformId(definition),
frequency: definition.history.settings.backfillFrequency,
syncDelay: definition.history.settings.backfillSyncDelay,
});
}
const generateTransformPutRequest = ({
definition,
filter,
transformId,
frequency,
syncDelay,
}: {
definition: EntityDefinition;
transformId: string;
filter: QueryDslQueryContainer[];
frequency?: string;
syncDelay?: string;
}) => {
return {
transform_id: transformId,
_meta: {
definitionVersion: definition.version,
managed: definition.managed,
},
defer_validation: true,
source: {
index: definition.indexPatterns,
...(filter.length > 0 && {
query: {
bool: {
filter,
},
},
}),
},
dest: {
index: `${generateHistoryIndexName({ id: 'noop' } as EntityDefinition)}`,
pipeline: generateHistoryIngestPipelineId(definition),
},
frequency: frequency || ENTITY_DEFAULT_HISTORY_FREQUENCY,
sync: {
time: {
field: definition.history.settings.syncField || definition.history.timestampField,
delay: syncDelay || ENTITY_DEFAULT_HISTORY_SYNC_DELAY,
},
},
settings: {
deduce_mappings: false,
unattended: true,
},
pivot: {
group_by: {
...definition.identityFields.reduce(
(acc, id) => ({
...acc,
[`entity.identity.${id.field}`]: {
terms: { field: id.field, missing_bucket: id.optional },
},
}),
{}
),
['@timestamp']: {
date_histogram: {
field: definition.history.timestampField,
fixed_interval: definition.history.interval,
},
},
},
aggs: {
...generateHistoryMetricAggregations(definition),
...generateHistoryMetadataAggregations(definition),
'entity.lastSeenTimestamp': {
max: {
field: definition.history.timestampField,
},
},
},
},
};
};

97
x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts
View file

@ -5,44 +5,97 @@
* 2.0.
*/
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import { EntityDefinition } from '@kbn/entities-schema';
import {
QueryDslQueryContainer,
TransformPutTransformRequest,
} from '@elastic/elasticsearch/lib/api/types';
import { getElasticsearchQueryOrThrow } from '../helpers/get_elasticsearch_query_or_throw';
import { generateLatestMetricAggregations } from './generate_metric_aggregations';
import {
ENTITY_DEFAULT_LATEST_FREQUENCY,
ENTITY_DEFAULT_LATEST_SYNC_DELAY,
} from '../../../../common/constants_entities';
import {
generateHistoryIndexName,
generateLatestIndexName,
generateLatestIngestPipelineId,
generateLatestTransformId,
generateLatestIngestPipelineId,
generateLatestIndexName,
} from '../helpers/generate_component_id';
import { generateIdentityAggregations } from './generate_identity_aggregations';
import { generateLatestMetadataAggregations } from './generate_metadata_aggregations';
import { generateLatestMetricAggregations } from './generate_metric_aggregations';
export function generateLatestTransform(
definition: EntityDefinition
): TransformPutTransformRequest {
const filter: QueryDslQueryContainer[] = [];
if (definition.filter) {
filter.push(getElasticsearchQueryOrThrow(definition.filter));
}
if (definition.identityFields.some(({ optional }) => !optional)) {
definition.identityFields
.filter(({ optional }) => !optional)
.forEach(({ field }) => {
filter.push({ exists: { field } });
});
}
filter.push({
range: {
[definition.latest.timestampField]: {
gte: `now-${definition.latest.lookbackPeriod}`,
},
},
});
return generateTransformPutRequest({
definition,
filter,
transformId: generateLatestTransformId(definition),
frequency: definition.latest.settings?.frequency ?? ENTITY_DEFAULT_LATEST_FREQUENCY,
syncDelay: definition.latest.settings?.syncDelay ?? ENTITY_DEFAULT_LATEST_SYNC_DELAY,
});
}
const generateTransformPutRequest = ({
definition,
filter,
transformId,
frequency,
syncDelay,
}: {
definition: EntityDefinition;
transformId: string;
filter: QueryDslQueryContainer[];
frequency: string;
syncDelay: string;
}) => {
return {
transform_id: generateLatestTransformId(definition),
transform_id: transformId,
_meta: {
definitionVersion: definition.version,
managed: definition.managed,
},
defer_validation: true,
source: {
index: `${generateHistoryIndexName(definition)}.*`,
index: definition.indexPatterns,
...(filter.length > 0 && {
query: {
bool: {
filter,
},
},
}),
},
dest: {
index: `${generateLatestIndexName({ id: 'noop' } as EntityDefinition)}`,
pipeline: generateLatestIngestPipelineId(definition),
},
frequency: definition.latest?.settings?.frequency ?? ENTITY_DEFAULT_LATEST_FREQUENCY,
frequency,
sync: {
time: {
field: definition.latest?.settings?.syncField ?? 'event.ingested',
delay: definition.latest?.settings?.syncDelay ?? ENTITY_DEFAULT_LATEST_SYNC_DELAY,
field: definition.latest.settings?.syncField || definition.latest.timestampField,
delay: syncDelay,
},
},
settings: {
@ -51,25 +104,25 @@ export function generateLatestTransform(
},
pivot: {
group_by: {
['entity.id']: {
terms: { field: 'entity.id' },
},
...definition.identityFields.reduce(
(acc, id) => ({
...acc,
[`entity.identity.${id.field}`]: {
terms: { field: id.field, missing_bucket: id.optional },
},
}),
{}
),
},
aggs: {
...generateLatestMetricAggregations(definition),
...generateLatestMetadataAggregations(definition),
...generateIdentityAggregations(definition),
'entity.lastSeenTimestamp': {
max: {
field: 'entity.lastSeenTimestamp',
},
},
'entity.firstSeenTimestamp': {
min: {
field: '@timestamp',
field: definition.latest.timestampField,
},
},
},
},
};
}
};

144
x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts
View file

@ -7,134 +7,22 @@
import { entityDefinitionSchema } from '@kbn/entities-schema';
import { rawEntityDefinition } from '../helpers/fixtures/entity_definition';
import {
generateHistoryMetadataAggregations,
generateLatestMetadataAggregations,
} from './generate_metadata_aggregations';
import { generateLatestMetadataAggregations } from './generate_metadata_aggregations';
describe('Generate Metadata Aggregations for history and latest', () => {
describe('generateHistoryMetadataAggregations()', () => {
it('should generate metadata aggregations for string format', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: ['host.name'],
});
expect(generateHistoryMetadataAggregations(definition)).toEqual({
'entity.metadata.host.name': {
terms: {
field: 'host.name',
size: 1000,
},
},
});
});
it('should generate metadata aggregations for object format with only source', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [{ source: 'host.name' }],
});
expect(generateHistoryMetadataAggregations(definition)).toEqual({
'entity.metadata.host.name': {
terms: {
field: 'host.name',
size: 1000,
},
},
});
});
it('should generate metadata aggregations for object format with source and aggregation', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }],
});
expect(generateHistoryMetadataAggregations(definition)).toEqual({
'entity.metadata.host.name': {
terms: {
field: 'host.name',
size: 10,
},
},
});
});
it('should generate metadata aggregations for object format with source, aggregation, and destination', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [
{
source: 'host.name',
aggregation: { type: 'terms', limit: 20 },
destination: 'hostName',
},
],
});
expect(generateHistoryMetadataAggregations(definition)).toEqual({
'entity.metadata.hostName': {
terms: {
field: 'host.name',
size: 20,
},
},
});
});
it('should generate metadata aggregations for terms and top_value', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [
{
source: 'host.name',
aggregation: { type: 'terms', limit: 10 },
destination: 'hostName',
},
{
source: 'agent.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
destination: 'agentName',
},
],
});
expect(generateHistoryMetadataAggregations(definition)).toEqual({
'entity.metadata.hostName': {
terms: {
field: 'host.name',
size: 10,
},
},
'entity.metadata.agentName': {
filter: {
exists: {
field: 'agent.name',
},
},
aggs: {
top_value: {
top_metrics: {
metrics: { field: 'agent.name' },
sort: { '@timestamp': 'desc' },
},
},
},
},
});
});
});
describe('generateLatestMetadataAggregations()', () => {
it('should generate metadata aggregations for string format', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: ['host.name'],
});
expect(generateLatestMetadataAggregations(definition)).toEqual({
'entity.metadata.host.name': {
filter: {
range: {
'@timestamp': {
gte: 'now-360s',
gte: 'now-10m',
},
},
},
@ -142,7 +30,7 @@ describe('Generate Metadata Aggregations for history and latest', () => {
data: {
terms: {
field: 'host.name',
size: 1000,
size: 10,
},
},
},
@ -160,7 +48,7 @@ describe('Generate Metadata Aggregations for history and latest', () => {
filter: {
range: {
'@timestamp': {
gte: 'now-360s',
gte: 'now-10m',
},
},
},
@ -168,7 +56,7 @@ describe('Generate Metadata Aggregations for history and latest', () => {
data: {
terms: {
field: 'host.name',
size: 1000,
size: 10,
},
},
},
@ -179,14 +67,16 @@ describe('Generate Metadata Aggregations for history and latest', () => {
it('should generate metadata aggregations for object format with source and aggregation', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }],
metadata: [
{ source: 'host.name', aggregation: { type: 'terms', limit: 10, lookbackPeriod: '1h' } },
],
});
expect(generateLatestMetadataAggregations(definition)).toEqual({
'entity.metadata.host.name': {
filter: {
range: {
'@timestamp': {
gte: 'now-360s',
gte: 'now-1h',
},
},
},
@ -218,14 +108,14 @@ describe('Generate Metadata Aggregations for history and latest', () => {
filter: {
range: {
'@timestamp': {
gte: 'now-360s',
gte: 'now-10m',
},
},
},
aggs: {
data: {
terms: {
field: 'hostName',
field: 'host.name',
size: 10,
},
},
@ -255,14 +145,14 @@ describe('Generate Metadata Aggregations for history and latest', () => {
filter: {
range: {
'@timestamp': {
gte: 'now-360s',
gte: 'now-10m',
},
},
},
aggs: {
data: {
terms: {
field: 'hostName',
field: 'host.name',
size: 10,
},
},
@ -275,13 +165,13 @@ describe('Generate Metadata Aggregations for history and latest', () => {
{
range: {
'@timestamp': {
gte: 'now-360s',
gte: 'now-10m',
},
},
},
{
exists: {
field: 'agentName',
field: 'agent.name',
},
},
],
@ -291,7 +181,7 @@ describe('Generate Metadata Aggregations for history and latest', () => {
top_value: {
top_metrics: {
metrics: {
field: 'agentName',
field: 'agent.name',
},
sort: {
'@timestamp': 'desc',

106
x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts
View file

@ -6,26 +6,51 @@
*/
import { EntityDefinition } from '@kbn/entities-schema';
import { calculateOffset } from '../helpers/calculate_offset';
export function generateHistoryMetadataAggregations(definition: EntityDefinition) {
export function generateLatestMetadataAggregations(definition: EntityDefinition) {
if (!definition.metadata) {
return {};
}
return definition.metadata.reduce((aggs, metadata) => {
const lookbackPeriod = metadata.aggregation.lookbackPeriod || definition.latest.lookbackPeriod;
let agg;
if (metadata.aggregation.type === 'terms') {
agg = {
terms: {
field: metadata.source,
size: metadata.aggregation.limit,
filter: {
range: {
'@timestamp': {
gte: `now-${lookbackPeriod}`,
},
},
},
aggs: {
data: {
terms: {
field: metadata.source,
size: metadata.aggregation.limit,
},
},
},
};
} else if (metadata.aggregation.type === 'top_value') {
agg = {
filter: {
exists: {
field: metadata.source,
bool: {
must: [
{
range: {
'@timestamp': {
gte: `now-${lookbackPeriod}`,
},
},
},
{
exists: {
field: metadata.source,
},
},
],
},
},
aggs: {
@ -47,70 +72,3 @@ export function generateHistoryMetadataAggregations(definition: EntityDefinition
};
}, {});
}
export function generateLatestMetadataAggregations(definition: EntityDefinition) {
if (!definition.metadata) {
return {};
}
const offsetInSeconds = `${calculateOffset(definition)}s`;
return definition.metadata.reduce((aggs, metadata) => {
let agg;
if (metadata.aggregation.type === 'terms') {
agg = {
filter: {
range: {
'@timestamp': {
gte: `now-${offsetInSeconds}`,
},
},
},
aggs: {
data: {
terms: {
field: metadata.destination,
size: metadata.aggregation.limit,
},
},
},
};
} else if (metadata.aggregation.type === 'top_value') {
agg = {
filter: {
bool: {
must: [
{
range: {
'@timestamp': {
gte: `now-${metadata.aggregation.lookbackPeriod ?? offsetInSeconds}`,
},
},
},
{
exists: {
field: metadata.destination,
},
},
],
},
},
aggs: {
top_value: {
top_metrics: {
metrics: {
field: metadata.destination,
},
sort: metadata.aggregation.sort,
},
},
},
};
}
return {
...aggs,
[`entity.metadata.${metadata.destination}`]: agg,
};
}, {});
}

30
x-pack/plugins/entity_manager/server/lib/entities/transform/generate_metric_aggregations.ts
View file

@ -104,41 +104,15 @@ function buildMetricEquation(keyMetric: KeyMetric) {
};
}
export function generateHistoryMetricAggregations(definition: EntityDefinition) {
if (!definition.metrics) {
return {};
}
return definition.metrics.reduce((aggs, keyMetric) => {
return {
...aggs,
...buildMetricAggregations(keyMetric, definition.history.timestampField),
[`entity.metrics.${keyMetric.name}`]: buildMetricEquation(keyMetric),
};
}, {});
}
export function generateLatestMetricAggregations(definition: EntityDefinition) {
if (!definition.metrics) {
return {};
}
return definition.metrics.reduce((aggs, keyMetric) => {
return {
...aggs,
[`_${keyMetric.name}`]: {
top_metrics: {
metrics: [{ field: `entity.metrics.${keyMetric.name}` }],
sort: [{ '@timestamp': 'desc' }],
},
},
[`entity.metrics.${keyMetric.name}`]: {
bucket_script: {
buckets_path: {
value: `_${keyMetric.name}[entity.metrics.${keyMetric.name}]`,
},
script: 'params.value',
},
},
...buildMetricAggregations(keyMetric, definition.latest.timestampField),
[`entity.metrics.${keyMetric.name}`]: buildMetricEquation(keyMetric),
};
}, {});
}

16
x-pack/plugins/entity_manager/server/lib/entities/transform/validate_transform_ids.ts
View file

@ -7,26 +7,14 @@
import { EntityDefinition } from '@kbn/entities-schema';
import { EntityDefinitionIdInvalid } from '../errors/entity_definition_id_invalid';
import {
generateHistoryBackfillTransformId,
generateHistoryTransformId,
generateLatestTransformId,
} from '../helpers/generate_component_id';
import { generateLatestTransformId } from '../helpers/generate_component_id';
const TRANSFORM_ID_MAX_LENGTH = 64;
export function validateDefinitionCanCreateValidTransformIds(definition: EntityDefinition) {
const historyTransformId = generateHistoryTransformId(definition);
const latestTransformId = generateLatestTransformId(definition);
const historyBackfillTransformId = generateHistoryBackfillTransformId(definition);
const spareChars =
TRANSFORM_ID_MAX_LENGTH -
Math.max(
historyTransformId.length,
latestTransformId.length,
historyBackfillTransformId.length
);
const spareChars = TRANSFORM_ID_MAX_LENGTH - latestTransformId.length;
if (spareChars < 0) {
throw new EntityDefinitionIdInvalid(

18
x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts
View file

@ -11,14 +11,10 @@ import { EntityDefinition } from '@kbn/entities-schema';
import { Logger } from '@kbn/logging';
import { deleteEntityDefinition } from './delete_entity_definition';
import { deleteIndices } from './delete_index';
import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline';
import { deleteIngestPipelines } from './delete_ingest_pipeline';
import { findEntityDefinitions } from './find_entity_definition';
import {
generateHistoryIndexTemplateId,
generateLatestIndexTemplateId,
} from './helpers/generate_component_id';
import { deleteTemplate } from '../manage_index_templates';
import { deleteTemplates } from '../manage_index_templates';
import { stopTransforms } from './stop_transforms';
@ -40,19 +36,13 @@ export async function uninstallEntityDefinition({
await stopTransforms(esClient, definition, logger);
await deleteTransforms(esClient, definition, logger);
await Promise.all([
deleteHistoryIngestPipeline(esClient, definition, logger),
deleteLatestIngestPipeline(esClient, definition, logger),
]);
await deleteIngestPipelines(esClient, definition, logger);
if (deleteData) {
await deleteIndices(esClient, definition, logger);
}
await Promise.all([
deleteTemplate({ esClient, logger, name: generateHistoryIndexTemplateId(definition) }),
deleteTemplate({ esClient, logger, name: generateLatestIndexTemplateId(definition) }),
]);
await deleteTemplates(esClient, definition, logger);
await deleteEntityDefinition(soClient, definition);
}

2
x-pack/plugins/entity_manager/server/lib/entity_client.ts
View file

@ -41,7 +41,7 @@ export class EntityClient {
});
if (!installOnly) {
await startTransforms(this.options.esClient, definition, this.options.logger);
await startTransforms(this.options.esClient, installedDefinition, this.options.logger);
}
return installedDefinition;

39
x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
import {
ClusterPutComponentTemplateRequest,
IndicesPutIndexTemplateRequest,
@ -15,6 +16,7 @@ import { entitiesLatestBaseComponentTemplateConfig } from '../templates/componen
import { entitiesEntityComponentTemplateConfig } from '../templates/components/entity';
import { entitiesEventComponentTemplateConfig } from '../templates/components/event';
import { retryTransientEsErrors } from './entities/helpers/retry';
import { generateEntitiesLatestIndexTemplateConfig } from './entities/templates/entities_latest_template';
interface TemplateManagementOptions {
esClient: ElasticsearchClient;
@ -67,14 +69,27 @@ interface DeleteTemplateOptions {
export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) {
try {
await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger });
const result = await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), {
logger,
});
logger.debug(() => `Installed entity manager index template: ${JSON.stringify(template)}`);
return result;
} catch (error: any) {
logger.error(`Error updating entity manager index template: ${error.message}`);
throw error;
}
}
export async function createAndInstallTemplates(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
): Promise<Array<{ type: 'template'; id: string }>> {
const template = generateEntitiesLatestIndexTemplateConfig(definition);
await upsertTemplate({ esClient, template, logger });
return [{ type: 'template', id: template.name }];
}
export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) {
try {
await retryTransientEsErrors(
@ -87,6 +102,28 @@ export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateO
}
}
export async function deleteTemplates(
esClient: ElasticsearchClient,
definition: EntityDefinition,
logger: Logger
) {
try {
await Promise.all(
(definition.installedComponents ?? [])
.filter(({ type }) => type === 'template')
.map(({ id }) =>
retryTransientEsErrors(
() => esClient.indices.deleteIndexTemplate({ name: id }, { ignore: [404] }),
{ logger }
)
)
);
} catch (error: any) {
logger.error(`Error deleting entity manager index template: ${error.message}`);
throw error;
}
}
export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) {
try {
await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), {

5
x-pack/plugins/entity_manager/server/routes/enablement/disable.ts
View file

@ -51,8 +51,8 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({
}),
handler: async ({ context, response, params, logger, server }) => {
try {
const esClient = (await context.core).elasticsearch.client.asCurrentUser;
const canDisable = await canDisableEntityDiscovery(esClient);
const esClientAsCurrentUser = (await context.core).elasticsearch.client.asCurrentUser;
const canDisable = await canDisableEntityDiscovery(esClientAsCurrentUser);
if (!canDisable) {
return response.forbidden({
body: {
@ -62,6 +62,7 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({
});
}
const esClient = (await context.core).elasticsearch.client.asSecondaryAuthUser;
const soClient = (await context.core).savedObjects.getClient({
includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
});

9
x-pack/plugins/entity_manager/server/routes/enablement/enable.ts
View file

@ -80,8 +80,10 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({
});
}
const esClient = (await context.core).elasticsearch.client.asCurrentUser;
const canEnable = await canEnableEntityDiscovery(esClient);
const core = await context.core;
const esClientAsCurrentUser = core.elasticsearch.client.asCurrentUser;
const canEnable = await canEnableEntityDiscovery(esClientAsCurrentUser);
if (!canEnable) {
return response.forbidden({
body: {
@ -91,7 +93,7 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({
});
}
const soClient = (await context.core).savedObjects.getClient({
const soClient = core.savedObjects.getClient({
includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
});
const existingApiKey = await readEntityDiscoveryAPIKey(server);
@ -117,6 +119,7 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({
await saveEntityDiscoveryAPIKey(soClient, apiKey);
const esClient = core.elasticsearch.client.asSecondaryAuthUser;
const installedDefinitions = await installBuiltInEntityDefinitions({
esClient,
soClient,

30
x-pack/plugins/entity_manager/server/routes/entities/reset.ts
View file

@ -12,25 +12,13 @@ import { EntitySecurityException } from '../../lib/entities/errors/entity_securi
import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error';
import { readEntityDefinition } from '../../lib/entities/read_entity_definition';
import {
deleteHistoryIngestPipeline,
deleteLatestIngestPipeline,
} from '../../lib/entities/delete_ingest_pipeline';
import { deleteIngestPipelines } from '../../lib/entities/delete_ingest_pipeline';
import { deleteIndices } from '../../lib/entities/delete_index';
import {
createAndInstallHistoryIngestPipeline,
createAndInstallLatestIngestPipeline,
} from '../../lib/entities/create_and_install_ingest_pipeline';
import {
createAndInstallHistoryBackfillTransform,
createAndInstallHistoryTransform,
createAndInstallLatestTransform,
} from '../../lib/entities/create_and_install_transform';
import { createAndInstallIngestPipelines } from '../../lib/entities/create_and_install_ingest_pipeline';
import { createAndInstallTransforms } from '../../lib/entities/create_and_install_transform';
import { startTransforms } from '../../lib/entities/start_transforms';
import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found';
import { isBackfillEnabled } from '../../lib/entities/helpers/is_backfill_enabled';
import { createEntityManagerServerRoute } from '../create_entity_manager_server_route';
import { deleteTransforms } from '../../lib/entities/delete_transforms';
import { stopTransforms } from '../../lib/entities/stop_transforms';
@ -51,18 +39,12 @@ export const resetEntityDefinitionRoute = createEntityManagerServerRoute({
await stopTransforms(esClient, definition, logger);
await deleteTransforms(esClient, definition, logger);
await deleteHistoryIngestPipeline(esClient, definition, logger);
await deleteLatestIngestPipeline(esClient, definition, logger);
await deleteIngestPipelines(esClient, definition, logger);
await deleteIndices(esClient, definition, logger);
// Recreate everything
await createAndInstallHistoryIngestPipeline(esClient, definition, logger);
await createAndInstallLatestIngestPipeline(esClient, definition, logger);
await createAndInstallHistoryTransform(esClient, definition, logger);
if (isBackfillEnabled(definition)) {
await createAndInstallHistoryBackfillTransform(esClient, definition, logger);
}
await createAndInstallLatestTransform(esClient, definition, logger);
await createAndInstallIngestPipelines(esClient, definition, logger);
await createAndInstallTransforms(esClient, definition, logger);
await startTransforms(esClient, definition, logger);
return response.ok({ body: { acknowledged: true } });

33
x-pack/plugins/entity_manager/server/saved_objects/entity_definition.ts
View file

@ -5,11 +5,36 @@
* 2.0.
*/
import { SavedObjectModelDataBackfillFn } from '@kbn/core-saved-objects-server';
import { SavedObject, SavedObjectsType } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import {
generateHistoryIndexTemplateId,
generateHistoryIngestPipelineId,
generateHistoryTransformId,
generateLatestIndexTemplateId,
generateLatestIngestPipelineId,
generateLatestTransformId,
} from '../lib/entities/helpers/generate_component_id';
export const SO_ENTITY_DEFINITION_TYPE = 'entity-definition';
export const backfillInstalledComponents: SavedObjectModelDataBackfillFn<
EntityDefinition,
EntityDefinition
> = (savedObject) => {
const definition = savedObject.attributes;
definition.installedComponents = [
{ type: 'transform', id: generateHistoryTransformId(definition) },
{ type: 'transform', id: generateLatestTransformId(definition) },
{ type: 'ingest_pipeline', id: generateHistoryIngestPipelineId(definition) },
{ type: 'ingest_pipeline', id: generateLatestIngestPipelineId(definition) },
{ type: 'template', id: generateHistoryIndexTemplateId(definition) },
{ type: 'template', id: generateLatestIndexTemplateId(definition) },
];
return savedObject;
};
export const entityDefinition: SavedObjectsType = {
name: SO_ENTITY_DEFINITION_TYPE,
hidden: false,
@ -64,5 +89,13 @@ export const entityDefinition: SavedObjectsType = {
},
],
},
'3': {
changes: [
{
type: 'data_backfill',
backfillFn: backfillInstalledComponents,
},
],
},
},
};

31
x-pack/plugins/entity_manager/server/templates/components/helpers.test.ts
View file

@ -1,31 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
import { getCustomHistoryTemplateComponents, getCustomLatestTemplateComponents } from './helpers';
describe('helpers', () => {
it('getCustomLatestTemplateComponents should return template component in the right sort order', () => {
const result = getCustomLatestTemplateComponents({ id: 'test' } as EntityDefinition);
expect(result).toEqual([
'test@platform',
'test-latest@platform',
'test@custom',
'test-latest@custom',
]);
});
it('getCustomHistoryTemplateComponents should return template component in the right sort order', () => {
const result = getCustomHistoryTemplateComponents({ id: 'test' } as EntityDefinition);
expect(result).toEqual([
'test@platform',
'test-history@platform',
'test@custom',
'test-history@custom',
]);
});
});

35
x-pack/plugins/entity_manager/server/templates/components/helpers.ts
View file

@ -1,35 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
import { isBuiltinDefinition } from '../../lib/entities/helpers/is_builtin_definition';
export const getCustomLatestTemplateComponents = (definition: EntityDefinition) => {
if (isBuiltinDefinition(definition)) {
return [];
}
return [
`${definition.id}@platform`, // @platform goes before so it can be overwritten by custom
`${definition.id}-latest@platform`,
`${definition.id}@custom`,
`${definition.id}-latest@custom`,
];
};
export const getCustomHistoryTemplateComponents = (definition: EntityDefinition) => {
if (isBuiltinDefinition(definition)) {
return [];
}
return [
`${definition.id}@platform`, // @platform goes before so it can be overwritten by custom
`${definition.id}-history@platform`,
`${definition.id}@custom`,
`${definition.id}-history@custom`,
];
};

1
x-pack/plugins/entity_manager/tsconfig.json
View file

@ -34,5 +34,6 @@
"@kbn/zod-helpers",
"@kbn/encrypted-saved-objects-plugin",
"@kbn/licensing-plugin",
"@kbn/core-saved-objects-server",
]
}

1
x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts
View file

@ -61,7 +61,6 @@ export async function getEntitiesWithSource({
identityFields: entity?.entity.identityFields,
id: entity?.entity.id,
definitionId: entity?.entity.definitionId,
firstSeenTimestamp: entity?.entity.firstSeenTimestamp,
lastSeenTimestamp: entity?.entity.lastSeenTimestamp,
displayName: entity?.entity.displayName,
metrics: entity?.entity.metrics,

6
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/definition.ts
View file

@ -27,9 +27,8 @@ export const buildHostEntityDefinition = (space: string): EntityDefinition =>
'host.type',
'host.architecture',
],
history: {
latest: {
timestampField: '@timestamp',
interval: '1m',
},
version: '1.0.0',
managed: true,
@ -44,9 +43,8 @@ export const buildUserEntityDefinition = (space: string): EntityDefinition =>
identityFields: ['user.name'],
displayNameTemplate: '{{user.name}}',
metadata: ['user.email', 'user.full_name', 'user.hash', 'user.id', 'user.name', 'user.roles'],
history: {
latest: {
timestampField: '@timestamp',
interval: '1m',
},
version: '1.0.0',
managed: true,

18
x-pack/test/api_integration/apis/entity_manager/definitions.ts
View file

@ -8,10 +8,7 @@
import semver from 'semver';
import expect from '@kbn/expect';
import { entityLatestSchema } from '@kbn/entities-schema';
import {
entityDefinition as mockDefinition,
entityDefinitionWithBackfill as mockBackfillDefinition,
} from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures';
import { entityDefinition as mockDefinition } from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures';
import { PartialConfig, cleanup, generate } from '@kbn/data-forge';
import { generateLatestIndexName } from '@kbn/entityManager-plugin/server/lib/entities/helpers/generate_component_id';
import { FtrProviderContext } from '../../ftr_provider_context';
@ -33,8 +30,9 @@ export default function ({ getService }: FtrProviderContext) {
describe('Entity definitions', () => {
describe('definitions installations', () => {
it('can install multiple definitions', async () => {
const mockDefinitionDup = { ...mockDefinition, id: 'mock_definition_dup' };
await installDefinition(supertest, { definition: mockDefinition });
await installDefinition(supertest, { definition: mockBackfillDefinition });
await installDefinition(supertest, { definition: mockDefinitionDup });
const { definitions } = await getInstalledDefinitions(supertest);
expect(definitions.length).to.eql(2);
@ -49,7 +47,7 @@ export default function ({ getService }: FtrProviderContext) {
expect(
definitions.some(
(definition) =>
definition.id === mockBackfillDefinition.id &&
definition.id === mockDefinitionDup.id &&
definition.state.installed === true &&
definition.state.running === true
)
@ -57,7 +55,7 @@ export default function ({ getService }: FtrProviderContext) {
await Promise.all([
uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true }),
uninstallDefinition(supertest, { id: mockBackfillDefinition.id, deleteData: true }),
uninstallDefinition(supertest, { id: mockDefinitionDup.id, deleteData: true }),
]);
});
@ -89,7 +87,7 @@ export default function ({ getService }: FtrProviderContext) {
id: mockDefinition.id,
update: {
version: incVersion!,
history: {
latest: {
timestampField: '@updatedTimestampField',
},
},
@ -99,7 +97,7 @@ export default function ({ getService }: FtrProviderContext) {
definitions: [updatedDefinition],
} = await getInstalledDefinitions(supertest);
expect(updatedDefinition.version).to.eql(incVersion);
expect(updatedDefinition.history.timestampField).to.eql('@updatedTimestampField');
expect(updatedDefinition.latest.timestampField).to.eql('@updatedTimestampField');
await uninstallDefinition(supertest, { id: mockDefinition.id });
});
@ -114,7 +112,7 @@ export default function ({ getService }: FtrProviderContext) {
id: mockDefinition.id,
update: {
version: '1.0.0',
history: {
latest: {
timestampField: '@updatedTimestampField',
},
},

12
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts
View file

@ -27,10 +27,7 @@ export default ({ getService }: FtrProviderContext) => {
it('should have installed the expected user resources', async () => {
await utils.initEntityEngineForEntityType('user');
const expectedTransforms = [
'entities-v1-history-ea_default_user_entity_store',
'entities-v1-latest-ea_default_user_entity_store',
];
const expectedTransforms = ['entities-v1-latest-ea_default_user_entity_store'];
await utils.expectTransformsExist(expectedTransforms);
});
@ -38,10 +35,7 @@ export default ({ getService }: FtrProviderContext) => {
it('should have installed the expected host resources', async () => {
await utils.initEntityEngineForEntityType('host');
const expectedTransforms = [
'entities-v1-history-ea_default_host_entity_store',
'entities-v1-latest-ea_default_host_entity_store',
];
const expectedTransforms = ['entities-v1-latest-ea_default_host_entity_store'];
await utils.expectTransformsExist(expectedTransforms);
});
@ -173,7 +167,6 @@ export default ({ getService }: FtrProviderContext) => {
})
.expect(200);
await utils.expectTransformNotFound('entities-v1-history-ea_host_entity_store');
await utils.expectTransformNotFound('entities-v1-latest-ea_host_entity_store');
});
@ -187,7 +180,6 @@ export default ({ getService }: FtrProviderContext) => {
})
.expect(200);
await utils.expectTransformNotFound('entities-v1-history-ea_user_entity_store');
await utils.expectTransformNotFound('entities-v1-latest-ea_user_entity_store');
});
});

10
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts
View file

@ -38,10 +38,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
it('should have installed the expected user resources', async () => {
await utils.initEntityEngineForEntityType('user');
const expectedTransforms = [
`entities-v1-history-ea_${namespace}_user_entity_store`,
`entities-v1-latest-ea_${namespace}_user_entity_store`,
];
const expectedTransforms = [`entities-v1-latest-ea_${namespace}_user_entity_store`];
await utils.expectTransformsExist(expectedTransforms);
});
@ -49,10 +46,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
it('should have installed the expected host resources', async () => {
await utils.initEntityEngineForEntityType('host');
const expectedTransforms = [
`entities-v1-history-ea_${namespace}_host_entity_store`,
`entities-v1-latest-ea_${namespace}_host_entity_store`,
];
const expectedTransforms = [`entities-v1-latest-ea_${namespace}_host_entity_store`];
await utils.expectTransformsExist(expectedTransforms);
});