github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[eem] top_value metadata aggregation (#188243)

Browse source
This commit is contained in:
Kevin Lacabane 2024-09-16 03:01:41 +02:00 • committed by GitHub
parent 8bffd61805
commit 1e6b13c8e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 346 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

22
x-pack/packages/kbn-entities-schema/src/schema/__snapshots__/common.test.ts.snap generated
View file

@ -67,7 +67,7 @@ Object {
"limit"
],
"code": "custom",
"message": "limit should be greater than 1"
"message": "limit for terms aggregation should be greater than 1"
}
]],
"success": false,
@ -77,8 +77,11 @@ Object {
exports[`schemas metadataSchema should parse successfully with a source and desitination 1`] = `
Object {
"data": Object {
"aggregation": Object {
"limit": 1000,
"type": "terms",
},
"destination": "hostName",
"limit": 1000,
"source": "host.name",
},
"success": true,
@ -88,8 +91,11 @@ Object {
exports[`schemas metadataSchema should parse successfully with an valid string 1`] = `
Object {
"data": Object {
"aggregation": Object {
"limit": 1000,
"type": "terms",
},
"destination": "host.name",
"limit": 1000,
"source": "host.name",
},
"success": true,
@ -99,8 +105,11 @@ Object {
exports[`schemas metadataSchema should parse successfully with just a source 1`] = `
Object {
"data": Object {
"aggregation": Object {
"limit": 1000,
"type": "terms",
},
"destination": "host.name",
"limit": 1000,
"source": "host.name",
},
"success": true,
@ -110,8 +119,11 @@ Object {
exports[`schemas metadataSchema should parse successfully with valid object 1`] = `
Object {
"data": Object {
"aggregation": Object {
"limit": 1000,
"type": "terms",
},
"destination": "hostName",
"limit": 1000,
"source": "host.name",
},
"success": true,

34
x-pack/packages/kbn-entities-schema/src/schema/common.test.ts
View file

@ -28,7 +28,7 @@ describe('schemas', () => {
const result = metadataSchema.safeParse({
source: 'host.name',
destination: 'host.name',
limit: 0,
aggregation: { type: 'terms', limit: 0 },
});
expect(result.success).toBeFalsy();
expect(result).toMatchSnapshot();
@ -52,11 +52,41 @@ describe('schemas', () => {
const result = metadataSchema.safeParse({
source: 'host.name',
destination: 'hostName',
size: 1,
});
expect(result.success).toBeTruthy();
expect(result).toMatchSnapshot();
});
it('should default to terms aggregation when none provided', () => {
const result = metadataSchema.safeParse({
source: 'host.name',
destination: 'hostName',
});
expect(result.success).toBeTruthy();
expect(result.data).toEqual({
source: 'host.name',
destination: 'hostName',
aggregation: { type: 'terms', limit: 1000 },
});
});
it('should parse supported aggregations', () => {
const result = metadataSchema.safeParse({
source: 'host.name',
destination: 'hostName',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
});
expect(result.success).toBeTruthy();
});
it('should reject unsupported aggregation', () => {
const result = metadataSchema.safeParse({
source: 'host.name',
destination: 'hostName',
aggregation: { type: 'unknown_agg', limit: 10 },
});
expect(result.success).toBeFalsy();
});
});
describe('durationSchema', () => {

28
x-pack/packages/kbn-entities-schema/src/schema/common.ts
View file

@ -84,24 +84,40 @@ export const keyMetricSchema = z.object({
export type KeyMetric = z.infer<typeof keyMetricSchema>;
export const metadataAggregation = z.union([
z.object({ type: z.literal('terms'), limit: z.number().default(1000) }),
z.object({
type: z.literal('top_value'),
sort: z.record(z.string(), z.union([z.literal('asc'), z.literal('desc')])),
lookbackPeriod: z.optional(durationSchema),
}),
]);
export const metadataSchema = z
.object({
source: z.string(),
destination: z.optional(z.string()),
limit: z.optional(z.number().default(1000)),
aggregation: z
.optional(metadataAggregation)
.default({ type: z.literal('terms').value, limit: 1000 }),
})
.or(
z.string().transform((value) => ({
source: value,
destination: value,
aggregation: { type: z.literal('terms').value, limit: 1000 },
}))
)
.transform((metadata) => ({
...metadata,
destination: metadata.destination ?? metadata.source,
limit: metadata.limit ?? 1000,
}))
.or(z.string().transform((value) => ({ source: value, destination: value, limit: 1000 })))
.superRefine((value, ctx) => {
if (value.limit < 1) {
if (value.aggregation.type === 'terms' && value.aggregation.limit < 1) {
ctx.addIssue({
path: ['limit'],
code: z.ZodIssueCode.custom,
message: 'limit should be greater than 1',
message: 'limit for terms aggregation should be greater than 1',
});
}
if (value.source.length === 0) {
@ -120,6 +136,8 @@ export const metadataSchema = z
}
});
export type MetadataField = z.infer<typeof metadataSchema>;
export const identityFieldsSchema = z
.object({
field: z.string(),

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts
View file

@ -46,7 +46,7 @@ export const builtInServicesFromLogsEntityDefinition: EntityDefinition =
displayNameTemplate: '{{service.name}}{{#service.environment}}:{{.}}{{/service.environment}}',
metadata: [
{ source: '_index', destination: 'sourceIndex' },
{ source: 'agent.name', limit: 100 },
{ source: 'agent.name', aggregation: { type: 'terms', limit: 100 } },
'data_stream.type',
'service.environment',
'service.name',

43
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1 } from '@kbn/entities-schema';
import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1, MetadataField } from '@kbn/entities-schema';
import {
initializePathScript,
cleanScript,
@ -13,10 +13,19 @@ import {
import { generateHistoryIndexName } from '../helpers/generate_component_id';
import { isBuiltinDefinition } from '../helpers/is_builtin_definition';
function mapDestinationToPainless(field: string) {
function getMetadataSourceField({ aggregation, destination, source }: MetadataField) {
if (aggregation.type === 'terms') {
return `ctx.entity.metadata.${destination}.keySet()`;
} else if (aggregation.type === 'top_value') {
return `ctx.entity.metadata.${destination}.top_value["${source}"]`;
}
}
function mapDestinationToPainless(metadata: MetadataField) {
const field = metadata.destination;
return `
${initializePathScript(field)}
ctx.${field} = ctx.entity.metadata.${field}.keySet();
ctx.${field} = ${getMetadataSourceField(metadata)};
`;
}
@ -25,15 +34,27 @@ function createMetadataPainlessScript(definition: EntityDefinition) {
return '';
}
return definition.metadata.reduce((acc, def) => {
const destination = def.destination;
return definition.metadata.reduce((acc, metadata) => {
const { destination, source } = metadata;
const optionalFieldPath = destination.replaceAll('.', '?.');
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath} != null) {
${mapDestinationToPainless(destination)}
}
`;
return `${acc}\n${next}`;
if (metadata.aggregation.type === 'terms') {
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath} != null) {
${mapDestinationToPainless(metadata)}
}
`;
return `${acc}\n${next}`;
} else if (metadata.aggregation.type === 'top_value') {
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${source}"] != null) {
${mapDestinationToPainless(metadata)}
}
`;
return `${acc}\n${next}`;
}
return acc;
}, '');
}

43
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1 } from '@kbn/entities-schema';
import { EntityDefinition, ENTITY_SCHEMA_VERSION_V1, MetadataField } from '@kbn/entities-schema';
import {
initializePathScript,
cleanScript,
@ -13,10 +13,19 @@ import {
import { generateLatestIndexName } from '../helpers/generate_component_id';
import { isBuiltinDefinition } from '../helpers/is_builtin_definition';
function mapDestinationToPainless(field: string) {
function getMetadataSourceField({ aggregation, destination, source }: MetadataField) {
if (aggregation.type === 'terms') {
return `ctx.entity.metadata.${destination}.data.keySet()`;
} else if (aggregation.type === 'top_value') {
return `ctx.entity.metadata.${destination}.top_value["${destination}"]`;
}
}
function mapDestinationToPainless(metadata: MetadataField) {
const field = metadata.destination;
return `
${initializePathScript(field)}
ctx.${field} = ctx.entity.metadata.${field}.data.keySet();
ctx.${field} = ${getMetadataSourceField(metadata)};
`;
}
@ -25,15 +34,27 @@ function createMetadataPainlessScript(definition: EntityDefinition) {
return '';
}
return definition.metadata.reduce((acc, def) => {
const destination = def.destination || def.source;
return definition.metadata.reduce((acc, metadata) => {
const destination = metadata.destination;
const optionalFieldPath = destination.replaceAll('.', '?.');
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) {
${mapDestinationToPainless(destination)}
}
`;
return `${acc}\n${next}`;
if (metadata.aggregation.type === 'terms') {
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) {
${mapDestinationToPainless(metadata)}
}
`;
return `${acc}\n${next}`;
} else if (metadata.aggregation.type === 'top_value') {
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath}?.top_value["${destination}"] != null) {
${mapDestinationToPainless(metadata)}
}
`;
return `${acc}\n${next}`;
}
return acc;
}, '');
}

139
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.test.ts
View file

@ -44,10 +44,10 @@ describe('Generate Metadata Aggregations for history and latest', () => {
});
});
it('should generate metadata aggregations for object format with source and limit', () => {
it('should generate metadata aggregations for object format with source and aggregation', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [{ source: 'host.name', limit: 10 }],
metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }],
});
expect(generateHistoryMetadataAggregations(definition)).toEqual({
'entity.metadata.host.name': {
@ -59,11 +59,44 @@ describe('Generate Metadata Aggregations for history and latest', () => {
});
});
it('should generate metadata aggregations for object format with source, limit, and destination', () => {
it('should generate metadata aggregations for object format with source, aggregation, and destination', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }],
metadata: [
{
source: 'host.name',
aggregation: { type: 'terms', limit: 20 },
destination: 'hostName',
},
],
});
expect(generateHistoryMetadataAggregations(definition)).toEqual({
'entity.metadata.hostName': {
terms: {
field: 'host.name',
size: 20,
},
},
});
});
it('should generate metadata aggregations for terms and top_value', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [
{
source: 'host.name',
aggregation: { type: 'terms', limit: 10 },
destination: 'hostName',
},
{
source: 'agent.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
destination: 'agentName',
},
],
});
expect(generateHistoryMetadataAggregations(definition)).toEqual({
'entity.metadata.hostName': {
terms: {
@ -71,6 +104,21 @@ describe('Generate Metadata Aggregations for history and latest', () => {
size: 10,
},
},
'entity.metadata.agentName': {
filter: {
exists: {
field: 'agent.name',
},
},
aggs: {
top_value: {
top_metrics: {
metrics: { field: 'agent.name' },
sort: { '@timestamp': 'desc' },
},
},
},
},
});
});
});
@ -128,10 +176,10 @@ describe('Generate Metadata Aggregations for history and latest', () => {
});
});
it('should generate metadata aggregations for object format with source and limit', () => {
it('should generate metadata aggregations for object format with source and aggregation', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [{ source: 'host.name', limit: 10 }],
metadata: [{ source: 'host.name', aggregation: { type: 'terms', limit: 10 } }],
});
expect(generateLatestMetadataAggregations(definition)).toEqual({
'entity.metadata.host.name': {
@ -154,10 +202,16 @@ describe('Generate Metadata Aggregations for history and latest', () => {
});
});
it('should generate metadata aggregations for object format with source, limit, and destination', () => {
it('should generate metadata aggregations for object format with source, aggregation, and destination', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [{ source: 'host.name', limit: 10, destination: 'hostName' }],
metadata: [
{
source: 'host.name',
aggregation: { type: 'terms', limit: 10 },
destination: 'hostName',
},
],
});
expect(generateLatestMetadataAggregations(definition)).toEqual({
'entity.metadata.hostName': {
@ -179,5 +233,74 @@ describe('Generate Metadata Aggregations for history and latest', () => {
},
});
});
it('should generate metadata aggregations for terms and top_value', () => {
const definition = entityDefinitionSchema.parse({
...rawEntityDefinition,
metadata: [
{
source: 'host.name',
aggregation: { type: 'terms', limit: 10 },
destination: 'hostName',
},
{
source: 'agent.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
destination: 'agentName',
},
],
});
expect(generateLatestMetadataAggregations(definition)).toEqual({
'entity.metadata.hostName': {
filter: {
range: {
'@timestamp': {
gte: 'now-360s',
},
},
},
aggs: {
data: {
terms: {
field: 'hostName',
size: 10,
},
},
},
},
'entity.metadata.agentName': {
filter: {
bool: {
must: [
{
range: {
'@timestamp': {
gte: 'now-360s',
},
},
},
{
exists: {
field: 'agentName',
},
},
],
},
},
aggs: {
top_value: {
top_metrics: {
metrics: {
field: 'agentName',
},
sort: {
'@timestamp': 'desc',
},
},
},
},
},
});
});
});
});

100
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_metadata_aggregations.ts
View file

@ -6,25 +6,46 @@
*/
import { EntityDefinition } from '@kbn/entities-schema';
import { ENTITY_DEFAULT_METADATA_LIMIT } from '../../../../common/constants_entities';
import { calculateOffset } from '../helpers/calculate_offset';
export function generateHistoryMetadataAggregations(definition: EntityDefinition) {
if (!definition.metadata) {
return {};
}
return definition.metadata.reduce(
(aggs, metadata) => ({
...aggs,
[`entity.metadata.${metadata.destination ?? metadata.source}`]: {
return definition.metadata.reduce((aggs, metadata) => {
let agg;
if (metadata.aggregation.type === 'terms') {
agg = {
terms: {
field: metadata.source,
size: metadata.limit ?? ENTITY_DEFAULT_METADATA_LIMIT,
size: metadata.aggregation.limit,
},
},
}),
{}
);
};
} else if (metadata.aggregation.type === 'top_value') {
agg = {
filter: {
exists: {
field: metadata.source,
},
},
aggs: {
top_value: {
top_metrics: {
metrics: {
field: metadata.source,
},
sort: metadata.aggregation.sort,
},
},
},
};
}
return {
...aggs,
[`entity.metadata.${metadata.destination}`]: agg,
};
}, {});
}
export function generateLatestMetadataAggregations(definition: EntityDefinition) {
@ -32,29 +53,64 @@ export function generateLatestMetadataAggregations(definition: EntityDefinition)
return {};
}
const offsetInSeconds = calculateOffset(definition);
const offsetInSeconds = `${calculateOffset(definition)}s`;
return definition.metadata.reduce(
(aggs, metadata) => ({
...aggs,
[`entity.metadata.${metadata.destination}`]: {
return definition.metadata.reduce((aggs, metadata) => {
let agg;
if (metadata.aggregation.type === 'terms') {
agg = {
filter: {
range: {
'@timestamp': {
gte: `now-${offsetInSeconds}s`,
gte: `now-${offsetInSeconds}`,
},
},
},
aggs: {
data: {
terms: {
field: metadata.destination ?? metadata.source,
size: metadata.limit ?? ENTITY_DEFAULT_METADATA_LIMIT,
field: metadata.destination,
size: metadata.aggregation.limit,
},
},
},
},
}),
{}
);
};
} else if (metadata.aggregation.type === 'top_value') {
agg = {
filter: {
bool: {
must: [
{
range: {
'@timestamp': {
gte: `now-${metadata.aggregation.lookbackPeriod ?? offsetInSeconds}`,
},
},
},
{
exists: {
field: metadata.destination,
},
},
],
},
},
aggs: {
top_value: {
top_metrics: {
metrics: {
field: metadata.destination,
},
sort: metadata.aggregation.sort,
},
},
},
};
}
return {
...aggs,
[`entity.metadata.${metadata.destination}`]: agg,
};
}, {});
}