github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[EEM] Remove duplicates from latest data set (#187699)

Browse source 
By only grouping on `entity.id` we should be able to remove duplicates
in the latest indices.
This PR also removes the values found for `entity.identityFields` and
replaces it with a list of those field names.
This PR also lifts the values for the identity fields to the root of the
document.
This PR removes the `displayName` from the historical documents.

### How to test

Source data:
```
PUT index_a
{
  "mappings": {
    "properties": {
      "a": {
        "type": "keyword"
      },
      "@timestamp": {
        "type": "date"
      }
    }
  }
}

PUT index_b
{
  "mappings": {
    "properties": {
      "b": {
        "type": "keyword"
      },
      "@timestamp": {
        "type": "date"
      }
    }
  }
}

POST index_a/_doc
{
  "a": "same",
  "@timestamp": "2024-07-05T12:33:06.162Z"
}

POST index_b/_doc
{
  "b": "same",
  "@timestamp": "2024-07-05T12:33:06.162Z"
}
```

Entity definition:
```
POST kbn:/internal/api/entities/definition
{
  "id": "bucket_key",
  "name": "Bucket key",
  "type": "service",
  "indexPatterns": [
    "index_*"
  ],
  "timestampField": "@timestamp",
  "lookback": "5m",
  "identityFields": [
    {
      "field": "a",
      "optional": true
    },
    {
      "field": "b",
      "optional": true
    }
  ],
  "displayNameTemplate": "{{a}}{{b}}",
  "history": {
    "timestampField": "@timestamp",
    "interval": "5m"
  }
}
```

### Change in the format of the resulting documents
```
"identityFields": {
  "a": null,
  "b": "same"
},
```
=>
```
"identityFields": [
  "a",
  "b"
],
```
This commit is contained in:
Milton Hultgren 2024-07-12 16:59:18 +02:00 • committed by GitHub
parent 25a4e242a2
commit 66e3f08c1d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 574 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

214
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/ingest_pipeline_script_processor_helpers.test.ts Normal file
View file

@ -0,0 +1,214 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { initializePathScript, cleanScript } from './ingest_pipeline_script_processor_helpers';
describe('Ingest Pipeline script processor helpers', () => {
describe('initializePathScript', () => {
it('initializes a single depth field', () => {
expect(initializePathScript('someField')).toMatchInlineSnapshot(`
"
if (ctx.someField == null) {
ctx.someField = new HashMap();
}
"
`);
});
it('initializes a multi depth field', () => {
expect(initializePathScript('some.nested.field')).toMatchInlineSnapshot(`
"
if (ctx.some == null) {
ctx.some = new HashMap();
}
if (ctx.some.nested == null) {
ctx.some.nested = new HashMap();
}
if (ctx.some.nested.field == null) {
ctx.some.nested.field = new HashMap();
}
"
`);
});
});
describe('cleanScript', () => {
it('removes duplicate empty lines and does basic indentation', () => {
const mostlyCleanScript = `
// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx["entity"]["identity"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx["entity"]["identity"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(":");
}
// Assign the slo.instanceId
ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown";
}
`;
expect(cleanScript(mostlyCleanScript)).toMatchInlineSnapshot(`
"// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx[\\"entity\\"][\\"identity\\"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(\\":\\");
}
// Assign the slo.instanceId
ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\";
}"
`);
const messyScript = `
if (someThing) {
${initializePathScript('some.whatever')}
${initializePathScript('some.else.whatever')}
ctx.some.thing.else = whatever;
if (nothing) {
more.stuff = otherStuff;
}
}
`;
expect(cleanScript(messyScript)).toMatchInlineSnapshot(`
"if (someThing) {
if (ctx.some == null) {
ctx.some = new HashMap();
}
if (ctx.some.whatever == null) {
ctx.some.whatever = new HashMap();
}
if (ctx.some == null) {
ctx.some = new HashMap();
}
if (ctx.some.else == null) {
ctx.some.else = new HashMap();
}
if (ctx.some.else.whatever == null) {
ctx.some.else.whatever = new HashMap();
}
ctx.some.thing.else = whatever;
if (nothing) {
more.stuff = otherStuff;
}
}"
`);
});
it('does not change any non white space character', () => {
const mostlyCleanScript = `
// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx["entity"]["identity"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx["entity"]["identity"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(":");
}
// Assign the slo.instanceId
ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown";
}
`;
function nonWhiteSpaceCharacters(string: string) {
return string
.split('')
.filter((character) => character !== ' ' && character !== '\t' && character !== '\n')
.join('');
}
expect(nonWhiteSpaceCharacters(cleanScript(mostlyCleanScript))).toEqual(
nonWhiteSpaceCharacters(mostlyCleanScript)
);
});
});
});

51
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/ingest_pipeline_script_processor_helpers.ts Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { first, last } from 'lodash';
export function initializePathScript(field: string) {
return field.split('.').reduce((acc, _part, currentIndex, parts) => {
const currentSegment = parts.slice(0, currentIndex + 1).join('.');
const next = `
if (ctx.${currentSegment} == null) {
ctx.${currentSegment} = new HashMap();
}
`;
return `${acc}\n${next}`;
}, '');
}
export function cleanScript(script: string) {
const codeLines = script
.split('\n')
.map((line) => line.trim())
.filter((line) => line !== '');
let cleanedScript = '';
let indentLevel = 0;
for (let i = 0; i < codeLines.length; i++) {
if (i === 0) {
cleanedScript += `${codeLines[i]}\n`;
continue;
}
const previousLine = i === 0 ? null : codeLines[i - 1];
const currentLine = codeLines[i];
if (last(previousLine) === '{') {
indentLevel++;
} else if (first(currentLine) === '}') {
indentLevel--;
}
const indent = new Array(indentLevel).fill(' ').join('');
cleanedScript += `${indent}${currentLine}\n`;
}
return cleanedScript.trim();
}

129
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap generated
View file

@ -34,50 +34,46 @@ Array [
},
Object {
"set": Object {
"field": "entity.displayName",
"value": "{{entity.identityFields.log.logger}}{{#entity.identityFields.event.category}}:{{.}}{{/entity.identityFields.event.category}}",
"field": "entity.identityFields",
"value": Array [
"log.logger",
"event.category",
],
},
},
Object {
"script": Object {
"description": "Generated the entity.id field",
"source": "
// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx[\\"entity\\"][\\"identityFields\\"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx[\\"entity\\"][\\"identityFields\\"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(\\":\\");
}
// Assign the slo.instanceId
ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\";
}
",
"source": "// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
// Iterate through the values
for(Object value: subject.values()) {
// If the value is a HashMap, recurse
if (value instanceof HashMap) {
values.addAll(collectValues((HashMap) value));
} else {
values.add(String.valueOf(value));
}
}
return values;
}
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx[\\"entity\\"][\\"identity\\"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx[\\"entity\\"][\\"identity\\"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
Collections.sort(sortedValues);
// Create comma delimited string
for(String instanceValue: sortedValues) {
entityId.append(instanceValue);
entityId.append(\\":\\");
}
// Assign the entity.id
ctx[\\"entity\\"][\\"id\\"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : \\"unknown\\";
}",
},
},
Object {
@ -92,21 +88,38 @@ Array [
Object {
"script": Object {
"source": "if (ctx.entity?.metadata?.tags != null) {
ctx[\\"tags\\"] = ctx.entity.metadata.tags.keySet();
if (ctx.tags == null) {
ctx.tags = new HashMap();
}
ctx.tags = ctx.entity.metadata.tags.keySet();
}
if (ctx.entity?.metadata?.host?.name != null) {
if(ctx.host == null) ctx[\\"host\\"] = new HashMap();
ctx[\\"host\\"][\\"name\\"] = ctx.entity.metadata.host.name.keySet();
if (ctx.host == null) {
ctx.host = new HashMap();
}
if (ctx.host.name == null) {
ctx.host.name = new HashMap();
}
ctx.host.name = ctx.entity.metadata.host.name.keySet();
}
if (ctx.entity?.metadata?.host?.os?.name != null) {
if(ctx.host == null) ctx[\\"host\\"] = new HashMap();
if(ctx.host.os == null) ctx[\\"host\\"][\\"os\\"] = new HashMap();
ctx[\\"host\\"][\\"os\\"][\\"name\\"] = ctx.entity.metadata.host.os.name.keySet();
if (ctx.host == null) {
ctx.host = new HashMap();
}
if (ctx.host.os == null) {
ctx.host.os = new HashMap();
}
if (ctx.host.os.name == null) {
ctx.host.os.name = new HashMap();
}
ctx.host.os.name = ctx.entity.metadata.host.os.name.keySet();
}
if (ctx.entity?.metadata?.sourceIndex != null) {
ctx[\\"sourceIndex\\"] = ctx.entity.metadata.sourceIndex.keySet();
}
",
if (ctx.sourceIndex == null) {
ctx.sourceIndex = new HashMap();
}
ctx.sourceIndex = ctx.entity.metadata.sourceIndex.keySet();
}",
},
},
Object {
@ -115,6 +128,26 @@ if (ctx.entity?.metadata?.sourceIndex != null) {
"ignore_missing": true,
},
},
Object {
"set": Object {
"field": "log.logger",
"if": "ctx.entity?.identity?.log?.logger != null",
"value": "{{entity.identity.log.logger}}",
},
},
Object {
"set": Object {
"field": "event.category",
"if": "ctx.entity?.identity?.event?.category != null",
"value": "{{entity.identity.event.category}}",
},
},
Object {
"remove": Object {
"field": "entity.identity",
"ignore_missing": true,
},
},
Object {
"date_index_name": Object {
"date_formats": Array [

80
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap generated
View file

@ -32,24 +32,50 @@ Array [
"value": "v1",
},
},
Object {
"set": Object {
"field": "entity.identityFields",
"value": Array [
"log.logger",
"event.category",
],
},
},
Object {
"script": Object {
"source": "if (ctx.entity?.metadata?.tags.data != null) {
ctx[\\"tags\\"] = ctx.entity.metadata.tags.data.keySet();
if (ctx.tags == null) {
ctx.tags = new HashMap();
}
ctx.tags = ctx.entity.metadata.tags.data.keySet();
}
if (ctx.entity?.metadata?.host?.name.data != null) {
if(ctx.host == null) ctx[\\"host\\"] = new HashMap();
ctx[\\"host\\"][\\"name\\"] = ctx.entity.metadata.host.name.data.keySet();
if (ctx.host == null) {
ctx.host = new HashMap();
}
if (ctx.host.name == null) {
ctx.host.name = new HashMap();
}
ctx.host.name = ctx.entity.metadata.host.name.data.keySet();
}
if (ctx.entity?.metadata?.host?.os?.name.data != null) {
if(ctx.host == null) ctx[\\"host\\"] = new HashMap();
if(ctx.host.os == null) ctx[\\"host\\"][\\"os\\"] = new HashMap();
ctx[\\"host\\"][\\"os\\"][\\"name\\"] = ctx.entity.metadata.host.os.name.data.keySet();
if (ctx.host == null) {
ctx.host = new HashMap();
}
if (ctx.host.os == null) {
ctx.host.os = new HashMap();
}
if (ctx.host.os.name == null) {
ctx.host.os.name = new HashMap();
}
ctx.host.os.name = ctx.entity.metadata.host.os.name.data.keySet();
}
if (ctx.entity?.metadata?.sourceIndex.data != null) {
ctx[\\"sourceIndex\\"] = ctx.entity.metadata.sourceIndex.data.keySet();
}
",
if (ctx.sourceIndex == null) {
ctx.sourceIndex = new HashMap();
}
ctx.sourceIndex = ctx.entity.metadata.sourceIndex.data.keySet();
}",
},
},
Object {
@ -58,6 +84,42 @@ if (ctx.entity?.metadata?.sourceIndex.data != null) {
"ignore_missing": true,
},
},
Object {
"script": Object {
"if": "ctx.entity.identity.log?.logger != null && ctx.entity.identity.log.logger.size() != 0",
"source": "if (ctx.log == null) {
ctx.log = new HashMap();
}
if (ctx.log.logger == null) {
ctx.log.logger = new HashMap();
}
ctx.log.logger = ctx.entity.identity.log.logger.keySet().toArray()[0];",
},
},
Object {
"script": Object {
"if": "ctx.entity.identity.event?.category != null && ctx.entity.identity.event.category.size() != 0",
"source": "if (ctx.event == null) {
ctx.event = new HashMap();
}
if (ctx.event.category == null) {
ctx.event.category = new HashMap();
}
ctx.event.category = ctx.entity.identity.event.category.keySet().toArray()[0];",
},
},
Object {
"remove": Object {
"field": "entity.identity",
"ignore_missing": true,
},
},
Object {
"set": Object {
"field": "entity.displayName",
"value": "{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}",
},
},
Object {
"set": Object {
"field": "_index",

75
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts
View file

@ -7,42 +7,46 @@
import { EntityDefinition } from '@kbn/entities-schema';
import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities';
import {
initializePathScript,
cleanScript,
} from '../helpers/ingest_pipeline_script_processor_helpers';
import { generateHistoryIndexName } from '../helpers/generate_component_id';
function createIdTemplate(definition: EntityDefinition) {
return definition.identityFields.reduce((template, id) => {
return template.replaceAll(id.field, `entity.identityFields.${id.field}`);
}, definition.displayNameTemplate);
}
function mapDestinationToPainless(destination: string) {
const fieldParts = destination.split('.');
return fieldParts.reduce((acc, _part, currentIndex, parts) => {
if (currentIndex + 1 === parts.length) {
return `${acc}\n ctx${parts
.map((s) => `["${s}"]`)
.join('')} = ctx.entity.metadata.${destination}.keySet();`;
}
return `${acc}\n if(ctx.${parts.slice(0, currentIndex + 1).join('.')} == null) ctx${parts
.slice(0, currentIndex + 1)
.map((s) => `["${s}"]`)
.join('')} = new HashMap();`;
}, '');
function mapDestinationToPainless(field: string) {
return `
${initializePathScript(field)}
ctx.${field} = ctx.entity.metadata.${field}.keySet();
`;
}
function createMetadataPainlessScript(definition: EntityDefinition) {
if (!definition.metadata) {
return '';
}
return definition.metadata.reduce((script, def) => {
return definition.metadata.reduce((acc, def) => {
const destination = def.destination || def.source;
return `${script}if (ctx.entity?.metadata?.${destination.replaceAll(
'.',
'?.'
)} != null) {${mapDestinationToPainless(destination)}\n}\n`;
const optionalFieldPath = destination.replaceAll('.', '?.');
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath} != null) {
${mapDestinationToPainless(destination)}
}
`;
return `${acc}\n${next}`;
}, '');
}
function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) {
return definition.identityFields.map((key) => ({
set: {
if: `ctx.entity?.identity?.${key.field.replaceAll('.', '?.')} != null`,
field: key.field,
value: `{{entity.identity.${key.field}}}`,
},
}));
}
export function generateHistoryProcessors(definition: EntityDefinition) {
return [
{
@ -77,14 +81,14 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
},
{
set: {
field: 'entity.displayName',
value: createIdTemplate(definition),
field: 'entity.identityFields',
value: definition.identityFields.map((identityField) => identityField.field),
},
},
{
script: {
description: 'Generated the entity.id field',
source: `
source: cleanScript(`
// This function will recursively collect all the values of a HashMap of HashMaps
Collection collectValues(HashMap subject) {
Collection values = new ArrayList();
@ -103,9 +107,9 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
// Create the string builder
StringBuilder entityId = new StringBuilder();
if (ctx["entity"]["identityFields"] != null) {
if (ctx["entity"]["identity"] != null) {
// Get the values as a collection
Collection values = collectValues(ctx["entity"]["identityFields"]);
Collection values = collectValues(ctx["entity"]["identity"]);
// Convert to a list and sort
List sortedValues = new ArrayList(values);
@ -117,10 +121,10 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
entityId.append(":");
}
// Assign the slo.instanceId
// Assign the entity.id
ctx["entity"]["id"] = entityId.length() > 0 ? entityId.substring(0, entityId.length() - 1) : "unknown";
}
`,
`),
},
},
{
@ -136,7 +140,7 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
}))
: []),
...(definition.metadata != null
? [{ script: { source: createMetadataPainlessScript(definition) } }]
? [{ script: { source: cleanScript(createMetadataPainlessScript(definition)) } }]
: []),
{
remove: {
@ -144,6 +148,13 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
ignore_missing: true,
},
},
...liftIdentityFieldsToDocumentRoot(definition),
{
remove: {
field: 'entity.identity',
ignore_missing: true,
},
},
{
date_index_name: {
field: '@timestamp',

71
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts
View file

@ -7,36 +7,49 @@
import { EntityDefinition } from '@kbn/entities-schema';
import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities';
import {
initializePathScript,
cleanScript,
} from '../helpers/ingest_pipeline_script_processor_helpers';
import { generateLatestIndexName } from '../helpers/generate_component_id';
function mapDestinationToPainless(destination: string) {
const fieldParts = destination.split('.');
return fieldParts.reduce((acc, _part, currentIndex, parts) => {
if (currentIndex + 1 === parts.length) {
return `${acc}\n ctx${parts
.map((s) => `["${s}"]`)
.join('')} = ctx.entity.metadata.${destination}.data.keySet();`;
}
return `${acc}\n if(ctx.${parts.slice(0, currentIndex + 1).join('.')} == null) ctx${parts
.slice(0, currentIndex + 1)
.map((s) => `["${s}"]`)
.join('')} = new HashMap();`;
}, '');
function mapDestinationToPainless(field: string) {
return `
${initializePathScript(field)}
ctx.${field} = ctx.entity.metadata.${field}.data.keySet();
`;
}
function createMetadataPainlessScript(definition: EntityDefinition) {
if (!definition.metadata) {
return '';
}
return definition.metadata.reduce((script, def) => {
return definition.metadata.reduce((acc, def) => {
const destination = def.destination || def.source;
return `${script}if (ctx.entity?.metadata?.${destination.replaceAll(
'.',
'?.'
)}.data != null) {${mapDestinationToPainless(destination)}\n}\n`;
const optionalFieldPath = destination.replaceAll('.', '?.');
const next = `
if (ctx.entity?.metadata?.${optionalFieldPath}.data != null) {
${mapDestinationToPainless(destination)}
}
`;
return `${acc}\n${next}`;
}, '');
}
function liftIdentityFieldsToDocumentRoot(definition: EntityDefinition) {
return definition.identityFields.map((identityField) => {
const optionalFieldPath = identityField.field.replaceAll('.', '?.');
const assignValue = `ctx.${identityField.field} = ctx.entity.identity.${identityField.field}.keySet().toArray()[0];`;
return {
script: {
if: `ctx.entity.identity.${optionalFieldPath} != null && ctx.entity.identity.${identityField.field}.size() != 0`,
source: cleanScript(`${initializePathScript(identityField.field)}\n${assignValue}`),
},
};
});
}
export function generateLatestProcessors(definition: EntityDefinition) {
return [
{
@ -69,13 +82,19 @@ export function generateLatestProcessors(definition: EntityDefinition) {
value: ENTITY_SCHEMA_VERSION_V1,
},
},
{
set: {
field: 'entity.identityFields',
value: definition.identityFields.map((identityField) => identityField.field),
},
},
...(definition.staticFields != null
? Object.keys(definition.staticFields).map((field) => ({
set: { field, value: definition.staticFields![field] },
}))
: []),
...(definition.metadata != null
? [{ script: { source: createMetadataPainlessScript(definition) } }]
? [{ script: { source: cleanScript(createMetadataPainlessScript(definition)) } }]
: []),
{
remove: {
@ -83,6 +102,20 @@ export function generateLatestProcessors(definition: EntityDefinition) {
ignore_missing: true,
},
},
...liftIdentityFieldsToDocumentRoot(definition),
{
remove: {
field: 'entity.identity',
ignore_missing: true,
},
},
{
// This must happen AFTER we lift the identity fields into the root of the document
set: {
field: 'entity.displayName',
value: definition.displayNameTemplate,
},
},
{
set: {
field: '_index',

4
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap generated
View file

@ -101,13 +101,13 @@ Object {
"fixed_interval": "1m",
},
},
"entity.identityFields.event.category": Object {
"entity.identity.event.category": Object {
"terms": Object {
"field": "event.category",
"missing_bucket": true,
},
},
"entity.identityFields.log.logger": Object {
"entity.identity.log.logger": Object {
"terms": Object {
"field": "log.logger",
"missing_bucket": false,

29
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap generated
View file

@ -47,6 +47,18 @@ Object {
"field": "@timestamp",
},
},
"entity.identity.event.category": Object {
"terms": Object {
"field": "event.category",
"size": 1,
},
},
"entity.identity.log.logger": Object {
"terms": Object {
"field": "log.logger",
"size": 1,
},
},
"entity.lastSeenTimestamp": Object {
"max": Object {
"field": "entity.lastSeenTimestamp",
@ -138,28 +150,11 @@ Object {
},
},
"group_by": Object {
"entity.displayName": Object {
"terms": Object {
"field": "entity.displayName.keyword",
},
},
"entity.id": Object {
"terms": Object {
"field": "entity.id",
},
},
"entity.identityFields.event.category": Object {
"terms": Object {
"field": "entity.identityFields.event.category",
"missing_bucket": true,
},
},
"entity.identityFields.log.logger": Object {
"terms": Object {
"field": "entity.identityFields.log.logger",
"missing_bucket": false,
},
},
},
},
"settings": Object {

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts
View file

@ -69,7 +69,7 @@ export function generateHistoryTransform(
...definition.identityFields.reduce(
(acc, id) => ({
...acc,
[`entity.identityFields.${id.field}`]: {
[`entity.identity.${id.field}`]: {
terms: { field: id.field, missing_bucket: id.optional },
},
}),

23
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_identity_aggregations.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
export function generateIdentityAggregations(definition: EntityDefinition) {
return definition.identityFields.reduce(
(aggs, identityField) => ({
...aggs,
[`entity.identity.${identityField.field}`]: {
terms: {
field: identityField.field,
size: 1,
},
},
}),
{}
);
}

22
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_latest_transform.ts
View file

@ -5,19 +5,20 @@
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import { EntityDefinition } from '@kbn/entities-schema';
import {
ENTITY_DEFAULT_LATEST_FREQUENCY,
ENTITY_DEFAULT_LATEST_SYNC_DELAY,
} from '../../../../common/constants_entities';
import { generateLatestMetadataAggregations } from './generate_metadata_aggregations';
import {
generateHistoryIndexName,
generateLatestTransformId,
generateLatestIngestPipelineId,
generateLatestIndexName,
generateLatestIngestPipelineId,
generateLatestTransformId,
} from '../helpers/generate_component_id';
import { generateIdentityAggregations } from './generate_identity_aggregations';
import { generateLatestMetadataAggregations } from './generate_metadata_aggregations';
import { generateLatestMetricAggregations } from './generate_metric_aggregations';
export function generateLatestTransform(
@ -53,22 +54,11 @@ export function generateLatestTransform(
['entity.id']: {
terms: { field: 'entity.id' },
},
['entity.displayName']: {
terms: { field: 'entity.displayName.keyword' },
},
...definition.identityFields.reduce(
(acc, id) => ({
...acc,
[`entity.identityFields.${id.field}`]: {
terms: { field: `entity.identityFields.${id.field}`, missing_bucket: id.optional },
},
}),
{}
),
},
aggs: {
...generateLatestMetricAggregations(definition),
...generateLatestMetadataAggregations(definition),
...generateIdentityAggregations(definition),
'entity.lastSeenTimestamp': {
max: {
field: 'entity.lastSeenTimestamp',

16
x-pack/plugins/observability_solution/entity_manager/server/templates/components/base_latest.ts
View file

@ -20,6 +20,22 @@ export const entitiesLatestBaseComponentTemplateConfig: ClusterPutComponentTempl
template: {
mappings: {
properties: {
entity: {
properties: {
displayName: {
type: 'text',
fields: {
keyword: {
ignore_above: 1024,
type: 'keyword',
},
},
},
firstSeenTimestamp: {
type: 'date',
},
},
},
labels: {
type: 'object',
},

13
x-pack/plugins/observability_solution/entity_manager/server/templates/components/entity.ts
View file

@ -29,15 +29,6 @@ export const entitiesEntityComponentTemplateConfig: ClusterPutComponentTemplateR
ignore_above: 1024,
type: 'keyword',
},
displayName: {
type: 'text',
fields: {
keyword: {
ignore_above: 1024,
type: 'keyword',
},
},
},
definitionId: {
ignore_above: 1024,
type: 'keyword',
@ -53,8 +44,8 @@ export const entitiesEntityComponentTemplateConfig: ClusterPutComponentTemplateR
lastSeenTimestamp: {
type: 'date',
},
firstSeenTimestamp: {
type: 'date',
identityFields: {
type: 'keyword',
},
},
},