[EEM] Fix ingest pipeline to not overwrite existing parents in metadata (#186672)

## Summary

This fixes #186668 by adding a check to see if a `HashMap` exists before
creating a new one. This happens when the ingest pipeline converts the
output from a transfrom's `terms` sub-aggregation output to a list of
terms.

### Testing

Use Syntrace to generate data with this command:
```
node scripts/synthtrace packages/kbn-apm-synthtrace/src/scenarios/continuous_rollups.ts --live
```
Then create this Entity definition:
```JSON
POST kbn:/internal/api/entities/definition
{
  "id": "apm-countinuous-rollups",
  "name": "Services for APM",
  "type": "service",
  "indexPatterns": ["logs-*", "metrics-*", "traces-*"],
  "history": {
    "timestampField": "@timestamp",
    "interval": "1m"
  },
  "identityFields": ["service.name", "service.environment"],
  "displayNameTemplate": "{{service.name}}:{{service.environment}}",
  "metadata": [
    "host.name",
    "transaction.name",
    "transaction.type",
    "observer.version"
  ],
  "metrics": [
    {
      "name": "latency",
      "equation": "A",
      "metrics": [
        {
          "name": "A",
          "aggregation": "avg",
          "field": "transaction.duration.histogram"
        }
      ]
    },
    {
      "name": "throughput",
      "equation": "A / 5",
      "metrics": [
        {
          "name": "A",
          "aggregation": "doc_count"
        }
      ]
    },
    {
      "name": "failedTransRate",
      "equation": "A / B",
      "metrics": [
        {
          "name": "A",
          "aggregation": "doc_count",
          "filter": "event.outcome: \"failure\""
        },
        {
          "name": "B",
          "aggregation": "doc_count",
          "filter": "event.outcome: *"
        }
      ]
    }
  ]
}
```
Query the history with:
```JSON
POST .entities-observability.history-v1.*/_search?track_total_hits=true
{
  "sort": [
    {
      "@timestamp": {
        "order": "desc"
      }
    }
  ]
}
```
and query the latest with:
```JSON
POST .entities-observability.latest-v1.*/_search?track_total_hits=true
{}
```

You should see `transaction.name` with a list of transactions and
`transaction.type` with a list of the different transaction types.

Co-authored-by: Kevin Lacabane <kevin.lacabane@elastic.co>
This commit is contained in:
Chris Cowan 2024-06-24 13:18:20 -06:00 committed by GitHub
parent c13419e166
commit 9009cb8cf2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 38 additions and 5 deletions

View file

@ -17,7 +17,7 @@ export const entityDefinition = entityDefinitionSchema.parse({
},
identityFields: ['log.logger', { field: 'event.category', optional: true }],
displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}',
metadata: ['tags', 'host.name'],
metadata: ['tags', 'host.name', 'host.os.name'],
metrics: [
{
name: 'logRate',

View file

@ -83,9 +83,14 @@ Array [
ctx[\\"tags\\"] = ctx.entity.metadata.tags.keySet();
}
if (ctx.entity?.metadata?.host?.name != null) {
ctx[\\"host\\"] = new HashMap();
if(ctx.host == null) ctx[\\"host\\"] = new HashMap();
ctx[\\"host\\"][\\"name\\"] = ctx.entity.metadata.host.name.keySet();
}
if (ctx.entity?.metadata?.host?.os?.name != null) {
if(ctx.host == null) ctx[\\"host\\"] = new HashMap();
if(ctx.host.os == null) ctx[\\"host\\"][\\"os\\"] = new HashMap();
ctx[\\"host\\"][\\"os\\"][\\"name\\"] = ctx.entity.metadata.host.os.name.keySet();
}
",
},
},

View file

@ -26,9 +26,14 @@ Array [
ctx[\\"tags\\"] = ctx.entity.metadata.tags.data.keySet();
}
if (ctx.entity?.metadata?.host?.name.data != null) {
ctx[\\"host\\"] = new HashMap();
if(ctx.host == null) ctx[\\"host\\"] = new HashMap();
ctx[\\"host\\"][\\"name\\"] = ctx.entity.metadata.host.name.data.keySet();
}
if (ctx.entity?.metadata?.host?.os?.name.data != null) {
if(ctx.host == null) ctx[\\"host\\"] = new HashMap();
if(ctx.host.os == null) ctx[\\"host\\"][\\"os\\"] = new HashMap();
ctx[\\"host\\"][\\"os\\"][\\"name\\"] = ctx.entity.metadata.host.os.name.data.keySet();
}
",
},
},

View file

@ -22,7 +22,7 @@ function mapDestinationToPainless(destination: string, source: string) {
.map((s) => `["${s}"]`)
.join('')} = ctx.entity.metadata.${source}.keySet();`;
}
return `${acc}\n ctx${parts
return `${acc}\n if(ctx.${parts.slice(0, currentIndex + 1).join('.')} == null) ctx${parts
.slice(0, currentIndex + 1)
.map((s) => `["${s}"]`)
.join('')} = new HashMap();`;

View file

@ -16,7 +16,7 @@ function mapDestinationToPainless(destination: string, source: string) {
.map((s) => `["${s}"]`)
.join('')} = ctx.entity.metadata.${source}.data.keySet();`;
}
return `${acc}\n ctx${parts
return `${acc}\n if(ctx.${parts.slice(0, currentIndex + 1).join('.')} == null) ctx${parts
.slice(0, currentIndex + 1)
.map((s) => `["${s}"]`)
.join('')} = new HashMap();`;

View file

@ -49,6 +49,12 @@ Object {
"size": 1000,
},
},
"entity.metadata.host.os.name": Object {
"terms": Object {
"field": "host.os.name",
"size": 1000,
},
},
"entity.metadata.tags": Object {
"terms": Object {
"field": "tags",

View file

@ -65,6 +65,23 @@ Object {
},
},
},
"entity.metadata.host.os.name": Object {
"aggs": Object {
"data": Object {
"terms": Object {
"field": "host.os.name",
"size": 1000,
},
},
},
"filter": Object {
"range": Object {
"event.ingested": Object {
"gte": "now-1m",
},
},
},
},
"entity.metadata.tags": Object {
"aggs": Object {
"data": Object {