github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Automatic Import] Remove fields with @ from the script processor (#201548)

Browse source 
## Summary

This PR filters the fields containing `@` in date type from `script`
processor.

## Before this PR


![image](https://github.com/user-attachments/assets/a733d81f-aaaf-4787-b974-1e5d35ff4b8f)

```json
    {
      "script": {
        "tag": "script_convert_array_to_string",
        "description": "Ensures the date processor does not receive an array value.",
        "lang": "painless",
        "source": "if (ctx.varonis?.varonis_alerts?.@timestamp != null &&\n    ctx.varonis.varonis_alerts.@timestamp instanceof ArrayList){\n    ctx.varonis.varonis_alerts.@timestamp = ctx.varonis.varonis_alerts.@timestamp[0];\n}\n"
      }
    },
    {
      "date": {
        "if": "ctx.varonis?.varonis_alerts?.@timestamp != null",
        "tag": "date_processor_varonis.varonis_alerts.@timestamp",
        "field": "varonis.varonis_alerts.@timestamp",
        "target_field": "event.start",
        "formats": [
          "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
          "ISO8601"
        ]
      }
    },
```

## After this PR

```json
      "date": {
        "if": "ctx.varonis?.varonis_alerts?.@timestamp != null",
        "tag": "date_processor_varonis.varonis_alerts.@timestamp",
        "field": "varonis.varonis_alerts.@timestamp",
        "target_field": "event.start",
        "formats": [
          "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
          "ISO8601"
        ]
      }
    },
```

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
This commit is contained in:
Bharat Pasupula 2024-11-25 14:00:14 +01:00 • committed by GitHub
parent 92fc653480
commit 8964dc92c7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 692 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

381
x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { EcsMappingState } from '../../server/types';
import { SamplesFormatName } from '../../common';
export const ecsMappingExpectedResults = {
@ -480,3 +481,383 @@ export const ecsTestState = {
combinedSamples: '{"test1": "test1"}',
additionalProcessors: [],
};
export const ecsPipelineState: EcsMappingState = {
lastExecutedChain: 'validateMappings',
rawSamples: [],
additionalProcessors: [],
prefixedSamples: [
'{"xdfsfs":{"ds":{"ei":0,"event":"cert.create","uid":"efd326fc-dd13-4df8-erre-3102c2d717d3","code":"TC000I","time":"2024-02-24T06:56:50.648137154Z","cluster_name":"teleport.ericbeahan.com","cert_type":"user","identity":{"user":"teleport-admin","roles":["access","editor"],"logins":["root","ubuntu","ec2-user","-teleport-internal-join"],"expires":"2024-02-24T06:56:50.648137154Z","route_to_cluster":"teleport.ericbeahan.com","traits":{"aws_role_arns":null,"azure_identities":null,"db_names":null,"db_roles":null,"db_users":null,"gcp_service_accounts":null,"host_user_gid":[""],"host_user_uid":[""],"kubernetes_groups":null,"kubernetes_users":null,"logins":["root","ubuntu","ec2-user"],"windows_logins":null},"teleport_cluster":"teleport.ericbeahan.com","client_ip":"1.2.3.4","prev_identity_expires":"0001-01-01T00:00:00Z","private_key_policy":"none"}}}}',
'{"xdfsfs":{"ds":{"ei":0,"event":"session.start","uid":"fff30583-13be-49e8-b159-32952c6ea34f","code":"T2000I","time":"2024-02-23T18:56:57.648137154Z","cluster_name":"teleport.ericbeahan.com","user":"teleport-admin","login":"ec2-user","user_kind":1,"sid":"293fda2d-2266-4d4d-b9d1-bd5ea9dd9fc3","private_key_policy":"none","namespace":"default","server_id":"face0091-2bf1-54er-a16a-f1514b4119f4","server_hostname":"ip-172-31-8-163.us-east-2.compute.internal","server_labels":{"hostname":"ip-172-31-8-163.us-east-2.compute.internal","teleport.internal/resource-id":"dccb2999-9fb8-4169-aded-ec7a1c0a26de"},"addr.remote":"1.2.3.4:50339","proto":"ssh","size":"80:25","initial_command":[""],"session_recording":"node"}}}',
],
combinedSamples:
'{\n "xdfsfs": {\n "ds": {\n "identity": {\n "client_ip": "1.2.3.4",\n "prev_identity_expires": "0001-01-01T00:00:00Z",\n "private_key_policy": "none"\n },\n "user": "teleport-admin",\n "login": "ec2-user",\n "user_kind": 1,\n "sid": "293fda2d-2266-4d4d-b9d1-bd5ea9dd9fc3",\n "private_key_policy": "none",\n "namespace": "default",\n "server_id": "face0091-2bf1-43fd-a16a-f1514b4119f4",\n "server_hostname": "ip-172-31-8-163.us-east-2.compute.internal",\n "server_labels": {\n "hostname": "ip-172-31-8-163.us-east-2.compute.internal",\n "teleport.internal/resource-id": "dccb2999-9fb8-4169-aded-ec7a1c0a26de"\n },\n "addr.remote": "1.2.3.4:50339",\n "proto": "ssh",\n "size": "80:25",\n "initial_command": [\n ""\n ],\n "session_recording": "node"\n }\n }\n}',
sampleChunks: [],
exAnswer:
'{\n "crowdstrike": {\n "falcon": {\n "metadata": {\n "customerIDString": null,\n "offset": null,\n "eventType": {\n "target": "event.code",\n "confidence": 0.94,\n "type": "string",\n "date_formats": []\n },\n "eventCreationTime": {\n "target": "event.created",\n "confidence": 0.85,\n "type": "date",\n "date_formats": [\n "UNIX"\n ]\n },\n "version": null,\n "event": {\n "DeviceId": null,\n "CustomerId": null,\n "Ipv": {\n "target": "network.type",\n "confidence": 0.99,\n "type": "string",\n "date_formats": []\n }\n }\n }\n }\n }\n}',
packageName: 'xdfsfs',
dataStreamName: 'ds',
finalized: false,
currentMapping: {
xdfsfs: {
ds: {
identity: {
client_ip: {
target: 'client.ip',
confidence: 0.95,
type: 'string',
date_formats: [],
},
prev_identity_expires: {
target: 'event.end',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
},
private_key_policy: null,
},
user: {
target: 'user.name',
confidence: 0.9,
type: 'string',
date_formats: [],
},
login: {
target: 'user.id',
confidence: 0.8,
type: 'string',
date_formats: [],
},
user_kind: null,
sid: {
target: 'event.id',
confidence: 0.85,
type: 'string',
date_formats: [],
},
private_key_policy: null,
namespace: null,
server_id: {
target: 'host.id',
confidence: 0.9,
type: 'string',
date_formats: [],
},
server_hostname: {
target: 'host.hostname',
confidence: 0.95,
type: 'string',
date_formats: [],
},
server_labels: {
hostname: null,
'teleport.internal/resource-id': null,
},
'addr.remote': {
target: 'source.address',
confidence: 0.9,
type: 'string',
date_formats: [],
},
proto: {
target: 'network.protocol',
confidence: 0.95,
type: 'string',
date_formats: [],
},
size: null,
initial_command: null,
session_recording: null,
},
},
},
chunkMapping: {
xdfsfs: {
ds: {
ei: null,
event: {
target: 'event.action',
confidence: 0.9,
type: 'string',
date_formats: [],
},
uid: {
target: 'event.id',
confidence: 0.95,
type: 'string',
date_formats: [],
},
code: {
target: 'event.code',
confidence: 0.9,
type: 'string',
date_formats: [],
},
time: {
target: 'event.created',
confidence: 0.95,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
},
cluster_name: {
target: 'cloud.account.name',
confidence: 0.8,
type: 'string',
date_formats: [],
},
cert_type: null,
identity: {
user: {
target: 'user.name',
confidence: 0.95,
type: 'string',
date_formats: [],
},
roles: {
target: 'user.roles',
confidence: 0.9,
type: 'string',
date_formats: [],
},
logins: null,
expires: {
target: 'user.changes.name',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
},
route_to_cluster: null,
traits: {
aws_role_arns: null,
azure_identities: null,
db_names: null,
db_roles: null,
db_users: null,
gcp_service_accounts: null,
host_user_gid: null,
host_user_uid: null,
kubernetes_groups: null,
kubernetes_users: null,
logins: null,
windows_logins: null,
},
teleport_cluster: null,
client_ip: {
target: 'client.ip',
confidence: 0.95,
type: 'string',
date_formats: [],
},
prev_identity_expires: {
target: 'event.end',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
},
private_key_policy: null,
},
user: {
target: 'user.name',
confidence: 0.9,
type: 'string',
date_formats: [],
},
login: {
target: 'user.id',
confidence: 0.8,
type: 'string',
date_formats: [],
},
user_kind: null,
sid: {
target: 'event.id',
confidence: 0.85,
type: 'string',
date_formats: [],
},
private_key_policy: null,
namespace: null,
server_id: {
target: 'host.id',
confidence: 0.9,
type: 'string',
date_formats: [],
},
server_hostname: {
target: 'host.hostname',
confidence: 0.95,
type: 'string',
date_formats: [],
},
server_labels: {
hostname: null,
'teleport.internal/resource-id': null,
},
'addr.remote': {
target: 'source.address',
confidence: 0.9,
type: 'string',
date_formats: [],
},
proto: {
target: 'network.protocol',
confidence: 0.95,
type: 'string',
date_formats: [],
},
size: null,
initial_command: null,
session_recording: null,
},
},
},
finalMapping: {
xdfsfs: {
ds: {
ei: null,
event: {
target: 'event.action',
confidence: 0.9,
type: 'string',
date_formats: [],
},
uid: {
target: 'event.id',
confidence: 0.95,
type: 'string',
date_formats: [],
},
code: {
target: 'event.code',
confidence: 0.9,
type: 'string',
date_formats: [],
},
'@timestamp': {
target: '@timestamp',
confidence: 0.95,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
},
cluster_name: {
target: 'cloud.account.name',
confidence: 0.8,
type: 'string',
date_formats: [],
},
cert_type: null,
identity: {
user: {
target: 'user.name',
confidence: 0.95,
type: 'string',
date_formats: [],
},
roles: {
target: 'user.roles',
confidence: 0.9,
type: 'string',
date_formats: [],
},
logins: null,
expires: {
target: 'user.changes.name',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"],
},
route_to_cluster: null,
traits: {
aws_role_arns: null,
azure_identities: null,
db_names: null,
db_roles: null,
db_users: null,
gcp_service_accounts: null,
host_user_gid: null,
host_user_uid: null,
kubernetes_groups: null,
kubernetes_users: null,
logins: null,
windows_logins: null,
},
teleport_cluster: null,
client_ip: {
target: 'client.ip',
confidence: 0.95,
type: 'string',
date_formats: [],
},
prev_identity_expires: {
target: 'event.end',
confidence: 0.7,
type: 'date',
date_formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'"],
},
private_key_policy: null,
},
user: {
target: 'user.name',
confidence: 0.9,
type: 'string',
date_formats: [],
},
login: {
target: 'user.id',
confidence: 0.8,
type: 'string',
date_formats: [],
},
user_kind: null,
sid: null,
private_key_policy: null,
namespace: null,
server_id: {
target: 'host.id',
confidence: 0.9,
type: 'string',
date_formats: [],
},
server_hostname: {
target: 'host.hostname',
confidence: 0.95,
type: 'string',
date_formats: [],
},
server_labels: {
hostname: null,
'teleport.internal/resource-id': null,
},
'addr.remote': {
target: 'source.address',
confidence: 0.9,
type: 'string',
date_formats: [],
},
proto: {
target: 'network.protocol',
confidence: 0.95,
type: 'string',
date_formats: [],
},
size: null,
initial_command: null,
session_recording: null,
},
},
},
useFinalMapping: true,
hasTriedOnce: true,
currentPipeline: {},
duplicateFields: [],
missingKeys: [],
invalidEcsFields: [],
results: {},
samplesFormat: {
name: 'json',
json_path: [],
},
ecsVersion: '8.11.0',
ecs: '',
chunkSize: 0,
};

306
x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.test.ts Normal file
View file

@ -0,0 +1,306 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ecsPipelineState } from '../../../__jest__/fixtures/ecs_mapping';
import type { EcsMappingState } from '../../types';
import { createPipeline } from './pipeline';
const state: EcsMappingState = ecsPipelineState;
describe('Testing pipeline templates', () => {
it('handle pipeline creation', async () => {
const pipeline = createPipeline(state);
expect(pipeline.processors).toEqual([
{
set: { field: 'ecs.version', tag: 'set_ecs_version', value: '8.11.0' },
},
{
set: {
field: 'originalMessage',
copy_from: 'message',
tag: 'copy_original_message',
},
},
{
rename: {
field: 'originalMessage',
target_field: 'event.original',
tag: 'rename_message',
ignore_missing: true,
if: 'ctx.event?.original == null',
},
},
{
remove: {
field: 'originalMessage',
ignore_missing: true,
tag: 'remove_copied_message',
if: 'ctx.event?.original != null',
},
},
{
remove: { field: 'message', ignore_missing: true, tag: 'remove_message' },
},
{
json: {
field: 'event.original',
tag: 'json_original',
target_field: 'xdfsfs.ds',
},
},
{
rename: {
field: 'xdfsfs.ds.event',
target_field: 'event.action',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.uid',
target_field: 'event.id',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.code',
target_field: 'event.code',
ignore_missing: true,
},
},
{
date: {
field: 'xdfsfs.ds.@timestamp',
target_field: '@timestamp',
formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'", 'ISO8601'],
tag: 'date_processor_xdfsfs.ds.@timestamp',
if: 'ctx.xdfsfs?.ds?.@timestamp != null',
},
},
{
rename: {
field: 'xdfsfs.ds.cluster_name',
target_field: 'cloud.account.name',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.identity.user',
target_field: 'user.name',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.identity.roles',
target_field: 'user.roles',
ignore_missing: true,
},
},
{
script: {
description: 'Ensures the date processor does not receive an array value.',
tag: 'script_convert_array_to_string',
lang: 'painless',
source:
'if (ctx.xdfsfs?.ds?.identity?.expires != null &&\n' +
' ctx.xdfsfs.ds.identity.expires instanceof ArrayList){\n' +
' ctx.xdfsfs.ds.identity.expires = ctx.xdfsfs.ds.identity.expires[0];\n' +
'}\n',
},
},
{
date: {
field: 'xdfsfs.ds.identity.expires',
target_field: 'user.changes.name',
formats: ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'", 'ISO8601'],
tag: 'date_processor_xdfsfs.ds.identity.expires',
if: 'ctx.xdfsfs?.ds?.identity?.expires != null',
},
},
{
convert: {
field: 'xdfsfs.ds.identity.client_ip',
target_field: 'client.ip',
ignore_missing: true,
ignore_failure: true,
type: 'ip',
},
},
{
script: {
description: 'Ensures the date processor does not receive an array value.',
tag: 'script_convert_array_to_string',
lang: 'painless',
source:
'if (ctx.xdfsfs?.ds?.identity?.prev_identity_expires != null &&\n' +
' ctx.xdfsfs.ds.identity.prev_identity_expires instanceof ArrayList){\n' +
' ctx.xdfsfs.ds.identity.prev_identity_expires = ctx.xdfsfs.ds.identity.prev_identity_expires[0];\n' +
'}\n',
},
},
{
date: {
field: 'xdfsfs.ds.identity.prev_identity_expires',
target_field: 'event.end',
formats: ["yyyy-MM-dd'T'HH:mm:ss'Z'", 'ISO8601'],
tag: 'date_processor_xdfsfs.ds.identity.prev_identity_expires',
if: 'ctx.xdfsfs?.ds?.identity?.prev_identity_expires != null',
},
},
{
rename: {
field: 'xdfsfs.ds.user',
target_field: 'user.name',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.login',
target_field: 'user.id',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.server_id',
target_field: 'host.id',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.server_hostname',
target_field: 'host.hostname',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.addr.remote',
target_field: 'source.address',
ignore_missing: true,
},
},
{
rename: {
field: 'xdfsfs.ds.proto',
target_field: 'network.protocol',
ignore_missing: true,
},
},
{
script: {
description: 'Drops null/empty values recursively.',
tag: 'script_drop_null_empty_values',
lang: 'painless',
source:
'boolean dropEmptyFields(Object object) {\n' +
' if (object == null || object == "") {\n' +
' return true;\n' +
' } else if (object instanceof Map) {\n' +
' ((Map) object).values().removeIf(value -> dropEmptyFields(value));\n' +
' return (((Map) object).size() == 0);\n' +
' } else if (object instanceof List) {\n' +
' ((List) object).removeIf(value -> dropEmptyFields(value));\n' +
' return (((List) object).length == 0);\n' +
' }\n' +
' return false;\n' +
'}\n' +
'dropEmptyFields(ctx);\n',
},
},
{
geoip: {
field: 'source.ip',
tag: 'geoip_source_ip',
target_field: 'source.geo',
ignore_missing: true,
},
},
{
geoip: {
ignore_missing: true,
database_file: 'GeoLite2-ASN.mmdb',
field: 'source.ip',
tag: 'geoip_source_asn',
target_field: 'source.as',
properties: ['asn', 'organization_name'],
},
},
{
rename: {
field: 'source.as.asn',
tag: 'rename_source_as_asn',
target_field: 'source.as.number',
ignore_missing: true,
},
},
{
rename: {
field: 'source.as.organization_name',
tag: 'rename_source_as_organization_name',
target_field: 'source.as.organization.name',
ignore_missing: true,
},
},
{
geoip: {
field: 'destination.ip',
tag: 'geoip_destination_ip',
target_field: 'destination.geo',
ignore_missing: true,
},
},
{
geoip: {
database_file: 'GeoLite2-ASN.mmdb',
field: 'destination.ip',
tag: 'geoip_destination_asn',
target_field: 'destination.as',
properties: ['asn', 'organization_name'],
ignore_missing: true,
},
},
{
rename: {
field: 'destination.as.asn',
tag: 'rename_destination_as_asn',
target_field: 'destination.as.number',
ignore_missing: true,
},
},
{
rename: {
field: 'destination.as.organization_name',
tag: 'rename_destination_as_organization_name',
target_field: 'destination.as.organization.name',
ignore_missing: true,
},
},
{
remove: {
field: ['xdfsfs.ds.identity.client_ip'],
ignore_missing: true,
tag: 'remove_fields',
},
},
{
remove: {
field: 'event.original',
tag: 'remove_original_event',
if: 'ctx?.tags == null || !(ctx.tags.contains("preserve_original_event"))',
ignore_failure: true,
ignore_missing: true,
},
},
]);
});
});

3
x-pack/plugins/integration_assistant/server/graphs/ecs/pipeline.ts
View file

@ -186,6 +186,9 @@ export function createPipeline(state: EcsMappingState): IngestPipeline {
const env = new Environment(new FileSystemLoader(templatesPath), {
autoescape: false,
});
env.addFilter('includes', function (str, substr) {
return str.includes(substr);
});
env.addFilter('startswith', function (str, prefix) {
return str.startsWith(prefix);
});

3
x-pack/plugins/integration_assistant/server/templates/pipeline.yml.njk
View file

@ -35,6 +35,7 @@ processors:
target_field: {% if value.target_field | startswith('@') %}"{{ value.target_field }}"{% else %}{{ value.target_field }}{% endif %}
ignore_missing: true{% endif %}
{% if key == 'date' %}
{% if not value.field | includes('.@') %} {# Leaving fields of type 'test.log.@timestamp' #}
- script:
description: Ensures the date processor does not receive an array value.
tag: script_convert_array_to_string
@ -43,7 +44,7 @@ processors:
if (ctx.{% endraw %}{{ value.field.replaceAll('.', '?.') }}{% raw %} != null &&
ctx.{% endraw %}{{ value.field }}{% raw %} instanceof ArrayList){
ctx.{% endraw %}{{ value.field }}{% raw %} = ctx.{% endraw %}{{ value.field }}{% raw %}[0];
}{% endraw %}
}{% endraw %}{% endif %}
- {{ key }}:
field: {{ value.field }}
target_field: {% if value.target_field | startswith('@') %}"{{ value.target_field }}"{% else %}{{ value.target_field }}{% endif %}