Added test configuration files for #7403

Fixes #8711
This commit is contained in:
Benjamin M. Hughes 2017-11-22 14:41:03 +00:00 committed by Armin Braun
parent 0c31360445
commit 055de8e9c3
18 changed files with 2012 additions and 0 deletions

View file

@ -0,0 +1,10 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
input {
beats {
port => 5044
tags => [ "beats" ]
id => "input_beats"
}
}

View file

@ -0,0 +1,13 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
input {
udp {
port => 9995
type => "netflow"
codec => netflow {
versions => [5, 9]
}
id => "input_udp_netflow"
}
}

View file

@ -0,0 +1,11 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
input {
snmptrap {
tags => ["SNMP Trap", "Ready"]
type => snmptrap
community => "public"
id => "input_snmptrap"
}
}

View file

@ -0,0 +1,15 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
input {
tcp {
port => 8514
type => "ciscosyslog"
id => "input_tcp_cisco-syslog"
}
udp {
port => 8514
type => "cisco-syslog"
id => "input_udp_cisco-syslog"
}
}

View file

@ -0,0 +1,10 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
input {
syslog {
port => 5514
type => syslog
id => "input_syslog"
}
}

View file

@ -0,0 +1,24 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
filter {
if [type] == "syslog" {
mutate {
add_tag => "syslog"
id => "filter_mutate_add_syslog_tag"
}
if [logsource] =~ "pfsense.*" {
mutate {
add_tag => ["pfsense", "firewall"]
id => "filter_mutate_add_pfsense_tag"
}
}
if [logsource] =~ "freenas.*" {
if [program] == "snmpd" {
drop {
id => "filter_drop_freenas_snmpd"
}
}
}
}
}

View file

@ -0,0 +1,37 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
filter {
if ("syslog" in [tags] or "pfsense" in [tags]) {
if [program] =~ /^dhcpd$/ {
mutate {
add_tag => [ "dhcp" ]
id => "filter_mutate_add_dhcp_tag"
}
if [message] =~ /^DHCPACK|^DHCPREQUEST|^DHCPOFFER/ {
grok {
patterns_dir => [ "/etc/logstash/patterns" ]
patterns_files_glob => "dhcpd"
match => { "message" => "%{DHCPACK}" }
id => "filter_grok_dhcpack"
}
}
if [message] =~ /^DHCPDISCOVER/ {
grok {
patterns_dir => [ "/etc/logstash/patterns" ]
patterns_files_glob => "dhcpd"
match => { "message" => "%{DHCPDISCOVER}" }
id => "filter_grok_dhcpdiscover"
}
}
if [message] =~ /^DHCPINFORM/ {
grok {
patterns_dir => [ "/etc/logstash/patterns" ]
patterns_files_glob => "dhcpd"
match => { "message" => "%{DHCPINFORM}" }
id => "filter_grok_dhcpinform"
}
}
}
}
}

View file

@ -0,0 +1,164 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
# pfSense
filter {
if [type] == "syslog" and "pfsense" in [tags] {
grok {
match => { "message" => "(?<msg>.*)" }
id => "filter_grok_pfsense_message"
}
mutate {
gsub => [ "timestamp", " ", " " ]
replace => { "message" => "%{msg}" }
replace => { "host" => "%{logsource}" }
remove_field => [ "msg" ]
id => "filter_mutate_pfsense_tidy_message"
}
dns {
resolve => [ "host" ]
action => "replace"
id => "filter_dns_pfsense_host"
}
if [program] == "filterlog" {
mutate {
remove_field => [ "msg" ]
replace => { "type" => "filterlog" }
id => "filter_mutate_pfsense_filterlog_remove_fields"
}
grok {
patterns_dir => [ "/etc/logstash/patterns" ]
patterns_files_glob => "pfsense*"
match => { "message" => "%{LOG_DATA}%{IP_SPECIFIC_DATA}%{IP_DATA}(%{PROTOCOL_DATA})?" }
id => "filter_grok_pfsense_filterlog"
}
if "_grokparsefailure" not in [tags] {
geoip {
add_tag => [ "GeoIP_src" ]
source => "src_ip"
target => "geoip_src_ip"
id => "filter_geoip_pfsense_filterlog_src"
}
geoip {
add_tag => [ "GeoIP_dest" ]
source => "dest_ip"
target => "geoip_dest_ip"
id => "filter_geoip_pfsense_filterlog_dest"
}
mutate {
add_field => { "src_host" => "%{src_ip}" }
add_field => { "dest_host" => "%{dest_ip}" }
lowercase => [ "%{proto}" ]
id => "filter_mutate_pfsense_filterlog_add_fields"
}
dns {
action => "replace"
reverse => [ "src_host" ]
id => "filter_dns_pfsense_filterlog_src"
}
dns {
action => "replace"
reverse => [ "dest_host" ]
id => "filter_dns_pfsense_filterlog_dest"
}
}
}
if [program] == "suricata" {
mutate {
add_tag => [ "suricata" ]
id => "filter_mutate_pfsense_suricata_tag"
}
}
}
# Snort
if [program] == "snort" {
mutate {
add_tag => [ "snort" ]
add_field => { "src_host" => "%{src_ip}" }
add_field => { "dest_host" => "%{dest_ip}" }
id => "filter_mutate_pfsense_snort_tags"
}
grok {
match => { "message" => ["\[%{INT:ids_gid}\:%{INT:ids_sid}\:%{INT:ids_rev}\].%{GREEDYDATA:ids_alert}.\[Classification\: %{DATA:ids_classification}\].*\[Priority\: %{INT:ids_priority}].*{%{WORD:ids_proto}}.*%{IP:src_ip}:%{INT:src_port} \-\>.*%{IP:dest_ip}:%{INT:dest_port}", "\[%{INT:ids_gid}\:%{INT:ids_sid}\:%{INT:ids_rev}\].%{GREEDYDATA:ids_alert}.\[Classification\: %{DATA:ids_classification}\].*\[Priority\: %{INT:ids_priority}].*\{PROTO:%{WORD:ids_proto}.*%{IP:src_ip} \-\>.*%{IP:dest_ip}}" ] }
id => "filter_grok_pfsense_snort"
}
translate {
field => "ids_priority"
destination => "ids_priority_full"
dictionary => [
"1", "High",
"2", "Medium",
"3", "Low"
]
id => "filter_translate_pfsense_snort"
}
geoip {
add_tag => [ "GeoIP_src" ]
source => "src_ip"
target => "geoip_src_ip"
id => "filter_geoip_pfsense_snort_src"
}
geoip {
add_tag => [ "GeoIP_dst" ]
source => "dest_ip"
target => "geoip_dest_ip"
id => "filter_geoip_pfsense_snort_dest"
}
dns {
action => "replace"
reverse => [ "src_host" ]
id => "filter_dns_pfsense_snort_src"
}
dns {
action => "replace"
reverse => [ "dest_host" ]
id => "filter_dns_pfsense_snort_dest"
}
if [ids_signature] {
if [ids_alert] =~ /^GPL/ {
mutate {
add_tag => [ "Snort-ET-sig" ]
add_field => { "ids_rule_type" => "Emerging Threats" }
id => "filter_mutate_pfsense_snort_ids_gpl"
}
}
if [ids_alert] =~ /^ET/ {
mutate {
add_tag => [ "Snort-ET-sig" ]
add_field => { "ids_rule_type" => "Emerging Threats" }
id => "filter_mutate_pfsense_snort_ids_et"
}
}
if "Snort-ET-sig" not in [tags] {
mutate {
add_tag => [ "Snort-sig" ]
add_field => { "ids_rule_type" => "Snort" }
id => "filter_mutate_pfsense_snort_ids_sig"
}
}
}
if "Snort-sig" in [tags] {
if [ids_gid] == "1" {
mutate {
add_field => { "Signature_Info" => "http://rootedyour.com/snortsid?sid=%{ids_sid}" }
id => "filter_mutate_pfsense_snort_ids_gid_1"
}
}
if [ids_gid] != "1" {
mutate {
add_field => { "Signature_Info" => "http://rootedyour.com/snortsid?sid=%{ids_gid}-%{ids_sid}" }
id => "filter_mutate_pfsense_snort_ids_gid_not_1"
}
}
if "Snort-ET-sig" in [tags] {
mutate {
add_field => { "Signature_Info" => "http://doc.emergingthreats.net/bin/view/Main/%{ids_sid}" }
id => "filter_mutate_pfsense_snort_ids_et_sig"
}
}
}
}
}

View file

@ -0,0 +1,103 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
filter {
if [type] == "syslog" {
if "_grokparsefailure_sysloginput" in [tags] {
drop {
id => "filter_drop_syslogintput_syslog"
}
}
date {
match => [ "timestamp", "MMM d HH:mm:ss", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "MMMM d'th' yyyy',' HH:mm:ss.SSS" , "MMMM dd'th' yyyy',' HH:mm:ss.SSS" ]
locale => "en_GB"
timezone => "Europe/London"
id => "filter_date_syslog"
}
syslog_pri {
id => "filter_syslog_pri_syslog"
}
mutate {
add_field => [ "received_from", "%{host}" ]
replace => [ "host", "%{logsource}"]
replace => [ "timestamp", "%{@timestamp}" ]
id => "filter_mutate_syslog_add_fields"
}
cidr {
add_tag => [ "host_is_ip" ]
address => [ "%{host}" ]
network => [ "0.0.0.0/0" ]
id => "filter_cidr_syslog_host"
}
if "host_is_ip" not in [tags] {
dns {
resolve => [ "host" ]
action => "replace"
remove_tag => [ "host_is_ip" ]
id => "filter_dns_syslog_host"
}
}
cidr {
add_tag => [ "logsource_is_ip" ]
address => [ "%{logsource}" ]
network => [ "0.0.0.0/0" ]
id => "filter_cidr_syslog_logsource"
}
if ("logsource_is_ip" in [tags]) {
dns {
reverse => [ "logsource" ]
action => "replace"
remove_tag => [ "host_is_ip" ]
id => "filter_dns_syslog_logsource"
}
}
}
if [type] == "local-syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
id => "filter_grok_syslog_local-syslog"
}
if "_grokparsefailure" not in [tags] {
syslog_pri {
id => "filter_syslog_pri_syslog_local-syslog"
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
locale => "en"
timezone => "Europe/London"
id => "filter_date_syslog_local-syslog"
}
mutate {
replace => [ "@source_host", "%{syslog_hostname}" ]
replace => [ "@message", "%{syslog_message}" ]
id => "filter_mutate_syslog_local-syslog_add_fields"
}
mutate {
remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
id => "filter_mutate_syslog_local-syslog_remove_fields"
}
}
}
if [type] == "beats-syslog" {
syslog_pri {
id => "filter_syslog_pri_syslog_beats-syslog"
}
cidr {
add_tag => [ "host_is_ip" ]
address => [ "%{host}" ]
network => [ "0.0.0.0/0" ]
id => "filter_cidr_syslog_beats-syslog"
}
if "host_is_ip" not in [tags] {
dns {
resolve => [ "host" ]
action => "replace"
id => "filter_dns_syslog_beats-syslog_host"
}
}
}
}

View file

@ -0,0 +1,22 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
filter {
if [type] == "syslog" and [program] == "fail2ban" {
date {
match => [ "timestamp", "MMM d HH:mm:ss", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "MMMM d'th' yyyy',' HH:mm:ss.SSS" , "MMMM dd'th' yyyy',' HH:mm:ss.SSS", "ISO8601" ]
locale => "en_GB"
timezone => "Europe/London"
id => "filter_date_syslog-programs"
}
syslog_pri {
id => "filter_syslog_pri_syslog-programs"
}
geoip {
add_tag => [ "GeoIP_src" ]
source => "src_ip"
target => "geoip_src_ip"
id => "filter_geoip_src_syslog-programs"
}
}
}

View file

@ -0,0 +1,11 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
filter {
if [type] == "apache-access" {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
id => "filter_grok_apache-access"
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,49 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
filter {
if [type] == "iptables" {
grok {
patterns_dir => [ "/etc/logstash/patterns" ]
patterns_files_glob => "iptables*"
match => { "message" => "%{IPTABLES}" }
id => "filter_grok_iptables"
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
locale => "en"
timezone => "Europe/London"
id => "filter_date_iptables"
}
mutate {
add_tag => [ "firewall", "iptables" ]
id => "filter_mutate_iptables_add_tag"
}
if "_grokparsefailure" not in [tags] {
mutate {
copy => { "host" => "host_ip" }
id => "filter_mutate_iptables_copy_host_ip"
}
dns {
resolve => [ "host_ip" ]
action => "replace"
add_tag => [ "dns_host_lookup" ]
id => "filter_dns_iptables_host_ip"
}
geoip {
add_tag => [ "geoip_src" ]
source => "src_ip"
target => "geoip_src_ip"
tag_on_failure => [ "_geoip_lookup_failure_src" ]
id => "filter_geoip_iptables_src"
}
geoip {
add_tag => [ "geoip_dst" ]
source => "dst_ip"
target => "geoip_dst_ip"
tag_on_failure => [ "_geoip_lookup_failure_dst" ]
id => "filter_geoip_iptables_dst"
}
}
}
}

View file

@ -0,0 +1,58 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
filter {
if [type] == "nginx-access" {
mutate {
add_tag => ["nginx"]
id => "filter_mutate_nginx-access_add_tag"
}
grok {
match => { "message" => "%{NGINXACCESS}" }
patterns_dir => [ "/etc/logstash/patterns" ]
id => "filter_grok_nginx-access"
}
if "_grokparsefailure" not in [tags] {
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
locale => "en_GB"
timezone => "Europe/London"
id => "filter_date_nginx-access"
}
geoip {
add_tag => [ "GeoIP_Client" ]
source => "clientip"
target => "geoip"
id => "filter_geoip_nginx-access_clientip"
}
mutate {
add_field => { "clienthost" => "%{clientip}" }
remove_field => [ "timestamp" ]
id => "filter_mutate_nginx-access_add_clienthost"
}
dns {
action => "replace"
reverse => [ "clienthost" ]
id => "filter_dns_nginx-access_clienthost"
}
cidr {
add_tag => [ "host_is_ip" ]
address => [ "%{host}" ]
network => [ "0.0.0.0/0" ]
id => "filter_cidr_nginx-access_host_ip"
}
if "host_is_ip" not in [tags] {
dns {
resolve => [ "host" ]
action => "replace"
# remove_tag => [ "host_is_ip" ]
id => "filter_dns_nginx-access_host_ip"
}
}
mutate {
add_field => [ "received_at", "%{@timestamp}" ]
id => "filter_mutate_nginx-access_add_receieved_at"
}
}
}
}

View file

@ -0,0 +1,42 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
# Netflow
filter {
if [type] == "netflow" {
geoip {
add_tag => [ "GeoIP_src" ]
source => "[netflow][ipv4_src_addr]"
target => "[geoip][src_ip]"
id => "filter_geoip_netflow_src"
}
geoip {
add_tag => [ "GeoIP_dst" ]
source => "[netflow][ipv4_dst_addr]"
target => "[geoip][dest_ip]"
id => "filter_geoip_netflow_dest"
}
translate {
field => "[netflow][protocol]"
destination => "[netflow][protocol_name]"
override => "true"
dictionary => [ "1", "ICMP",
"2", "IGMP",
"6", "TCP",
"9", "IGMP",
"17", "UDP",
"41", "IPv6-Encap",
"43", "IPv6-Route",
"44", "IPv6-Frag",
"47", "GRE",
"50", "ESP",
"58", "IPv6-ICMP",
"88", "EIGRP",
"89", "OSPF-IGP",
"115", "L2TP",
"124", "ISIS" ]
fallback => "No protocol name found"
id => "filter_translate_netflow_protocol_name"
}
}
}

View file

@ -0,0 +1,29 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
# output {
# if [type] == "cisco-syslog" {
# file {
# path => "/var/log/logstash/cisco_debug_events-%{+YYYY-MM-dd}"
# id => "output_file_debug_cisco-syslog"
# }
# }
# }
output {
if [type] == "snmptrap" {
file {
path => "/var/log/logstash/snmptrap_debug_events-%{+YYYY-MM-dd}"
id => "output_file_debug_snmptrap"
}
}
}
output {
if [type] == "firewall" {
file {
path => "/var/log/logstash/firewall_debug_events-%{+YYYY-MM-dd}"
id => "output_file_debug_firewall"
}
}
}

View file

@ -0,0 +1,87 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
output {
if [type] != "snmptrap" {
if [type] == "netflow" {
elasticsearch {
hosts => ["localhost:9200"]
index => "netflow-%{+YYYY.MM.dd}"
template => "/etc/logstash/templates/netflow-template-es6x.json"
template_name => "netflow"
template_overwrite => true
id => "output_elasticsearch_netflow"
}
} else if [@metadata][beat] == "winlogbeat" {
elasticsearch {
hosts => ["localhost:9200"]
index => "winlogbeat-%{+YYYY.MM.dd}"
template => "/etc/logstash/templates/winlogbeat-template-es6x.json"
template_name => "winlogbeat"
template_overwrite => true
id => "output_elasticsearch_winlogbeat"
}
} else if [@metadata][beat] == "metricbeat" {
elasticsearch {
hosts => ["localhost:9200"]
index => "metricbeat-%{+YYYY.MM.dd}"
template => "/etc/logstash/templates/metricbeat-template-es6x.json"
template_name => "metricbeat"
template_overwrite => true
id => "output_elasticsearch_metricbeat"
}
} else if [@metadata][beat] == "packetbeat" {
elasticsearch {
hosts => ["localhost:9200"]
index => "packetbeat-%{+YYYY.MM.dd}"
template => "/etc/logstash/templates/packetbeat-template-es6x.json"
template_name => "packetbeat"
template_overwrite => true
id => "output_elasticsearch_packetbeat"
}
} else if [type] == "iptables" {
elasticsearch {
hosts => ["localhost:9200"]
index => "iptables-%{+YYYY.MM.dd}"
template => "/etc/logstash/templates/iptables-template-es6x.json"
template_name => "iptables"
template_overwrite => true
id => "output_elasticsearch_iptables"
}
} else if [type] == "filterlog" {
elasticsearch {
hosts => ["localhost:9200"]
index => "filterlog-%{+YYYY.MM.dd}"
template => "/etc/logstash/templates/filterlog-template-es6x.json"
template_name => "filterlog"
template_overwrite => true
id => "output_elasticsearch_filterlog"
}
} else if [type] == "cisco-syslog" {
elasticsearch {
hosts => ["localhost:9200"]
index => "cisco-syslog-%{+YYYY.MM.dd}"
template => "/etc/logstash/templates/cisco-syslog-template-es6x.json"
template_name => "cisco-syslog"
template_overwrite => true
id => "output_elasticsearch_cisco-syslog"
}
} else if [type] == "nginx-access" {
elasticsearch {
hosts => ["localhost:9200"]
index => "nginx-access-%{+YYYY.MM.dd}"
template => "/etc/logstash/templates/nginx-access-template-es6x.json"
template_name => "nginx-access"
template_overwrite => true
id => "output_elasticsearch_nginx-access"
}
} else {
elasticsearch {
hosts => ["localhost:9200"]
template => "/etc/logstash/templates/elasticsearch-template-es6x.json"
template_overwrite => true
id => "output_elasticsearch_logstash"
}
}
}
}

View file

@ -0,0 +1,23 @@
# Generated by Chef for REDACTED
# Do NOT modify this file by hand.
#
output {
if [type] == "syslog" and "_grokparsefailure" in [tags] {
file {
path => "/var/log/logstash/failed_syslog_events-%{+YYYY-MM-dd}"
id => "output_file_syslog_failure"
}
}
if [type] == "cisco-syslog" and "_grokparsefailure" in [tags] {
file {
path => "/var/log/logstash/failed_cisco_events-%{+YYYY-MM-dd}"
id => "output_file_cisco-syslog_failure"
}
}
if [type] == "nginx-access" and "_grokparsefailure" in [tags] {
file {
path => "/var/log/logstash/failed_nginx_events-%{+YYYY-MM-dd}"
id => "output_file_nginx-access_failure"
}
}
}