github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-06-27 17:08:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

Remove the Arcsight module and the modules framework (#16794)

Browse source 
Remove all module related code
- remove arcsight module
- remove module framework
- remove module tests
- remove module configs
This commit is contained in:
kaisecheng 2024-12-19 14:28:54 +00:00 • committed by GitHub
parent 03ddf12893
commit 05789744d2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
369 changed files with 30 additions and 7785 deletions
Download patch file Download diff file Expand all files Collapse all files

4
build.gradle
View file

@ -334,7 +334,6 @@ tasks.register("assembleTarDistribution") {
inputs.files fileTree("${projectDir}/bin")
inputs.files fileTree("${projectDir}/config")
inputs.files fileTree("${projectDir}/lib")
inputs.files fileTree("${projectDir}/modules")
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
inputs.files fileTree("${projectDir}/logstash-core/lib")
inputs.files fileTree("${projectDir}/logstash-core/src")
@ -351,7 +350,6 @@ tasks.register("assembleOssTarDistribution") {
inputs.files fileTree("${projectDir}/bin")
inputs.files fileTree("${projectDir}/config")
inputs.files fileTree("${projectDir}/lib")
inputs.files fileTree("${projectDir}/modules")
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
inputs.files fileTree("${projectDir}/logstash-core/lib")
inputs.files fileTree("${projectDir}/logstash-core/src")
@ -366,7 +364,6 @@ tasks.register("assembleZipDistribution") {
inputs.files fileTree("${projectDir}/bin")
inputs.files fileTree("${projectDir}/config")
inputs.files fileTree("${projectDir}/lib")
inputs.files fileTree("${projectDir}/modules")
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
inputs.files fileTree("${projectDir}/logstash-core/lib")
inputs.files fileTree("${projectDir}/logstash-core/src")
@ -383,7 +380,6 @@ tasks.register("assembleOssZipDistribution") {
inputs.files fileTree("${projectDir}/bin")
inputs.files fileTree("${projectDir}/config")
inputs.files fileTree("${projectDir}/lib")
inputs.files fileTree("${projectDir}/modules")
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
inputs.files fileTree("${projectDir}/logstash-core/lib")
inputs.files fileTree("${projectDir}/logstash-core/src")

32
config/logstash.yml
View file

@ -181,38 +181,6 @@
#
# api.auth.basic.password_policy.mode: WARN
#
# ------------ Module Settings ---------------
# Define modules here. Modules definitions must be defined as an array.
# The simple way to see this is to prepend each `name` with a `-`, and keep
# all associated variables under the `name` they are associated with, and
# above the next, like this:
#
# modules:
# - name: MODULE_NAME
# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE
# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE
# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE
# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE
#
# Module variable names must be in the format of
#
# var.PLUGIN_TYPE.PLUGIN_NAME.KEY
#
# modules:
#
# ------------ Cloud Settings ---------------
# Define Elastic Cloud settings here.
# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy
# and it may have an label prefix e.g. staging:dXMtZ...
# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host'
# cloud.id: <identifier>
#
# Format of cloud.auth is: <user>:<pass>
# This is optional
# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password'
# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password'
# cloud.auth: elastic:<password>
#
# ------------ Queuing Settings --------------
#
# Internal queuing model, "memory" for legacy in-memory based queuing and

6
docker/data/logstash/env2yaml/env2yaml.go
View file

@ -17,11 +17,12 @@ package main
import (
"errors"
"fmt"
"gopkg.in/yaml.v2"
"io/ioutil"
"log"
"os"
"strings"
"gopkg.in/yaml.v2"
)
var validSettings = []string{
@ -67,7 +68,6 @@ var validSettings = []string{
"log.level",
"log.format",
"log.format.json.fix_duplicate_message_fields",
"modules",
"metric.collect",
"path.logs",
"path.plugins",
@ -126,8 +126,6 @@ var validSettings = []string{
"xpack.management.elasticsearch.ssl.cipher_suites",
"xpack.geoip.download.endpoint",
"xpack.geoip.downloader.enabled",
"cloud.id",
"cloud.auth",
}
// Given a setting name, return a downcased version with delimiters removed.

130
logstash-core/lib/logstash/config/modules_common.rb
View file

@ -1,130 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/elasticsearch_client"
require "logstash/modules/kibana_client"
require "logstash/modules/elasticsearch_importer"
require "logstash/modules/kibana_importer"
require "logstash/modules/settings_merger"
module LogStash module Config
class ModulesCommon # extracted here for bwc with 5.x
include LogStash::Util::Loggable
MODULES_MAX_PIPELINES = 1
def self.pipeline_configs(settings)
pipelines = []
plugin_modules = LogStash::PLUGIN_REGISTRY.plugins_with_type(:modules)
cli_settings = settings.get("modules.cli")
yml_settings = settings.get("modules")
modules_array = if !(cli_settings.empty? && yml_settings.empty?)
LogStash::Modules::SettingsMerger.merge(cli_settings, yml_settings)
elsif cli_settings.empty?
yml_settings
else
cli_settings
end
if modules_array.empty?
# no specified modules
return pipelines
end
logger.debug("Specified modules", :modules_array => modules_array.to_s)
module_names = modules_array.collect {|module_hash| module_hash["name"]}
if module_names.size > MODULES_MAX_PIPELINES
error_message = I18n.t("logstash.modules.configuration.modules-too-many-specified", :max => MODULES_MAX_PIPELINES, :specified_modules => module_names.join(', '))
raise LogStash::ConfigLoadingError, error_message
end
if module_names.length > module_names.uniq.length
duplicate_modules = module_names.group_by(&:to_s).select { |_, v| v.size > 1 }.keys
raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-must-be-unique", :duplicate_modules => duplicate_modules)
end
available_module_names = plugin_modules.map(&:module_name)
specified_and_available_names = module_names & available_module_names
if (specified_and_available_names).empty?
i18n_opts = {:specified_modules => module_names, :available_modules => available_module_names}
raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-unavailable", **i18n_opts)
end
specified_and_available_names.each do |module_name|
connect_fail_args = {}
begin
module_settings = settings.clone
module_hash = modules_array.find {|m| m["name"] == module_name}
current_module = plugin_modules.find { |allmodules| allmodules.module_name == module_name }
enabled = current_module.is_enabled?(module_settings)
unless enabled
logger.warn("The #{module_name} module is not enabled. Please check the logs for additional information.")
next
end
alt_name = "module-#{module_name}"
pipeline_id = alt_name
module_settings.set("pipeline.id", pipeline_id)
LogStash::Modules::SettingsMerger.merge_cloud_settings(module_hash, module_settings)
LogStash::Modules::SettingsMerger.merge_kibana_auth!(module_hash)
current_module.with_settings(module_hash)
config_test = settings.get("config.test_and_exit")
module_setup = settings.get("modules_setup")
# Only import data if it's not a config test and --setup is true
if !config_test && module_setup
logger.info("Setting up the #{module_name} module")
esclient = LogStash::ElasticsearchClient.build(module_hash)
kbnclient = LogStash::Modules::KibanaClient.new(module_hash)
esconnected = esclient.can_connect?
kbnconnected = kbnclient.can_connect?
if esconnected && kbnconnected
current_module.add_kibana_version(kbnclient.version_parts)
current_module.import(
LogStash::Modules::ElasticsearchImporter.new(esclient),
LogStash::Modules::KibanaImporter.new(kbnclient)
)
else
connect_fail_args[:module_name] = module_name
connect_fail_args[:elasticsearch_hosts] = esclient.host_settings
connect_fail_args[:kibana_hosts] = kbnclient.host_settings
end
else
logger.info("Starting the #{module_name} module")
end
config_string = current_module.config_string
pipelines << {"pipeline_id" => pipeline_id, "alt_name" => alt_name, "config_string" => config_string, "settings" => module_settings}
rescue => e
new_error = LogStash::ConfigLoadingError.new(I18n.t("logstash.modules.configuration.parse-failed", :error => e.message))
new_error.set_backtrace(e.backtrace)
raise new_error
end
if !connect_fail_args.empty?
raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.elasticsearch_connection_failed", **connect_fail_args)
end
end
pipelines
end
end
end end

32
logstash-core/lib/logstash/config/source/base.rb
View file

@ -71,37 +71,5 @@ module LogStash module Config module Source
def config_path?
!(config_path.nil? || config_path.empty?)
end
def modules_cli_setting
@settings.get_setting("modules.cli")
end
def modules_cli
modules_cli_setting.value
end
def modules_cli?
!(modules_cli.nil? || modules_cli.empty?)
end
def modules_setting
@settings.get_setting("modules")
end
def modules
modules_setting.value
end
def modules?
!(modules.nil? || modules.empty?)
end
def both_module_configs?
modules_cli? && modules?
end
def modules_defined?
modules_cli? || modules?
end
end
end end end

2
logstash-core/lib/logstash/config/source/local.rb
View file

@ -179,7 +179,7 @@ module LogStash module Config module Source
def match?
# see basic settings predicates and getters defined in the base class
(config_string? || config_path?) && !(modules_cli? || modules?) && !automatic_reload_with_config_string?
(config_string? || config_path?) && !automatic_reload_with_config_string?
end
def config_conflict?

68
logstash-core/lib/logstash/config/source/modules.rb
View file

@ -1,68 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/config/source/base"
require "logstash/config/modules_common"
module LogStash module Config module Source
class Modules < Base
include LogStash::Util::Loggable
def pipeline_configs
if config_conflict? # double check
raise ConfigurationError, @conflict_messages.join(", ")
end
pipelines = LogStash::Config::ModulesCommon.pipeline_configs(@settings)
pipelines.map do |hash|
org.logstash.config.ir.PipelineConfig.new(self.class, hash["pipeline_id"].to_sym,
org.logstash.common.SourceWithMetadata.new("module", hash["alt_name"], 0, 0, hash["config_string"]),
hash["settings"])
end
end
def match?
# see basic settings predicates and getters defined in the base class
(modules_cli? || modules?) && !(config_string? || config_path?) && !automatic_reload_with_modules?
end
def config_conflict?
@conflict_messages.clear
# Make note that if modules are configured in both cli and logstash.yml that cli module
# settings will overwrite the logstash.yml modules settings
if modules_cli? && modules?
logger.info(I18n.t("logstash.runner.cli-module-override"))
end
if automatic_reload_with_modules?
@conflict_messages << I18n.t("logstash.runner.reload-with-modules")
end
# Check if config (-f or -e) and modules are configured
if (modules_cli? || modules?) && (config_string? || config_path?)
@conflict_messages << I18n.t("logstash.runner.config-module-exclusive")
end
@conflict_messages.any?
end
private
def automatic_reload_with_modules?
(modules_cli? || modules?) && config_reload_automatic?
end
end
end end end

6
logstash-core/lib/logstash/config/source/multi_local.rb
View file

@ -48,7 +48,7 @@ module LogStash module Config module Source
end
def match?
if modules_cli? || modules? || config_string? || config_path?
if config_string? || config_path?
return false
end
detect_pipelines if !@detect_pipelines_called
@ -62,7 +62,7 @@ module LogStash module Config module Source
def config_conflict?
@conflict_messages.clear
# are there any auto-reload conflicts?
if !(modules_cli? || modules? || config_string? || config_path?)
if !(config_string? || config_path?)
detect_pipelines if !@detect_pipelines_called
if @detected_marker.nil?
@conflict_messages << I18n.t("logstash.runner.config-pipelines-failed-read", :path => pipelines_yaml_location)
@ -74,7 +74,7 @@ module LogStash module Config module Source
@conflict_messages << @detected_marker.message
end
else
do_warning? && logger.warn("Ignoring the 'pipelines.yml' file because modules or command line options are specified")
do_warning? && logger.warn("Ignoring the 'pipelines.yml' file because command line options are specified")
end
@conflict_messages.any?
end

1
logstash-core/lib/logstash/config/source_loader.rb
View file

@ -16,7 +16,6 @@
# under the License.
require "logstash/config/source/local"
require "logstash/config/source/modules"
require "logstash/config/source/multi_local"
require "thread"
require "set"

157
logstash-core/lib/logstash/elasticsearch_client.rb
View file

@ -1,157 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "elasticsearch"
require "elasticsearch/transport/transport/http/manticore"
require 'logstash/util/manticore_ssl_config_helper'
require 'logstash/util/password'
module LogStash class ElasticsearchClient
include LogStash::Util::Loggable
class Response
# duplicated here from Elasticsearch::Transport::Transport::Response
# to create a normalised response across different client IMPL
attr_reader :status, :body, :headers
def initialize(status, body, headers = {})
@status, @body, @headers = status, body, headers
@body = body.force_encoding('UTF-8') if body.respond_to?(:force_encoding)
end
end
def self.build(settings)
new(RubyClient.new(settings, logger))
end
class RubyClient
include LogStash::Util::ManticoreSSLConfigHelper
def initialize(settings, logger)
@settings = settings
@logger = logger
@client_args = client_args
ssl_options = manticore_ssl_options_from_config('elasticsearch', settings)
@client_args[:ssl] = ssl_options
username = @settings["var.elasticsearch.username"]
if username
password = @settings["var.elasticsearch.password"]
if password.is_a?(LogStash::Util::Password)
password = password.value
end
@client_args[:transport_options] = { :headers => { "Authorization" => 'Basic ' + Base64.encode64("#{username}:#{password}").chomp } }
end
@client = Elasticsearch::Client.new(@client_args)
end
def can_connect?
begin
head(SecureRandom.hex(32).prepend('_'))
rescue Elasticsearch::Transport::Transport::Errors::BadRequest
true
rescue Manticore::SocketException
false
end
end
def host_settings
@client_args[:hosts]
end
def delete(path)
begin
normalize_response(@client.perform_request('DELETE', path, {}, nil))
rescue Exception => e
if e.class.to_s =~ /NotFound/ || e.message =~ /Not\s*Found|404/i
Response.new(404, "", {})
else
raise e
end
end
end
def put(path, content)
normalize_response(@client.perform_request('PUT', path, {}, content))
end
def head(path)
begin
normalize_response(@client.perform_request('HEAD', path, {}, nil))
rescue Exception => e
if is_404_error?(e)
Response.new(404, "", {})
else
raise e
end
end
end
private
def is_404_error?(error)
error.class.to_s =~ /NotFound/ || error.message =~ /Not\s*Found|404/i
end
def normalize_response(response)
Response.new(response.status, response.body, response.headers)
end
def client_args
{
:transport_class => Elasticsearch::Transport::Transport::HTTP::Manticore,
:hosts => [*unpack_hosts],
# :logger => @logger, # silence the client logging
}
end
def unpack_hosts
setting = @settings.fetch("var.elasticsearch.hosts", "localhost:9200")
if setting.is_a?(String)
return setting.split(',').map(&:strip)
end
setting
end
end
def initialize(client)
@client = client
end
def delete(path)
@client.delete(path)
end
def put(path, content)
@client.put(path, content)
end
def head(path)
@client.head(path)
end
def can_connect?
@client.can_connect?
end
def host_settings
@client.host_settings
end
end end

8
logstash-core/lib/logstash/environment.rb
View file

@ -20,7 +20,6 @@ require "logstash/config/cpu_core_strategy"
require "logstash/settings"
require "logstash/util/cloud_setting_id"
require "logstash/util/cloud_setting_auth"
require "logstash/util/modules_setting_array"
require "socket"
require "stud/temporary"
@ -39,13 +38,6 @@ module LogStash
Setting::NullableString.new("path.config", nil, false),
Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
Setting::NullableString.new("config.string", nil, false),
Setting::Modules.new("modules.cli", LogStash::Util::ModulesSettingArray, []),
Setting::Modules.new("modules", LogStash::Util::ModulesSettingArray, []),
Setting.new("modules_list", Array, []),
Setting.new("modules_variable_list", Array, []),
Setting::Modules.new("cloud.id", LogStash::Util::CloudSettingId),
Setting::Modules.new("cloud.auth", LogStash::Util::CloudSettingAuth),
Setting::Boolean.new("modules_setup", false),
Setting::Boolean.new("config.test_and_exit", false),
Setting::Boolean.new("config.reload.automatic", false),
Setting::TimeValue.new("config.reload.interval", "3s"), # in seconds

87
logstash-core/lib/logstash/modules/cli_parser.rb
View file

@ -1,87 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
module LogStash module Modules class CLIParser
include LogStash::Util::Loggable
attr_reader :output
def initialize(module_names, module_variables)
@output = []
# The #compact here catches instances when module_variables may be nil or
# [nil] and sets it to []
parse_it(module_names, Array(module_variables).compact)
end
def parse_modules(module_list)
parsed_modules = []
module_list.each do |module_value|
# Calling --modules but not filling it results in [nil], so skip that.
next if module_value.nil?
# Catch if --modules was launched empty but an option/flag (-something)
# follows immediately after
if module_value.start_with?('-')
raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-invalid-name", :module_name => module_value)
end
parsed_modules.concat module_value.split(',')
end
parsed_modules
end
def get_kv(module_name, unparsed)
# Ensure that there is at least 1 equals sign in our variable string
# Using String#partition to split on the first '='
# This hackery is to catch the possibility of an equals (`=`) sign
# in a passphrase, which might result in an incomplete key. The
# portion before the first `=` should always be the key, leaving
# the rest to be the value
k, op, rest = unparsed.partition('=')
if rest.size.zero?
raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-variables-malformed", :rawvar => (module_name + '.' + unparsed))
end
return k.strip, rest.strip
end
def name_splitter(unparsed)
# It must have at least `modulename.something`
module_name, dot, rest = unparsed.partition('.')
if rest.count('.') >= 1
return module_name, rest
else
raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-variables-malformed", :rawvar => unparsed)
end
end
def parse_vars(module_name, vars_list)
module_hash = {"name" => module_name}
vars_list.each do |unparsed|
extracted_name, modvar = name_splitter(unparsed)
next if extracted_name != module_name
k, v = get_kv(extracted_name, modvar)
module_hash[k] = v
end
module_hash
end
def parse_it(module_list, module_variable_list)
if module_list.is_a?(Array)
parse_modules(module_list).each do |module_name|
@output << parse_vars(module_name, module_variable_list)
end
end
end
end end end

35
logstash-core/lib/logstash/modules/elasticsearch_config.rb
View file

@ -1,35 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require_relative "elasticsearch_resource"
module LogStash module Modules class ElasticsearchConfig
attr_reader :index_name
# We name it `modul` here because `module` has meaning in Ruby.
def initialize(modul, settings)
@directory = ::File.join(modul.directory, "elasticsearch")
@name = modul.module_name
@settings = settings
@full_path = ::File.join(@directory, "#{@name}.json")
@index_name = @settings.fetch("elasticsearch.template_path", "_template")
end
def resources
[ElasticsearchResource.new(@index_name, "not-used", @full_path)]
end
end end end

49
logstash-core/lib/logstash/modules/elasticsearch_importer.rb
View file

@ -1,49 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
module LogStash module Modules class ElasticsearchImporter
include LogStash::Util::Loggable
def initialize(client)
@client = client
end
def put(resource, overwrite = true)
path = resource.import_path
logger.debug("Attempting PUT", :url_path => path, :file_path => resource.content_path)
if !overwrite && content_exists?(path)
logger.debug("Found existing Elasticsearch resource.", :resource => path)
return
end
put_overwrite(path, resource.content)
end
private
def put_overwrite(path, content)
if content_exists?(path)
response = @client.delete(path)
end
# hmmm, versioning?
@client.put(path, content).status == 201
end
def content_exists?(path)
response = @client.head(path)
response.status >= 200 && response.status < 300
end
end end end # class LogStash::Modules::Importer

25
logstash-core/lib/logstash/modules/elasticsearch_resource.rb
View file

@ -1,25 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require_relative "resource_base"
module LogStash module Modules class ElasticsearchResource
include ResourceBase
def import_path
base + "/" + content_id
end
end end end

50
logstash-core/lib/logstash/modules/file_reader.rb
View file

@ -1,50 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/json"
module LogStash module Modules class FileReader
# stub these methods for testing
include LogStash::Util::Loggable
def self.read(path)
begin
::File.read(path)
rescue => e
logger.error("Error when reading file from path", :path => path)
""
end
end
def self.read_json(path)
json = read(path)
begin
LogStash::Json.load(json)
rescue => e
logger.error("Error when parsing json from path", :path => path)
return {}
end
end
def self.glob(path)
files = Dir.glob(path)
if files.empty?
logger.warn("No files found for glob", :pattern => path)
end
files
end
end end end

39
logstash-core/lib/logstash/modules/kibana_base.rb
View file

@ -1,39 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/json"
module LogStash module Modules class KibanaBase
attr_reader :import_path, :content
def initialize(import_path, content)
@import_path, @content = import_path, content
end
def import(client)
raise NotImplementedError, "#{self.class.name} needs to implement `#import`"
end
def to_s
import_path
end
def content_as_object
return content unless content.is_a?(String)
LogStash::Json.load(content) rescue nil
end
end end end

163
logstash-core/lib/logstash/modules/kibana_client.rb
View file

@ -1,163 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/json"
require "manticore"
require 'logstash/util/manticore_ssl_config_helper'
require 'logstash/util/password'
module LogStash module Modules class KibanaClient
include LogStash::Util::Loggable
include LogStash::Util::ManticoreSSLConfigHelper
class Response
# to create a custom response with body as an Object (Hash or Array)
attr_reader :status, :body, :headers
def initialize(status, body, headers = {})
@status, @body, @headers = status, body, headers
@body = body.is_a?(String) ? LogStash::Json.load(body) : body
end
def succeeded?
@status >= 200 && @status < 300
end
def failed?
!succeeded?
end
end
SCHEME_REGEX = /^https?$/
attr_reader :version, :endpoint
def initialize(settings, client = nil) # allow for test mock injection
@settings = settings
client_options = {
request_timeout: 5,
connect_timeout: 5,
socket_timeout: 5,
pool_max: 10,
pool_max_per_route: 2
}
ssl_options = manticore_ssl_options_from_config('kibana', settings)
ssl_enabled = ssl_options.any?
client_options[:ssl] = ssl_options
@host = @settings.fetch("var.kibana.host", "localhost:5601")
implicit_scheme, colon_slash_slash, host = @host.partition("://")
explicit_scheme = @settings["var.kibana.scheme"]
@scheme = "http"
if !colon_slash_slash.empty?
if !explicit_scheme.nil? && implicit_scheme != explicit_scheme
# both are set and not the same - error
msg = sprintf("Detected differing Kibana host schemes as sourced from var.kibana.host: '%s' and var.kibana.scheme: '%s'", implicit_scheme, explicit_scheme)
raise ArgumentError.new(msg)
end
@scheme = implicit_scheme
@host = host
elsif !explicit_scheme.nil?
@scheme = explicit_scheme
end
if SCHEME_REGEX.match(@scheme).nil?
msg = sprintf("Kibana host scheme given is invalid, given value: '%s' - acceptable values: 'http', 'https'", @scheme)
raise ArgumentError.new(msg)
end
if ssl_enabled && @scheme != "https"
@scheme = "https"
end
@endpoint = "#{@scheme}://#{@host}"
@client = client || Manticore::Client.new(client_options)
@http_options = {:headers => {'Content-Type' => 'application/json'}}
username = @settings["var.kibana.username"]
if username
password = @settings["var.kibana.password"]
if password.is_a?(LogStash::Util::Password)
password = password.value
end
@http_options[:headers]['Authorization'] = 'Basic ' + Base64.encode64("#{username}:#{password}").chomp
end
# e.g. {"name":"Elastics-MacBook-Pro.local","version":{"number":"6.0.0-beta1","build_hash":"41e69","build_number":15613,"build_snapshot":true}..}
@version = "0.0.0"
response = get("api/status")
if response.succeeded?
status = response.body
if status["version"].is_a?(Hash)
@version = status["version"]["number"]
if status["version"]["build_snapshot"]
@version.concat("-SNAPSHOT")
end
end
end
@http_options[:headers]['kbn-version'] = @version
end
def version_parts
@version.split(/[.-]/)
end
def host_settings
"[\"#{@host}\"]"
end
def get(relative_path)
# e.g. api/kibana/settings
safely(:get, relative_path, @http_options)
end
# content will be converted to a json string
def post(relative_path, content, headers = nil)
body = content.is_a?(String) ? content : LogStash::Json.dump(content)
options = {:body => body}.merge(headers || @http_options)
safely(:post, relative_path, options)
end
def head(relative_path)
safely(:head, relative_path, @http_options)
end
def can_connect?
head("api/status").succeeded?
end
private
def safely(method_sym, relative_path, options = {})
begin
resp = @client.http(method_sym, full_url(relative_path), options).call
Response.new(resp.code, resp.body, resp.headers)
rescue Manticore::ManticoreException => e
logger.error("Error when executing Kibana client request", :error => e)
body = {"statusCode" => 0, "error" => e.message}
Response.new(0, body, {})
end
end
def full_url(relative)
"#{@endpoint}/#{relative}"
end
end end end

135
logstash-core/lib/logstash/modules/kibana_config.rb
View file

@ -1,135 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require_relative "file_reader"
require_relative "kibana_settings"
require_relative "kibana_dashboards"
require_relative "kibana_resource"
module LogStash module Modules class KibanaConfig
include LogStash::Util::Loggable
ALLOWED_DIRECTORIES = ["search", "visualization"]
attr_reader :index_name # not used when importing via kibana but for BWC with ElasticsearchConfig
# We name it `modul` here because `module` has meaning in Ruby.
def initialize(modul, settings)
build_versioned_directory(modul)
@name = modul.module_name
@settings = settings
@index_name = "kibana"
@pattern_name = "#{@name}-*"
@kibana_settings = [
KibanaSettings::Setting.new("defaultIndex", @pattern_name)
]
end
def dashboards
# there can be more than one dashboard to load
filenames = FileReader.read_json(dynamic("dashboard"))
filenames.map do |filename|
KibanaResource.new(@index_name, "dashboard", dynamic("dashboard", filename))
end
end
def index_pattern
[KibanaResource.new(@index_name, "index-pattern", dynamic("index-pattern"), nil, @pattern_name)]
end
def resources
list = index_pattern
dashboards.each do |board|
list << board
extract_panels_into(board, list)
end
list.concat(extract_saved_searches_into(list))
[
KibanaSettings.new("api/kibana/settings", @kibana_settings),
KibanaDashboards.new("api/kibana/dashboards/import", list)
]
end
private
def build_versioned_directory(modul)
# try to detect which directory holds the config for the kibana version
base_dir = ::File.join(modul.directory, "kibana")
maj, min, patch = modul.kibana_version_parts
version_dir = "#{maj}.#{min}.#{patch}"
@directory = ::File.join(base_dir, version_dir)
return if ::File.directory?(@directory)
version_dir = "#{maj}.#{min}.x"
@directory = ::File.join(base_dir, version_dir)
return if ::File.directory?(@directory)
version_dir = "#{maj}.x"
@directory = ::File.join(base_dir, version_dir)
unless ::File.directory?(@directory)
logger.error("Cannot find kibana version sub-directory", :module => @name, :base_directory => base_dir)
end
end
def dynamic(dynamic_folder, filename = @name)
::File.join(@directory, dynamic_folder, "#{filename}.json")
end
def extract_panels_into(dashboard, list)
dash = dashboard.content_as_object
if !dash.is_a?(Hash)
logger.warn("Kibana dashboard JSON is not an Object", :module => @name)
return
end
panelsjson = dash["panelsJSON"]
if panelsjson.nil?
logger.info("No panelJSON key found in kibana dashboard", :module => @name)
return
end
begin
panels = LogStash::Json.load(panelsjson)
rescue => e
logger.error("JSON parse error when reading kibana panelsJSON", :module => @name)
return
end
panels.each do |panel|
panel_type = panel["type"]
if ALLOWED_DIRECTORIES.member?(panel_type)
list << KibanaResource.new(@index_name, panel_type, dynamic(panel_type, panel["id"]))
else
logger.warn("panelJSON contained unknown type", :type => panel_type)
end
end
end
def extract_saved_searches_into(list)
result = [] # must not add to list while iterating
list.each do |resource|
content = resource.content_as_object
next if content.nil?
next unless content.keys.include?("savedSearchId")
saved_search = content["savedSearchId"]
next if saved_search.nil?
ss_resource = KibanaResource.new(@index_name, "search", dynamic("search", saved_search))
next if list.member?(ss_resource) || result.member?(ss_resource)
result << ss_resource
end
result
end
end end end

50
logstash-core/lib/logstash/modules/kibana_dashboards.rb
View file

@ -1,50 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require_relative "kibana_base"
module LogStash module Modules class KibanaDashboards < KibanaBase
include LogStash::Util::Loggable
attr_reader :import_path, :content
# content is a list of kibana file resources
def initialize(import_path, content)
@import_path, @content = import_path, content
end
def import(client)
# e.g. curl "http://localhost:5601/api/kibana/dashboards/import"
# extract and prepare all objects
objects = []
content.each do |resource|
hash = {
"id" => resource.content_id,
"type" => resource.content_type,
"version" => 1,
"attributes" => resource.content_as_object
}
objects << hash
end
body = {"version": client.version, "objects": objects}
response = client.post(import_path, body)
if response.failed?
logger.error("Attempted POST failed", :url_path => import_path, :response => response.body)
end
response
end
end end end

30
logstash-core/lib/logstash/modules/kibana_importer.rb
View file

@ -1,30 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
module LogStash module Modules class KibanaImporter
include LogStash::Util::Loggable
def initialize(client)
@client = client
end
def put(via_kibana)
path = via_kibana.import_path
logger.debug("Attempting POST", :url_path => path, :content => via_kibana.content)
via_kibana.import(@client)
end
end end end

25
logstash-core/lib/logstash/modules/kibana_resource.rb
View file

@ -1,25 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require_relative "resource_base"
module LogStash module Modules class KibanaResource
include ResourceBase
def import_path
base + "/" + content_type + "/" + content_id
end
end end end

55
logstash-core/lib/logstash/modules/kibana_settings.rb
View file

@ -1,55 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require_relative "kibana_base"
module LogStash module Modules class KibanaSettings < KibanaBase
include LogStash::Util::Loggable
class Setting
attr_reader :name, :value
def initialize(name, value)
@name, @value = name, value
end
end
attr_reader :import_path, :content
# content is an array of Setting required for this module
def initialize(import_path, content)
@import_path, @content = import_path, content
end
def import(client)
# e.g. curl "http://localhost:5601/api/kibana/settings"
# 6.0.0-beta1 -> {"settings":{"buildNum":{"userValue":15613},"defaultIndex":{"userValue":"arcsight-*"}}}
# 5.4 -> {"settings":{"defaultIndex":{"userValue":"cef-*"},"metrics:max_buckets":{"userValue":"600000"}}}
# array of Setting objects
# The POST api body { "changes": { "defaultIndex": "arcsight-*", "metrics:max_buckets": "400" } }
settings = {}
content.each do |setting|
settings[setting.name] = "#{setting.value}"
end
body = {"changes" => settings}
response = client.post(import_path, body)
if response.failed?
logger.error("Attempted POST failed", :url_path => import_path, :response => response.body)
end
response
end
end end end

149
logstash-core/lib/logstash/modules/logstash_config.rb
View file

@ -1,149 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require_relative "file_reader"
require "logstash/settings"
module LogStash module Modules class LogStashConfig
include LogStash::Util::Loggable
# We name it `modul` here because `module` has meaning in Ruby.
def initialize(modul, settings)
@directory = ::File.join(modul.directory, "logstash")
@name = modul.module_name
@settings = settings
end
def template
::File.join(@directory, "#{@name}.conf.erb")
end
def configured_inputs(default = [], aliases = {})
name = "var.inputs"
values = get_setting(LogStash::Setting::SplittableStringArray.new(name, String, default))
aliases.each { |k, v| values << v if values.include?(k) }
aliases.invert.each { |k, v| values << v if values.include?(k) }
values.flatten.uniq
end
def alias_settings_keys!(aliases)
aliased_settings = alias_matching_keys(aliases, @settings)
@settings = alias_matching_keys(aliases.invert, aliased_settings)
end
def array_to_string(array)
"[#{array.collect { |i| "'#{i}'" }.join(", ")}]"
end
def csv_string(array)
"'#{array.join(',')}'"
end
def get_setting(setting_class)
raw_value = @settings[setting_class.name]
# If we dont check for NIL, the Settings class will try to coerce the value
# and most of the it will fails when a NIL value is explicitly set.
# This will be fixed once we wrap the plugins settings into a Settings class
setting_class.set(raw_value) unless raw_value.nil?
setting_class.value
end
def setting(name, default)
# by default we use the more permissive setting which is a `NullableString`
# This is fine because the end format of the logstash configuration is a string representation
# of the pipeline. There is a good reason why I think we should use the settings classes, we
# can `preprocess` a template and generate a configuration from the defined settings
# validate the values and replace them in the template.
case default
when String
get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
when Numeric
get_setting(LogStash::Setting::Numeric.new(name, default))
when true, false
get_setting(LogStash::Setting::Boolean.new(name, default))
else
get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
end
end
def has_setting?(name)
@settings.key?(name)
end
def raw_setting(name)
@settings[name]
end
def fetch_raw_setting(name, default)
@settings.fetch(name, default)
end
def elasticsearch_output_config(type_string = nil, index_suffix = "-%{+YYYY.MM.dd}")
hosts = array_to_string(get_setting(LogStash::Setting::SplittableStringArray.new("var.elasticsearch.hosts", String, ["localhost:9200"])))
index = "#{@name}#{index_suffix}"
user = @settings["var.elasticsearch.username"]
password = @settings["var.elasticsearch.password"]
lines = ["hosts => #{hosts}", "index => \"#{index}\""]
lines.push(user ? "user => \"#{user}\"" : nil)
lines.push(password ? "password => \"#{password.value}\"" : nil)
lines.push(type_string ? "document_type => #{type_string}" : nil)
lines.push("ssl_enabled => #{@settings.fetch('var.elasticsearch.ssl.enabled', false)}")
if cacert = @settings["var.elasticsearch.ssl.certificate_authority"]
lines.push("ssl_certificate_authorities => \"#{cacert}\"") if cacert
end
# NOTE: the first line should be indented in the conf.erb
<<-CONF
elasticsearch {
#{lines.compact.join("\n ")}
manage_template => false
}
CONF
end
def config_string
# process the template and settings
# send back as a string
renderer = ERB.new(FileReader.read(template))
renderer.result(binding)
end
private
# For a first version we are copying the values of the original hash,
# this might become problematic if we users changes the values of the
# settings in the template, which could result in an inconsistent view of the original data
#
# For v1 of the feature I think its an OK compromise, v2 we have a more advanced hash that
# support alias.
def alias_matching_keys(aliases, target)
aliased_target = target.dup
aliases.each do |matching_key_prefix, new_key_prefix|
target.each do |k, v|
re = /^#{matching_key_prefix}\./
if k =~ re
alias_key = k.gsub(re, "#{new_key_prefix}.")
# If the user setup the same values twices with different values lets just halt.
raise "Cannot create an alias, the destination key has already a value set: original key: #{k}, alias key: #{alias_key}" if (!aliased_target[alias_key].nil? && aliased_target[alias_key] != v)
aliased_target[alias_key] = v unless v.nil?
end
end
end
aliased_target
end
end end end

53
logstash-core/lib/logstash/modules/resource_base.rb
View file

@ -1,53 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/json"
require_relative "file_reader"
module LogStash module Modules module ResourceBase
attr_reader :base, :content_type, :content_path, :content_id
def initialize(base, content_type, content_path, content = nil, content_id = nil)
@base, @content_type, @content_path = base, content_type, content_path
@content_id = content_id || ::File.basename(@content_path, ".*")
# content at this time will be a JSON string
@content = content
if !@content.nil?
@content_as_object = LogStash::Json.load(@content) rescue {}
end
end
def content
@content ||= FileReader.read(@content_path)
end
def to_s
"#{base}, #{content_type}, #{content_path}, #{content_id}"
end
def content_as_object
@content_as_object ||= FileReader.read_json(@content_path) rescue nil
end
def <=>(other)
to_s <=> other.to_s
end
def ==(other)
to_s == other.to_s
end
end end end

69
logstash-core/lib/logstash/modules/scaffold.rb
View file

@ -1,69 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "erb"
require_relative "elasticsearch_config"
require_relative "kibana_config"
require_relative "logstash_config"
module LogStash module Modules class Scaffold
include LogStash::Util::Loggable
attr_reader :directory, :module_name, :kibana_version_parts
attr_reader :kibana_configuration, :logstash_configuration, :elasticsearch_configuration
def initialize(name, directory)
@module_name = name
@directory = directory # this is the 'configuration folder in the GEM root.'
@kibana_version_parts = "6.0.0".split('.') # this is backup in case kibana client fails to connect
logger.debug("Found module", :module_name => name, :directory => directory)
end
def add_kibana_version(version_parts)
@kibana_version_parts = version_parts
end
def import(import_engine, kibana_import_engine)
@elasticsearch_configuration.resources.each do |resource|
import_engine.put(resource)
end
@kibana_configuration.resources.each do |resource|
kibana_import_engine.put(resource)
end
end
def with_settings(module_settings)
@logstash_configuration = LogStashConfig.new(self, module_settings)
@kibana_configuration = KibanaConfig.new(self, module_settings)
@elasticsearch_configuration = ElasticsearchConfig.new(self, module_settings)
self
end
def config_string()
# settings should be set earlier by the caller.
# settings should be the subset from the YAML file with a structure like
# {"name" => "plugin name", "k1" => "v1", "k2" => "v2"}, etc.
return nil if @logstash_configuration.nil?
@logstash_configuration.config_string
end
# subclass may override
def is_enabled?(settings)
true
end
end end end # class LogStash::Modules::Scaffold

94
logstash-core/lib/logstash/modules/settings_merger.rb
View file

@ -1,94 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/util"
module LogStash module Modules module SettingsMerger
include LogStash::Util::Loggable
extend self
# cli_settings Array or LogStash::Util::ModulesSettingArray
# yml_settings Array or LogStash::Util::ModulesSettingArray
def merge(cli_settings, yml_settings)
# both args are arrays of hashes, e.g.
# [{"name"=>"mod1", "var.input.tcp.port"=>"3333"}, {"name"=>"mod2"}]
# [{"name"=>"mod1", "var.input.tcp.port"=>2222, "var.kibana.username"=>"rupert", "var.kibana.password"=>"fotherington"}, {"name"=>"mod3", "var.input.tcp.port"=>4445}]
merged = []
# union and group_by preserves order
# union will also coalesce identical hashes
# this "|" operator is provided to Java List by RubyJavaIntegration
union_of_settings = (cli_settings | yml_settings)
grouped_by_name = union_of_settings.group_by {|e| e["name"]}
grouped_by_name.each do |_, array|
if array.size == 2
merged << array.last.merge(array.first)
else
merged.concat(array)
end
end
merged
end
def merge_cloud_settings(module_settings, logstash_settings)
cloud_id = logstash_settings.get("cloud.id")
cloud_auth = logstash_settings.get("cloud.auth")
if cloud_id.nil?
if cloud_auth.nil?
return # user did not specify cloud settings
else
raise ArgumentError.new("Cloud Auth without Cloud Id")
end
end
if logger.debug?
settings_copy = LogStash::Util.deep_clone(module_settings)
end
module_settings["var.kibana.scheme"] = cloud_id.kibana_scheme
module_settings["var.kibana.host"] = cloud_id.kibana_host
# elasticsearch client does not use scheme, it URI parses the host setting
module_settings["var.elasticsearch.hosts"] = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
unless cloud_auth.nil?
module_settings["var.elasticsearch.username"] = cloud_auth.username
module_settings["var.elasticsearch.password"] = cloud_auth.password
module_settings["var.kibana.username"] = cloud_auth.username
module_settings["var.kibana.password"] = cloud_auth.password
end
if logger.debug?
format_module_settings(settings_copy, module_settings).each {|line| logger.debug(line)}
end
end
def merge_kibana_auth!(module_settings)
module_settings["var.kibana.username"] = module_settings["var.elasticsearch.username"] if module_settings["var.kibana.username"].nil?
module_settings["var.kibana.password"] = module_settings["var.elasticsearch.password"] if module_settings["var.kibana.password"].nil?
end
def format_module_settings(settings_before, settings_after)
output = []
output << "-------- Module Settings ---------"
settings_after.each do |setting_name, setting|
setting_before = settings_before.fetch(setting_name, "")
line = "#{setting_name}: '#{setting}'"
if setting_before != setting
line.concat(", was: '#{setting_before}'")
end
output << line
end
output << "-------- Module Settings ---------"
output
end
end end end

33
logstash-core/lib/logstash/modules/util.rb
View file

@ -1,33 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require_relative "scaffold"
# This module function should be used when gems or
# x-pack defines modules in their folder structures.
module LogStash module Modules module Util
def self.register_local_modules(path)
modules_path = ::File.join(path, "modules")
::Dir.foreach(modules_path) do |item|
# Ignore unix relative path ids
next if item == '.' or item == '..'
# Ignore non-directories
next if !::File.directory?(::File.join(modules_path, ::File::Separator, item))
LogStash::PLUGIN_REGISTRY.add(:modules, item, Scaffold.new(item, ::File.join(modules_path, item, "configuration")))
end
end
end end end

1
logstash-core/lib/logstash/plugins/registry.rb
View file

@ -17,7 +17,6 @@
require "rubygems/package"
require "logstash/plugin"
require "logstash/modules/scaffold"
require "logstash/codecs/base"
require "logstash/filters/base"
require "logstash/outputs/base"

34
logstash-core/lib/logstash/runner.rb
View file

@ -34,7 +34,6 @@ end
require "clamp"
require "logstash-core/logstash-core"
require "logstash/environment"
require "logstash/modules/cli_parser"
require "logstash/util/settings_helper"
require "logstash/util/jackson"
@ -46,7 +45,6 @@ require "logstash/patches/clamp"
require "logstash/settings"
require "logstash/version"
require 'logstash/plugins'
require "logstash/modules/util"
require "logstash/bootstrap_check/default_config"
require 'logstash/deprecation_message'
@ -92,31 +90,6 @@ class LogStash::Runner < Clamp::StrictCommand
:default => LogStash::SETTINGS.get_default("config.field_reference.escape_style"),
:attribute_name => "config.field_reference.escape_style"
# Module settings
option ["--modules"], "MODULES",
I18n.t("logstash.runner.flag.modules"),
:multivalued => true,
:attribute_name => "modules_list"
option ["-M", "--modules.variable"], "MODULES_VARIABLE",
I18n.t("logstash.runner.flag.modules_variable"),
:multivalued => true,
:attribute_name => "modules_variable_list"
option ["--setup"], :flag,
I18n.t("logstash.runner.flag.modules_setup"),
:default => LogStash::SETTINGS.get_default("modules_setup"),
:attribute_name => "modules_setup"
option ["--cloud.id"], "CLOUD_ID",
I18n.t("logstash.runner.flag.cloud_id"),
:attribute_name => "cloud.id"
option ["--cloud.auth"], "CLOUD_AUTH",
I18n.t("logstash.runner.flag.cloud_auth"),
:attribute_name => "cloud.auth"
# Pipeline settings
option ["--pipeline.id"], "ID",
I18n.t("logstash.runner.flag.pipeline-id"),
@ -266,7 +239,6 @@ class LogStash::Runner < Clamp::StrictCommand
# Default we check local sources: `-e`, `-f` and the logstash.yml options.
@source_loader = LogStash::Config::SourceLoader.new(@settings)
@source_loader.add_source(LogStash::Config::Source::Local.new(@settings))
@source_loader.add_source(LogStash::Config::Source::Modules.new(@settings))
@source_loader.add_source(LogStash::Config::Source::MultiLocal.new(@settings))
super(*args)
@ -318,8 +290,6 @@ class LogStash::Runner < Clamp::StrictCommand
jvmArgs = ManagementFactory.getRuntimeMXBean().getInputArguments()
logger.info "JVM bootstrap flags: #{jvmArgs}"
# Add local modules to the registry before everything else
LogStash::Modules::Util.register_local_modules(LogStash::Environment::LOGSTASH_HOME)
# Verify the Jackson defaults
LogStash::Util::Jackson.verify_jackson_overrides
@ -340,10 +310,6 @@ class LogStash::Runner < Clamp::StrictCommand
return start_shell(setting("interactive"), binding) if setting("interactive")
module_parser = LogStash::Modules::CLIParser.new(setting("modules_list"), setting("modules_variable_list"))
# Now populate Setting for modules.list with our parsed array.
@settings.set("modules.cli", module_parser.output)
begin
@bootstrap_checks.each { |bootstrap| bootstrap.check(@settings) }
rescue LogStash::BootstrapCheckError => e

24
logstash-core/lib/logstash/settings.rb
View file

@ -800,30 +800,6 @@ module LogStash
end
end
class Modules < Coercible
def initialize(name, klass, default = nil)
super(name, klass, default, false)
end
def set(value)
coerced_value = coerce(value)
@wrapped_setting.set(coerced_value)
coerced_value
end
def coerce(value)
if value.is_a?(@klass)
return value
end
@klass.new(value)
end
protected
def validate(value)
coerce(value)
end
end
# @see Setting#nullable
# @api internal
class Nullable < SimpleDelegator

86
logstash-core/lib/logstash/util/manticore_ssl_config_helper.rb
View file

@ -1,86 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
module LogStash; module Util; module ManticoreSSLConfigHelper
extend self
##
# Extract Manticore-style SSL directives from the given configuration.
#
# @param namespace [String] a string namespace (e.g., `kibana` in `var.kibana.ssl.*`)
# @param settings [Hash<String,Object>] a collection of Manticore-friendly SSL directives.
# if SSL explicitly disabled, an _empty_ hash will be returned.
#
# @return [Hash<Symbol,Object>]
def manticore_ssl_options_from_config(namespace, settings)
ssl_settings = strip_prefix(settings, "var.#{namespace}.ssl.")
# boolean settings may be strings if set through the cli
# or booleans if set through the yaml file, so we use .to_s
if ssl_settings.include?('enabled') && !coerce_boolean(ssl_settings['enabled'])
logger.warn('SSL explicitly disabled; other SSL settings will be ignored') if logger && ssl_settings.size > 1
return {}
end
{
:verify => ssl_settings.fetch('verification_mode', :strict).to_sym,
:ca_file => ssl_settings.fetch('certificate_authority', nil),
:client_cert => ssl_settings.fetch('certificate', nil),
:client_key => ssl_settings.fetch('key', nil),
}
end
private
##
# Returns the subset of the hash whose keys match the given prefix, with the prefix removed
#
# @param hash [Hash<String,Object>]
# @param prefix [String]
# @return [Hash<String,Object>]
def strip_prefix(hash, prefix)
hash.each_with_object({}) do |(key, value), memo|
next unless key.start_with?(prefix)
unprefixed_key = key[prefix.length..-1]
memo[unprefixed_key] = value
end
end
##
# Coerces the non-nil input to boolean
#
# @param value [Boolean,String,Integer]
# @return [Boolean]
def coerce_boolean(value)
case value
when true, "true", "T", 1 then true
when false, "false", "F", 0 then false
else
fail("Boolean value required, received `#{value}`")
end
end
##
# Adapter to enable logging via the including class' `#logger` method or `@logger` instance variable
#
# @return [Logger,nil]
def logger
return super if defined?(super)
@logger
end
end end end

20
logstash-core/lib/logstash/util/modules_setting_array.rb
View file

@ -1,20 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
module LogStash; module Util
java_import "org.logstash.util.ModulesSettingArray"
end; end

64
logstash-core/locales/en.yml
View file

@ -67,33 +67,11 @@ en:
logging:
unrecognized_option: |-
unrecognized option [%{option}]
modules:
configuration:
parse-failed: |-
Failed to parse the module configuration: [%{error}]
modules-must-be-unique: >-
Only a single instance of any module can be run at a time. Duplicate
modules: %{duplicate_modules}
modules-invalid-name: >-
Invalid module name: %{module_name}
modules-variables-malformed: >-
Failed to parse module variable %{rawvar}. Must be in -M
"MODULE_NAME.KEY.SUBKEY=VALUE" format
modules-unavailable: >-
The modules specified are not available yet.
Specified modules: %{specified_modules}
Available modules: %{available_modules}
elasticsearch_connection_failed: >-
Failed to import module configurations to Elasticsearch and/or Kibana.
Module: %{module_name} has Elasticsearch hosts: %{elasticsearch_hosts} and Kibana hosts: %{kibana_hosts}
modules-too-many-specified: >-
Too many modules specified. Maximum allowed: %{max}, specified: %{specified_modules}
runner:
short-help: |-
usage:
bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG]
bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
@ -108,14 +86,6 @@ en:
the '-f yourlogstash.conf' flag?
config-string-path-exclusive:
Settings 'path.config' (-f) and 'config.string' (-e) can't be used simultaneously.
config-module-exclusive: >-
Settings 'path.config' (-f) or 'config.string' (-e) can't be used in conjunction with
(--modules) or the "modules:" block in the logstash.yml file.
reload-with-modules: >-
Configuration reloading can't be used with command-line or logstash.yml specified modules.
cli-module-override: >-
Both command-line and logstash.yml modules configurations detected.
Using command-line module configuration to override logstash.yml module configuration.
config-pipelines-failed-read: >-
Failed to read pipelines yaml file. Location: %{path}
config-pipelines-failed-read-with-exception: >-
@ -244,40 +214,6 @@ en:
HTML-style ampersand-hash encoding notation
representing decimal unicode codepoints
(`[` is `&#91;`; `]` is `&#93;`).
modules: |+
Load Logstash modules.
Modules can be defined using multiple instances
'--modules module1 --modules module2',
or comma-separated syntax
'--modules=module1,module2'
Cannot be used in conjunction with '-e' or '-f'
Use of '--modules' will override modules declared
in the 'logstash.yml' file.
modules_variable: |+
Load variables for module template.
Multiple instances of '-M' or
'--modules.variable' are supported.
Ignored if '--modules' flag is not used.
Should be in the format of
'-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
as in
'-M "example.var.filter.mutate.fieldname=fieldvalue"'
modules_setup: |+
Load index template into Elasticsearch, and saved searches,
index-pattern, visualizations, and dashboards into Kibana when
running modules.
cloud_id: |+
Sets the elasticsearch and kibana host settings for
module connections in Elastic Cloud.
Your Elastic Cloud User interface or the Cloud support
team should provide this.
Add an optional label prefix '<label>:' to help you
identify multiple cloud.ids.
e.g. 'staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy'
cloud_auth: |+
Sets the elasticsearch and kibana username and password
for module connections in Elastic Cloud
e.g. 'username:<password>'
configtest: |+
Check configuration for valid syntax and then exit.
api_enabled: |+

4
logstash-core/spec/logstash/config/source/local_spec.rb
View file

@ -305,9 +305,7 @@ describe LogStash::Config::Source::Local do
let(:settings) do
mock_settings(
"config.string" => "#{filter_block} #{output_block}",
"path.config" => config_file,
"modules.cli" => [],
"modules" => []
"path.config" => config_file
)
end

53
logstash-core/spec/logstash/elasticsearch_client_spec.rb
View file

@ -1,53 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/elasticsearch_client"
describe LogStash::ElasticsearchClient do
describe LogStash::ElasticsearchClient::RubyClient do
let(:settings) { {} }
let(:logger) { nil }
describe "ssl option handling" do
context "when using a string for ssl.enabled" do
let(:settings) do
{ "var.elasticsearch.ssl.enabled" => "true" }
end
it "should set the ssl options" do
expect(Elasticsearch::Client).to receive(:new) do |args|
expect(args[:ssl]).to_not be_empty
end
described_class.new(settings, logger)
end
end
context "when using a boolean for ssl.enabled" do
let(:settings) do
{ "var.elasticsearch.ssl.enabled" => true }
end
it "should set the ssl options" do
expect(Elasticsearch::Client).to receive(:new) do |args|
expect(args[:ssl]).to_not be_empty
end
described_class.new(settings, logger)
end
end
end
end
end

142
logstash-core/spec/logstash/modules/cli_parser_spec.rb
View file

@ -1,142 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "spec_helper"
require "logstash/modules/cli_parser"
describe LogStash::Modules::CLIParser do
subject { LogStash::Modules::CLIParser.new(module_names, module_variables) }
let(:logger) { double("logger") }
let(:module_name) { "foo" }
let(:module_names) { [module_name, "bar"] }
let(:proto_key_value) { "var.input.stdin.type=example" }
let(:proto_mod_vars) { module_name + "." + proto_key_value }
let(:module_variables) { [proto_mod_vars] }
let(:expected_output) { { "name" => module_name, "var.input.stdin.type" => "example" } }
describe ".parse_modules" do
let(:module1) { "module1" }
let(:module2) { "module2" }
let(:csv_modules) { "#{module1},#{module2}" }
let(:list_with_csv) { [module_name, csv_modules] }
let(:post_parse) { [module_name, module1, module2] }
context "when it receives an array without a csv entry" do
it "return the array unaltered" do
expect(subject.parse_modules(module_names)).to eq(module_names)
end
end
context "when it receives an empty array" do
it "return an empty array" do
expect(subject.parse_modules([])).to eq([])
end
end
context "when it receives an array with a csv entry" do
it "return the original array with the csv values split into elements" do
expect(subject.parse_modules(list_with_csv)).to eq(post_parse)
end
end
context "when it receives an array with a bad csv entry" do
let(:bad_modules) { ["-Minvalid", module1] }
it "raise a LogStash::ConfigLoadingError exception" do
expect { subject.parse_modules(bad_modules) }.to raise_error LogStash::ConfigLoadingError
end
end
context "when it receives a nil value in an array" do
let(:array_with_nil) { list_with_csv << nil }
it "skip it" do
expect(subject.parse_modules(array_with_nil)).to eq(post_parse)
end
end
end
describe ".get_kv" do
context "when it receives a valid string" do
let(:expected_key) { "var.input.stdin.type" }
let(:expected_value) { "example" }
let(:unparsed) { expected_key + "=" + expected_value }
it "split it into a key value pair" do
expect(subject.get_kv(module_name, unparsed)).to eq([expected_key, expected_value])
end
end
context "when it receives an invalid string" do
let(:bad_example) { "var.fail" }
it "raise a LogStash::ConfigLoadingError exception" do
expect { subject.get_kv(module_name, bad_example) }.to raise_error LogStash::ConfigLoadingError
end
end
end
describe ".name_splitter" do
context "when it receives a valid string" do
let(:expected) { "var.input.stdin.type=example" }
it "split the module name from the rest of the string" do
expect(subject.name_splitter(proto_mod_vars)).to eq([module_name, expected])
end
end
context "when it receives an invalid string" do
let(:bad_example) { "var.fail" }
it "raise a LogStash::ConfigLoadingError exception" do
expect { subject.name_splitter(bad_example) }.to raise_error LogStash::ConfigLoadingError
end
end
end
describe ".parse_vars" do
context "when it receives a vars_list with valid strings" do
it "return a hash with the module name and associated variables as key value pairs" do
expect(subject.parse_vars(module_name, module_variables)).to eq(expected_output)
end
end
context "when it receives a string that doesn't start with module_name" do
let(:has_unrelated) { module_variables << "bar.var.input.stdin.type=different" }
it "skips it" do
expect(subject.parse_vars(module_name, has_unrelated)).to eq(expected_output)
end
end
context "when it receives an empty vars_list" do
let(:name_only) { { "name" => module_name } }
it "return a hash with only 'name => module_name'" do
expect(subject.parse_vars(module_name, [])).to eq(name_only)
end
end
end
describe ".parse_it" do
context "when it receives a valid module_list and module_variable_list" do
let(:module_names) { [module_name]}
it "@output is array of hashes with the module name and associated variables as key value pairs" do
expect(subject.output).to eq([expected_output])
end
end
context "when it receives a non-array value for module_list" do
let(:module_names) { "string value" }
it "return an empty array" do
expect(subject.output).to eq([])
end
end
end
end

105
logstash-core/spec/logstash/modules/kibana_client_spec.rb
View file

@ -1,105 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "spec_helper"
require "logstash/modules/kibana_client"
module LogStash module Modules
KibanaTestResponse = Struct.new(:code, :body, :headers)
class KibanaTestClient
def http(method, endpoint, options)
self
end
def call
KibanaTestResponse.new(200, '{"version":{"number":"1.2.3","build_snapshot":false}}', {})
end
end
describe KibanaClient do
let(:settings) { Hash.new }
let(:test_client) { KibanaTestClient.new }
let(:kibana_host) { "https://foo.bar:4321" }
subject(:kibana_client) { described_class.new(settings, test_client) }
context "when supplied with conflicting scheme data" do
let(:settings) { {"var.kibana.scheme" => "http", "var.kibana.host" => kibana_host} }
it "a new instance will throw an error" do
expect {described_class.new(settings, test_client)}.to raise_error(ArgumentError, /Detected differing Kibana host schemes as sourced from var\.kibana\.host: 'https' and var\.kibana\.scheme: 'http'/)
end
end
context "when supplied with invalid schemes" do
["httpd", "ftp", "telnet"].each do |uri_scheme|
it "a new instance will throw an error" do
re = /Kibana host scheme given is invalid, given value: '#{uri_scheme}' - acceptable values: 'http', 'https'/
expect {described_class.new({"var.kibana.scheme" => uri_scheme}, test_client)}.to raise_error(ArgumentError, re)
end
end
end
context "when supplied with the scheme in the host only" do
let(:settings) { {"var.kibana.host" => kibana_host} }
it "has a version and an endpoint" do
expect(kibana_client.version).to eq("1.2.3")
expect(kibana_client.endpoint).to eq("https://foo.bar:4321")
end
end
context "when supplied with the scheme in the scheme setting" do
let(:settings) { {"var.kibana.scheme" => "https", "var.kibana.host" => "foo.bar:4321"} }
it "has a version and an endpoint" do
expect(kibana_client.version).to eq("1.2.3")
expect(kibana_client.endpoint).to eq(kibana_host)
end
end
context "when supplied with a no scheme host setting and ssl is enabled" do
let(:settings) { {"var.kibana.ssl.enabled" => "true", "var.kibana.host" => "foo.bar:4321"} }
it "has a version and an endpoint" do
expect(kibana_client.version).to eq("1.2.3")
expect(kibana_client.endpoint).to eq(kibana_host)
end
end
describe "ssl option handling" do
context "when using a string for ssl.enabled" do
let(:settings) do
{ "var.kibana.ssl.enabled" => "true" }
end
it "should set the ssl options" do
expect(Manticore::Client).to receive(:new) do |args|
expect(args[:ssl]).to_not be_empty
end.and_return(test_client)
described_class.new(settings)
end
end
context "when using a boolean for ssl.enabled" do
let(:settings) do
{ "var.kibana.ssl.enabled" => true }
end
it "should set the ssl options" do
expect(Manticore::Client).to receive(:new) do |args|
expect(args[:ssl]).to_not be_empty
end.and_return(test_client)
described_class.new(settings)
end
end
end
end
end end

91
logstash-core/spec/logstash/modules/logstash_config_spec.rb
View file

@ -1,91 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/modules/logstash_config"
describe LogStash::Modules::LogStashConfig do
let(:module_name) { "testing" }
let(:mod) { instance_double("module", :directory => Stud::Temporary.directory, :module_name => module_name) }
let(:settings) { {"var.logstash.testing.pants" => "fancy", "var.elasticsearch.password" => LogStash::Util::Password.new('correct_horse_battery_staple') }}
subject { described_class.new(mod, settings) }
describe "configured inputs" do
context "when no inputs is send" do
it "returns the default" do
expect(subject.configured_inputs(["kafka"])).to include("kafka")
end
end
context "when inputs are send" do
let(:settings) { { "var.inputs" => "tcp" } }
it "returns the configured inputs" do
expect(subject.configured_inputs(["kafka"])).to include("tcp")
end
context "when alias is specified" do
let(:settings) { { "var.inputs" => "smartconnector" } }
it "returns the configured inputs" do
expect(subject.configured_inputs(["kafka"], { "smartconnector" => "tcp" })).to include("tcp", "smartconnector")
end
end
end
end
describe "array to logstash array string" do
it "return an escaped string" do
expect(subject.array_to_string(["hello", "ninja"])).to eq("['hello', 'ninja']")
end
end
describe 'elasticsearch_config_output' do
let(:args) { nil }
let(:config) { subject.elasticsearch_output_config(*args) }
it 'should put the password in correctly' do
expect(config).to include("password => \"correct_horse_battery_staple\"")
end
it 'appends the timestamp expression to the index name' do
expect(config).to include("index => \"#{module_name}-%{+YYYY.MM.dd}\"")
end
context "when index_suffix is customized" do
let(:custom_suffix) { "-new_suffix" }
let(:args) { ["my_custom", custom_suffix] }
it 'the index name uses the custom suffix instead' do
expect(config).to include("index => \"#{module_name}#{custom_suffix}\"")
end
end
end
describe "alias modules options" do
let(:alias_table) do
{ "var.logstash.testing" => "var.logstash.better" }
end
before do
subject.alias_settings_keys!(alias_table)
end
it "allow to retrieve settings" do
expect(subject.setting("var.logstash.better.pants", "dont-exist")).to eq("fancy")
end
it "allow to retrieve settings with the original name" do
expect(subject.setting("var.logstash.testing.pants", "dont-exist")).to eq("fancy")
end
end
end

248
logstash-core/spec/logstash/modules/scaffold_spec.rb
View file

@ -1,248 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "logstash/elasticsearch_client"
require "logstash/modules/kibana_client"
require "logstash/modules/kibana_config"
require "logstash/modules/scaffold"
require "logstash/modules/elasticsearch_importer"
require "logstash/modules/kibana_importer"
require_relative "../../support/helpers"
describe LogStash::Modules::Scaffold do
let(:base_dir) { "gem-home" }
let(:mname) { "foo" }
subject(:test_module) { described_class.new(mname, base_dir) }
let(:module_settings) do
{
"var.elasticsearch.hosts" => "es.mycloud.com:9200",
"var.elasticsearch.user" => "foo",
"var.elasticsearch.password" => "password",
"var.input.tcp.port" => 5606,
}
end
let(:dashboard_hash) do
{
"hits" => 0,
"timeRestore" => false,
"description" => "",
"title" => "Filebeat Apache2 Dashboard",
"uiStateJSON" => "{}",
"panelsJSON" => '[{"col":1,"id":"foo-c","panelIndex":1,"row":1,"size_x":12,"size_y":3,"type":"visualization"},{"id":"foo-d","type":"search","panelIndex":7,"size_x":12,"size_y":3,"col":1,"row":11,"columns":["apache2.error.client","apache2.error.level","apache2.error.module","apache2.error.message"],"sort":["@timestamp","desc"]}]',
"optionsJSON" => "{}",
"version" => 1,
"kibanaSavedObjectMeta" => {
"searchSourceJSON" => "{}"
}
}
end
let(:viz_hash) do
{
"visState" => "",
"description" => "",
"title" => "foo-c",
"uiStateJSON" => "",
"version" => 1,
"savedSearchId" => "foo-e",
"kibanaSavedObjectMeta" => {}
}
end
let(:index_pattern_hash) do
{
"title" => "foo-*",
"timeFieldName" => "time",
"fieldFormatMap" => "{some map}",
"fields" => "[some array]"
}
end
context "logstash operation" do
let(:ls_conf) do
<<-ERB
input {
tcp {
port => <%= setting("var.input.tcp.port", 45) %>
host => <%= setting("var.input.tcp.host", "localhost") %>
type => <%= setting("var.input.tcp.type", "server") %>
}
}
filter {
}
output {
<%= elasticsearch_output_config() %>
}
ERB
end
before do
allow(LogStash::Modules::FileReader).to receive(:read).and_return(ls_conf)
end
it "provides a logstash config" do
expect(test_module.logstash_configuration).to be_nil
test_module.with_settings(LogStash::Util::ModulesSettingArray.new([module_settings]).first)
expect(test_module.logstash_configuration).not_to be_nil
config_string = test_module.config_string
expect(config_string).to include("port => 5606")
expect(config_string).to include("hosts => ['es.mycloud.com:9200']")
end
end
context "elasticsearch operation" do
it "provides the elasticsearch mapping file paths" do
test_module.with_settings(module_settings)
expect(test_module.elasticsearch_configuration).not_to be_nil
files = test_module.elasticsearch_configuration.resources
expect(files.size).to eq(1)
expect(files.first).to be_a(LogStash::Modules::ElasticsearchResource)
expect(files.first.content_path).to eq("gem-home/elasticsearch/foo.json")
expect(files.first.import_path).to eq("_template/foo")
end
end
context "kibana operation" do
before do
# allow(LogStash::Modules::FileReader).to receive(:read_json).and_return({})
allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/dashboard/foo.json").and_return(["Foo-Dashboard"])
allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/dashboard/Foo-Dashboard.json").and_return(dashboard_hash)
allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/visualization/foo-c.json").and_return(viz_hash)
allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/search/foo-d.json").and_return({"d" => "search"})
allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/search/foo-e.json").and_return({"e" => "search"})
allow(LogStash::Modules::FileReader).to receive(:read_json).with("gem-home/kibana/6.x/index-pattern/foo.json").and_return(index_pattern_hash)
end
it "provides a list of importable files" do
expect(test_module.kibana_configuration).to be_nil
test_module.with_settings(module_settings)
expect(test_module.kibana_configuration).not_to be_nil
resources = test_module.kibana_configuration.resources
expect(resources.size).to eq(2)
resource1 = resources[0]
resource2 = resources[1]
expect(resource1).to be_a(LogStash::Modules::KibanaSettings)
expect(resource2).to be_a(LogStash::Modules::KibanaDashboards)
expect(resource1.import_path).to eq("api/kibana/settings")
expect(resource1.content).to be_a(Array)
expect(resource1.content.size).to eq(1)
test_object = resource1.content[0]
expect(test_object).to be_a(LogStash::Modules::KibanaSettings::Setting)
expect(test_object.name).to eq("defaultIndex")
expect(test_object.value).to eq("foo-*")
expect(resource2.import_path).to eq("api/kibana/dashboards/import")
expect(resource2.content).to be_a(Array)
expect(resource2.content.size).to eq(5)
expect(resource2.content.map {|o| o.class}.uniq).to eq([LogStash::Modules::KibanaResource])
test_object = resource2.content[0]
expect(test_object.content_id).to eq("foo-*")
expect(test_object.content_type).to eq("index-pattern")
expect(test_object.content_as_object).to eq(index_pattern_hash)
test_object = resource2.content[1]
expect(test_object.content_id).to eq("Foo-Dashboard")
expect(test_object.content_type).to eq("dashboard")
expect(test_object.content_as_object).to eq(dashboard_hash)
test_object = resource2.content[2]
expect(test_object.content_id).to eq("foo-c") #<- the panels can contain items from other folders
expect(test_object.content_type).to eq("visualization")
expect(test_object.content_as_object).to eq(viz_hash)
expect(test_object.content_as_object["savedSearchId"]).to eq("foo-e")
test_object = resource2.content[3]
expect(test_object.content_id).to eq("foo-d") #<- the panels can contain items from other folders
expect(test_object.content_type).to eq("search")
expect(test_object.content_as_object).to eq("d" => "search")
test_object = resource2.content[4]
expect(test_object.content_id).to eq("foo-e") # <- the visualization can contain items from the search folder
expect(test_object.content_type).to eq("search")
expect(test_object.content_as_object).to eq("e" => "search")
end
end
context "importing to elasticsearch stubbed client" do
let(:mname) { "tester" }
let(:base_dir) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "modules_test_files", "modules", "#{mname}", "configuration")) }
let(:response) { double(:response) }
let(:client) { double(:client) }
let(:kbnclient) { double(:kbnclient) }
let(:paths) { [] }
let(:expected_paths) { ["_template/tester", "api/kibana/settings", "api/kibana/dashboards/import"] }
let(:contents) { [] }
let(:expected_objects) do
[
"index-pattern tester-*",
"dashboard FW-Dashboard",
"visualization FW-Viz-1",
"visualization FW-Viz-2",
"search Search-Tester"
]
end
before do
allow(response).to receive(:status).and_return(404)
allow(client).to receive(:head).and_return(response)
allow(kbnclient).to receive(:version).and_return("9.8.7-6")
end
it "calls the import method" do
expect(client).to receive(:put).once do |path, content|
paths << path
LogStash::ElasticsearchClient::Response.new(201, "", {})
end
expect(kbnclient).to receive(:post).twice do |path, content|
paths << path
contents << content
LogStash::Modules::KibanaClient::Response.new(201, "", {})
end
test_module.with_settings(module_settings)
test_module.import(LogStash::Modules::ElasticsearchImporter.new(client), LogStash::Modules::KibanaImporter.new(kbnclient))
expect(paths).to eq(expected_paths)
expect(contents[0]).to eq({"changes" => {"defaultIndex" => "tester-*"}})
second_kbn_post = contents[1]
expect(second_kbn_post[:version]).to eq("9.8.7-6")
expect(second_kbn_post[:objects]).to be_a(Array)
expect(second_kbn_post[:objects].size).to eq(5)
objects_types_ids = second_kbn_post[:objects].map {|h| "#{h["type"]} #{h["id"]}"}
expect(objects_types_ids).to eq(expected_objects)
end
end
context "import 4 realz", :skip => "integration" do
let(:mname) { "cef" }
let(:base_dir) { File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "modules_test_files", "#{mname}")) }
let(:module_settings) do
{
"var.elasticsearch.hosts" => "localhost:9200",
"var.elasticsearch.user" => "foo",
"var.elasticsearch.password" => "password",
"var.input.tcp.port" => 5606,
}
end
it "puts stuff in ES" do
test_module.with_settings(module_settings)
client = LogStash::ElasticsearchClient.build(module_settings)
import_engine = LogStash::Modules::Importer.new(client)
test_module.import(import_engine)
expect(1).to eq(1)
end
end
end

156
logstash-core/spec/logstash/modules/settings_merger_spec.rb
View file

@ -1,156 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "spec_helper"
require "logstash/util/cloud_setting_id"
require "logstash/util/cloud_setting_auth"
require "logstash/modules/settings_merger"
require "logstash/util/password"
require "logstash/util/modules_setting_array"
class SubstituteSettingsForRSpec
def initialize(hash = {}) @hash = hash; end
def put(key, value) @hash[key] = value; end
def get(key) @hash[key]; end
end
describe LogStash::Modules::SettingsMerger do
describe "#merge" do
let(:cli) { LogStash::Util::ModulesSettingArray.new [{"name" => "mod1", "var.input.tcp.port" => "3333"}, {"name" => "mod2"}] }
let(:yml) {[{"name" => "mod1", "var.input.tcp.port" => 2222, "var.kibana.username" => "rupert", "var.kibana.password" => "fotherington"}, {"name" => "mod3", "var.input.tcp.port" => 4445}]}
subject(:results) { described_class.merge(cli, yml) }
it "merges cli overwriting any common fields in yml" do
expect(results).to be_a(Array)
expect(results.size).to eq(3)
expect(results[0]["name"]).to eq("mod1")
expect(results[0]["var.input.tcp.port"]).to eq("3333")
expect(results[0]["var.kibana.username"]).to eq("rupert")
expect(results[1]["name"]).to eq("mod2")
expect(results[2]["name"]).to eq("mod3")
expect(results[2]["var.input.tcp.port"]).to eq(4445)
end
end
describe "#merge_kibana_auth" do
before do
described_class.merge_kibana_auth!(mod_settings)
end
context 'only elasticsearch username and password is set' do
let(:mod_settings) { {"name" => "mod1", "var.input.tcp.port" => 2222, "var.elasticsearch.username" => "rupert", "var.elasticsearch.password" => "fotherington" } }
it "sets kibana username and password" do
expect(mod_settings["var.elasticsearch.username"]).to eq("rupert")
expect(mod_settings["var.elasticsearch.password"]).to eq("fotherington")
expect(mod_settings["var.kibana.username"]).to eq("rupert")
expect(mod_settings["var.kibana.password"]).to eq("fotherington")
end
end
context 'elasticsearch and kibana usernames and passwords are set' do
let(:mod_settings) { {"name" => "mod1", "var.input.tcp.port" => 2222, "var.elasticsearch.username" => "rupert", "var.elasticsearch.password" => "fotherington",
"var.kibana.username" => "davey", "var.kibana.password" => "stott"} }
it "keeps existing kibana username and password" do
expect(mod_settings["var.elasticsearch.username"]).to eq("rupert")
expect(mod_settings["var.elasticsearch.password"]).to eq("fotherington")
expect(mod_settings["var.kibana.username"]).to eq("davey")
expect(mod_settings["var.kibana.password"]).to eq("stott")
end
end
end
describe "#merge_cloud_settings" do
let(:cloud_id) { LogStash::Util::CloudSettingId.new("label:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy") }
let(:cloud_auth) { LogStash::Util::CloudSettingAuth.new("elastix:bigwhoppingfairytail") }
let(:mod_settings) { {} }
context "when both are supplied" do
let(:expected_table) do
{
"var.kibana.scheme" => "https",
"var.kibana.host" => "identifier.us-east-1.aws.found.io:443",
"var.elasticsearch.hosts" => "https://notareal.us-east-1.aws.found.io:443",
"var.elasticsearch.username" => "elastix",
"var.kibana.username" => "elastix"
}
end
let(:ls_settings) { SubstituteSettingsForRSpec.new({"cloud.id" => cloud_id, "cloud.auth" => cloud_auth}) }
before do
described_class.merge_cloud_settings(mod_settings, ls_settings)
end
it "adds entries to module settings" do
expected_table.each do |key, expected|
expect(mod_settings[key]).to eq(expected)
end
expect(mod_settings["var.elasticsearch.password"].value).to eq("bigwhoppingfairytail")
expect(mod_settings["var.kibana.password"].value).to eq("bigwhoppingfairytail")
end
end
context "when cloud.id is supplied" do
let(:expected_table) do
{
"var.kibana.scheme" => "https",
"var.kibana.host" => "identifier.us-east-1.aws.found.io:443",
"var.elasticsearch.hosts" => "https://notareal.us-east-1.aws.found.io:443",
}
end
let(:ls_settings) { SubstituteSettingsForRSpec.new({"cloud.id" => cloud_id}) }
before do
described_class.merge_cloud_settings(mod_settings, ls_settings)
end
it "adds entries to module settings" do
expected_table.each do |key, expected|
expect(mod_settings[key]).to eq(expected)
end
end
end
context "when only cloud.auth is supplied" do
let(:ls_settings) { SubstituteSettingsForRSpec.new({"cloud.auth" => cloud_auth}) }
it "should raise an error" do
expect { described_class.merge_cloud_settings(mod_settings, ls_settings) }.to raise_exception(ArgumentError)
end
end
context "when neither cloud.id nor cloud.auth is supplied" do
let(:ls_settings) { SubstituteSettingsForRSpec.new() }
it "should do nothing" do
expect(mod_settings).to be_empty
end
end
end
describe "#format_module_settings" do
let(:before_hash) { {"foo" => "red", "bar" => "blue", "qux" => "pink"} }
let(:after_hash) { {"foo" => "red", "bar" => "steel-blue", "baz" => LogStash::Util::Password.new("cyan"), "qux" => nil} }
subject(:results) { described_class.format_module_settings(before_hash, after_hash) }
it "yields an array of formatted lines for ease of logging" do
expect(results.size).to eq(after_hash.size + 2)
expect(results.first).to eq("-------- Module Settings ---------")
expect(results.last).to eq("-------- Module Settings ---------")
expect(results[1]).to eq("foo: 'red'")
expect(results[2]).to eq("bar: 'steel-blue', was: 'blue'")
expect(results[3]).to eq("baz: '<password>', was: ''")
expect(results[4]).to eq("qux: '', was: 'pink'")
end
end
end

1
logstash-core/spec/logstash/plugins/registry_spec.rb
View file

@ -126,7 +126,6 @@ describe LogStash::Plugins::Registry do
registry.add(:filter, "simple_plugin", simple_plugin)
expect(registry.plugins_with_type(:filter)).to include(simple_plugin)
expect(registry.plugins_with_type(:modules)).to match([])
end
end
end

117
logstash-core/spec/logstash/runner_spec.rb
View file

@ -22,9 +22,6 @@ require "stud/trap"
require "stud/temporary"
require "logstash/util/java_version"
require "logstash/config/source_loader"
require "logstash/config/modules_common"
require "logstash/modules/util"
require "logstash/elasticsearch_client"
require "json"
require "webmock/rspec"
require_relative "../support/helpers"
@ -60,10 +57,6 @@ describe LogStash::Runner do
allow(agent).to receive(:shutdown)
end
after(:each) do
LogStash::SETTINGS.get_value("modules_list").clear
end
describe "argument precedence" do
let(:config) { "input {} output {}" }
let(:cli_args) { ["-e", config, "-w", "20"] }
@ -348,116 +341,6 @@ describe LogStash::Runner do
end
end
describe "logstash modules" do
before(:each) do
test_modules_dir = File.expand_path(File.join(File.dirname(__FILE__), "..", "modules_test_files"))
LogStash::Modules::Util.register_local_modules(test_modules_dir)
end
describe "--config.test_and_exit" do
subject { LogStash::Runner.new("") }
let(:args) { ["-t", "--modules", module_string] }
context "with a good configuration" do
let(:module_string) { "tester" }
it "should exit successfully" do
expect(logger).not_to receive(:fatal)
expect(subject.run(args)).to eq(0)
end
end
context "with a bad configuration" do
let(:module_string) { "rlwekjhrewlqrkjh" }
it "should fail by returning a bad exit code" do
expect(logger).to receive(:fatal)
expect(subject.run(args)).to eq(1)
end
end
end
describe "--modules" do
let(:args) { ["--modules", module_string, "--setup"] }
context "with an available module specified but no connection to elasticsearch" do
let(:module_string) { "tester" }
before do
expect(logger).to receive(:fatal) do |msg, hash|
expect(msg).to eq("An unexpected error occurred!")
expect(hash).to be_a_config_loading_error_hash(
/Failed to import module configurations to Elasticsearch and\/or Kibana. Module: tester has/)
end
expect(LogStash::Agent).to receive(:new) do |settings, source_loader|
pipelines = LogStash::Config::ModulesCommon.pipeline_configs(settings)
expect(pipelines).to eq([])
agent
end
end
it "should log fatally and return a bad exit code" do
expect(subject.run("bin/logstash", args)).to eq(1)
end
end
context "with an available module specified and a mocked connection to elasticsearch" do
let(:module_string) { "tester" }
let(:kbn_version) { "6.0.0" }
let(:esclient) { double(:esclient) }
let(:kbnclient) { double(:kbnclient) }
let(:response) { double(:response) }
before do
allow(response).to receive(:status).and_return(404)
allow(esclient).to receive(:head).and_return(response)
allow(esclient).to receive(:can_connect?).and_return(true)
allow(kbnclient).to receive(:version).and_return(kbn_version)
allow(kbnclient).to receive(:version_parts).and_return(kbn_version.split('.'))
allow(kbnclient).to receive(:can_connect?).and_return(true)
allow(LogStash::ElasticsearchClient).to receive(:build).and_return(esclient)
allow(LogStash::Modules::KibanaClient).to receive(:new).and_return(kbnclient)
expect(esclient).to receive(:put).once do |path, content|
LogStash::ElasticsearchClient::Response.new(201, "", {})
end
expect(kbnclient).to receive(:post).twice do |path, content|
LogStash::Modules::KibanaClient::Response.new(201, "", {})
end
expect(LogStash::Agent).to receive(:new) do |settings, source_loader|
pipelines = LogStash::Config::ModulesCommon.pipeline_configs(settings)
expect(pipelines).not_to be_empty
module_pipeline = pipelines.first
expect(module_pipeline).to include("pipeline_id", "config_string")
expect(module_pipeline["pipeline_id"]).to include('tester')
expect(module_pipeline["config_string"]).to include('index => "tester-')
agent
end
expect(logger).not_to receive(:fatal)
expect(logger).not_to receive(:error)
end
it "should not terminate logstash" do
expect(subject.run("bin/logstash", args)).to be_nil
end
end
context "with an unavailable module specified" do
let(:module_string) { "fancypants" }
before do
expect(logger).to receive(:fatal) do |msg, hash|
expect(msg).to eq("An unexpected error occurred!")
expect(hash).to be_a_config_loading_error_hash(
/The modules specified are not available yet. Specified modules: \["fancypants"\] Available modules:/)
end
expect(LogStash::Agent).to receive(:new) do |settings, source_loader|
pipelines = LogStash::Config::ModulesCommon.pipeline_configs(settings)
expect(pipelines).to eq([])
agent
end
end
it "should log fatally and return a bad exit code" do
expect(subject.run("bin/logstash", args)).to eq(1)
end
end
end
end
describe 'jackson defaults' do
subject { LogStash::Runner.new("") }
let(:args) { ["-e", "input {} output {}"] }

143
logstash-core/spec/logstash/settings/modules_spec.rb
View file

@ -1,143 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
require "spec_helper"
require "logstash/settings"
require "logstash/util/cloud_setting_id"
require "logstash/util/cloud_setting_auth"
require "logstash/util/modules_setting_array"
require "java"
describe LogStash::Setting::Modules do
describe "Modules.Cli" do
subject { described_class.new("mycloudid", LogStash::Util::ModulesSettingArray, []) }
context "when given an array of hashes that contains a password key" do
let(:secret) { 'some_secret'}
it "should convert password Strings to Password" do
source = [{"var.kibana.password" => secret}]
setting = subject.set(source)
expect(setting).to be_a(java.util.ArrayList)
expect(setting.class).to eq(LogStash::Util::ModulesSettingArray)
expect(setting.first.fetch("var.kibana.password")).to be_a(LogStash::Util::Password)
expect(setting.first.fetch("var.kibana.password").value).to eq(secret)
end
it 'should not wrap values that are already passwords' do
source = [{"var.kibana.password" => LogStash::Util::Password.new(secret)}]
setting = subject.set(source)
expect(setting).to be_a(java.util.ArrayList)
expect(setting.class).to eq(LogStash::Util::ModulesSettingArray)
expect(setting.first.fetch("var.kibana.password")).to be_a(LogStash::Util::Password)
expect(setting.first.fetch("var.kibana.password").value).to eq(secret)
end
end
end
describe "Cloud.Id" do
subject { described_class.new("mycloudid", LogStash::Util::CloudSettingId) }
context "when given a string which is not a cloud id" do
it "should raise an exception" do
expect { subject.set("foobarbaz") }.to raise_error(ArgumentError, /Cloud Id.*is invalid/)
end
end
context "when given a string which is empty" do
it "should raise an exception" do
expect { subject.set("") }.to raise_error(ArgumentError, /Cloud Id.*is invalid/)
end
end
context "when given a string which is has environment prefix only" do
it "should raise an exception" do
expect { subject.set("testing:") }.to raise_error(ArgumentError, /Cloud Id.*is invalid/)
end
end
context "when given a badly formatted encoded id" do
it "should not raise an error" do
encoded = Base64.urlsafe_encode64("foo$$bal")
expect { subject.set(encoded) }.to raise_error(ArgumentError, "Cloud Id, after decoding, is invalid. Format: '<segment1>$<segment2>$<segment3>'. Received: \"foo$$bal\".")
end
end
context "when given a nil" do
it "should not raise an error" do
expect { subject.set(nil) }.to_not raise_error
end
end
context "when given a string which is an unlabelled cloud id" do
it "should set a LogStash::Util::CloudId instance" do
expect { subject.set("dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy") }.to_not raise_error
expect(subject.value.elasticsearch_host).to eq("notareal.us-east-1.aws.found.io:443")
expect(subject.value.kibana_host).to eq("identifier.us-east-1.aws.found.io:443")
expect(subject.value.label).to eq("")
end
end
context "when given a string which is a labelled cloud id" do
it "should set a LogStash::Util::CloudId instance" do
expect { subject.set("staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy") }.to_not raise_error
expect(subject.value.elasticsearch_host).to eq("notareal.us-east-1.aws.found.io:443")
expect(subject.value.kibana_host).to eq("identifier.us-east-1.aws.found.io:443")
expect(subject.value.label).to eq("staging")
end
end
end
describe "Cloud.Auth" do
subject { described_class.new("mycloudauth", LogStash::Util::CloudSettingAuth) }
context "when given a string without a separator or a password" do
it "should raise an exception" do
expect { subject.set("foobarbaz") }.to raise_error(ArgumentError, /Cloud Auth username and password format should be/)
end
end
context "when given a string without a password" do
it "should raise an exception" do
expect { subject.set("foo:") }.to raise_error(ArgumentError, /Cloud Auth username and password format should be/)
end
end
context "when given a string without a username" do
it "should raise an exception" do
expect { subject.set(":bar") }.to raise_error(ArgumentError, /Cloud Auth username and password format should be/)
end
end
context "when given a string which is empty" do
it "should raise an exception" do
expect { subject.set("") }.to raise_error(ArgumentError, /Cloud Auth username and password format should be/)
end
end
context "when given a nil" do
it "should not raise an error" do
expect { subject.set(nil) }.to_not raise_error
end
end
context "when given a string which is a cloud auth" do
it "should set the string" do
expect { subject.set("frodo:baggins") }.to_not raise_error
expect(subject.value.username).to eq("frodo")
expect(subject.value.password.value).to eq("baggins")
expect(subject.value.to_s).to eq("frodo:<password>")
end
end
end
end

2
logstash-core/spec/logstash/webserver_spec.rb
View file

@ -21,6 +21,8 @@ require "socket"
require "spec_helper"
require "open-uri"
require "webmock/rspec"
require "faraday"
require "ostruct"
def block_ports(range)
servers = []

1
logstash-core/spec/modules_test_files/modules/tester/configuration/README.txt
View file

@ -1 +0,0 @@
This is not a real working module, i.e. one that will work in kibana.

51
logstash-core/spec/modules_test_files/modules/tester/configuration/elasticsearch/tester.json
View file

@ -1,51 +0,0 @@
{
"order": 100,
"template": "tester-*",
"mappings": {
"_default_": {
"dynamic": true,
"dynamic_templates": [
{
"string_fields": {
"mapping": {
"type": "keyword"
},
"match_mapping_type": "string",
"match": "*"
}
}
],
"_all": {
"enabled": true
},
"properties": {
"f1": {
"type": "integer"
},
"f2": {
"type": "integer"
},
"f3": {
"type": "ip"
},
"f4": {
"format": "epoch_millis||epoch_second||date_time||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"f6": {
"type": "integer"
},
"f7": {
"type": "integer"
},
"f8": {
"type": "integer"
},
"f9": {
"type": "integer"
},
}
}
},
"aliases": {}
}

20
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/5.x/dashboard/FW-Dashboard.json
View file

@ -1,20 +0,0 @@
{
"title": "FW - Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"id\":\"FW-Viz-1\",\"type\":\"visualization\",\"panelIndex\":12,\"size_x\":4,\"size_y\":3,\"col\":9,\"row\":6},{\"id\":\"FW-Viz-2\",\"type\":\"visualization\",\"panelIndex\":13,\"size_x\":4,\"size_y\":3,\"col\":5,\"row\":6},{\"id\":\"Search-Tester\",\"type\":\"search\",\"panelIndex\":14,\"size_x\":6,\"size_y\":3,\"col\":1,\"row\":13,\"columns\":[\"sevCode\",\"name\",\"deviceVendor\",\"deviceProduct\",\"categoryDeviceType\",\"categoryBehavior\",\"categoryOutcome\",\"sourceAddress\",\"sourcePort\",\"sourceHostName\",\"destinationAddress\",\"destinationPort\",\"destinationHostName\",\"sourceUserName\",\"destinationUserName\"],\"sort\":[\"@timestamp\",\"desc\"]}]",
"optionsJSON": "{\"darkTheme\":false}",
"uiStateJSON": "{\"P-1\":{\"vis\":{\"legendOpen\":true,\"colors\":{\"/Success\":\"#629E51\",\"/Failure\":\"#BF1B00\"}}},\"P-2\":{\"vis\":{\"legendOpen\":true}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-4\":{\"vis\":{\"legendOpen\":true,\"colors\":{\"/Success\":\"#629E51\",\"/Failure\":\"#BF1B00\",\"Check Point\":\"#C15C17\",\"CISCO\":\"#EF843C\",\"NetScreen\":\"#F9BA8F\"}}},\"P-5\":{\"mapCenter\":[46.195042108660154,-56.42578125]},\"P-6\":{\"mapCenter\":[15.961329081596647,-0.3515625],\"mapZoom\":1},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-12\":{\"vis\":{\"legendOpen\":false}}}",
"version": 1,
"timeRestore": true,
"timeTo": "now",
"timeFrom": "now-1h",
"refreshInterval": {
"display": "Off",
"pause": false,
"value": 0
},
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
}
}

1
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/5.x/dashboard/tester.json
View file

@ -1 +0,0 @@
["FW-Dashboard"]

7
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/5.x/index-pattern/tester.json
View file

@ -1,7 +0,0 @@
{
"title":"tester-*",
"timeFieldName":"startTime",
"notExpandable":true,
"fieldFormatMap": "{\"f3\":{\"id\":\"bytes\"},\"f4\":{\"id\":\"bytes\"},\"f5\":{\"id\":\"bytes\"}}",
"fields": "[{\"name\":\"f1\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f2\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f3\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f3\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f5\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f6\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f7\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f8\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false,\"searchable\":false,\"aggregatable\":false},{\"name\":\"f9\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false,\"searchable\":true,\"aggregatable\":true}]"
}

19
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/5.x/search/Search-Tester.json
View file

@ -1,19 +0,0 @@
{
"title": "Firewall Events",
"description": "",
"hits": 0,
"columns": [
"f1",
"f2",
"f3",
"f4"
],
"sort": [
"f4",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"tester-*\",\"query\":{\"query_string\":{\"query\":\"f3:\\\"Firewall\\\"\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
}
}

11
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/5.x/visualization/FW-Viz-1.json
View file

@ -1,11 +0,0 @@
{
"title": "FW Viz 1",
"visState": "{\"title\":\"FW - Area by Outcome\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"overlap\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"categoryOutcome\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"colors\":{\"/Success\":\"#629E51\",\"/Failure\":\"#BF1B00\"}}}",
"description": "",
"savedSearchId": "Search-Tester",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/5.x/visualization/FW-Viz-2.json
View file

@ -1,11 +0,0 @@
{
"title": "FW Viz 2",
"visState": "{\"title\":\"FW - Count by Source, Destination Address and Ports\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"square root\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":\"7\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Overall Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"count\",\"schema\":\"radius\",\"params\":{}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sourceAddress\",\"customLabel\":\"Source Address\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"destinationAddress\",\"customLabel\":\"Destination Address\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"destinationPort\",\"customLabel\":\"Destination / Service Ports\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"colors\":{\"Overall Count\":\"#BF1B00\",\"Source Address\":\"#E0752D\",\"Destination Address\":\"#E5AC0E\",\"Device Address\":\"#447EBC\",\"Service Port\":\"#447EBC\",\"Destination / Service Ports\":\"#447EBC\"}}}",
"description": "",
"savedSearchId": "Search-Tester",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

20
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/6.x/dashboard/FW-Dashboard.json
View file

@ -1,20 +0,0 @@
{
"title": "FW - Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"id\":\"FW-Viz-1\",\"type\":\"visualization\",\"panelIndex\":12,\"size_x\":4,\"size_y\":3,\"col\":9,\"row\":6},{\"id\":\"FW-Viz-2\",\"type\":\"visualization\",\"panelIndex\":13,\"size_x\":4,\"size_y\":3,\"col\":5,\"row\":6},{\"id\":\"Search-Tester\",\"type\":\"search\",\"panelIndex\":14,\"size_x\":6,\"size_y\":3,\"col\":1,\"row\":13,\"columns\":[\"sevCode\",\"name\",\"deviceVendor\",\"deviceProduct\",\"categoryDeviceType\",\"categoryBehavior\",\"categoryOutcome\",\"sourceAddress\",\"sourcePort\",\"sourceHostName\",\"destinationAddress\",\"destinationPort\",\"destinationHostName\",\"sourceUserName\",\"destinationUserName\"],\"sort\":[\"@timestamp\",\"desc\"]}]",
"optionsJSON": "{\"darkTheme\":false}",
"uiStateJSON": "{\"P-1\":{\"vis\":{\"legendOpen\":true,\"colors\":{\"/Success\":\"#629E51\",\"/Failure\":\"#BF1B00\"}}},\"P-2\":{\"vis\":{\"legendOpen\":true}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-4\":{\"vis\":{\"legendOpen\":true,\"colors\":{\"/Success\":\"#629E51\",\"/Failure\":\"#BF1B00\",\"Check Point\":\"#C15C17\",\"CISCO\":\"#EF843C\",\"NetScreen\":\"#F9BA8F\"}}},\"P-5\":{\"mapCenter\":[46.195042108660154,-56.42578125]},\"P-6\":{\"mapCenter\":[15.961329081596647,-0.3515625],\"mapZoom\":1},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-12\":{\"vis\":{\"legendOpen\":false}}}",
"version": 1,
"timeRestore": true,
"timeTo": "now",
"timeFrom": "now-1h",
"refreshInterval": {
"display": "Off",
"pause": false,
"value": 0
},
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
}
}

1
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/6.x/dashboard/tester.json
View file

@ -1 +0,0 @@
["FW-Dashboard"]

7
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/6.x/index-pattern/tester.json
View file

@ -1,7 +0,0 @@
{
"title":"tester-*",
"timeFieldName":"startTime",
"notExpandable":true,
"fieldFormatMap": "{\"f3\":{\"id\":\"bytes\"},\"f4\":{\"id\":\"bytes\"},\"f5\":{\"id\":\"bytes\"}}",
"fields": "[{\"name\":\"f1\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f2\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f3\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f3\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f5\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f6\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f7\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true,\"searchable\":true,\"aggregatable\":true},{\"name\":\"f8\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false,\"searchable\":false,\"aggregatable\":false},{\"name\":\"f9\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false,\"searchable\":true,\"aggregatable\":true}]"
}

19
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/6.x/search/Search-Tester.json
View file

@ -1,19 +0,0 @@
{
"title": "Firewall Events",
"description": "",
"hits": 0,
"columns": [
"f1",
"f2",
"f3",
"f4"
],
"sort": [
"f4",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"tester-*\",\"query\":{\"query_string\":{\"query\":\"f3:\\\"Firewall\\\"\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
}
}

11
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/6.x/visualization/FW-Viz-1.json
View file

@ -1,11 +0,0 @@
{
"title": "FW Viz 1",
"visState": "{\"title\":\"FW - Area by Outcome\",\"type\":\"area\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"smoothLines\":true,\"scale\":\"linear\",\"interpolate\":\"linear\",\"mode\":\"overlap\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"categoryOutcome\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"colors\":{\"/Success\":\"#629E51\",\"/Failure\":\"#BF1B00\"}}}",
"description": "",
"savedSearchId": "Search-Tester",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
logstash-core/spec/modules_test_files/modules/tester/configuration/kibana/6.x/visualization/FW-Viz-2.json
View file

@ -1,11 +0,0 @@
{
"title": "FW Viz 2",
"visState": "{\"title\":\"FW - Count by Source, Destination Address and Ports\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"top\",\"showCircles\":true,\"smoothLines\":true,\"interpolate\":\"linear\",\"scale\":\"square root\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":\"7\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Overall Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"count\",\"schema\":\"radius\",\"params\":{}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sourceAddress\",\"customLabel\":\"Source Address\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"destinationAddress\",\"customLabel\":\"Destination Address\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"destinationPort\",\"customLabel\":\"Destination / Service Ports\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"colors\":{\"Overall Count\":\"#BF1B00\",\"Source Address\":\"#E0752D\",\"Destination Address\":\"#E5AC0E\",\"Device Address\":\"#447EBC\",\"Service Port\":\"#447EBC\",\"Destination / Service Ports\":\"#447EBC\"}}}",
"description": "",
"savedSearchId": "Search-Tester",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

34
logstash-core/spec/modules_test_files/modules/tester/configuration/logstash/tester.conf.erb
View file

@ -1,34 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
input {
tcp {
codec => json
port => <%= setting("var.input.tcp.port", 5000) %>
type => syslog
}
}
filter {
date {
match => [ "logdate", "ISO8601" ]
}
}
output {
<%= elasticsearch_output_config() %>
}

81
logstash-core/src/main/java/org/logstash/util/ModulesSettingArray.java
View file

@ -1,81 +0,0 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.logstash.util;
import co.elastic.logstash.api.Password;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;
/**
* Wrapper of Logstash modules settings, with ability to replace password fields with
* the obfuscator {@link Password} implementation.
* */
public final class ModulesSettingArray extends ArrayList<Map<String, Object>> {
private static final long serialVersionUID = 4094949366274116593L;
public ModulesSettingArray(Collection<? extends Map<String, Object>> original) {
super(wrapPasswords(original));
}
private static Collection<Map<String, Object>> wrapPasswords(Collection<? extends Map<String, Object>> original) {
return original.stream()
.map(ModulesSettingArray::wrapPasswordsInSettings)
.collect(Collectors.toList());
}
private static Map<String, Object> wrapPasswordsInSettings(Map<String, Object> settings) {
// Insertion order is important. The Map object passed into is usually a org.jruby.RubyHash, which preserves
// the insertion order, during the scan. Here we need to keep the same order, because tests on modules
// expects a precise order of keys. It's important to have stable tests.
final Map<String, Object> acc = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : settings.entrySet()) {
if (entry.getKey().endsWith("password") && !(entry.getValue() instanceof Password)) {
acc.put(entry.getKey(), new Password((String) entry.getValue()));
} else {
acc.put(entry.getKey(), entry.getValue());
}
}
return acc;
}
public Map<String, Object> getFirst() {
try {
return get(0);
} catch (IndexOutOfBoundsException ex) {
return null;
}
}
public Map<String, Object> getLast() {
try {
return get(size() - 1);
} catch (IndexOutOfBoundsException ex) {
return null;
}
}
}

2
modules/README.md
View file

@ -1,2 +0,0 @@
This file keeps `/modules` non empty for build.
Keep the file until Arcsight module is removed.

1
rakelib/artifacts.rake
View file

@ -33,7 +33,6 @@ namespace "artifact" do
"bin/**/*",
"config/**/*",
"data",
"modules/**/*",
"lib/bootstrap/**/*",
"lib/pluginmanager/**/*",

97
rakelib/modules.rake
View file

@ -1,97 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
namespace "modules" do
def unpacker(src_file, dest_dir)
puts "Reading #{src_file}"
array = JSON.parse(IO.read(src_file))
if !array.is_a?(Array)
raise "#{src_file} does not contain a JSON array as the first object"
end
array.each do |hash|
values = hash.values_at("_id", "_type", "_source")
if values.any?(&:nil?)
puts "#{src_file} contains a JSON object that does not have _id, _type and _source fields"
next
end
id, subfolder, source = values
filename = "#{id}.json"
partial_path = ::File.join(dest_dir, subfolder)
FileUtils.mkdir_p(partial_path)
full_path = ::File.join(partial_path, filename)
FileUtils.rm_f(full_path)
content = JSON.pretty_generate(source) + "\n"
puts "Writing #{full_path}"
IO.write(full_path, content)
end
end
def collector(dashboard_dir, module_name)
file_paths = Dir.glob(::File.join(dashboard_dir, "*.json"))
filenames = file_paths.map do |file_path|
filename = File.basename(file_path, ".json")
next if filename == module_name
puts "Adding #{filename}"
filename
end.compact
full_path = ::File.join(dashboard_dir, "#{module_name}.json")
FileUtils.rm_f(full_path)
content = JSON.pretty_generate(filenames) + "\n"
puts "Writing #{full_path}"
IO.write(full_path, content)
end
desc "Unpack kibana resources in a JSON array to individual files"
task "unpack", :src_file, :dest_dir do |task, args|
unpacker(args[:src_file], args[:dest_dir])
puts "Done"
end
desc "Collect all dashboards filenames into the module dashboard structure e.g. dashboard/cef.json"
task "make_dashboard_json", :dashboard_dir, :module_name do |task, args|
collector(args[:dashboard_dir], args[:module_name])
puts "Done"
end
desc "Unpack all kibana resources from a folder of JSON files."
# from Kibana / saved objects -> export all
# rake modules:unpack_all[my_module,/User/me/Downloads/my_module,/User/me/workspace/logstash/modules/my_module/6.x/configuration/kibana]
# Note - you can not currently export index-patterns from the UI, see: https://github.com/elastic/kibana/issues/4288
# To get the index-pattern, you need to pull from the .kibana index and manually update
# curl -XGET 'https://<user>:<password>@<es-host>:<es-port>/.kibana/_search?q=type:index-pattern&size=100&pretty' -o idx-patterns.json
task "unpack_all", :module_name, :kibana_source_dir, :dest_dir do |task, args|
module_name = args[:module_name]
kibana_source_dir = args[:kibana_source_dir]
dest_dir = args[:dest_dir]
Dir.glob(::File.join(kibana_source_dir, "*.json")).each do |file_path|
unpacker(file_path, dest_dir)
end
dashboard_dir = ::File.join(dest_dir, "dashboard")
collector(dashboard_dir, module_name)
puts "Done"
end
end

4
x-pack/lib/config_management/bootstrap_check.rb
View file

@ -29,10 +29,6 @@ module LogStash
raise LogStash::BootstrapCheckError, "You cannot use -t since Elasticsearch is configured as the config store"
end
if !settings.get("modules.cli").empty? || !settings.get("modules").empty?
raise LogStash::BootstrapCheckError, "You cannot use --modules since Elasticsearch is configured as the config store"
end
interval = settings.get("xpack.management.logstash.poll_interval")
# override core settings, so the agent will trigger the auto reload

2
x-pack/lib/config_management/hooks.rb
View file

@ -7,7 +7,6 @@ require "config_management/elasticsearch_source"
require "logstash/config/source_loader"
require "logstash/config/source/local"
require "logstash/config/source/multi_local"
require "logstash/config/source/modules"
module LogStash
module ConfigManagement
@ -36,7 +35,6 @@ module LogStash
logger.debug("Removing the `Logstash::Config::Source::Local` and replacing it with `ElasticsearchSource`")
runner.source_loader.remove_source(LogStash::Config::Source::Local)
runner.source_loader.remove_source(LogStash::Config::Source::MultiLocal)
runner.source_loader.remove_source(LogStash::Config::Source::Modules)
source = LogStash::ConfigManagement::ElasticsearchSource.new(runner.settings)
runner.source_loader.add_source(source)
end

63
x-pack/lib/helpers/elasticsearch_options.rb
View file

@ -40,12 +40,6 @@ module LogStash module Helpers
"ssl.key" => "ssl_key",
}
# Retrieve elasticsearch options from either specific settings, or modules if the setting is not there and the
# feature supports falling back to modules if the feature is not specified in logstash.yml
def es_options_from_settings_or_modules(feature, settings)
only_modules_configured?(feature, settings) ? es_options_from_modules(settings) : es_options_from_settings(feature, settings)
end
# Populate the Elasticsearch options from LogStashSettings file, based on the feature that is being used.
# @return Hash
def es_options_from_settings(feature, settings)
@ -126,42 +120,6 @@ module LogStash module Helpers
hosts.all? {|host| host.match?(HTTPS_SCHEME)}
end
# Elasticsearch settings can be extracted from the modules settings inside the configuration.
# Few options will be supported, however - the modules security configuration is
# different to
def es_options_from_modules(settings)
module_settings = extract_module_settings(settings)
if module_settings.empty?
return nil
end
opts = {}
setting = LogStash::Setting::SplittableStringArray.new("var.elasticsearch.hosts", String, ["localhost:9200"])
raw_value = module_settings[setting.name]
setting.set(raw_value) unless raw_value.nil?
opts['hosts'] = setting.value
opts['user'] = module_settings['var.elasticsearch.username']
password = module_settings['var.elasticsearch.password']
opts['password'] = password.value unless password.nil?
# Sniffing is not supported for modules.
opts['sniffing'] = false
if cacert = module_settings["var.elasticsearch.ssl.certificate_authority"]
opts['ssl_certificate_authorities'] = cacert
opts['ssl_enabled'] = true
end
opts
end
# Determine whether only modules have been configured, and not monitoring
# @param String feature to be checked
# @param Logstash::Settings Logstash settings
def only_modules_configured?(feature, settings)
modules_configured?(settings) && !feature_configured?(feature, settings)
end
# If no settings are configured, then assume that the feature has not been configured.
def feature_configured?(feature, settings)
ES_SETTINGS.each do |option|
@ -170,27 +128,6 @@ module LogStash module Helpers
false
end
def modules_configured?(settings)
!extract_module_settings(settings).nil?
end
# Read module settings from yaml file. This should be refactored in Logstash proper to allow for DRY-ing up
# these settings
def extract_module_settings(settings)
cli_settings = settings.get("modules.cli")
yml_settings = settings.get("modules")
modules_array = if !(cli_settings.empty? && yml_settings.empty?)
LogStash::Modules::SettingsMerger.merge(cli_settings, yml_settings)
elsif cli_settings.empty?
yml_settings
else
cli_settings
end
LogStash::Modules::SettingsMerger.merge_cloud_settings(modules_array.first, settings) unless modules_array.empty?
# As only one module is supported in the initial rollout, use the first one found
modules_array.first
end
private
def validate_authentication!(feature, settings, prefix)

76
x-pack/lib/modules/module_license_checker.rb
View file

@ -1,76 +0,0 @@
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License;
# you may not use this file except in compliance with the Elastic License.
require 'license_checker/licensed'
require 'helpers/elasticsearch_options'
java_import java.util.concurrent.TimeUnit
module LogStash
module LicenseChecker
class ModuleLicenseChecker
include LogStash::LicenseChecker::Licensed
include LogStash::Helpers::ElasticsearchOptions
include LogStash::Util::Loggable
def initialize(module_name, valid_licenses)
@module_name = module_name
@feature = "#{@module_name}_module"
@valid_licenses = valid_licenses
@setup = false
end
# returns true if license is valid, false otherwise
def check(settings)
setup(settings) unless @setup
license_check
end
private
def setup(settings)
@es_options = es_options_from_modules(settings) || {}
#TODO: reduce the refresh period and handle if a license expires while running
setup_license_checker(@feature, 3650, TimeUnit::DAYS)
@setup = true
end
def populate_license_state(xpack_info, is_serverless)
if xpack_info.failed?
{
:state => :error,
:log_level => :error,
:log_message => "Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster."
}
elsif !xpack_info.installed?
{
:state => :error,
:log_level => :error,
:log_message => "X-Pack is installed on Logstash but not on Elasticsearch. Please install X-Pack on Elasticsearch to use the #{@module_name} module."
}
elsif !xpack_info.license_available?
{
:state => :error,
:log_level => :error,
:log_message => "The #{@module_name} module is not available: License information is currently unavailable. Please make sure you have added your production elasticsearch connection information."
}
elsif !xpack_info.license_one_of?(@valid_licenses)
{
:state => :error,
:log_level => :error,
:log_message => "The #{@module_name} module is not available: #{xpack_info.license_type} is not a valid license for this feature."
}
elsif !xpack_info.license_active?
{
:state => :ok,
:log_level => :warn,
:log_message => "The #{@module_name} module requires an active license."
}
else
{:state => :ok, :log_level => :info, :log_message => "The #{@module_name} module License OK"}
end
end
end
end
end

22
x-pack/lib/modules/xpack_scaffold.rb
View file

@ -1,22 +0,0 @@
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License;
# you may not use this file except in compliance with the Elastic License.
require "logstash/modules/scaffold"
require "modules/module_license_checker"
module LogStash
module Modules
class XpackScaffold < LogStash::Modules::Scaffold
def initialize(name, directory, valid_licenses)
super(name, directory)
@license_checker = LogStash::LicenseChecker::ModuleLicenseChecker.new(name, valid_licenses)
end
def is_enabled?(settings)
@license_checker.check(settings)
end
end
end
end

2
x-pack/lib/monitoring/internal_pipeline_source.rb
View file

@ -18,7 +18,7 @@ module LogStash module Monitoring
@pipeline_config = pipeline_config
@settings = settings
@agent = agent
@es_options = es_options_from_settings_or_modules(FEATURE, @settings)
@es_options = es_options_from_settings(FEATURE, @settings)
setup_license_checker(FEATURE)
end

2
x-pack/lib/monitoring/monitoring.rb
View file

@ -216,7 +216,7 @@ module LogStash
else
opt = retrieve_collection_settings(settings, "xpack.")
end
es_settings = es_options_from_settings_or_modules('monitoring', settings)
es_settings = es_options_from_settings('monitoring', settings)
data = TemplateData.new(LogStash::SETTINGS.get("node.uuid"), API_VERSION,
es_settings,
opt[:collection_interval], opt[:collection_timeout_interval],

14
x-pack/lib/x-pack/logstash_registry.rb
View file

@ -3,18 +3,12 @@
# you may not use this file except in compliance with the Elastic License.
require "logstash/runner" # needed for LogStash::XPACK_PATH
xpack_modules = ["arcsight"]
xpack_modules.each do |name|
$LOAD_PATH << File.join(LogStash::XPACK_PATH, "modules", name, "lib")
end
require "logstash/plugins/registry"
require "logstash/modules/util"
require "monitoring/monitoring"
require "monitoring/inputs/metrics"
require "monitoring/outputs/elasticsearch_monitoring"
require "config_management/extension"
require "geoip_database_management/extension"
require "modules/xpack_scaffold"
LogStash::PLUGIN_REGISTRY.add(:input, "metrics", LogStash::Inputs::Metrics)
LogStash::PLUGIN_REGISTRY.add(:output, "elasticsearch_monitoring", LogStash::Outputs::ElasticSearchMonitoring)
@ -23,10 +17,4 @@ LogStash::PLUGIN_REGISTRY.add(:universal, "config_management", LogStash::ConfigM
LogStash::PLUGIN_REGISTRY.add(:universal, "geoip_database_management", LogStash::GeoipDatabaseManagement::Extension)
license_levels = Hash.new
license_levels.default = LogStash::LicenseChecker::LICENSE_TYPES
xpack_modules.each do |name|
path = File.join(File.dirname(__FILE__), "..", "..", "modules", name, "configuration")
LogStash::PLUGIN_REGISTRY.add(:modules, name,
LogStash::Modules::XpackScaffold.new(name, path, license_levels[name]))
end
license_levels.default = LogStash::LicenseChecker::LICENSE_TYPES

99
x-pack/modules/README.md
View file

@ -1,99 +0,0 @@
# Module settings and structure
## settings
### logstash.yml
```
modules:
- name: netflow
var.elasticsearch.hosts: "es.mycloud.com"
var.elasticsearch.username: "foo"
var.elasticsearch.password: "password"
var.kibana.host: "kb.mycloud.com"
var.kibana.username: "foo"
var.kibana.password: "password"
var.input.tcp.port: 5606
```
### command-line
```
bin/logstash \
--modules netflow \
-M "netflow.var.output.elasticsearch.host=es.mycloud.com" \
-M "netflow.var.output.elasticsearch.user=foo" \
-M "netflow.var.output.elasticsearch.password=password" \
-M "netflow.var.input.tcp.port=5606"
```
## Current Gem structure
```
GEM File structure
logstash-module-netflow
├── configuration
│ ├── elasticsearch
│ │ └── netflow.json
│ ├── kibana
│ │ ├── dashboard
│ │ │ └── netflow.json (contains '["dash1", "dash2"]')
│ │ │ └── dash1.json ("panelJSON" contains refs to visualization panels 1,2 and search 1)
│ │ │ └── dash2.json ("panelJSON" contains refs to visualization panel 3 and search 2)
│ │ ├── index-pattern
| | | └── netflow.json
│ │ ├── search
| | | └── search1.json
| | | └── search2.json
│ │ └── vizualization
| | | └── panel1.json
| | | └── panel2.json
| | | └── panel3.json
│ └── logstash
│ └── netflow.conf.erb
├── lib
│ └── logstash_registry.rb
└── logstash-module-netflow.gemspec
```
## Proposed multi-version Gem structure
```
GEM File structure
logstash-module-netflow
├── configuration
│ ├── elasticsearch
│ │ └── netflow.json
│ ├── kibana
│ │ ├── dashboard
│ │ │ └── netflow.json (contains '{"v5.5.0": ["dash1", "dash2"], "v6.0.4": ["dash1", "dash2"]')
│ │ │ └── v5.5.0
│ │ │ | └── dash1.json ("panelJSON" contains refs to visualization panels 1,2 and search 1)
│ │ │ | └── dash2.json ("panelJSON" contains refs to visualization panel 3 and search 2)
│ │ │ └── v6.0.4
│ │ │ └── dash1.json ("panelJSON" contains refs to visualization panels 1,2 and search 1)
│ │ │ └── dash2.json ("panelJSON" contains refs to visualization panel 3 and search 2)
│ │ ├── index-pattern
│ │ │ └── v5
| | | | └── netflow.json
│ │ │ └── v6
| | | └── netflow.json
│ │ ├── search
│ │ │ └── v5
| | | | └── search1.json
| | | | └── search2.json
│ │ │ └── v6
| | | └── search1.json
| | | └── search2.json
│ │ └── vizualization
│ │ │ └── v5
| | | | └── panel1.json
| | | | └── panel2.json
| | | | └── panel3.json
│ │ │ └── v6
| | | └── panel1.json
| | | └── panel2.json
| | | └── panel3.json
│ └── logstash
│ └── netflow.conf.erb
├── lib
│ └── logstash_registry.rb
└── logstash-module-netflow.gemspec
```

220
x-pack/modules/arcsight/configuration/elasticsearch/arcsight.json
View file

@ -1,220 +0,0 @@
{
"order": 0,
"index_patterns": "arcsight-*",
"mappings": {
"_meta": {
"version": "8.0.0"
},
"dynamic": true,
"dynamic_templates": [
{
"string_fields": {
"mapping": {
"type": "keyword"
},
"match_mapping_type": "string",
"match": "*"
}
}
],
"properties": {
"destinationPort": {
"type": "integer"
},
"flexDate1": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"sourcePort": {
"type": "integer"
},
"baseEventCount": {
"type": "integer"
},
"destinationAddress": {
"type": "ip"
},
"destinationProcessId": {
"type": "integer"
},
"oldFileSize": {
"type": "integer"
},
"destination": {
"dynamic": false,
"type": "object",
"properties": {
"city_name": {
"type": "keyword"
},
"country_name": {
"type": "keyword"
},
"country_code2": {
"type": "keyword"
},
"location": {
"type": "geo_point"
},
"region_name": {
"type": "keyword"
}
}
},
"source": {
"dynamic": false,
"type": "object",
"properties": {
"city_name": {
"type": "keyword"
},
"country_name": {
"type": "keyword"
},
"country_code2": {
"type": "keyword"
},
"location": {
"type": "geo_point"
},
"region_name": {
"type": "keyword"
}
}
},
"deviceReceiptTime": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"destinationTranslatedPort": {
"type": "integer"
},
"deviceTranslatedAddress": {
"type": "ip"
},
"deviceAddress": {
"type": "ip"
},
"agentReceiptTime": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"startTime": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"sourceProcessId": {
"type": "integer"
},
"bytesIn": {
"type": "integer"
},
"bytesOut": {
"type": "integer"
},
"severity": {
"type": "keyword"
},
"deviceProcessId": {
"type": "integer"
},
"agentAddress": {
"type": "ip"
},
"sourceAddress": {
"type": "ip"
},
"sourceTranslatedPort": {
"type": "integer"
},
"deviceCustomDate2": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"deviceCustomDate1": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"flexNumber1": {
"type": "long"
},
"deviceCustomFloatingPoint1": {
"type": "float"
},
"oldFileModificationTime": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"deviceCustomFloatingPoint2": {
"type": "float"
},
"oldFileCreateTime": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"deviceCustomFloatingPoint3": {
"type": "float"
},
"sourceTranslatedAddress": {
"type": "ip"
},
"deviceCustomFloatingPoint4": {
"type": "float"
},
"flexNumber2": {
"type": "long"
},
"fileCreateTime": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"fileModificationTime": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"fileSize": {
"type": "integer"
},
"destinationTranslatedAddress": {
"type": "ip"
},
"endTime": {
"format": "epoch_millis||epoch_second||date_time||MMM dd YYYY HH:mm:ss z||MMM dd yyyy HH:mm:ss",
"type": "date"
},
"deviceCustomNumber1": {
"type": "long"
},
"deviceDirection": {
"type": "integer"
},
"device": {
"dynamic": false,
"type": "object",
"properties": {
"city_name": {
"type": "keyword"
},
"country_name": {
"type": "keyword"
},
"country_code2": {
"type": "keyword"
},
"location": {
"type": "geo_point"
},
"region_name": {
"type": "keyword"
}
}
},
"deviceCustomNumber3": {
"type": "long"
},
"deviceCustomNumber2": {
"type": "long"
}
}
}
}

20
x-pack/modules/arcsight/configuration/kibana/5.x/dashboard/153e0bf0-752f-11e7-ae68-d756b92f3a9c.json
View file

@ -1,20 +0,0 @@
{
"title": "[ArcSight] Microsoft DNS Overview",
"hits": 0,
"description": "",
"panelsJSON": "[{\"col\":11,\"id\":\"4ee62420-7523-11e7-871d-5f0fb978413c\",\"panelIndex\":1,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"1de45d60-7523-11e7-9445-91c40765092f\",\"panelIndex\":3,\"row\":2,\"size_x\":10,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"d72d7940-7529-11e7-9445-91c40765092f\",\"panelIndex\":5,\"row\":9,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"1c54cda0-752c-11e7-9445-91c40765092f\",\"panelIndex\":6,\"row\":13,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"8cda1c30-752a-11e7-9445-91c40765092f\",\"panelIndex\":7,\"row\":9,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"aa57b050-7526-11e7-b440-f1d91dc5774d\",\"panelIndex\":9,\"row\":4,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"4303de60-752b-11e7-9445-91c40765092f\",\"panelIndex\":11,\"row\":15,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"31b85570-454a-11e7-86b6-95298e9da6dc\",\"panelIndex\":12,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"8f0161a0-752d-11e7-b440-f1d91dc5774d\",\"panelIndex\":13,\"row\":15,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ebfd45a0-75a4-11e7-b440-f1d91dc5774d\",\"panelIndex\":14,\"row\":6,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"bd1c82c0-75a7-11e7-871d-5f0fb978413c\",\"panelIndex\":15,\"row\":6,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":3,\"panelIndex\":16,\"type\":\"visualization\",\"id\":\"c658b300-7745-11e7-8fb2-417804dc0ec8\",\"col\":7,\"row\":19},{\"size_x\":6,\"size_y\":3,\"panelIndex\":17,\"type\":\"visualization\",\"id\":\"b1f98ce0-7745-11e7-8fb2-417804dc0ec8\",\"col\":1,\"row\":19}]",
"optionsJSON": "{\"darkTheme\":false}",
"uiStateJSON": "{\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-13\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-3\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-5\":{\"vis\":{\"defaultColors\":{\"0 - 18k\":\"rgb(247,251,255)\",\"18k - 36k\":\"rgb(227,238,249)\",\"36k - 54k\":\"rgb(208,225,242)\",\"54k - 72k\":\"rgb(182,212,233)\",\"72k - 90k\":\"rgb(148,196,223)\",\"90k - 108k\":\"rgb(107,174,214)\",\"108k - 126k\":\"rgb(74,152,201)\",\"126k - 144k\":\"rgb(46,126,188)\",\"144k - 162k\":\"rgb(23,100,171)\",\"162k - 180k\":\"rgb(8,74,145)\"},\"legendOpen\":false}},\"P-16\":{\"mapZoom\":1,\"mapCenter\":[12.211180191503997,0]},\"P-17\":{\"mapZoom\":1,\"mapCenter\":[-0.17578097424708533,0]}}",
"version": 1,
"timeRestore": true,
"timeTo": "now",
"timeFrom": "now-24h",
"refreshInterval": {
"display": "Off",
"pause": false,
"value": 0
},
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
}
}

20
x-pack/modules/arcsight/configuration/kibana/5.x/dashboard/37af0b40-398d-11e7-ae19-21fb91585845.json
View file

@ -1,20 +0,0 @@
{
"title": "[ArcSight] Network Overview Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"col\":1,\"id\":\"77cb1470-3989-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":1,\"row\":12,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"801fff70-395a-11e7-ae19-21fb91585845\",\"panelIndex\":2,\"row\":18,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"d6d526f0-395b-11e7-ae19-21fb91585845\",\"panelIndex\":5,\"row\":4,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"f57ea930-395d-11e7-ae19-21fb91585845\",\"panelIndex\":6,\"row\":16,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"161e27e0-3988-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":7,\"row\":2,\"size_x\":10,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"75582a90-3987-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":9,\"row\":6,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"e9c3ee00-3978-11e7-ae19-21fb91585845\",\"panelIndex\":11,\"row\":6,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ad802c10-3973-11e7-ae19-21fb91585845\",\"panelIndex\":13,\"row\":9,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"ec926660-396f-11e7-ae19-21fb91585845\",\"panelIndex\":15,\"row\":9,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"154ff7e0-3987-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":16,\"row\":6,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"5acb74d0-398b-11e7-ae19-21fb91585845\",\"panelIndex\":17,\"row\":14,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"7a043760-3990-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":18,\"row\":20,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"fd70bca0-398f-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":19,\"row\":20,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":11,\"id\":\"ed2f5570-3d5b-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":20,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"31b85570-454a-11e7-86b6-95298e9da6dc\",\"panelIndex\":21,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":6,\"panelIndex\":24,\"type\":\"visualization\",\"id\":\"45387480-3989-11e7-8b9d-ddc45b5f6d00\",\"col\":1,\"row\":24},{\"size_x\":6,\"size_y\":6,\"panelIndex\":25,\"type\":\"visualization\",\"id\":\"35ce1310-3989-11e7-8b9d-ddc45b5f6d00\",\"col\":7,\"row\":24}]",
"optionsJSON": "{\"darkTheme\":false}",
"uiStateJSON": "{\"P-11\":{\"vis\":{\"colors\":{\"/Attempt\":\"#0A50A1\",\"/Failure\":\"#BF1B00\",\"/Success\":\"#629E51\"}}},\"P-13\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-15\":{\"vis\":{\"defaultColors\":{\"0% - 17%\":\"rgb(255,255,204)\",\"17% - 34%\":\"rgb(255,230,146)\",\"34% - 50%\":\"rgb(254,191,90)\",\"50% - 67%\":\"rgb(253,141,60)\",\"67% - 84%\":\"rgb(244,61,37)\",\"84% - 100%\":\"rgb(202,8,35)\"},\"legendOpen\":false}},\"P-16\":{\"vis\":{\"colors\":{\"Anti-Virus\":\"#EF843C\",\"Content Security\":\"#7EB26D\",\"Firewall\":\"#E24D42\",\"Integrated Security\":\"#962D82\",\"Network-based IDS/IPS\":\"#1F78C1\",\"Operating System\":\"#1F78C1\",\"VPN\":\"#EAB839\"}}},\"P-18\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-7\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-8\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-9\":{\"vis\":{\"colors\":{\"/Attempt\":\"#0A50A1\",\"/Failure\":\"#BF1B00\",\"/Success\":\"#629E51\"}}},\"P-25\":{\"mapZoom\":1,\"mapCenter\":[-0.3515602939922709,0]},\"P-24\":{\"mapZoom\":1,\"mapCenter\":[-0.3515602939922709,0]}}",
"version": 1,
"timeRestore": true,
"timeTo": "now",
"timeFrom": "now-24h",
"refreshInterval": {
"display": "Off",
"pause": false,
"value": 0
},
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
}
}

20
x-pack/modules/arcsight/configuration/kibana/5.x/dashboard/64c92510-4555-11e7-83ea-67cb6920446d.json
View file

@ -1,20 +0,0 @@
{
"title": "[ArcSight] Endpoint OS Activity Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"col\":1,\"id\":\"c9e333a0-4550-11e7-86b6-95298e9da6dc\",\"panelIndex\":3,\"row\":8,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"463fc740-454e-11e7-86b6-95298e9da6dc\",\"panelIndex\":4,\"row\":2,\"size_x\":10,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"8f8d6230-454f-11e7-86b6-95298e9da6dc\",\"panelIndex\":5,\"row\":8,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"f0664070-4551-11e7-86b6-95298e9da6dc\",\"panelIndex\":7,\"row\":6,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"d8314510-454f-11e7-86b6-95298e9da6dc\",\"panelIndex\":8,\"row\":14,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2b369910-4553-11e7-83ea-67cb6920446d\",\"panelIndex\":9,\"row\":11,\"size_x\":6,\"size_y\":6,\"type\":\"visualization\"},{\"col\":7,\"id\":\"9141cc20-4553-11e7-83ea-67cb6920446d\",\"panelIndex\":10,\"row\":11,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"31b85570-454a-11e7-86b6-95298e9da6dc\",\"panelIndex\":11,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":11,\"id\":\"0e4558b0-4552-11e7-86b6-95298e9da6dc\",\"panelIndex\":12,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"47c2a140-454f-11e7-86b6-95298e9da6dc\",\"panelIndex\":13,\"row\":4,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"68180c80-4556-11e7-83ea-67cb6920446d\",\"panelIndex\":14,\"row\":17,\"size_x\":4,\"size_y\":5,\"type\":\"visualization\"},{\"col\":9,\"id\":\"08ee04d0-4556-11e7-83ea-67cb6920446d\",\"panelIndex\":15,\"row\":22,\"size_x\":4,\"size_y\":6,\"type\":\"visualization\"},{\"col\":1,\"id\":\"b897ce70-4556-11e7-83ea-67cb6920446d\",\"panelIndex\":16,\"row\":21,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"93531890-4556-11e7-83ea-67cb6920446d\",\"panelIndex\":17,\"row\":26,\"size_x\":8,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"a8ce0ef0-4556-11e7-83ea-67cb6920446d\",\"panelIndex\":18,\"row\":17,\"size_x\":8,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"82caeb10-4556-11e7-83ea-67cb6920446d\",\"panelIndex\":19,\"row\":24,\"size_x\":8,\"size_y\":2,\"type\":\"visualization\"}]",
"optionsJSON": "{\"darkTheme\":false}",
"uiStateJSON": "{\"P-13\":{\"vis\":{\"colors\":{\"Destination Users\":\"#E24D42\",\"Event Count\":\"#64B0C8\"}}},\"P-14\":{\"vis\":{\"legendOpen\":false}},\"P-15\":{\"vis\":{\"legendOpen\":false}},\"P-2\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-3\":{\"vis\":{\"colors\":{\"Count\":\"#64B0C8\",\"Destination User Names\":\"#E24D42\",\"Event Types\":\"#EF843C\"},\"legendOpen\":true}},\"P-4\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-5\":{\"vis\":{\"defaultColors\":{\"0 - 55k\":\"rgb(255,255,204)\",\"55k - 110k\":\"rgb(255,241,170)\",\"110k - 165k\":\"rgb(254,225,135)\",\"165k - 220k\":\"rgb(254,201,101)\",\"220k - 275k\":\"rgb(254,171,73)\",\"275k - 330k\":\"rgb(253,141,60)\",\"330k - 385k\":\"rgb(252,91,46)\",\"385k - 440k\":\"rgb(237,47,34)\",\"440k - 495k\":\"rgb(212,16,32)\",\"495k - 550k\":\"rgb(176,0,38)\"},\"legendOpen\":false}},\"P-8\":{\"vis\":{\"colors\":{\"/Attempt\":\"#447EBC\",\"/Failure\":\"#E24D42\",\"/Success\":\"#7EB26D\"}}},\"P-9\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
"version": 1,
"timeRestore": true,
"timeTo": "now",
"timeFrom": "now-24h",
"refreshInterval": {
"display": "Off",
"pause": false,
"value": 0
},
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
}
}

20
x-pack/modules/arcsight/configuration/kibana/5.x/dashboard/82051450-3e56-11e7-96c4-0d3a291ec93a.json
View file

@ -1,20 +0,0 @@
{
"title": "[ArcSight] Network Suspicious Activity Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"col\":1,\"id\":\"aa2ff0a0-3e4a-11e7-96c4-0d3a291ec93a\",\"panelIndex\":1,\"row\":8,\"size_x\":8,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"992c7bd0-3e4e-11e7-96c4-0d3a291ec93a\",\"panelIndex\":2,\"row\":11,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"f99c22e0-3e4e-11e7-96c4-0d3a291ec93a\",\"panelIndex\":3,\"row\":11,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e3888410-3e50-11e7-96c4-0d3a291ec93a\",\"panelIndex\":5,\"row\":6,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":9,\"id\":\"75582a90-3987-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":9,\"row\":8,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"0bdbb5a0-3e55-11e7-96c4-0d3a291ec93a\",\"panelIndex\":11,\"row\":4,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"afdba840-3e55-11e7-96c4-0d3a291ec93a\",\"panelIndex\":12,\"row\":14,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"bfa45650-3e55-11e7-96c4-0d3a291ec93a\",\"panelIndex\":13,\"row\":14,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"cd462cc0-3e55-11e7-96c4-0d3a291ec93a\",\"panelIndex\":14,\"row\":11,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"31b85570-454a-11e7-86b6-95298e9da6dc\",\"panelIndex\":15,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"col\":1,\"id\":\"161e27e0-3988-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":16,\"row\":2,\"size_x\":10,\"size_y\":2,\"type\":\"visualization\"},{\"col\":11,\"id\":\"ed2f5570-3d5b-11e7-8b9d-ddc45b5f6d00\",\"panelIndex\":17,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"}]",
"optionsJSON": "{\"darkTheme\":false}",
"uiStateJSON": "{\"P-1\":{\"vis\":{\"colors\":{\"Destination Addresses\":\"#E0752D\",\"Destination Ports\":\"#E24D42\"},\"legendOpen\":false}},\"P-16\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-17\":{\"vis\":{\"defaultColors\":{\"0 - 50\":\"rgb(255,255,204)\",\"100 - 200\":\"rgb(253,141,60)\",\"200 - 300\":\"rgb(227,27,28)\",\"300 - 400\":\"rgb(128,0,38)\",\"50 - 100\":\"rgb(254,217,118)\"}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-8\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-9\":{\"vis\":{\"colors\":{\"/Attempt\":\"#0A50A1\",\"/Failure\":\"#BF1B00\",\"/Success\":\"#629E51\"}}}}",
"version": 1,
"timeRestore": true,
"timeTo": "now",
"timeFrom": "now-24h",
"refreshInterval": {
"display": "Off",
"pause": false,
"value": 0
},
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
}
}

7
x-pack/modules/arcsight/configuration/kibana/5.x/dashboard/arcsight.json
View file

@ -1,7 +0,0 @@
[
"153e0bf0-752f-11e7-ae68-d756b92f3a9c",
"37af0b40-398d-11e7-ae19-21fb91585845",
"64c92510-4555-11e7-83ea-67cb6920446d",
"82051450-3e56-11e7-96c4-0d3a291ec93a",
"d2fa5030-3e5d-11e7-b212-897f1496dc0e"
]

20
x-pack/modules/arcsight/configuration/kibana/5.x/dashboard/d2fa5030-3e5d-11e7-b212-897f1496dc0e.json
View file

@ -1,20 +0,0 @@
{
"title": "[ArcSight] Endpoint Overview Dashboard",
"hits": 0,
"description": "",
"panelsJSON": "[{\"col\":11,\"id\":\"c53825b0-3e4b-11e7-af78-9fc514b4e118\",\"panelIndex\":1,\"row\":2,\"size_x\":2,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"e301a830-3e4d-11e7-af78-9fc514b4e118\",\"panelIndex\":2,\"row\":9,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"9de87d40-3e4e-11e7-af78-9fc514b4e118\",\"panelIndex\":3,\"row\":9,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"96af5bf0-3e50-11e7-af78-9fc514b4e118\",\"panelIndex\":5,\"row\":4,\"size_x\":12,\"size_y\":2,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ff476320-3e4a-11e7-af78-9fc514b4e118\",\"panelIndex\":6,\"row\":12,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"b74e59b0-3e5f-11e7-899c-f940f646009b\",\"panelIndex\":7,\"row\":2,\"size_x\":10,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"7c6875e0-3e61-11e7-899c-f940f646009b\",\"panelIndex\":8,\"row\":12,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"categoryDeviceGroup\",\"categoryTechnique\",\"categoryOutcome\",\"categorySignificance\",\"categoryObject\",\"categoryBehavior\",\"categoryDeviceType\"],\"id\":\"1d9ba830-3e47-11e7-af78-9fc514b4e118\",\"panelIndex\":9,\"row\":20,\"size_x\":12,\"size_y\":5,\"sort\":[\"deviceReceiptTime\",\"desc\"],\"type\":\"search\"},{\"col\":7,\"id\":\"cc8affd0-3e65-11e7-899c-f940f646009b\",\"panelIndex\":10,\"row\":15,\"size_x\":6,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"1bde8be0-3e68-11e7-899c-f940f646009b\",\"panelIndex\":11,\"row\":6,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"7c414c90-3e66-11e7-899c-f940f646009b\",\"panelIndex\":12,\"row\":15,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":7,\"id\":\"6fb90a30-3e6b-11e7-9d4a-89ea81333ea4\",\"panelIndex\":14,\"row\":6,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"31b85570-454a-11e7-86b6-95298e9da6dc\",\"panelIndex\":15,\"row\":1,\"size_x\":12,\"size_y\":1,\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":3,\"panelIndex\":17,\"type\":\"visualization\",\"id\":\"2a33c810-3e4d-11e7-af78-9fc514b4e118\",\"col\":7,\"row\":17}]",
"optionsJSON": "{\"darkTheme\":false}",
"uiStateJSON": "{\"P-11\":{\"vis\":{\"colors\":{\"Anti-Virus\":\"#EAB839\",\"Database\":\"#629E51\",\"Host-based IDS/IPS\":\"#E0752D\",\"Operating System\":\"#BF1B00\",\"Security Mangement\":\"#64B0C8\"}}},\"P-12\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-14\":{\"vis\":{\"colors\":{\"/Attempt\":\"#0A50A1\",\"/Failure\":\"#BF1B00\",\"/Informational\":\"#7EB26D\",\"/Informational/Warning\":\"#EF843C\",\"/Success\":\"#629E51\",\"Anti-Virus\":\"#EAB839\",\"Database\":\"#629E51\",\"Host-based IDS/IPS\":\"#E0752D\",\"Log Consolidator\":\"#E0F9D7\",\"Operating System\":\"#BF1B00\",\"Recon\":\"#BF1B00\",\"Security Mangement\":\"#64B0C8\"}}},\"P-2\":{\"vis\":{\"colors\":{\"/Attempt\":\"#0A50A1\",\"/Failure\":\"#BF1B00\",\"/Success\":\"#629E51\"}}},\"P-3\":{\"vis\":{\"colors\":{\"/Attempt\":\"#0A50A1\",\"/Failure\":\"#BF1B00\",\"/Success\":\"#629E51\"}}},\"P-7\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-8\":{\"vis\":{\"colors\":{\"/Attempt\":\"#0A50A1\",\"/Failure\":\"#BF1B00\",\"/Success\":\"#629E51\"}}},\"P-17\":{\"mapZoom\":1,\"mapCenter\":[12.897489183755892,0]}}",
"version": 1,
"timeRestore": true,
"timeTo": "now",
"timeFrom": "now-24h",
"refreshInterval": {
"display": "Off",
"pause": false,
"value": 0
},
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
}
}

7
x-pack/modules/arcsight/configuration/kibana/5.x/index-pattern/arcsight.json
View file

File diff suppressed because one or more lines are too long

35
x-pack/modules/arcsight/configuration/kibana/5.x/search/16a72e70-4543-11e7-9510-4b0b4978ab0e.json
View file

@ -1,35 +0,0 @@
{
"title": "DNS Events",
"description": "",
"hits": 0,
"columns": [
"deviceVendor",
"deviceProduct",
"applicationProtocol",
"categoryBehavior",
"categoryOutcome",
"destinationAddress",
"destinationDnsDomain",
"destinationPort",
"deviceCustomString1Label",
"deviceCustomString1",
"deviceCustomString3Label",
"deviceCustomString3",
"deviceCustomString4Label",
"deviceCustomString4",
"deviceEventCategory",
"deviceHostName",
"deviceSeverity",
"sourceAddress",
"sourcePort",
"transportProtocol"
],
"sort": [
"deviceReceiptTime",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"arcsight-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"deviceEventCategory:\\\"dns\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}

22
x-pack/modules/arcsight/configuration/kibana/5.x/search/1d9ba830-3e47-11e7-af78-9fc514b4e118.json
View file

@ -1,22 +0,0 @@
{
"title": "Endpoint Event Explorer [ArcSight]",
"description": "",
"hits": 0,
"columns": [
"categoryDeviceGroup",
"categoryTechnique",
"categoryOutcome",
"categorySignificance",
"categoryObject",
"categoryBehavior",
"categoryDeviceType"
],
"sort": [
"deviceReceiptTime",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"arcsight-*\",\"highlightAll\":true,\"version\":true,\"filter\":[],\"query\":{\"query_string\":{\"query\":\"categoryDeviceGroup:\\\"/Operating System\\\" OR categoryDeviceGroup:\\\"/IDS/Host\\\" OR categoryDeviceGroup:\\\"/Application\\\"\",\"analyze_wildcard\":true}}}"
}
}

30
x-pack/modules/arcsight/configuration/kibana/5.x/search/6315e7a0-34be-11e7-95dc-4f6090d732f6.json
View file

@ -1,30 +0,0 @@
{
"title": "Network Events [ArcSight]",
"description": "",
"hits": 0,
"columns": [
"priority",
"name",
"sourceAddress",
"sourcePort",
"destinationAddress",
"destinationPort",
"applicationProtocol",
"message",
"categoryBehavior",
"categoryOutcome",
"deviceAddress",
"deviceProduct",
"deviceVendor",
"categoryDeviceGroup",
"categoryDeviceType"
],
"sort": [
"deviceReceiptTime",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"arcsight-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"categoryDeviceGroup:\\\"/Firewall\\\" OR categoryDeviceGroup:\\\"/IDS/Network\\\" OR categoryDeviceGroup:\\\"/VPN\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}"
}
}

27
x-pack/modules/arcsight/configuration/kibana/5.x/search/7a2fc9c0-454d-11e7-86b6-95298e9da6dc.json
View file

@ -1,27 +0,0 @@
{
"title": "Endpoint - OS Events [ArcSight]",
"description": "",
"hits": 0,
"columns": [
"deviceVendor",
"deviceProduct",
"name",
"deviceEventClassId",
"deviceEventCategory",
"sourceUserName",
"destinationUserName",
"destinationHostName",
"categoryBehavior",
"categoryOutcome",
"sourceNtDomain",
"destinationNTDomain"
],
"sort": [
"deviceReceiptTime",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"arcsight-*\",\"highlightAll\":true,\"version\":true,\"filter\":[],\"query\":{\"query_string\":{\"query\":\"categoryDeviceGroup:\\\"/Operating System\\\"\",\"analyze_wildcard\":true}}}"
}
}

38
x-pack/modules/arcsight/configuration/kibana/5.x/search/bb1f4bc0-73fd-11e7-b4d0-0fc7dfb45744.json
View file

@ -1,38 +0,0 @@
{
"title": "Microsoft DNS Events [ArcSight]",
"description": "",
"hits": 0,
"columns": [
"deviceVendor",
"deviceProduct",
"categoryBehavior",
"categoryOutcome",
"destinationAddress",
"destinationPort",
"destinationHostName",
"deviceEventClassId",
"deviceCustomString1Label",
"deviceCustomString1",
"deviceCustomString2Label",
"deviceCustomString2",
"deviceCustomString3Label",
"deviceCustomString3",
"deviceCustomString4Label",
"deviceCustomString4",
"deviceEventCategory",
"deviceSeverity",
"sourceAddress",
"sourcePort",
"transportProtocol",
"bytesIn",
"requestUrl"
],
"sort": [
"deviceReceiptTime",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"arcsight-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"deviceProduct:\\\"DNS Trace Log\\\"\"}},\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/039815b0-4548-11e7-a94a-5d0a73686c64.json
View file

@ -1,11 +0,0 @@
{
"title": "Top Destination Domains by Source Address",
"visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Source Address(es)\",\"field\":\"sourceAddress\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":10},\"schema\":\"segment\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Destination Domain(s)\",\"field\":\"destinationDnsDomain\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Source Address(es)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"title\":\"Top Destination Domains by Source Address\",\"type\":\"histogram\"}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "16a72e70-4543-11e7-9510-4b0b4978ab0e",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/08ee04d0-4556-11e7-83ea-67cb6920446d.json
View file

@ -1,11 +0,0 @@
{
"title": "Top 10 Source Users by Destination Users [ArcSight]",
"visState": "{\"title\":\"Top 10 Source Users by Destination Users [ArcSight]\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"sourceUserName\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source Users\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"destinationUserName\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Destination Users\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "7a2fc9c0-454d-11e7-86b6-95298e9da6dc",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

10
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/08f8cf10-3e57-11e7-96c4-0d3a291ec93a.json
View file

@ -1,10 +0,0 @@
{
"title": "Firewall - Navigation",
"visState": "{\"title\":\"Firewall - Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### **Navigation Pane** ###\\n\\n[Firewall Devices Overview](#/dashboard/37af0b40-398d-11e7-ae19-21fb91585845)\\n\\n[Firewall Suspicious Activities](#/dashboard/82051450-3e56-11e7-96c4-0d3a291ec93a)\\n\\n[Endopint Overview](#dashboard/d2fa5030-3e5d-11e7-b212-897f1496dc0e)\"},\"aggs\":[],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}"
}
}

10
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/0bdbb5a0-3e55-11e7-96c4-0d3a291ec93a.json
View file

@ -1,10 +0,0 @@
{
"title": "Events by Source Addresses [ArcSight]",
"visState": "{\"title\":\"Events by Source Addresses [ArcSight]\",\"type\":\"metrics\",\"params\":{\"id\":\"e1a58ab0-3957-11e7-ae19-21fb91585845\",\"type\":\"timeseries\",\"series\":[{\"id\":\"8f58a280-395a-11e7-ae19-21fb91585845\",\"color\":\"rgba(211,49,21,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"8f58a281-395a-11e7-ae19-21fb91585845\",\"type\":\"count\"},{\"settings\":\"\",\"minimize\":0,\"window\":\"\",\"model\":\"simple\",\"sigma\":\"\",\"id\":\"140cf490-395b-11e7-ae19-21fb91585845\",\"type\":\"moving_average\",\"field\":\"8f58a281-395a-11e7-ae19-21fb91585845\"}],\"seperate_axis\":1,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"3\",\"point_size\":\"0\",\"fill\":\"0\",\"stacked\":\"none\",\"filter\":\"categoryDeviceGroup:\\\"/Firewall\\\" OR categoryDeviceGroup:\\\"/IDS/Network\\\" OR categoryDeviceGroup:\\\"/VPN\\\" \",\"terms_field\":\"deviceHostName\",\"terms_order_by\":null,\"label\":\"Events\",\"steps\":0,\"value_template\":\"{{value}}\"},{\"id\":\"3bb711b0-395b-11e7-ae19-21fb91585845\",\"color\":\"rgba(104,188,0,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"3bb711b1-395b-11e7-ae19-21fb91585845\",\"type\":\"count\"},{\"settings\":\"\",\"minimize\":0,\"window\":\"\",\"model\":\"simple\",\"id\":\"4b515cc0-395b-11e7-ae19-21fb91585845\",\"type\":\"moving_average\",\"field\":\"3bb711b1-395b-11e7-ae19-21fb91585845\"}],\"seperate_axis\":1,\"axis_position\":\"left\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":\"0\",\"point_size\":1,\"fill\":\"0.5\",\"stacked\":\"none\",\"terms_field\":\"sourceAddress\",\"terms_size\":\"10\",\"label\":\"Top Source Addresses\"}],\"time_field\":\"deviceReceiptTime\",\"index_pattern\":\"arcsight-*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"id\":\"e4772140-3957-11e7-ae19-21fb91585845\"}],\"background_color\":null,\"filter\":\"categoryDeviceGroup:\\\"/Firewall\\\" OR categoryDeviceGroup:\\\"/IDS/Network\\\" OR categoryDeviceGroup:\\\"/VPN\\\" \",\"background_color_rules\":[{\"id\":\"837bfbb0-395a-11e7-ae19-21fb91585845\"}],\"gauge_color_rules\":[{\"id\":\"8970f250-395a-11e7-ae19-21fb91585845\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\"},\"aggs\":[],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}"
}
}

10
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/0e4558b0-4552-11e7-86b6-95298e9da6dc.json
View file

@ -1,10 +0,0 @@
{
"title": "Endpoint - OS Average EPS [ArcSight]",
"visState": "{\"title\":\"Endpoint - OS Average EPS [ArcSight]\",\"type\":\"metrics\",\"params\":{\"id\":\"3f2cf630-3e4b-11e7-af78-9fc514b4e118\",\"type\":\"gauge\",\"series\":[{\"id\":\"3f2cf631-3e4b-11e7-af78-9fc514b4e118\",\"color\":\"rgba(0,156,224,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"3f2cf632-3e4b-11e7-af78-9fc514b4e118\",\"type\":\"count\"},{\"id\":\"2f12f3d0-7dc5-11e7-95f6-690ab80d4e85\",\"type\":\"cumulative_sum\",\"field\":\"3f2cf632-3e4b-11e7-af78-9fc514b4e118\"},{\"unit\":\"1s\",\"id\":\"330d7a00-7dc5-11e7-95f6-690ab80d4e85\",\"type\":\"derivative\",\"field\":\"2f12f3d0-7dc5-11e7-95f6-690ab80d4e85\"},{\"settings\":\"\",\"minimize\":0,\"window\":\"\",\"model\":\"simple\",\"id\":\"373fd910-7dc5-11e7-95f6-690ab80d4e85\",\"type\":\"moving_average\",\"field\":\"330d7a00-7dc5-11e7-95f6-690ab80d4e85\"}],\"seperate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Event Throughput\",\"offset_time\":\"1m\",\"value_template\":\"{{value}} / s\"}],\"time_field\":\"deviceReceiptTime\",\"index_pattern\":\"arcsight-*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"show_legend\":1,\"bar_color_rules\":[{\"id\":\"527ca820-3e4b-11e7-af78-9fc514b4e118\"}],\"gauge_color_rules\":[{\"id\":\"52cee6d0-3e4b-11e7-af78-9fc514b4e118\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"filter\":\"categoryDeviceGroup:\\\"/Operating System\\\"\"},\"aggs\":[],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}"
}
}

10
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/1439a5e0-3e69-11e7-899c-f940f646009b.json
View file

@ -1,10 +0,0 @@
{
"title": "Endpoint - Navigation",
"visState": "{\"title\":\"Endpoint - Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### **Navigation Pane** ###\\n\\n[Endopint Overview](#dashboard/d2fa5030-3e5d-11e7-b212-897f1496dc0e)\"},\"aggs\":[],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/154ff7e0-3987-11e7-8b9d-ddc45b5f6d00.json
View file

@ -1,11 +0,0 @@
{
"title": "Device Type Breakdown [ArcSight]",
"visState": "{\"title\":\"Device Type Breakdown [ArcSight]\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"categoryDeviceType\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Firewall Types\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "6315e7a0-34be-11e7-95dc-4f6090d732f6",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/161e27e0-3988-11e7-8b9d-ddc45b5f6d00.json
View file

@ -1,11 +0,0 @@
{
"title": "Device Metrics Overview [ArcSight]",
"visState": "{\"title\":\"Device Metrics Overview [ArcSight]\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"fontSize\":\"30\",\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"bgFill\":\"#000\",\"fontSize\":\"12\",\"labelColor\":false,\"subText\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":false},\"handleNoResults\":true,\"type\":\"gauge\"},\"aggs\":[{\"id\":\"8\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Event Count\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"deviceHostName\",\"customLabel\":\"Devices\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sourceAddress\",\"customLabel\":\"Sources\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"destinationAddress\",\"customLabel\":\"Destinations\"}},{\"id\":\"7\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"destinationPort\",\"customLabel\":\"Ports\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}",
"description": "",
"savedSearchId": "6315e7a0-34be-11e7-95dc-4f6090d732f6",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/1bde8be0-3e68-11e7-899c-f940f646009b.json
View file

@ -1,11 +0,0 @@
{
"title": "Device Types by Vendor [ArcSight]",
"visState": "{\"title\":\"Device Types by Vendor [ArcSight]\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"categoryDeviceType\",\"exclude\":\"Network-based IDS/IPS\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"deviceVendor\",\"exclude\":\"\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "1d9ba830-3e47-11e7-af78-9fc514b4e118",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/1c54cda0-752c-11e7-9445-91c40765092f.json
View file

@ -1,11 +0,0 @@
{
"title": "Top 10 Event Types [ArcSight]",
"visState": "{\"title\":\"Top 10 Event Types [ArcSight]\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"square root\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"deviceEventClassId\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "bb1f4bc0-73fd-11e7-b4d0-0fc7dfb45744",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/1de45d60-7523-11e7-9445-91c40765092f.json
View file

@ -1,11 +0,0 @@
{
"title": "DNS Metrics Overview [ArcSight]",
"visState": "{\"title\":\"DNS Metrics Overview [ArcSight]\",\"type\":\"metric\",\"params\":{\"addLegend\":false,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":100}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"bgFill\":\"#000\",\"fontSize\":\"32\",\"labelColor\":false,\"subText\":\"\"},\"type\":\"simple\",\"useRange\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"5\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Event Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"deviceCustomString1\",\"customLabel\":\"Threads\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"deviceCustomString2\",\"customLabel\":\"OpCodes\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"deviceEventClassId\",\"customLabel\":\"Activity Types\"}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}",
"description": "",
"savedSearchId": "bb1f4bc0-73fd-11e7-b4d0-0fc7dfb45744",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/2a33c810-3e4d-11e7-af78-9fc514b4e118.json
View file

@ -1,11 +0,0 @@
{
"title": "Top Destination Locations by Event [ArcSight]",
"visState": "{\"title\":\"Top Destination Locations by Event [ArcSight]\",\"type\":\"tile_map\",\"params\":{\"mapType\":\"Shaded Circle Markers\",\"isDesaturated\":true,\"addTooltip\":true,\"heatMaxZoom\":0,\"heatMinOpacity\":0.1,\"heatRadius\":25,\"heatBlur\":15,\"heatNormalizeData\":true,\"legendPosition\":\"bottomright\",\"mapZoom\":2,\"mapCenter\":[0,0],\"wms\":{\"enabled\":false,\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\",\"options\":{\"version\":\"1.3.0\",\"layers\":\"0\",\"format\":\"image/png\",\"transparent\":true,\"attribution\":\"Maps provided by USGS\",\"styles\":\"\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"destination.location\",\"autoPrecision\":true,\"useGeocentroid\":true,\"precision\":2}}],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "1d9ba830-3e47-11e7-af78-9fc514b4e118",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

11
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/2b369910-4553-11e7-83ea-67cb6920446d.json
View file

@ -1,11 +0,0 @@
{
"title": "Top 15 Event Types by Events [ArcSight]",
"visState": "{\"title\":\"Top 15 Event Types by Events [ArcSight]\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"categoryBehavior\",\"size\":15,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Event Types\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sourceUserName\",\"customLabel\":\"Source Users\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"destinationUserName\",\"customLabel\":\"Destination Users\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"sourceHostName\",\"customLabel\":\"Source Hosts\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"destinationHostName\",\"customLabel\":\"Destination Hosts\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}",
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
"description": "",
"savedSearchId": "7a2fc9c0-454d-11e7-86b6-95298e9da6dc",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}

10
x-pack/modules/arcsight/configuration/kibana/5.x/visualization/31b85570-454a-11e7-86b6-95298e9da6dc.json
View file

@ -1,10 +0,0 @@
{
"title": "ArcSight: Dashboard Navigation",
"visState": "{\"title\":\"ArcSight: Dashboard Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"[Network Overview](#/dashboard/37af0b40-398d-11e7-ae19-21fb91585845) | [Network Suspicious Activity](#/dashboard/82051450-3e56-11e7-96c4-0d3a291ec93a) | [Endpoint Overview](#dashboard/d2fa5030-3e5d-11e7-b212-897f1496dc0e) | [Endpoint OS Activity](#/dashboard/64c92510-4555-11e7-83ea-67cb6920446d) | [Microsoft DNS Overview](#/dashboard/153e0bf0-752f-11e7-ae68-d756b92f3a9c)\"},\"aggs\":[],\"listeners\":{}}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\"}},\"filter\":[]}"
}
}

Some files were not shown because too many files have changed in this diff Show more