github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 22:57:16 -04:00
Code Issues Projects Releases Packages Wiki Activity

fix database manager with multiple pipelines (#12862) (#12995)

Browse source 
This PR adds support to geoip multiple pipelines which have a single instance
to manage database download to avoid individual download per pipeline
This commit is contained in:
kaisecheng 2021-06-17 10:44:18 +02:00 • committed by GitHub
parent 854fad6339
commit 72e8dc8d9c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 739 additions and 517 deletions
Download patch file Download diff file Expand all files Collapse all files

2
logstash-core/logstash-core.gemspec
View file

@ -75,7 +75,7 @@ Gem::Specification.new do |gem|
gem.add_runtime_dependency "manticore", '~> 0.6' gem.add_runtime_dependency "manticore", '~> 0.6'
# xpack geoip database service # xpack geoip database service
gem.add_development_dependency 'logstash-filter-geoip', '~> 7.1' # package hierarchy change gem.add_development_dependency 'logstash-filter-geoip', '>= 7.2.1' # breaking change of DatabaseManager
gem.add_dependency 'faraday' #(MIT license) gem.add_dependency 'faraday' #(MIT license)
gem.add_dependency 'down', '~> 5.2.0' #(MIT license) gem.add_dependency 'down', '~> 5.2.0' #(MIT license)
gem.add_dependency 'tzinfo-data' #(MIT license) gem.add_dependency 'tzinfo-data' #(MIT license)

298
x-pack/lib/filters/geoip/database_manager.rb
View file

@ -13,6 +13,10 @@ require "stud/try"
require "down" require "down"
require "rufus/scheduler" require "rufus/scheduler"
require "date" require "date"
require "singleton"
require "concurrent"
require "thread"
java_import org.apache.logging.log4j.ThreadContext
# The mission of DatabaseManager is to ensure the plugin running an up-to-date MaxMind database and # The mission of DatabaseManager is to ensure the plugin running an up-to-date MaxMind database and
# thus users are compliant with EULA. # thus users are compliant with EULA.
@ -21,7 +25,7 @@ require "date"
# to keep track of versions and the number of days disconnects to the endpoint. # to keep track of versions and the number of days disconnects to the endpoint.
# Once a new database version release, DownloadManager downloads it, and GeoIP Filter uses it on-the-fly. # Once a new database version release, DownloadManager downloads it, and GeoIP Filter uses it on-the-fly.
# If the last update timestamp is 25 days ago, a warning message shows in the log; # If the last update timestamp is 25 days ago, a warning message shows in the log;
# if it was 30 days ago, the GeoIP Filter should shutdown in order to be compliant. # if it was 30 days ago, the GeoIP Filter should stop using EULA database in order to be compliant.
# There are online mode and offline mode in DatabaseManager. `online` is for automatic database update # There are online mode and offline mode in DatabaseManager. `online` is for automatic database update
# while `offline` is for static database path provided by users # while `offline` is for static database path provided by users
@ -29,140 +33,210 @@ module LogStash module Filters module Geoip class DatabaseManager
extend LogStash::Filters::Geoip::Util extend LogStash::Filters::Geoip::Util
include LogStash::Util::Loggable include LogStash::Util::Loggable
include LogStash::Filters::Geoip::Util include LogStash::Filters::Geoip::Util
include Singleton
#TODO remove vendor_path private
def initialize(geoip, database_path, database_type, vendor_path) def initialize
@geoip = geoip prepare_cc_db
self.class.prepare_cc_db cc_city_database_path = get_db_path(CITY, CC)
@mode = database_path.nil? ? :online : :offline cc_asn_database_path = get_db_path(ASN, CC)
@mode = :disabled # This is a temporary change that turns off the database manager until it is ready for general availability.
@database_type = database_type
@database_path = patch_database_path(database_path)
if @mode == :online prepare_metadata
logger.info "By not manually configuring a database path with `database =>`, you accepted and agreed MaxMind EULA. "\ city_database_path = @metadata.database_path(CITY)
"For more details please visit https://www.maxmind.com/en/geolite2/eula" asn_database_path = @metadata.database_path(ASN)
setup @triggered = false
@trigger_lock = Mutex.new
@states = { "#{CITY}" => DatabaseState.new(@metadata.is_eula(CITY),
Concurrent::Array.new,
city_database_path,
cc_city_database_path),
"#{ASN}" => DatabaseState.new(@metadata.is_eula(ASN),
Concurrent::Array.new,
asn_database_path,
cc_asn_database_path) }
@download_manager = DownloadManager.new(@metadata)
end
protected
# create data dir, path.data, for geoip if it doesn't exist
# copy CC databases to data dir
def prepare_cc_db
FileUtils::mkdir_p(get_data_dir_path)
unless ::File.exist?(get_db_path(CITY, CC)) && ::File.exist?(get_db_path(ASN, CC))
cc_database_paths = ::Dir.glob(::File.join(LogStash::Environment::LOGSTASH_HOME, "vendor", "**", "{GeoLite2-ASN,GeoLite2-City}.mmdb"))
cc_dir_path = get_dir_path(CC)
FileUtils.mkdir_p(cc_dir_path)
FileUtils.cp_r(cc_database_paths, cc_dir_path)
end
end
def prepare_metadata
@metadata = DatabaseMetadata.new
unless @metadata.exist?
@metadata.save_metadata(CITY, CC, false)
@metadata.save_metadata(ASN, CC, false)
end
# reset md5 to allow re-download when the database directory is deleted manually
DB_TYPES.each { |type| @metadata.reset_md5(type) if @metadata.database_path(type).nil? }
@metadata
end
# notice plugins to use the new database path
# update metadata timestamp for those dbs that has no update or a valid update
# do daily check and clean up
def execute_download_job
begin
pipeline_id = ThreadContext.get("pipeline.id")
ThreadContext.put("pipeline.id", nil)
updated_db = @download_manager.fetch_database
updated_db.each do |database_type, valid_download, dirname, new_database_path|
if valid_download
@metadata.save_metadata(database_type, dirname, true)
@states[database_type].is_eula = true
@states[database_type].is_expired = false
@states[database_type].database_path = new_database_path
notify_plugins(database_type, :update, new_database_path) do |db_type, ids|
logger.info("geoip plugin will use database #{new_database_path}",
:database_type => db_type, :pipeline_ids => ids) unless ids.empty?
end
end
end
updated_types = updated_db.map { |database_type, valid_download, dirname, new_database_path| database_type }
(DB_TYPES - updated_types).each { |unchange_type| @metadata.update_timestamp(unchange_type) }
rescue => e
logger.error(e.message, error_details(e, logger))
ensure
check_age
clean_up_database clean_up_database
execute_download_job ThreadContext.put("pipeline.id", pipeline_id)
end
end
def notify_plugins(database_type, action, *args)
plugins = @states[database_type].plugins.dup
ids = plugins.map { |plugin| plugin.execution_context.pipeline_id }.sort
yield database_type, ids
plugins.each { |plugin| plugin.update_filter(action, *args) if plugin }
end
# call expiry action if Logstash use EULA database and fail to touch the endpoint for 30 days in a row
def check_age(database_types = DB_TYPES)
database_types.map do |database_type|
next unless @states[database_type].is_eula
days_without_update = (::Date.today - ::Time.at(@metadata.check_at(database_type)).to_date).to_i
case
when days_without_update >= 30
was_expired = @states[database_type].is_expired
@states[database_type].is_expired = true
@states[database_type].database_path = nil
notify_plugins(database_type, :expire) do |db_type, ids|
unless was_expired
logger.error("The MaxMind database hasn't been updated from last 30 days. Logstash is unable to get newer version from internet. "\
"According to EULA, GeoIP plugin needs to stop using MaxMind database in order to be compliant. "\
"Please check the network settings and allow Logstash accesses the internet to download the latest database, "\
"or switch to offline mode (:database => PATH_TO_YOUR_DATABASE) to use a self-managed database "\
"which you can download from https://dev.maxmind.com/geoip/geoip2/geolite2/ ")
logger.warn("geoip plugin will stop filtering and will tag all events with the '_geoip_expired_database' tag.",
:database_type => db_type, :pipeline_ids => ids)
end
end
when days_without_update >= 25
logger.warn("The MaxMind database hasn't been updated for last #{days_without_update} days. "\
"Logstash will fail the GeoIP plugin in #{30 - days_without_update} days. "\
"Please check the network settings and allow Logstash accesses the internet to download the latest database ")
else
logger.trace("passed age check", :days_without_update => days_without_update)
end
end
end
# Clean up directories which are not mentioned in metadata and not CC database
def clean_up_database
protected_dirnames = (@metadata.dirnames + [CC]).uniq
existing_dirnames = ::Dir.children(get_data_dir_path)
.select { |f| ::File.directory? ::File.join(get_data_dir_path, f) }
(existing_dirnames - protected_dirnames).each do |dirname|
dir_path = get_dir_path(dirname)
FileUtils.rm_r(dir_path)
logger.info("#{dir_path} is deleted")
end
end
def trigger_download
return if @triggered
@trigger_lock.synchronize do
return if @triggered
execute_download_job
# check database update periodically. trigger `call` method # check database update periodically. trigger `call` method
@scheduler = Rufus::Scheduler.new({:max_work_threads => 1}) @scheduler = Rufus::Scheduler.new({:max_work_threads => 1})
@scheduler.every('24h', self) @scheduler.every('24h', self)
elsif @mode == :disabled @triggered = true
# The plan is to use CC database in Logstash 7.x and enable EULA database in 8
else
logger.info "GeoIP database path is configured manually so the plugin will not check for update. "\
"Keep in mind that if you are not using the database shipped with this plugin, "\
"please go to https://www.maxmind.com/en/geolite2/eula and understand the terms and conditions."
end end
end end
DEFAULT_DATABASE_FILENAME = %w{
GeoLite2-City.mmdb
GeoLite2-ASN.mmdb
}.map(&:freeze).freeze
public public
# create data dir, path.data, for geoip if it doesn't exist
# copy CC databases to data dir
def self.prepare_cc_db
FileUtils::mkdir_p(get_data_dir)
unless ::File.exist?(get_file_path(CITY_DB_NAME)) && ::File.exist?(get_file_path(ASN_DB_NAME))
cc_database_paths = ::Dir.glob(::File.join(LogStash::Environment::LOGSTASH_HOME, "vendor", "**", "{GeoLite2-ASN,GeoLite2-City}.mmdb"))
FileUtils.cp_r(cc_database_paths, get_data_dir)
end
end
def execute_download_job
begin
has_update, new_database_path = @download_manager.fetch_database
@database_path = new_database_path if has_update
@metadata.save_timestamp(@database_path)
has_update
rescue => e
logger.error(e.message, error_details(e, logger))
check_age
false
end
end
# scheduler callback # scheduler callback
def call(job, time) def call(job, time)
logger.debug "scheduler runs database update check" logger.debug "scheduler runs database update check"
ThreadContext.put("pipeline.id", nil)
execute_download_job
end
begin def subscribe_database_path(database_type, database_path, geoip_plugin)
if execute_download_job if database_path.nil?
@geoip.setup_filter(database_path) trigger_download
clean_up_database
logger.info "By not manually configuring a database path with `database =>`, you accepted and agreed MaxMind EULA. "\
"For more details please visit https://www.maxmind.com/en/geolite2/eula" if @states[database_type].is_eula
@states[database_type].plugins.push(geoip_plugin) unless @states[database_type].plugins.member?(geoip_plugin)
@trigger_lock.synchronize do
@states[database_type].database_path
end end
rescue DatabaseExpiryError => e
logger.error(e.message, error_details(e, logger))
@geoip.expire_action
end
end
def close
@scheduler.every_jobs.each(&:unschedule) if @scheduler
end
def database_path
@database_path
end
protected
# return a valid database path or default database path
def patch_database_path(database_path)
return database_path if file_exist?(database_path)
return database_path if database_path = get_file_path("#{DB_PREFIX}#{@database_type}.#{DB_EXT}") and file_exist?(database_path)
raise "You must specify 'database => ...' in your geoip filter (I looked for '#{database_path}')"
end
def check_age
return if @metadata.cc?
days_without_update = (::Date.today - ::Time.at(@metadata.updated_at).to_date).to_i
case
when days_without_update >= 30
raise DatabaseExpiryError, "The MaxMind database has been used for more than 30 days. Logstash is unable to get newer version from internet. "\
"According to EULA, GeoIP plugin needs to stop in order to be compliant. "\
"Please check the network settings and allow Logstash accesses the internet to download the latest database, "\
"or configure a database manually (:database => PATH_TO_YOUR_DATABASE) to use a self-managed database which you can download from https://dev.maxmind.com/geoip/geoip2/geolite2/ "
when days_without_update >= 25
logger.warn("The MaxMind database has been used for #{days_without_update} days without update. "\
"Logstash will fail the GeoIP plugin in #{30 - days_without_update} days. "\
"Please check the network settings and allow Logstash accesses the internet to download the latest database ")
else else
logger.trace("The MaxMind database hasn't updated", :days_without_update => days_without_update) logger.info "GeoIP database path is configured manually so the plugin will not check for update. "\
"Keep in mind that if you are not using the database shipped with this plugin, "\
"please go to https://www.maxmind.com/en/geolite2/eula and understand the terms and conditions."
database_path
end end
end end
# Clean up files .mmdb, .tgz which are not mentioned in metadata and not default database def unsubscribe_database_path(database_type, geoip_plugin)
def clean_up_database @states[database_type].plugins.delete(geoip_plugin) if geoip_plugin
if @metadata.exist? end
protected_filenames = (@metadata.database_filenames + DEFAULT_DATABASE_FILENAME).uniq
existing_filenames = ::Dir.glob(get_file_path("*.{#{DB_EXT},#{GZ_EXT}}"))
.map { |path| ::File.basename(path) }
(existing_filenames - protected_filenames).each do |filename| def database_path(database_type)
::File.delete(get_file_path(filename)) @states[database_type].database_path
logger.debug("old database #{filename} is deleted") end
end
class DatabaseState
attr_reader :is_eula, :plugins, :database_path, :cc_database_path, :is_expired
attr_writer :is_eula, :database_path, :is_expired
# @param is_eula [Boolean]
# @param plugins [Concurrent::Array]
# @param database_path [String]
# @param cc_database_path [String]
def initialize(is_eula, plugins, database_path, cc_database_path)
@is_eula = is_eula
@plugins = plugins
@database_path = database_path
@cc_database_path = cc_database_path
@is_expired = false
end end
end end
def setup
@metadata = DatabaseMetadata.new(@database_type)
@metadata.save_timestamp(@database_path) unless @metadata.exist?
@database_path = @metadata.database_path || @database_path
@download_manager = DownloadManager.new(@database_type, @metadata)
end
class DatabaseExpiryError < StandardError
end
end end end end end end end end

84
x-pack/lib/filters/geoip/database_metadata.rb
View file

@ -11,61 +11,83 @@ module LogStash module Filters module Geoip class DatabaseMetadata
include LogStash::Util::Loggable include LogStash::Util::Loggable
include LogStash::Filters::Geoip::Util include LogStash::Filters::Geoip::Util
def initialize(database_type) def initialize
@metadata_path = get_file_path("metadata.csv") @metadata_path = ::File.join(get_data_dir_path, "metadata.csv")
@database_type = database_type
end end
public public
# csv format: database_type, update_at, gz_md5, md5, filename # csv format: database_type, check_at, gz_md5, dirname, is_eula
def save_timestamp(database_path) def save_metadata(database_type, dirname, is_eula)
metadata = get_metadata(false) metadata = get_metadata(database_type, false)
metadata << [@database_type, Time.now.to_i, md5(get_gz_name(database_path)), md5(database_path), metadata << [database_type, Time.now.to_i, md5(get_gz_path(database_type, dirname)),
::File.basename(database_path)] dirname, is_eula]
update(metadata)
end
def update_timestamp(database_type)
update_each_row do |row|
row[Column::CHECK_AT] = Time.now.to_i if row[Column::DATABASE_TYPE].eql?(database_type)
row
end
end
def reset_md5(database_type)
update_each_row do |row|
row[Column::GZ_MD5] = "" if row[Column::DATABASE_TYPE].eql?(database_type)
row
end
end
def update_each_row(&block)
metadata = get_all.map do |row|
yield row
end
update(metadata)
end
def update(metadata)
metadata = metadata.sort_by { |row| row[Column::DATABASE_TYPE] }
::CSV.open @metadata_path, 'w' do |csv| ::CSV.open @metadata_path, 'w' do |csv|
metadata.each { |row| csv << row } metadata.each { |row| csv << row }
end end
logger.trace("metadata updated", :metadata => metadata)
logger.debug("metadata updated", :metadata => metadata)
end end
def get_all def get_all
file_exist?(@metadata_path)? ::CSV.read(@metadata_path, headers: false) : Array.new file_exist?(@metadata_path)? ::CSV.read(@metadata_path, headers: false) : Array.new
end end
# Give rows of metadata in default database type, or empty array # Give rows of metadata that match/exclude the type
def get_metadata(match_type = true) def get_metadata(database_type, match = true)
get_all.select { |row| row[Column::DATABASE_TYPE].eql?(@database_type) == match_type } get_all.select { |row| row[Column::DATABASE_TYPE].eql?(database_type) == match }
end end
# Return database path which has valid md5 # Return a valid database path
def database_path def database_path(database_type)
get_metadata.map { |metadata| [metadata, get_file_path(metadata[Column::FILENAME])] } get_metadata(database_type).map { |metadata| get_db_path(database_type, metadata[Column::DIRNAME]) }
.select { |metadata, path| file_exist?(path) && (md5(path) == metadata[Column::MD5]) } .select { |path| file_exist?(path) }
.map { |metadata, path| path }
.last .last
end end
def gz_md5 def gz_md5(database_type)
get_metadata.map { |metadata| metadata[Column::GZ_MD5] } get_metadata(database_type).map { |metadata| metadata[Column::GZ_MD5] }
.last || '' .last || ''
end end
def updated_at def check_at(database_type)
(get_metadata.map { |metadata| metadata[Column::UPDATE_AT] } (get_metadata(database_type).map { |metadata| metadata[Column::CHECK_AT] }
.last || 0).to_i .last || 0).to_i
end end
def cc? def is_eula(database_type)
filename = get_metadata.map { |metadata| metadata[Column::FILENAME] }.last || '' (get_metadata(database_type).map { |metadata| metadata[Column::IS_EULA] }
filename.eql?(CITY_DB_NAME) || filename.eql?(ASN_DB_NAME) .last || 'false') == 'true'
end end
# Return database related filenames in .mmdb .tgz # Return all dirname
def database_filenames def dirnames
get_all.flat_map { |metadata| [ metadata[Column::FILENAME], get_gz_name(metadata[Column::FILENAME]) ] } get_all.map { |metadata| metadata[Column::DIRNAME] }
end end
def exist? def exist?
@ -74,10 +96,10 @@ module LogStash module Filters module Geoip class DatabaseMetadata
class Column class Column
DATABASE_TYPE = 0 DATABASE_TYPE = 0
UPDATE_AT = 1 CHECK_AT = 1
GZ_MD5 = 2 GZ_MD5 = 2
MD5 = 3 DIRNAME = 3
FILENAME = 4 IS_EULA = 4
end end
end end end end end end end end

98
x-pack/lib/filters/geoip/download_manager.rb
View file

@ -18,8 +18,7 @@ module LogStash module Filters module Geoip class DownloadManager
include LogStash::Util::Loggable include LogStash::Util::Loggable
include LogStash::Filters::Geoip::Util include LogStash::Filters::Geoip::Util
def initialize(database_type, metadata) def initialize(metadata)
@database_type = database_type
@metadata = metadata @metadata = metadata
end end
@ -28,75 +27,70 @@ module LogStash module Filters module Geoip class DownloadManager
GEOIP_ENDPOINT = "#{GEOIP_HOST}#{GEOIP_PATH}".freeze GEOIP_ENDPOINT = "#{GEOIP_HOST}#{GEOIP_PATH}".freeze
public public
# Check available update and download it. Unzip and validate the file. # Check available update and download them. Unzip and validate the file.
# return [has_update, new_database_path] # if the download failed, valid_download return false
# return Array of [database_type, valid_download, dirname, new_database_path]
def fetch_database def fetch_database
has_update, database_info = check_update dirname = Time.now.to_i.to_s
check_update
if has_update .map do |database_type, db_info|
new_database_path = unzip download_database(database_info) begin
assert_database!(new_database_path) new_zip_path = download_database(database_type, dirname, db_info)
return [true, new_database_path] new_database_path = unzip(database_type, dirname, new_zip_path)
end assert_database!(new_database_path)
[database_type, true, dirname, new_database_path]
[false, nil] rescue => e
end logger.error(e.message, error_details(e, logger))
[database_type, false, nil, nil]
def database_name end
@database_name ||= "#{DB_PREFIX}#{@database_type}" end
end
def database_name_ext
@database_name_ext ||= "#{database_name}.#{DB_EXT}"
end end
private private
# Call infra endpoint to get md5 of latest database and verify with metadata # Call infra endpoint to get md5 of latest databases and verify with metadata
# return [has_update, server db info] # return Array of new database information [database_type, db_info]
def check_update def check_update
uuid = get_uuid uuid = get_uuid
res = rest_client.get("#{GEOIP_ENDPOINT}?key=#{uuid}&elastic_geoip_service_tos=agree") res = rest_client.get("#{GEOIP_ENDPOINT}?key=#{uuid}&elastic_geoip_service_tos=agree")
logger.debug("check update", :endpoint => GEOIP_ENDPOINT, :response => res.status) logger.debug("check update", :endpoint => GEOIP_ENDPOINT, :response => res.status)
dbs = JSON.parse(res.body) service_resp = JSON.parse(res.body)
target_db = dbs.select { |db| db['name'].eql?("#{database_name}.#{GZ_EXT}") }.first
has_update = @metadata.gz_md5 != target_db['md5_hash']
logger.info "new database version detected? #{has_update}"
[has_update, target_db] updated_db = DB_TYPES.map do |database_type|
db_info = service_resp.find { |db| db['name'].eql?("#{GEOLITE}#{database_type}.#{GZ_EXT}") }
has_update = @metadata.gz_md5(database_type) != db_info['md5_hash']
[database_type, has_update, db_info]
end
.select { |database_type, has_update, db_info| has_update }
.map { |database_type, has_update, db_info| [database_type, db_info] }
logger.info "new database version detected? #{!updated_db.empty?}"
updated_db
end end
def download_database(server_db) def download_database(database_type, dirname, db_info)
Stud.try(3.times) do Stud.try(3.times) do
new_database_zip_path = get_file_path("#{database_name}_#{Time.now.to_i}.#{GZ_EXT}") FileUtils.mkdir_p(get_dir_path(dirname))
Down.download(server_db['url'], destination: new_database_zip_path) zip_path = get_gz_path(database_type, dirname)
raise "the new download has wrong checksum" if md5(new_database_zip_path) != server_db['md5_hash']
logger.debug("new database downloaded in ", :path => new_database_zip_path) Down.download(db_info['url'], destination: zip_path)
new_database_zip_path raise "the new download has wrong checksum" if md5(zip_path) != db_info['md5_hash']
logger.debug("new database downloaded in ", :path => zip_path)
zip_path
end end
end end
# extract all files and folders from .tgz to path.data directory # extract all files and folders from .tgz to path.data directory
# existing files folders will be replaced # return dirname [String], new_database_path [String]
def unzip(zip_path) def unzip(database_type, dirname, zip_path)
new_database_path = zip_path[0...-(GZ_EXT.length)] + DB_EXT temp_path = ::File.join(get_dir_path(dirname), database_type)
temp_dir = Stud::Temporary.pathname LogStash::Util::Tar.extract(zip_path, temp_path)
FileUtils.cp_r(::File.join(temp_path, '.'), get_dir_path(dirname))
FileUtils.rm_r(temp_path)
LogStash::Util::Tar.extract(zip_path, temp_dir) get_db_path(database_type, dirname)
logger.debug("extract database to ", :path => temp_dir)
::Dir.each_child(temp_dir) do |file|
path = ::File.join(temp_dir, file)
if !::File.directory?(path) && database_name_ext.eql?(file)
FileUtils.cp(path, new_database_path)
else
FileUtils.cp_r(path, get_data_dir)
end
end
new_database_path
end end
# Make sure the path has usable database # Make sure the path has usable database

30
x-pack/lib/filters/geoip/util.rb
View file

@ -9,16 +9,29 @@ module LogStash module Filters
module Geoip module Geoip
GZ_EXT = 'tgz'.freeze GZ_EXT = 'tgz'.freeze
DB_EXT = 'mmdb'.freeze DB_EXT = 'mmdb'.freeze
DB_PREFIX = 'GeoLite2-'.freeze GEOLITE = 'GeoLite2-'.freeze
CITY_DB_NAME = "#{DB_PREFIX}City.#{DB_EXT}" CITY = "City".freeze
ASN_DB_NAME = "#{DB_PREFIX}ASN.#{DB_EXT}" ASN = "ASN".freeze
DB_TYPES = [ASN, CITY].freeze
CITY_DB_NAME = "#{GEOLITE}#{CITY}.#{DB_EXT}".freeze
ASN_DB_NAME = "#{GEOLITE}#{ASN}.#{DB_EXT}".freeze
DEFAULT_DB_NAMES = [CITY_DB_NAME, ASN_DB_NAME].freeze
CC = "CC".freeze
module Util module Util
def get_file_path(filename) def get_db_path(database_type, dirname)
::File.join(get_data_dir, filename) ::File.join(get_data_dir_path, dirname, "#{GEOLITE}#{database_type}.#{DB_EXT}")
end end
def get_data_dir def get_gz_path(database_type, dirname)
::File.join(get_data_dir_path, dirname, "#{GEOLITE}#{database_type}.#{GZ_EXT}")
end
def get_dir_path(dirname)
::File.join(get_data_dir_path, dirname)
end
def get_data_dir_path
::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip") ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
end end
@ -30,11 +43,6 @@ module LogStash module Filters
file_exist?(file_path) ? Digest::MD5.hexdigest(::File.read(file_path)): "" file_exist?(file_path) ? Digest::MD5.hexdigest(::File.read(file_path)): ""
end end
# replace *.mmdb to *.tgz
def get_gz_name(filename)
filename[0...-(DB_EXT.length)] + GZ_EXT
end
def error_details(e, logger) def error_details(e, logger)
error_details = { :cause => e.cause } error_details = { :cause => e.cause }
error_details[:backtrace] = e.backtrace if logger.debug? error_details[:backtrace] = e.backtrace if logger.debug?

359
x-pack/spec/filters/geoip/database_manager_spec.rb
View file

@ -13,7 +13,7 @@ describe LogStash::Filters::Geoip do
let(:mock_download_manager) { double("download_manager") } let(:mock_download_manager) { double("download_manager") }
let(:mock_scheduler) { double("scheduler") } let(:mock_scheduler) { double("scheduler") }
let(:db_manager) do let(:db_manager) do
manager = LogStash::Filters::Geoip::DatabaseManager.new(mock_geoip_plugin, default_city_db_path, "City", get_vendor_path) manager = Class.new(LogStash::Filters::Geoip::DatabaseManager).instance
manager.instance_variable_set(:@metadata, mock_metadata) manager.instance_variable_set(:@metadata, mock_metadata)
manager.instance_variable_set(:@download_manager, mock_download_manager) manager.instance_variable_set(:@download_manager, mock_download_manager)
manager.instance_variable_set(:@scheduler, mock_scheduler) manager.instance_variable_set(:@scheduler, mock_scheduler)
@ -21,208 +21,265 @@ describe LogStash::Filters::Geoip do
end end
let(:logger) { double("Logger") } let(:logger) { double("Logger") }
before(:each) do CITY = LogStash::Filters::Geoip::CITY
LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db ASN = LogStash::Filters::Geoip::ASN
CC = LogStash::Filters::Geoip::CC
before do
stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', "https://somewhere.dev")
allow(mock_geoip_plugin).to receive(:update_filter)
end end
context "patch database" do after do
it "use input path" do delete_file(metadata_path, get_dir_path(second_dirname))
path = db_manager.send(:patch_database_path, default_asn_db_path)
expect(path).to eq(default_asn_db_path)
end
it "use CC license database as default" do
path = db_manager.send(:patch_database_path, "")
expect(path).to eq(default_city_db_path)
end
it "failed when default database is missing" do
expect(db_manager).to receive(:file_exist?).and_return(false, false)
expect { db_manager.send(:patch_database_path, "") }.to raise_error /I looked for/
end
end end
context "md5" do context "initialize" do
it "return md5 if file exists" do it "should set the initial state to cc database" do
str = db_manager.send(:md5, default_city_db_path) states = db_manager.instance_variable_get(:@states)
expect(str).not_to eq("") expect(states[CITY].is_eula).to be_falsey
expect(str).not_to be_nil expect(states[CITY].database_path).to eql(states[CITY].cc_database_path)
expect(::File.exist?(states[CITY].cc_database_path)).to be_truthy
expect(states[ASN].is_eula).to be_falsey
expect(states[ASN].database_path).to eql(states[ASN].cc_database_path)
expect(::File.exist?(states[ASN].cc_database_path)).to be_truthy
end end
it "return empty str if file not exists" do context "when metadata exists" do
file = Stud::Temporary.file.path + "/invalid" before do
str = db_manager.send(:md5, file) copy_cc(get_dir_path(second_dirname))
expect(str).to eq("") rewrite_temp_metadata(metadata_path, [city2_metadata])
end
end
context "check age" do
it "should raise error when 30 days has passed" do
expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i)
expect(mock_metadata).to receive(:cc?).and_return(false)
expect{ db_manager.send(:check_age) }.to raise_error /be compliant/
end
it "should give warning after 25 days" do
expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 26)).to_i)
expect(mock_metadata).to receive(:cc?).and_return(false)
expect(mock_geoip_plugin).to receive(:terminate_filter).never
if db_manager.instance_variable_get(:@mode) != :disabled
expect(LogStash::Filters::Geoip::DatabaseManager).to receive(:logger).at_least(:once).and_return(logger)
expect(logger).to receive(:warn)
expect(logger).to receive(:info)
end end
db_manager.send(:check_age) it "should use database record in metadata" do
states = db_manager.instance_variable_get(:@states)
expect(states[CITY].is_eula).to be_truthy
expect(states[CITY].database_path).to include second_dirname
end
end
context "when metadata exists but database is deleted manually" do
let(:db_manager) { Class.new(LogStash::Filters::Geoip::DatabaseManager).instance }
before do
rewrite_temp_metadata(metadata_path, [city2_metadata])
end
it "should return nil database path" do
states = db_manager.instance_variable_get(:@states)
expect(states[CITY].is_eula).to be_truthy
expect(states[CITY].database_path).to be_nil
end
end end
end end
context "execute download job" do context "execute download job" do
it "should be false if no update" do let(:valid_city_fetch) { [CITY, true, second_dirname, second_city_db_path] }
original = db_manager.instance_variable_get(:@database_path) let(:valid_asn_fetch) { [ASN, true, second_dirname, second_asn_db_path] }
expect(mock_download_manager).to receive(:fetch_database).and_return([false, nil]) let(:invalid_city_fetch) { [CITY, false, nil, nil] }
allow(mock_metadata).to receive(:save_timestamp)
expect(db_manager.send(:execute_download_job)).to be_falsey context "plugin is set" do
expect(db_manager.instance_variable_get(:@database_path)).to eq(original) let(:db_manager) do
manager = Class.new(LogStash::Filters::Geoip::DatabaseManager).instance
manager.instance_variable_set(:@metadata, mock_metadata)
manager.instance_variable_set(:@download_manager, mock_download_manager)
manager.instance_variable_set(:@scheduler, mock_scheduler)
manager.instance_variable_get(:@states)[CITY].plugins.push(mock_geoip_plugin)
manager.instance_variable_get(:@states)[CITY].is_eula = true
manager.instance_variable_get(:@states)[ASN].plugins.push(mock_geoip_plugin)
manager.instance_variable_get(:@states)[ASN].is_eula = true
manager
end
it "should update states when new downloads are valid" do
expect(mock_download_manager).to receive(:fetch_database).and_return([valid_city_fetch, valid_asn_fetch])
expect(mock_metadata).to receive(:save_metadata).at_least(:twice)
allow(mock_geoip_plugin).to receive_message_chain('execution_context.pipeline_id').and_return('pipeline_1', 'pipeline_2')
expect(mock_geoip_plugin).to receive(:update_filter).with(:update, instance_of(String)).at_least(:twice)
expect(mock_metadata).to receive(:update_timestamp).never
expect(db_manager).to receive(:check_age)
expect(db_manager).to receive(:clean_up_database)
db_manager.send(:execute_download_job)
expect(db_manager.database_path(CITY)).to match /#{second_dirname}\/#{default_city_db_name}/
expect(db_manager.database_path(ASN)).to match /#{second_dirname}\/#{default_asn_db_name}/
end
end end
it "should return true if update" do it "should update single state when new downloads are partially valid" do
original = db_manager.instance_variable_get(:@database_path) expect(mock_download_manager).to receive(:fetch_database).and_return([invalid_city_fetch, valid_asn_fetch])
expect(mock_download_manager).to receive(:fetch_database).and_return([true, "NEW_PATH"]) expect(mock_metadata).to receive(:save_metadata).with(ASN, second_dirname, true).at_least(:once)
allow(mock_metadata).to receive(:save_timestamp) expect(mock_metadata).to receive(:update_timestamp).never
expect(db_manager).to receive(:check_age)
expect(db_manager).to receive(:clean_up_database)
expect(db_manager.send(:execute_download_job)).to be_truthy db_manager.send(:execute_download_job)
expect(db_manager.instance_variable_get(:@database_path)).not_to eq(original) expect(db_manager.database_path(CITY)).to match /#{CC}\/#{default_city_db_name}/
expect(db_manager.database_path(ASN)).to match /#{second_dirname}\/#{default_asn_db_name}/
end end
it "should raise error when 30 days has passed" do it "should update single state and single metadata timestamp when one database got update" do
allow(mock_download_manager).to receive(:fetch_database).and_raise("boom") expect(mock_download_manager).to receive(:fetch_database).and_return([valid_asn_fetch])
expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i) expect(mock_metadata).to receive(:save_metadata).with(ASN, second_dirname, true).at_least(:once)
expect(mock_metadata).to receive(:cc?).and_return(false) expect(mock_metadata).to receive(:update_timestamp).with(CITY).at_least(:once)
expect(db_manager).to receive(:check_age)
expect(db_manager).to receive(:clean_up_database)
expect{ db_manager.send(:execute_download_job) }.to raise_error /be compliant/ db_manager.send(:execute_download_job)
expect(db_manager.database_path(CITY)).to match /#{CC}\/#{default_city_db_name}/
expect(db_manager.database_path(ASN)).to match /#{second_dirname}\/#{default_asn_db_name}/
end end
it "should update metadata timestamp for the unchange (no update)" do
expect(mock_download_manager).to receive(:fetch_database).and_return([])
expect(mock_metadata).to receive(:save_metadata).never
expect(mock_metadata).to receive(:update_timestamp).at_least(:twice)
expect(db_manager).to receive(:check_age)
expect(db_manager).to receive(:clean_up_database)
it "should return false when 25 days has passed" do db_manager.send(:execute_download_job)
allow(mock_download_manager).to receive(:fetch_database).and_raise("boom") expect(db_manager.database_path(CITY)).to match /#{CC}\/#{default_city_db_name}/
expect(db_manager.database_path(ASN)).to match /#{CC}\/#{default_asn_db_name}/
end
expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 25)).to_i) it "should not update metadata when fetch database throw exception" do
expect(mock_metadata).to receive(:cc?).and_return(false) expect(mock_download_manager).to receive(:fetch_database).and_raise('boom')
expect(db_manager).to receive(:check_age)
expect(db_manager).to receive(:clean_up_database)
expect(mock_metadata).to receive(:save_metadata).never
expect(db_manager.send(:execute_download_job)).to be_falsey db_manager.send(:execute_download_job)
end end
end end
context "scheduler call" do context "check age" do
it "should call plugin expiry action when raise error and last update > 30 days" do context "eula database" do
allow(mock_download_manager).to receive(:fetch_database).and_raise("boom") let(:db_manager) do
expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i) manager = Class.new(LogStash::Filters::Geoip::DatabaseManager).instance
expect(mock_metadata).to receive(:cc?).and_return(false) manager.instance_variable_set(:@metadata, mock_metadata)
expect(mock_geoip_plugin).to receive(:expire_action) manager.instance_variable_set(:@download_manager, mock_download_manager)
db_manager.send(:call, nil, nil) manager.instance_variable_set(:@scheduler, mock_scheduler)
manager.instance_variable_get(:@states)[CITY].plugins.push(mock_geoip_plugin)
manager.instance_variable_get(:@states)[CITY].is_eula = true
manager.instance_variable_get(:@states)[ASN].plugins.push(mock_geoip_plugin)
manager.instance_variable_get(:@states)[ASN].is_eula = true
manager
end
it "should give warning after 25 days" do
expect(mock_metadata).to receive(:check_at).and_return((Time.now - (60 * 60 * 24 * 26)).to_i).at_least(:twice)
expect(mock_geoip_plugin).to receive(:update_filter).never
allow(LogStash::Filters::Geoip::DatabaseManager).to receive(:logger).at_least(:once).and_return(logger)
expect(logger).to receive(:warn).at_least(:twice)
db_manager.send(:check_age)
end
it "should log error and update plugin filter when 30 days has passed" do
expect(mock_metadata).to receive(:check_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i).at_least(:twice)
allow(mock_geoip_plugin).to receive_message_chain('execution_context.pipeline_id').and_return('pipeline_1', 'pipeline_2')
expect(mock_geoip_plugin).to receive(:update_filter).with(:expire).at_least(:twice)
db_manager.send(:check_age)
end
end end
it "should not call plugin setup when database is up to date" do context "cc database" do
allow(mock_download_manager).to receive(:fetch_database).and_return([false, nil]) it "should not give warning after 25 days" do
expect(mock_metadata).to receive(:save_timestamp) expect(mock_geoip_plugin).to receive(:update_filter).never
allow(mock_geoip_plugin).to receive(:setup_filter).never expect(logger).to receive(:warn).never
db_manager.send(:call, nil, nil)
end
it "should call scheduler when has update" do db_manager.send(:check_age)
allow(db_manager).to receive(:execute_download_job).and_return(true) end
allow(mock_geoip_plugin).to receive(:setup_filter).once
allow(db_manager).to receive(:clean_up_database).once it "should not log error when 30 days has passed" do
db_manager.send(:call, nil, nil) expect(logger).to receive(:error).never
expect(mock_geoip_plugin).to receive(:update_filter).never
db_manager.send(:check_age)
end
end end
end end
context "clean up database" do context "clean up database" do
let(:asn00) { get_file_path("GeoLite2-ASN_000000000.mmdb") } let(:dirname) { "0123456789" }
let(:asn00gz) { get_file_path("GeoLite2-ASN_000000000.tgz") } let(:dirname2) { "9876543210" }
let(:city00) { get_file_path("GeoLite2-City_000000000.mmdb") } let(:dir_path) { get_dir_path(dirname) }
let(:city00gz) { get_file_path("GeoLite2-City_000000000.tgz") } let(:dir_path2) { get_dir_path(dirname2) }
let(:city44) { get_file_path("GeoLite2-City_4444444444.mmdb") } let(:asn00) { get_file_path(dirname, default_asn_db_name) }
let(:city44gz) { get_file_path("GeoLite2-City_4444444444.tgz") } let(:city00) { get_file_path(dirname, default_city_db_name) }
let(:asn02) { get_file_path(dirname2, default_asn_db_name) }
let(:city02) { get_file_path(dirname2, default_city_db_name) }
before(:each) do before(:each) do
[asn00, asn00gz, city00, city00gz, city44, city44gz].each { |file_path| ::File.delete(file_path) if ::File.exist?(file_path) } FileUtils.mkdir_p [dir_path, dir_path2]
end
it "should not delete when metadata file doesn't exist" do
expect(mock_metadata).to receive(:exist?).and_return(false)
allow(mock_geoip_plugin).to receive(:database_filenames).never
db_manager.send(:clean_up_database)
end end
it "should delete file which is not in metadata" do it "should delete file which is not in metadata" do
[asn00, asn00gz, city00, city00gz, city44, city44gz].each { |file_path| FileUtils.touch(file_path) } FileUtils.touch [asn00, city00, asn02, city02]
expect(mock_metadata).to receive(:exist?).and_return(true) expect(mock_metadata).to receive(:dirnames).and_return([dirname])
expect(mock_metadata).to receive(:database_filenames).and_return(["GeoLite2-City_4444444444.mmdb"])
db_manager.send(:clean_up_database) db_manager.send(:clean_up_database)
[asn00, asn00gz, city00, city00gz, city44gz].each { |file_path| expect(::File.exist?(file_path)).to be_falsey }
[default_city_db_path, default_asn_db_path, city44].each { |file_path| expect(::File.exist?(file_path)).to be_truthy }
end
it "should keep the default database" do [asn02, city02].each { |file_path| expect(::File.exist?(file_path)).to be_falsey }
expect(mock_metadata).to receive(:exist?).and_return(true) [get_dir_path(CC), asn00, city00].each { |file_path| expect(::File.exist?(file_path)).to be_truthy }
expect(mock_metadata).to receive(:database_filenames).and_return(["GeoLite2-City_4444444444.mmdb"])
db_manager.send(:clean_up_database)
[default_city_db_path, default_asn_db_path].each { |file_path| expect(::File.exist?(file_path)).to be_truthy }
end end
end end
context "setup metadata" do context "subscribe database path" do
let(:db_metadata) do it "should return user input path" do
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City") path = db_manager.subscribe_database_path(CITY, default_city_db_path, mock_geoip_plugin)
dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) expect(db_manager.instance_variable_get(:@states)[CITY].plugins.size).to eq(0)
dbm expect(path).to eq(default_city_db_path)
end end
let(:temp_metadata_path) { db_metadata.instance_variable_get(:@metadata_path) } it "should return database path in state if no user input" do
expect(db_manager.instance_variable_get(:@states)[CITY].plugins.size).to eq(0)
before(:each) do allow(db_manager).to receive(:trigger_download)
expect(::File.empty?(temp_metadata_path)).to be_truthy path = db_manager.subscribe_database_path(CITY, nil, mock_geoip_plugin)
allow(LogStash::Filters::Geoip::DatabaseMetadata).to receive(:new).and_return(db_metadata) expect(db_manager.instance_variable_get(:@states)[CITY].plugins.size).to eq(1)
expect(path).to eq(default_city_db_path)
end end
after(:each) do context "when eula database is expired" do
::File.delete(second_city_db_path) if ::File.exist?(second_city_db_path) let(:db_manager) do
end manager = Class.new(LogStash::Filters::Geoip::DatabaseManager).instance
manager.instance_variable_set(:@download_manager, mock_download_manager)
manager.instance_variable_set(:@scheduler, mock_scheduler)
manager
end
it "create metadata when file is missing" do before do
db_manager.send(:setup) rewrite_temp_metadata(metadata_path, [city_expired_metadata])
expect(db_manager.instance_variable_get(:@database_path)).to eql(default_city_db_path) end
expect(db_metadata.database_path).to eql(default_city_db_path)
expect(::File.exist?(temp_metadata_path)).to be_truthy
expect(::File.empty?(temp_metadata_path)).to be_falsey
end
it "manager should use database path in metadata" do it "should return nil" do
write_temp_metadata(temp_metadata_path, city2_metadata) allow(mock_download_manager).to receive(:fetch_database).and_raise("boom")
copy_city_database(second_city_db_name) expect(db_manager.instance_variable_get(:@states)[CITY].plugins.size).to eq(0)
expect(db_metadata).to receive(:save_timestamp).never path = db_manager.subscribe_database_path(CITY, nil, mock_geoip_plugin)
expect(db_manager.instance_variable_get(:@states)[CITY].plugins.size).to eq(1)
db_manager.send(:setup) expect(path).to be_nil
filename = db_manager.instance_variable_get(:@database_path).split('/').last end
expect(filename).to match /#{second_city_db_name}/
end
it "ignore database_path in metadata if md5 does not match" do
write_temp_metadata(temp_metadata_path, ["City","","","INVALID_MD5",second_city_db_name])
copy_city_database(second_city_db_name)
expect(db_metadata).to receive(:save_timestamp).never
db_manager.send(:setup)
filename = db_manager.instance_variable_get(:@database_path).split('/').last
expect(filename).to match /#{default_city_db_name}/
end end
end end
context "unsubscribe" do
let(:db_manager) do
manager = Class.new(LogStash::Filters::Geoip::DatabaseManager).instance
manager.instance_variable_set(:@metadata, mock_metadata)
manager.instance_variable_set(:@download_manager, mock_download_manager)
manager.instance_variable_set(:@scheduler, mock_scheduler)
manager.instance_variable_get(:@states)[CITY].plugins.push(mock_geoip_plugin)
manager.instance_variable_get(:@states)[CITY].is_eula = true
manager
end
it "should remove plugin in state" do
db_manager.unsubscribe_database_path(CITY, mock_geoip_plugin)
expect(db_manager.instance_variable_get(:@states)[CITY].plugins.size).to eq(0)
end
end
end end
end end

150
x-pack/spec/filters/geoip/database_metadata_spec.rb
View file

@ -6,22 +6,20 @@ require_relative 'test_helper'
require "filters/geoip/database_metadata" require "filters/geoip/database_metadata"
require "filters/geoip/database_manager" require "filters/geoip/database_manager"
require "stud/temporary" require "stud/temporary"
require "fileutils"
describe LogStash::Filters::Geoip do describe LogStash::Filters::Geoip do
describe 'DatabaseMetadata', :aggregate_failures do describe 'DatabaseMetadata', :aggregate_failures do
let(:database_type) { LogStash::Filters::Geoip::CITY }
let(:dbm) do let(:dbm) do
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City") dbm = LogStash::Filters::Geoip::DatabaseMetadata.new
dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
dbm dbm
end end
let(:temp_metadata_path) { dbm.instance_variable_get(:@metadata_path) } let(:temp_metadata_path) { dbm.instance_variable_get(:@metadata_path) }
let(:logger) { double("Logger") } let(:logger) { double("Logger") }
before(:each) do
LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db
end
context "get all" do context "get all" do
it "return multiple rows" do it "return multiple rows" do
write_temp_metadata(temp_metadata_path, city2_metadata) write_temp_metadata(temp_metadata_path, city2_metadata)
@ -34,65 +32,68 @@ describe LogStash::Filters::Geoip do
it "return metadata" do it "return metadata" do
write_temp_metadata(temp_metadata_path, city2_metadata) write_temp_metadata(temp_metadata_path, city2_metadata)
city = dbm.get_metadata city = dbm.get_metadata(database_type)
expect(city.size).to eq(2) expect(city.size).to eq(2)
asn = dbm.get_metadata(false) asn = dbm.get_metadata("ASN")
expect(asn.size).to eq(1) expect(asn.size).to eq(1)
end end
it "return empty array when file is missing" do it "return empty array when file is missing" do
metadata = dbm.get_metadata metadata = dbm.get_metadata(database_type)
expect(metadata.size).to eq(0) expect(metadata.size).to eq(0)
end end
it "return empty array when an empty file exist" do it "return empty array when an empty file exist" do
FileUtils.touch(temp_metadata_path) FileUtils.touch(temp_metadata_path)
metadata = dbm.get_metadata metadata = dbm.get_metadata(database_type)
expect(metadata.size).to eq(0) expect(metadata.size).to eq(0)
end end
end end
context "save timestamp" do context "save timestamp" do
before do
::File.open(default_city_gz_path, "w") { |f| f.write "make a non empty file" }
end
after do
delete_file(default_city_gz_path)
end
it "write the current time" do it "write the current time" do
dbm.save_timestamp(default_city_db_path) write_temp_metadata(temp_metadata_path)
dbm.save_metadata(database_type, second_dirname, true)
metadata = dbm.get_metadata.last expect(dbm.get_metadata(database_type).size).to eq(1)
expect(dbm.get_all.size).to eq(2)
metadata = dbm.get_metadata(database_type).last
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::DATABASE_TYPE]).to eq("City") expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::DATABASE_TYPE]).to eq("City")
past = metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::UPDATE_AT] past = metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::CHECK_AT]
expect(Time.now.to_i - past.to_i).to be < 100 expect(Time.now.to_i - past.to_i).to be < 100
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).not_to be_empty
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).to eq(md5(default_city_gz_path)) expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).to eq(md5(default_city_gz_path))
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_city_db_md5) expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::DIRNAME]).to eq(second_dirname)
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::FILENAME]).to eq(default_city_db_name) expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::IS_EULA]).to eq("true")
end end
end end
context "database path" do context "database path" do
before do
copy_cc(get_dir_path("CC"))
copy_cc(get_dir_path(second_dirname))
end
it "return the default city database path" do it "return the default city database path" do
write_temp_metadata(temp_metadata_path) write_temp_metadata(temp_metadata_path)
expect(dbm.database_path).to eq(default_city_db_path) expect(dbm.database_path(database_type)).to eq(default_city_db_path)
end end
it "return the last database path with valid md5" do context "when the database exist" do
write_temp_metadata(temp_metadata_path, city2_metadata) it "return the last database path with valid md5" do
write_temp_metadata(temp_metadata_path, city2_metadata)
expect(dbm.database_path).to eq(default_city_db_path) expect(dbm.database_path(database_type)).to eq(second_city_db_path)
end
end end
context "with ASN database type" do context "with ASN database type" do
let(:database_type) { "ASN" }
let(:dbm) do let(:dbm) do
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN") dbm = LogStash::Filters::Geoip::DatabaseMetadata.new
dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
dbm dbm
end end
@ -100,13 +101,14 @@ describe LogStash::Filters::Geoip do
it "return the default asn database path" do it "return the default asn database path" do
write_temp_metadata(temp_metadata_path) write_temp_metadata(temp_metadata_path)
expect(dbm.database_path).to eq(default_asn_db_path) expect(dbm.database_path(database_type)).to eq(default_asn_db_path)
end end
end end
context "with invalid database type" do context "with invalid database type" do
let(:database_type) { "???" }
let(:dbm) do let(:dbm) do
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???") dbm = LogStash::Filters::Geoip::DatabaseMetadata.new
dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
dbm dbm
end end
@ -114,38 +116,30 @@ describe LogStash::Filters::Geoip do
it "return nil if md5 not matched" do it "return nil if md5 not matched" do
write_temp_metadata(temp_metadata_path) write_temp_metadata(temp_metadata_path)
expect(dbm.database_path).to be_nil expect(dbm.database_path(database_type)).to be_nil
end end
end end
end end
context "gz md5" do context "gz md5" do
it "should give the last gz md5" do it "should give the last gz md5" do
write_temp_metadata(temp_metadata_path, ["City","","SOME_GZ_MD5","SOME_MD5",second_city_db_name]) write_temp_metadata(temp_metadata_path, ["City","","SOME_GZ_MD5","SOME_MD5",second_dirname])
expect(dbm.gz_md5).to eq("SOME_GZ_MD5") expect(dbm.gz_md5(database_type)).to eq("SOME_GZ_MD5")
end end
it "should give empty string if metadata is empty" do it "should give empty string if metadata is empty" do
expect(dbm.gz_md5).to eq("") expect(dbm.gz_md5(database_type)).to eq("")
end end
end end
context "updated at" do context "updated at" do
it "should give the last update timestamp" do it "should give the last update timestamp" do
write_temp_metadata(temp_metadata_path, ["City","1611690807","SOME_GZ_MD5","SOME_MD5",second_city_db_name]) write_temp_metadata(temp_metadata_path, ["City","1611690807","SOME_GZ_MD5",second_dirname,true])
expect(dbm.updated_at).to eq(1611690807) expect(dbm.check_at(database_type)).to eq(1611690807)
end end
it "should give 0 if metadata is empty" do it "should give 0 if metadata is empty" do
expect(dbm.updated_at).to eq(0) expect(dbm.check_at(database_type)).to eq(0)
end
end
context "database filenames" do
it "should give filename in .mmdb .tgz" do
write_temp_metadata(temp_metadata_path)
expect(dbm.database_filenames).to match_array([default_city_db_name, default_asn_db_name,
'GeoLite2-City.tgz', 'GeoLite2-ASN.tgz'])
end end
end end
@ -161,15 +155,69 @@ describe LogStash::Filters::Geoip do
end end
end end
context "is cc" do context "is eula" do
it "should return true if database is CC" do it "should give boolean false if database is CC" do
write_temp_metadata(temp_metadata_path) write_temp_metadata(temp_metadata_path)
expect(dbm.cc?).to be_truthy expect(dbm.is_eula(database_type)).to eq(false)
end end
it "should return false if database is EULA" do it "should give boolean true if database is EULA" do
write_temp_metadata(temp_metadata_path, ["City","1611690807","SOME_GZ_MD5","SOME_MD5",second_city_db_name]) write_temp_metadata(temp_metadata_path, city2_metadata)
expect(dbm.cc?).to be_falsey expect(dbm.is_eula(database_type)).to eq(true)
end
end
context "update timestamp" do
it "should update timestamp only for database type" do
write_temp_metadata(temp_metadata_path)
original = dbm.get_all
sleep(2)
dbm.update_timestamp(database_type)
updated = dbm.get_all
original.size.times do |i|
expect(original[i][LogStash::Filters::Geoip::DatabaseMetadata::Column::DATABASE_TYPE]).
to(eq(updated[i][LogStash::Filters::Geoip::DatabaseMetadata::Column::DATABASE_TYPE]))
expect(original[i][LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5])
.to(eq(updated[i][LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]))
expect(original[i][LogStash::Filters::Geoip::DatabaseMetadata::Column::DIRNAME])
.to(eq(updated[i][LogStash::Filters::Geoip::DatabaseMetadata::Column::DIRNAME]))
expect(original[i][LogStash::Filters::Geoip::DatabaseMetadata::Column::IS_EULA])
.to(eq(updated[i][LogStash::Filters::Geoip::DatabaseMetadata::Column::IS_EULA]))
end
# ASN
expect(original[0][LogStash::Filters::Geoip::DatabaseMetadata::Column::CHECK_AT])
.to(eq(updated[0][LogStash::Filters::Geoip::DatabaseMetadata::Column::CHECK_AT]))
# City
expect(original[1][LogStash::Filters::Geoip::DatabaseMetadata::Column::CHECK_AT])
.not_to(eq(updated[1][LogStash::Filters::Geoip::DatabaseMetadata::Column::CHECK_AT]))
end
end
context "reset md5" do
it "should reset md5 to empty string only" do
rewrite_temp_metadata(temp_metadata_path, [ ["ASN","1620246514","SOME MD5","1620246514",true],
["City","1620246514","SOME MD5","1620246514",true] ])
dbm.reset_md5(database_type)
row = dbm.get_metadata(database_type).last
expect(row[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).to be_empty
expect(row[LogStash::Filters::Geoip::DatabaseMetadata::Column::DIRNAME]).to eql("1620246514")
expect(row[LogStash::Filters::Geoip::DatabaseMetadata::Column::IS_EULA]).to be_truthy
end
end
context "dirnames" do
it "should reset md5 to empty string only" do
write_temp_metadata(temp_metadata_path, city2_metadata)
rewrite_temp_metadata(temp_metadata_path, [ ["ASN","1620246514","SOME MD5","CC",true],
city2_metadata ])
dirnames = dbm.dirnames
expect(dirnames).to match_array([second_dirname, "CC"])
end end
end end
end end

120
x-pack/spec/filters/geoip/download_manager_spec.rb
View file

@ -12,9 +12,10 @@ describe LogStash::Filters::Geoip do
describe 'DownloadManager', :aggregate_failures do describe 'DownloadManager', :aggregate_failures do
let(:mock_metadata) { double("database_metadata") } let(:mock_metadata) { double("database_metadata") }
let(:download_manager) do let(:download_manager) do
manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata) manager = LogStash::Filters::Geoip::DownloadManager.new(mock_metadata)
manager manager
end end
let(:database_type) { LogStash::Filters::Geoip::CITY }
let(:logger) { double("Logger") } let(:logger) { double("Logger") }
GEOIP_STAGING_HOST = "https://geoip.elastic.dev" GEOIP_STAGING_HOST = "https://geoip.elastic.dev"
@ -22,7 +23,6 @@ describe LogStash::Filters::Geoip do
before do before do
stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT) stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT)
LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db
end end
context "rest client" do context "rest client" do
@ -47,37 +47,34 @@ describe LogStash::Filters::Geoip do
allow(download_manager).to receive_message_chain("rest_client.get").and_return(mock_resp) allow(download_manager).to receive_message_chain("rest_client.get").and_return(mock_resp)
end end
it "should return has_update and db info when md5 does not match" do it "should return City db info when City md5 does not match" do
expect(mock_metadata).to receive(:gz_md5).and_return("") expect(mock_metadata).to receive(:gz_md5).and_return("8d57aec1958070f01042ac1ecd8ec2ab", "a123a45d67890a2bd02e5edd680f6703c")
has_update, info = download_manager.send(:check_update) updated_db = download_manager.send(:check_update)
expect(has_update).to be_truthy expect(updated_db.size).to eql(1)
type, info = updated_db[0]
expect(info).to have_key("md5_hash") expect(info).to have_key("md5_hash")
expect(info).to have_key("name") expect(info).to have_key("name")
expect(info).to have_key("provider") expect(info).to have_key("provider")
expect(info).to have_key("updated") expect(info).to have_key("updated")
expect(info).to have_key("url") expect(info).to have_key("url")
expect(info["name"]).to include("City") expect(type).to eql(database_type)
end end
it "should return false when md5 is the same" do it "should return empty array when md5 are the same" do
expect(mock_metadata).to receive(:gz_md5).and_return("89d225ac546310b1e7979502ac9ad11c") expect(mock_metadata).to receive(:gz_md5).and_return("8d57aec1958070f01042ac1ecd8ec2ab", "a195a73d4651a2bd02e5edd680f6703c")
has_update, info = download_manager.send(:check_update) updated_db = download_manager.send(:check_update)
expect(has_update).to be_falsey expect(updated_db.size).to eql(0)
end end
it "should return true when md5 does not match" do
expect(mock_metadata).to receive(:gz_md5).and_return("bca2a8bad7e5e4013dc17343af52a841")
has_update, info = download_manager.send(:check_update)
expect(has_update).to be_truthy
end
end end
context "download database" do context "download database" do
let(:db_info) do let(:db_info) do
{ {
"age" => 297221,
"md5_hash" => md5_hash, "md5_hash" => md5_hash,
"name" => filename, "name" => filename,
"provider" => "maxmind", "provider" => "maxmind",
@ -87,30 +84,34 @@ describe LogStash::Filters::Geoip do
end end
let(:md5_hash) { SecureRandom.hex } let(:md5_hash) { SecureRandom.hex }
let(:filename) { "GeoLite2-City.tgz"} let(:filename) { "GeoLite2-City.tgz"}
let(:dirname) { "0123456789" }
it "should raise error if md5 does not match" do it "should raise error if md5 does not match" do
allow(Down).to receive(:download) allow(Down).to receive(:download)
expect{ download_manager.send(:download_database, db_info) }.to raise_error /wrong checksum/ expect{ download_manager.send(:download_database, database_type, dirname, db_info) }.to raise_error /wrong checksum/
end end
it "should download file and return zip path" do it "should download file and return zip path" do
expect(download_manager).to receive(:md5).and_return(md5_hash) expect(download_manager).to receive(:md5).and_return(md5_hash)
path = download_manager.send(:download_database, db_info) new_zip_path = download_manager.send(:download_database, database_type, dirname, db_info)
expect(path).to match /GeoLite2-City_\d+\.tgz/ expect(new_zip_path).to match /GeoLite2-City\.tgz/
expect(::File.exist?(path)).to be_truthy expect(::File.exist?(new_zip_path)).to be_truthy
delete_file(path)
end end
end end
context "unzip" do context "unzip" do
let(:copyright_path) { get_file_path('COPYRIGHT.txt') } let(:dirname) { Time.now.to_i.to_s }
let(:license_path) { get_file_path('LICENSE.txt') } let(:copyright_path) { get_file_path(dirname, 'COPYRIGHT.txt') }
let(:readme_path) { get_file_path('README.txt') } let(:license_path) { get_file_path(dirname, 'LICENSE.txt') }
let(:folder_path) { get_file_path('inner') } let(:readme_path) { get_file_path(dirname, 'README.txt') }
let(:folder_more_path) { ::File.join(get_file_path('inner'), 'more.txt') } let(:folder_path) { get_file_path(dirname, 'inner') }
let(:folder_less_path) { ::File.join(get_file_path('inner'), 'less.txt') } let(:folder_more_path) { ::File.join(folder_path, 'more.txt') }
let(:folder_less_path) { ::File.join(folder_path, 'less.txt') }
before do
FileUtils.mkdir_p(get_dir_path(dirname))
end
after do after do
file_path = ::File.expand_path("./fixtures/sample.mmdb", ::File.dirname(__FILE__)) file_path = ::File.expand_path("./fixtures/sample.mmdb", ::File.dirname(__FILE__))
@ -119,11 +120,11 @@ describe LogStash::Filters::Geoip do
end end
it "should extract all files in tarball" do it "should extract all files in tarball" do
path = ::File.expand_path("./fixtures/sample.tgz", ::File.dirname(__FILE__)) zip_path = ::File.expand_path("./fixtures/sample.tgz", ::File.dirname(__FILE__))
unzip_db_path = download_manager.send(:unzip, path) new_db_path = download_manager.send(:unzip, database_type, dirname, zip_path)
expect(unzip_db_path).to match /\.mmdb/ expect(new_db_path).to match /GeoLite2-#{database_type}\.mmdb/
expect(::File.exist?(unzip_db_path)).to be_truthy expect(::File.exist?(new_db_path)).to be_truthy
expect(::File.exist?(copyright_path)).to be_truthy expect(::File.exist?(copyright_path)).to be_truthy
expect(::File.exist?(license_path)).to be_truthy expect(::File.exist?(license_path)).to be_truthy
expect(::File.exist?(readme_path)).to be_truthy expect(::File.exist?(readme_path)).to be_truthy
@ -134,6 +135,10 @@ describe LogStash::Filters::Geoip do
end end
context "assert database" do context "assert database" do
before do
copy_cc(get_dir_path("CC"))
end
it "should raise error if file is invalid" do it "should raise error if file is invalid" do
expect{ download_manager.send(:assert_database!, "Gemfile") }.to raise_error /failed to load database/ expect{ download_manager.send(:assert_database!, "Gemfile") }.to raise_error /failed to load database/
end end
@ -144,31 +149,38 @@ describe LogStash::Filters::Geoip do
end end
context "fetch database" do context "fetch database" do
it "should be false if no update" do it "should return array of db which has valid download" do
expect(download_manager).to receive(:check_update).and_return([false, {}]) expect(download_manager).to receive(:check_update).and_return([[LogStash::Filters::Geoip::ASN, {}],
[LogStash::Filters::Geoip::CITY, {}]])
has_update, new_database_path = download_manager.send(:fetch_database)
expect(has_update).to be_falsey
expect(new_database_path).to be_nil
end
it "should raise error" do
expect(download_manager).to receive(:check_update).and_return([true, {}])
expect(download_manager).to receive(:download_database).and_raise('boom')
expect { download_manager.send(:fetch_database) }.to raise_error
end
it "should be true if got update" do
expect(download_manager).to receive(:check_update).and_return([true, {}])
allow(download_manager).to receive(:download_database) allow(download_manager).to receive(:download_database)
allow(download_manager).to receive(:unzip) allow(download_manager).to receive(:unzip).and_return("NEW_DATABASE_PATH")
allow(download_manager).to receive(:assert_database!) allow(download_manager).to receive(:assert_database!)
has_update, new_database_path = download_manager.send(:fetch_database) updated_db = download_manager.send(:fetch_database)
expect(has_update).to be_truthy expect(updated_db.size).to eql(2)
asn_type, asn_valid_download, asn_dirname, asn_path = updated_db[0]
city_type, city_valid_download, city_dirname, city_path = updated_db[1]
expect(asn_valid_download).to be_truthy
expect(asn_path).to eql("NEW_DATABASE_PATH")
expect(city_valid_download).to be_truthy
expect(city_path).to eql("NEW_DATABASE_PATH")
end
it "should return array of db which has invalid download" do
expect(download_manager).to receive(:check_update).and_return([[LogStash::Filters::Geoip::ASN, {}],
[LogStash::Filters::Geoip::CITY, {}]])
expect(download_manager).to receive(:download_database).and_raise('boom').at_least(:twice)
updated_db = download_manager.send(:fetch_database)
expect(updated_db.size).to eql(2)
asn_type, asn_valid_download, asn_path = updated_db[0]
city_type, city_valid_download, city_path = updated_db[1]
expect(asn_valid_download).to be_falsey
expect(asn_path).to be_nil
expect(city_valid_download).to be_falsey
expect(city_path).to be_nil
end end
end end

42
x-pack/spec/filters/geoip/fixtures/normal_resp.json
View file

@ -1,44 +1,26 @@
[ [
{ {
"md5_hash": "bcfc39b5677554e091dbb19cd5cea4b0", "age": 297221,
"name": "GeoLite2-ASN.mmdb.gz", "md5_hash": "8d57aec1958070f01042ac1ecd8ec2ab",
"provider": "maxmind",
"updated": 1615852860,
"url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-ASN.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=ada6463b28177577f4981cbe5f29708d0196ed71cea0bf3c0bf8e9965c8f9fd3d184be852c4e84f24b2896d8043a466039e15b5581ba4fc7aa37a15c85c79999674a0966b28f53b0c5a8b1220b428d3c1e958f20a61e06758426b7308f1ba1966b04a2bf86a5a9f96b88c05753b429574829344d3043de1f7d2b93cade7b57d53ac6d3bcb4e6d11405f6f2e7ff8c25d813e3917177b9438f686f10bc4a006aadc6a7dde2343c9bc0017487684ad64f59bb2d0b7b73b3c817f24c91bd9afd2f36725937c8938def67d5cf6df3a7705bb40098548b55a6777ef2cd8e26c32efaa1bd0474f7f24d5e386d90e87d8a3c3aa63203a78004bccf2ad65cc97b26e94675"
},
{
"md5_hash": "be4e335eb819af148fa4e365f176923d",
"name": "GeoLite2-ASN.tgz", "name": "GeoLite2-ASN.tgz",
"provider": "maxmind", "provider": "maxmind",
"updated": 1615939277, "updated": 1619740833,
"url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-ASN.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=8d8566fdf8167d9874966c16663a76bf8a678083c753fae0397de2eaffdb9f1d19ff36dd28bb2dc3bd9230dab5256a6d08d694574b9c50cae4b8614115ef9d3d638caf29eb18cefd7a7f0154e7baaeab4c565c828a2f050bbdbb8f5a9647d67d0748960b77846674097f76ea0d721cadda9fd99379ee604eba692c9274d238a1a3d56b7c29e236182cf5e91bae63b72d1c9a1ee7c598d7c5156683aa71a9776151bec83cb99f07f75a83483d620960fd97eca4e12c3789d72ac272912df74da1d63572609883157c6d2f115f7ab1be6b3e4503e7dd501946124f1250a299338529b8abc199afe52ff9d38904603b12b674149b85d7597e57502fda05c4b65a75" "url": "https://storage.googleapis.com/elastic-paisano-production/maxmind/GeoLite2-ASN.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-production%40elastic-apps-163815.iam.gserviceaccount.com%2F20210430%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210430T000140Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=7212ea23f94454c8008bff13f7a54507b7fa88055f88224ba0d2c8f65bf172c6dfd0844f329223f141b0e1a6e68060df2fe2e5bb1c0bb75aefe246535a30710198d351b36cf59bfc42b0217ae7c3d4ae3da307781dcd603db7768070ac755a45596b1ae77739092b2eeb75b30cf82c6631e02ee48caf1268d0268477fa57f0482511b84ef1f10814dd2aaf7888abfe7c264ebb68c6e477fb2510cb00ed9c469e8fd43e1738dc10fb3702b84c452a12919340bd6fbcbdd1bc75e47a2457c25fb082a280ecd8465cd2eaba38e3f3a1786ec16d06d76fbf6a4b7758665dd517a34ffd583a67a95eff7d7c2106243771bb6aa247e735aeb260b7ebc231c2f2d3404f"
}, },
{ {
"md5_hash": "6cd9be41557fd4c6dd0a8609a3f96bbe", "age": 222085,
"name": "GeoLite2-City.mmdb.gz", "md5_hash": "a195a73d4651a2bd02e5edd680f6703c",
"provider": "maxmind",
"updated": 1615420855,
"url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-City.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=630106105d8f476a6d4e7de9fd777d8c250391ce1fbc799c7c683efeb39b319e1263948bcd326dc15f3ee0c9578f1fc95e5afe2d6b026dfac00b1fe188961df8ce3a8e5e0d71355fc0ea4d7f957af2ce8bf433210b0224d7175122ce0c1ced64dc39d2db7a979c1d173b72da58441a2358f605b92b71355cf00af4fdaa20943f21827506756b52706daaf780f173fe9f37a41fd7fc5539bbc41e79110fc4b00b37334d37179efa78c0a2ccd20ef6a5faff3baf1b5c2dfb2ef0ebb7ae4ef949f986a3cfbc8df4885476aef4ba6c06012a83418623219b48ee7ff04a41ae2ff2f421fb85fcbc04255df174647d6b9302f15441a783252c7443edfa70ef5f44068a"
},
{
"md5_hash": "89d225ac546310b1e7979502ac9ad11c",
"name": "GeoLite2-City.tgz", "name": "GeoLite2-City.tgz",
"provider": "maxmind", "provider": "maxmind",
"updated": 1615939277, "updated": 1619740834,
"url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-City.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=3f5e84337ef78e8039ed391cddbcc92b0ceb3b946d4a7f60476f0633584cd3324356c9ead4bfc19f1c8776849a26b850c7e388386c5dfa8eccc2afe7e7c21d4c7fdd093cfae5c52899d9df5ffe13db6c29a0558329c8a8aecda058f9778dd23615471023fc77cc514d372d9786cbd256e27818883c1ee4b7edee75c393c89d57e94e58c2be2f9c8ee7354864b53f66d61c917eae296e071f84776e8c358218d890333fd376753a4c0f903581480629bca86d1abf3bc65efc7da30617c4847367d0ae24ba1ce0528feba3c3c3c38ecdd9a8d820d7f1a9141e30578822564c192181a97761858b9e06cc05f7db4143c89c402cbb888dcabc1f6559f4f701b79a7c" "url": "https://storage.googleapis.com/elastic-paisano-production/maxmind/GeoLite2-City.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-production%40elastic-apps-163815.iam.gserviceaccount.com%2F20210430%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210430T000140Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=9e27f47fe4f9b92a57e0844eb53b44c4ec8948ff85ddc830394261c2b6ddb9a4bd9b9c56df703d362843ed06a4a6ded5558bcc47ffee5edce9e07e47cc945493d7808b05c5e70001308da0750ed6157dddc00e4eea4b90736d1103b3b437d0bed9468f74902c68d62ca4e8460969fd6adc6c1faf227eb08c70943a3079f9f52350a94038a9229e05cdf38fcb3c0a849e4cbf523c74729b04e0a7e5ac51df1b4baaa3202564e550716a1025baedfcf3c57a057ec57f5cfd38be499cd2ebd4621085eefda750ce60dbab611c8c09dae1625505a6f62b6a7a9164345a6b61ba697a9b0c9d6cf436caac64d5b5da2e75e074eaf87694f67b5f72aa8bccff4297e99b"
}, },
{ {
"md5_hash": "03bef5fb1fdc877304da3391052246dc", "age": 222445,
"name": "GeoLite2-Country.mmdb.gz", "md5_hash": "8b2092c0fc23e620fe4d801f2feff708",
"provider": "maxmind",
"updated": 1615420855,
"url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-Country.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=18d3266f09a8b208573fa48ca9c30cf0041b69de4eac1656cafebcf737a9f2637b0be12f9df4dd26c07bc297a4070cd0248f8874d3d03bb3fc992f7110c1c0def845f182dcc6289d5fe4faa97daf98e3bdcd2e37405bae1f04e1b293c556c352a0c574f7a52f0f0ea92bcbfb5a74542be9e651453c79a0df1f7a84f2d48d5e704ee11df9a180f9c4c76a809c6a7edab7e36b4863556d815042b9cf43fe8bb1c60f432fcae56b1779d610e8b1388addc277b0259ac595eee34227fc9884065c7aaf44c8446c4f00849d3f8dad6eba9cc7213bac33ff166dc86c344fd14da736390615bc4d00de5ba007b0b1013f46b7e81b9827d32ae9e20f779a6580f97164f9"
},
{
"md5_hash": "c0e76a2e7e0f781028e849c2d389d8a1",
"name": "GeoLite2-Country.tgz", "name": "GeoLite2-Country.tgz",
"provider": "maxmind", "provider": "maxmind",
"updated": 1615939276, "updated": 1619740833,
"url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-Country.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=5eaf641191c25f111afed9c569e31a5369733b3723db365b76cfbf93a7b39fd77481fe07f93fc5be2fb9ef987ef6f1c32bcb863d9d2de0e74aeece8ff568c41573c8a465e9ec5301bdc77c75b2ab369f5352f2da3f5262ae889facaf27f1685584ca49fa3bf4556ed0a92b6a4b1f1985f62378c92467d73b0c66fd1ed04cb311b903343249aed6d3ba32d7b80f0be9a08816737016038306886dcffaf141932e5fb06dfe96ff1caf8ed37f6f8128a0bdc6abf9516aeac891a791656d14f4c37b31f4c86d5dba430d92402c78d8b53dcf4ec557f0f8b6c1fb59357ae1aa7f6310289fdf16c094028570431312ea35f2c00f8cd2dcef8b98d2af5ed3ee09a7fefd" "url": "https://storage.googleapis.com/elastic-paisano-production/maxmind/GeoLite2-Country.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-production%40elastic-apps-163815.iam.gserviceaccount.com%2F20210430%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210430T000140Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=33f24e6a7e4660eaa225668cf8424ec7f2b8219b43fe6c7b446ee91eb907ef17871f3a4dba63796e28916317c0aaa581d384b4eb89c4acd1b3a8eaa44cb6a6fd2e43449ba9edc4e20e5a65dc583a13ed1cddc05072b80f8529939d1300da6540dc044811839dd43b13889c89b9101d1e31fb2d050b2af9bf059ab5d403427f4fd206ef845b7b985c7316a6943873c278000cccc14192576a639a1c5eb714c3673eb6dc286af09be244f3e01f5306bcb20787847d543598a64287657d39a7721f91f0afeb41309221d717515956f773625aaf7a6e3f51e5480f3dda601b0068523c4a84ca75070147b4d95bbce3c40bbe13ec152325ce0ce1ea394174f318bde5"
} }
] ]

73
x-pack/spec/filters/geoip/test_helper.rb
View file

@ -6,16 +6,16 @@ require 'spec_helper'
require "digest" require "digest"
module GeoipHelper module GeoipHelper
def get_vendor_path def get_data_dir_path
::File.expand_path("vendor", ::File.dirname(__FILE__))
end
def get_data_dir
::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip") ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
end end
def get_file_path(filename) def get_dir_path(dirname)
::File.join(get_data_dir, filename) ::File.join(get_data_dir_path, dirname)
end
def get_file_path(dirname, filename)
::File.join(get_dir_path(dirname), filename)
end end
def md5(file_path) def md5(file_path)
@ -23,20 +23,19 @@ module GeoipHelper
end end
def default_city_db_path def default_city_db_path
get_file_path("GeoLite2-City.mmdb") ::File.join(get_data_dir_path, "CC", default_city_db_name )
end end
def default_city_gz_path def default_city_gz_path
get_file_path("GeoLite2-City.tgz") ::File.join(get_data_dir_path, "CC", "GeoLite2-City.tgz" )
end end
def default_asn_db_path def default_asn_db_path
get_file_path("GeoLite2-ASN.mmdb") ::File.join(get_data_dir_path, "CC", default_asn_db_name )
end end
def metadata_path def metadata_path
get_file_path("metadata.csv") ::File.join(get_data_dir_path, "metadata.csv")
end end
def default_city_db_name def default_city_db_name
@ -47,39 +46,58 @@ module GeoipHelper
"GeoLite2-ASN.mmdb" "GeoLite2-ASN.mmdb"
end end
def second_city_db_name def second_city_db_path
"GeoLite2-City_20200220.mmdb" ::File.join(get_data_dir_path, second_dirname, default_city_db_name )
end end
def second_city_db_path def second_asn_db_path
get_file_path("GeoLite2-City_20200220.mmdb") ::File.join(get_data_dir_path, second_dirname, default_asn_db_name )
end
def second_dirname
"20200220"
end
def create_default_city_gz
::File.open(default_city_gz_path, "w") { |f| f.write "make a non empty file" }
end end
def default_city_db_md5 def default_city_db_md5
md5(default_city_db_path) md5(default_city_db_path)
end end
def DEFAULT_ASN_DB_MD5 def default_asn_db_md5
md5(default_asn_db_path) md5(default_asn_db_path)
end end
def write_temp_metadata(temp_file_path, row = nil) def write_temp_metadata(temp_file_path, row = nil)
now = Time.now.to_i now = Time.now.to_i
city = md5(default_city_db_path) dirname = "CC"
asn = md5(default_asn_db_path)
metadata = [] metadata = []
metadata << ["ASN",now,"",asn,default_asn_db_name] metadata << ["ASN",now,"",dirname,false]
metadata << ["City",now,"",city,default_city_db_name] metadata << ["City",now,"",dirname,false]
metadata << row if row metadata << row if row
FileUtils.mkdir_p(::File.dirname(temp_file_path))
CSV.open temp_file_path, 'w' do |csv|
metadata.each { |row| csv << row }
end
end
def rewrite_temp_metadata(temp_file_path, metadata = [])
FileUtils.mkdir_p(::File.dirname(temp_file_path))
CSV.open temp_file_path, 'w' do |csv| CSV.open temp_file_path, 'w' do |csv|
metadata.each { |row| csv << row } metadata.each { |row| csv << row }
end end
end end
def city2_metadata def city2_metadata
["City",Time.now.to_i,"",md5(default_city_db_path),second_city_db_name] ["City",Time.now.to_i,"",second_dirname,true]
end
def city_expired_metadata
["City","1220227200","","1220227200",true]
end end
def copy_city_database(filename) def copy_city_database(filename)
@ -88,13 +106,20 @@ module GeoipHelper
end end
def delete_file(*filepaths) def delete_file(*filepaths)
filepaths.map { |filepath| ::File.delete(filepath) if ::File.exist?(filepath) } filepaths.map { |filepath| FileUtils.rm_r(filepath) if ::File.exist?(filepath) }
end end
def get_metadata_database_name def get_metadata_database_name
::File.exist?(metadata_path) ? ::File.read(metadata_path).split(",").last[0..-2] : nil ::File.exist?(metadata_path) ? ::File.read(metadata_path).split(",").last[0..-2] : nil
end end
def copy_cc(dir_path)
cc_database_paths = ::Dir.glob(::File.expand_path(
::File.join("..", "..", "..", "..", "..", "vendor", "**", "{GeoLite2-ASN,GeoLite2-City}.mmdb"),
__FILE__))
FileUtils.mkdir_p(dir_path)
FileUtils.cp_r(cc_database_paths, dir_path)
end
end end
RSpec.configure do |c| RSpec.configure do |c|