diff --git a/x-pack/lib/filters/geoip/database_manager.rb b/x-pack/lib/filters/geoip/database_manager.rb index 6d86f2b41..733671f59 100644 --- a/x-pack/lib/filters/geoip/database_manager.rb +++ b/x-pack/lib/filters/geoip/database_manager.rb @@ -26,12 +26,14 @@ require "date" # while `offline` is for static database path provided by users module LogStash module Filters module Geoip class DatabaseManager + extend LogStash::Filters::Geoip::Util include LogStash::Util::Loggable include LogStash::Filters::Geoip::Util + #TODO remove vendor_path def initialize(geoip, database_path, database_type, vendor_path) - @vendor_path = vendor_path @geoip = geoip + self.class.prepare_cc_db @mode = database_path.nil? ? :online : :offline @database_type = database_type @database_path = patch_database_path(database_path) @@ -61,6 +63,16 @@ module LogStash module Filters module Geoip class DatabaseManager public + # create data dir, path.data, for geoip if it doesn't exist + # copy CC databases to data dir + def self.prepare_cc_db + FileUtils::mkdir_p(get_data_dir) + unless ::File.exist?(get_file_path(CITY_DB_NAME)) && ::File.exist?(get_file_path(ASN_DB_NAME)) + cc_database_paths = ::Dir.glob(::File.join(LogStash::Environment::LOGSTASH_HOME, "vendor", "**", "{GeoLite2-ASN,GeoLite2-City}.mmdb")) + FileUtils.cp_r(cc_database_paths, get_data_dir) + end + end + def execute_download_job begin has_update, new_database_path = @download_manager.fetch_database @@ -138,12 +150,12 @@ module LogStash module Filters module Geoip class DatabaseManager end def setup - @metadata = DatabaseMetadata.new(@database_type, @vendor_path) + @metadata = DatabaseMetadata.new(@database_type) @metadata.save_timestamp(@database_path) unless @metadata.exist? @database_path = @metadata.database_path || @database_path - @download_manager = DownloadManager.new(@database_type, @metadata, @vendor_path) + @download_manager = DownloadManager.new(@database_type, @metadata) end class DatabaseExpiryError < StandardError diff --git a/x-pack/lib/filters/geoip/database_metadata.rb b/x-pack/lib/filters/geoip/database_metadata.rb index 1274d6d71..55fe773c2 100644 --- a/x-pack/lib/filters/geoip/database_metadata.rb +++ b/x-pack/lib/filters/geoip/database_metadata.rb @@ -11,8 +11,7 @@ module LogStash module Filters module Geoip class DatabaseMetadata include LogStash::Util::Loggable include LogStash::Filters::Geoip::Util - def initialize(database_type, vendor_path) - @vendor_path = vendor_path + def initialize(database_type) @metadata_path = get_file_path("metadata.csv") @database_type = database_type end diff --git a/x-pack/lib/filters/geoip/download_manager.rb b/x-pack/lib/filters/geoip/download_manager.rb index 62f4630a6..9d162ff6c 100644 --- a/x-pack/lib/filters/geoip/download_manager.rb +++ b/x-pack/lib/filters/geoip/download_manager.rb @@ -18,8 +18,7 @@ module LogStash module Filters module Geoip class DownloadManager include LogStash::Util::Loggable include LogStash::Filters::Geoip::Util - def initialize(database_type, metadata, vendor_path) - @vendor_path = vendor_path + def initialize(database_type, metadata) @database_type = database_type @metadata = metadata end @@ -78,7 +77,7 @@ module LogStash module Filters module Geoip class DownloadManager end end - # extract all files and folders from .tgz to vendor directory + # extract all files and folders from .tgz to path.data directory # existing files folders will be replaced def unzip(zip_path) new_database_path = zip_path[0...-(GZ_EXT.length)] + DB_EXT @@ -93,7 +92,7 @@ module LogStash module Filters module Geoip class DownloadManager if !::File.directory?(path) && database_name_ext.eql?(file) FileUtils.cp(path, new_database_path) else - FileUtils.cp_r(path, @vendor_path) + FileUtils.cp_r(path, get_data_dir) end end diff --git a/x-pack/lib/filters/geoip/util.rb b/x-pack/lib/filters/geoip/util.rb index 64d55abd7..55f681b86 100644 --- a/x-pack/lib/filters/geoip/util.rb +++ b/x-pack/lib/filters/geoip/util.rb @@ -10,10 +10,16 @@ module LogStash module Filters GZ_EXT = 'tgz'.freeze DB_EXT = 'mmdb'.freeze DB_PREFIX = 'GeoLite2-'.freeze + CITY_DB_NAME = "#{DB_PREFIX}City.#{DB_EXT}" + ASN_DB_NAME = "#{DB_PREFIX}ASN.#{DB_EXT}" module Util def get_file_path(filename) - ::File.join(@vendor_path, filename) + ::File.join(get_data_dir, filename) + end + + def get_data_dir + ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip") end def file_exist?(path) diff --git a/x-pack/spec/filters/geoip/database_manager_spec.rb b/x-pack/spec/filters/geoip/database_manager_spec.rb index e5f52c5a7..df648e39f 100644 --- a/x-pack/spec/filters/geoip/database_manager_spec.rb +++ b/x-pack/spec/filters/geoip/database_manager_spec.rb @@ -21,6 +21,10 @@ describe LogStash::Filters::Geoip do end let(:logger) { double("Logger") } + before(:each) do + LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db + end + context "patch database" do it "use input path" do path = db_manager.send(:patch_database_path, default_asn_db_path) @@ -168,7 +172,7 @@ describe LogStash::Filters::Geoip do context "setup metadata" do let(:db_metadata) do - dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path) + dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City") dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) dbm end diff --git a/x-pack/spec/filters/geoip/database_metadata_spec.rb b/x-pack/spec/filters/geoip/database_metadata_spec.rb index 83b78927f..41a28ae5e 100644 --- a/x-pack/spec/filters/geoip/database_metadata_spec.rb +++ b/x-pack/spec/filters/geoip/database_metadata_spec.rb @@ -4,19 +4,24 @@ require_relative 'test_helper' require "filters/geoip/database_metadata" +require "filters/geoip/database_manager" require "stud/temporary" describe LogStash::Filters::Geoip do describe 'DatabaseMetadata', :aggregate_failures do let(:dbm) do - dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path) + dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City") dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) dbm end let(:temp_metadata_path) { dbm.instance_variable_get(:@metadata_path) } let(:logger) { double("Logger") } + before(:each) do + LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db + end + context "get all" do it "return multiple rows" do write_temp_metadata(temp_metadata_path, city2_metadata) @@ -67,7 +72,7 @@ describe LogStash::Filters::Geoip do expect(Time.now.to_i - past.to_i).to be < 100 expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).not_to be_empty expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).to eq(md5(default_city_gz_path)) - expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_cith_db_md5) + expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_city_db_md5) expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::FILENAME]).to eq(default_city_db_name) end end @@ -87,7 +92,7 @@ describe LogStash::Filters::Geoip do context "with ASN database type" do let(:dbm) do - dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN", get_vendor_path) + dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN") dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) dbm end @@ -101,7 +106,7 @@ describe LogStash::Filters::Geoip do context "with invalid database type" do let(:dbm) do - dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???", get_vendor_path) + dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???") dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path) dbm end diff --git a/x-pack/spec/filters/geoip/download_manager_spec.rb b/x-pack/spec/filters/geoip/download_manager_spec.rb index 22e092c1d..5242ed412 100644 --- a/x-pack/spec/filters/geoip/download_manager_spec.rb +++ b/x-pack/spec/filters/geoip/download_manager_spec.rb @@ -5,13 +5,14 @@ require_relative 'test_helper' require 'fileutils' require "filters/geoip/download_manager" +require "filters/geoip/database_manager" describe LogStash::Filters::Geoip do describe 'DownloadManager', :aggregate_failures do let(:mock_metadata) { double("database_metadata") } let(:download_manager) do - manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata, get_vendor_path) + manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata) manager end let(:logger) { double("Logger") } @@ -21,6 +22,7 @@ describe LogStash::Filters::Geoip do before do stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT) + LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db end context "rest client" do diff --git a/x-pack/spec/filters/geoip/test_helper.rb b/x-pack/spec/filters/geoip/test_helper.rb index df138a07c..a47d19a94 100644 --- a/x-pack/spec/filters/geoip/test_helper.rb +++ b/x-pack/spec/filters/geoip/test_helper.rb @@ -10,8 +10,12 @@ module GeoipHelper ::File.expand_path("vendor", ::File.dirname(__FILE__)) end + def get_data_dir + ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip") + end + def get_file_path(filename) - ::File.join(get_vendor_path, filename) + ::File.join(get_data_dir, filename) end def md5(file_path) @@ -24,6 +28,7 @@ module GeoipHelper def default_city_gz_path get_file_path("GeoLite2-City.tgz") + end def default_asn_db_path @@ -50,7 +55,7 @@ module GeoipHelper get_file_path("GeoLite2-City_20200220.mmdb") end - def default_cith_db_md5 + def default_city_db_md5 md5(default_city_db_path) end @@ -89,6 +94,7 @@ module GeoipHelper def get_metadata_database_name ::File.exist?(metadata_path) ? ::File.read(metadata_path).split(",").last[0..-2] : nil end + end RSpec.configure do |c|