github-mirrors/logstash
1
0
Fork 0
mirror of https://github.com/elastic/logstash.git synced 2025-04-24 14:47:19 -04:00
Code Issues Projects Releases Packages Wiki Activity

change download path for geoip plugin (#12863) (#12875)

Browse source 
The database was downloaded in plugin/vendor
This PR changes the working directory to `path.data`
This commit is contained in:
kaisecheng 2021-05-10 14:28:44 +02:00 • committed by GitHub
parent b33692e417
commit 68b8fd3a48
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 51 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

18
x-pack/lib/filters/geoip/database_manager.rb
View file

@ -26,12 +26,14 @@ require "date"
# while `offline` is for static database path provided by users
module LogStash module Filters module Geoip class DatabaseManager
extend LogStash::Filters::Geoip::Util
include LogStash::Util::Loggable
include LogStash::Filters::Geoip::Util
#TODO remove vendor_path
def initialize(geoip, database_path, database_type, vendor_path)
@vendor_path = vendor_path
@geoip = geoip
self.class.prepare_cc_db
@mode = database_path.nil? ? :online : :offline
@database_type = database_type
@database_path = patch_database_path(database_path)
@ -61,6 +63,16 @@ module LogStash module Filters module Geoip class DatabaseManager
public
# create data dir, path.data, for geoip if it doesn't exist
# copy CC databases to data dir
def self.prepare_cc_db
FileUtils::mkdir_p(get_data_dir)
unless ::File.exist?(get_file_path(CITY_DB_NAME)) && ::File.exist?(get_file_path(ASN_DB_NAME))
cc_database_paths = ::Dir.glob(::File.join(LogStash::Environment::LOGSTASH_HOME, "vendor", "**", "{GeoLite2-ASN,GeoLite2-City}.mmdb"))
FileUtils.cp_r(cc_database_paths, get_data_dir)
end
end
def execute_download_job
begin
has_update, new_database_path = @download_manager.fetch_database
@ -138,12 +150,12 @@ module LogStash module Filters module Geoip class DatabaseManager
end
def setup
@metadata = DatabaseMetadata.new(@database_type, @vendor_path)
@metadata = DatabaseMetadata.new(@database_type)
@metadata.save_timestamp(@database_path) unless @metadata.exist?
@database_path = @metadata.database_path || @database_path
@download_manager = DownloadManager.new(@database_type, @metadata, @vendor_path)
@download_manager = DownloadManager.new(@database_type, @metadata)
end
class DatabaseExpiryError < StandardError

3
x-pack/lib/filters/geoip/database_metadata.rb
View file

@ -11,8 +11,7 @@ module LogStash module Filters module Geoip class DatabaseMetadata
include LogStash::Util::Loggable
include LogStash::Filters::Geoip::Util
def initialize(database_type, vendor_path)
@vendor_path = vendor_path
def initialize(database_type)
@metadata_path = get_file_path("metadata.csv")
@database_type = database_type
end

7
x-pack/lib/filters/geoip/download_manager.rb
View file

@ -18,8 +18,7 @@ module LogStash module Filters module Geoip class DownloadManager
include LogStash::Util::Loggable
include LogStash::Filters::Geoip::Util
def initialize(database_type, metadata, vendor_path)
@vendor_path = vendor_path
def initialize(database_type, metadata)
@database_type = database_type
@metadata = metadata
end
@ -78,7 +77,7 @@ module LogStash module Filters module Geoip class DownloadManager
end
end
# extract all files and folders from .tgz to vendor directory
# extract all files and folders from .tgz to path.data directory
# existing files folders will be replaced
def unzip(zip_path)
new_database_path = zip_path[0...-(GZ_EXT.length)] + DB_EXT
@ -93,7 +92,7 @@ module LogStash module Filters module Geoip class DownloadManager
if !::File.directory?(path) && database_name_ext.eql?(file)
FileUtils.cp(path, new_database_path)
else
FileUtils.cp_r(path, @vendor_path)
FileUtils.cp_r(path, get_data_dir)
end
end

8
x-pack/lib/filters/geoip/util.rb
View file

@ -10,10 +10,16 @@ module LogStash module Filters
GZ_EXT = 'tgz'.freeze
DB_EXT = 'mmdb'.freeze
DB_PREFIX = 'GeoLite2-'.freeze
CITY_DB_NAME = "#{DB_PREFIX}City.#{DB_EXT}"
ASN_DB_NAME = "#{DB_PREFIX}ASN.#{DB_EXT}"
module Util
def get_file_path(filename)
::File.join(@vendor_path, filename)
::File.join(get_data_dir, filename)
end
def get_data_dir
::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
end
def file_exist?(path)

6
x-pack/spec/filters/geoip/database_manager_spec.rb
View file

@ -21,6 +21,10 @@ describe LogStash::Filters::Geoip do
end
let(:logger) { double("Logger") }
before(:each) do
LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db
end
context "patch database" do
it "use input path" do
path = db_manager.send(:patch_database_path, default_asn_db_path)
@ -168,7 +172,7 @@ describe LogStash::Filters::Geoip do
context "setup metadata" do
let(:db_metadata) do
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path)
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City")
dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
dbm
end

13
x-pack/spec/filters/geoip/database_metadata_spec.rb
View file

@ -4,19 +4,24 @@
require_relative 'test_helper'
require "filters/geoip/database_metadata"
require "filters/geoip/database_manager"
require "stud/temporary"
describe LogStash::Filters::Geoip do
describe 'DatabaseMetadata', :aggregate_failures do
let(:dbm) do
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path)
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City")
dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
dbm
end
let(:temp_metadata_path) { dbm.instance_variable_get(:@metadata_path) }
let(:logger) { double("Logger") }
before(:each) do
LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db
end
context "get all" do
it "return multiple rows" do
write_temp_metadata(temp_metadata_path, city2_metadata)
@ -67,7 +72,7 @@ describe LogStash::Filters::Geoip do
expect(Time.now.to_i - past.to_i).to be < 100
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).not_to be_empty
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).to eq(md5(default_city_gz_path))
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_cith_db_md5)
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_city_db_md5)
expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::FILENAME]).to eq(default_city_db_name)
end
end
@ -87,7 +92,7 @@ describe LogStash::Filters::Geoip do
context "with ASN database type" do
let(:dbm) do
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN", get_vendor_path)
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN")
dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
dbm
end
@ -101,7 +106,7 @@ describe LogStash::Filters::Geoip do
context "with invalid database type" do
let(:dbm) do
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???", get_vendor_path)
dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???")
dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
dbm
end

4
x-pack/spec/filters/geoip/download_manager_spec.rb
View file

@ -5,13 +5,14 @@
require_relative 'test_helper'
require 'fileutils'
require "filters/geoip/download_manager"
require "filters/geoip/database_manager"
describe LogStash::Filters::Geoip do
describe 'DownloadManager', :aggregate_failures do
let(:mock_metadata) { double("database_metadata") }
let(:download_manager) do
manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata, get_vendor_path)
manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata)
manager
end
let(:logger) { double("Logger") }
@ -21,6 +22,7 @@ describe LogStash::Filters::Geoip do
before do
stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT)
LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db
end
context "rest client" do

10
x-pack/spec/filters/geoip/test_helper.rb
View file

@ -10,8 +10,12 @@ module GeoipHelper
::File.expand_path("vendor", ::File.dirname(__FILE__))
end
def get_data_dir
::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
end
def get_file_path(filename)
::File.join(get_vendor_path, filename)
::File.join(get_data_dir, filename)
end
def md5(file_path)
@ -24,6 +28,7 @@ module GeoipHelper
def default_city_gz_path
get_file_path("GeoLite2-City.tgz")
end
def default_asn_db_path
@ -50,7 +55,7 @@ module GeoipHelper
get_file_path("GeoLite2-City_20200220.mmdb")
end
def default_cith_db_md5
def default_city_db_md5
md5(default_city_db_path)
end
@ -89,6 +94,7 @@ module GeoipHelper
def get_metadata_database_name
::File.exist?(metadata_path) ? ::File.read(metadata_path).split(",").last[0..-2] : nil
end
end
RSpec.configure do |c|