change download path for geoip plugin (#12863) (#12875)

The database was downloaded in plugin/vendor
This PR changes the working directory to `path.data`

x-pack/lib/filters/geoip/database_manager.rb

@@ -26,12 +26,14 @@ require "date"
 #   while `offline` is for static database path provided by users
 
 module LogStash module Filters module Geoip class DatabaseManager
+  extend LogStash::Filters::Geoip::Util
   include LogStash::Util::Loggable
   include LogStash::Filters::Geoip::Util
 
+  #TODO remove vendor_path
   def initialize(geoip, database_path, database_type, vendor_path)
-    @vendor_path = vendor_path
     @geoip = geoip
+    self.class.prepare_cc_db
     @mode = database_path.nil? ? :online : :offline
     @database_type = database_type
     @database_path = patch_database_path(database_path)
@@ -61,6 +63,16 @@ module LogStash module Filters module Geoip class DatabaseManager
 
   public
 
+  # create data dir, path.data, for geoip if it doesn't exist
+  # copy CC databases to data dir
+  def self.prepare_cc_db
+    FileUtils::mkdir_p(get_data_dir)
+    unless ::File.exist?(get_file_path(CITY_DB_NAME)) && ::File.exist?(get_file_path(ASN_DB_NAME))
+      cc_database_paths = ::Dir.glob(::File.join(LogStash::Environment::LOGSTASH_HOME, "vendor", "**", "{GeoLite2-ASN,GeoLite2-City}.mmdb"))
+      FileUtils.cp_r(cc_database_paths, get_data_dir)
+    end
+  end
+
   def execute_download_job
     begin
       has_update, new_database_path = @download_manager.fetch_database
@@ -138,12 +150,12 @@ module LogStash module Filters module Geoip class DatabaseManager
   end
 
   def setup
-    @metadata = DatabaseMetadata.new(@database_type, @vendor_path)
+    @metadata = DatabaseMetadata.new(@database_type)
     @metadata.save_timestamp(@database_path) unless @metadata.exist?
 
     @database_path = @metadata.database_path || @database_path
 
-    @download_manager = DownloadManager.new(@database_type, @metadata, @vendor_path)
+    @download_manager = DownloadManager.new(@database_type, @metadata)
   end
 
   class DatabaseExpiryError < StandardError

x-pack/lib/filters/geoip/database_metadata.rb

@@ -11,8 +11,7 @@ module LogStash module Filters module Geoip class DatabaseMetadata
   include LogStash::Util::Loggable
   include LogStash::Filters::Geoip::Util
 
-  def initialize(database_type, vendor_path)
+  def initialize(database_type)
-    @vendor_path = vendor_path
     @metadata_path = get_file_path("metadata.csv")
     @database_type = database_type
   end

x-pack/lib/filters/geoip/download_manager.rb

@@ -18,8 +18,7 @@ module LogStash module Filters module Geoip class DownloadManager
   include LogStash::Util::Loggable
   include LogStash::Filters::Geoip::Util
 
-  def initialize(database_type, metadata, vendor_path)
+  def initialize(database_type, metadata)
-    @vendor_path = vendor_path
     @database_type = database_type
     @metadata = metadata
   end
@@ -78,7 +77,7 @@ module LogStash module Filters module Geoip class DownloadManager
     end
   end
 
-  # extract all files and folders from .tgz to vendor directory
+  # extract all files and folders from .tgz to path.data directory
   # existing files folders will be replaced
   def unzip(zip_path)
     new_database_path = zip_path[0...-(GZ_EXT.length)] + DB_EXT
@@ -93,7 +92,7 @@ module LogStash module Filters module Geoip class DownloadManager
       if !::File.directory?(path) && database_name_ext.eql?(file)
         FileUtils.cp(path, new_database_path)
       else
-        FileUtils.cp_r(path, @vendor_path)
+        FileUtils.cp_r(path, get_data_dir)
       end
     end
 

x-pack/lib/filters/geoip/util.rb

@@ -10,10 +10,16 @@ module LogStash module Filters
     GZ_EXT = 'tgz'.freeze
     DB_EXT = 'mmdb'.freeze
     DB_PREFIX = 'GeoLite2-'.freeze
+    CITY_DB_NAME = "#{DB_PREFIX}City.#{DB_EXT}"
+    ASN_DB_NAME = "#{DB_PREFIX}ASN.#{DB_EXT}"
 
     module Util
       def get_file_path(filename)
-        ::File.join(@vendor_path, filename)
+        ::File.join(get_data_dir, filename)
+      end
+
+      def get_data_dir
+        ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
       end
 
       def file_exist?(path)

x-pack/spec/filters/geoip/database_manager_spec.rb

@@ -21,6 +21,10 @@ describe LogStash::Filters::Geoip do
     end
     let(:logger) { double("Logger") }
 
+    before(:each) do
+      LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db
+    end
+
     context "patch database" do
       it "use input path" do
         path = db_manager.send(:patch_database_path, default_asn_db_path)
@@ -168,7 +172,7 @@ describe LogStash::Filters::Geoip do
 
     context "setup metadata" do
       let(:db_metadata) do
-        dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path)
+        dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City")
         dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
         dbm
       end

x-pack/spec/filters/geoip/database_metadata_spec.rb

@@ -4,19 +4,24 @@
 
 require_relative 'test_helper'
 require "filters/geoip/database_metadata"
+require "filters/geoip/database_manager"
 require "stud/temporary"
 
 describe LogStash::Filters::Geoip do
 
   describe 'DatabaseMetadata', :aggregate_failures do
     let(:dbm) do
-      dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path)
+      dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City")
       dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
       dbm
     end
     let(:temp_metadata_path) { dbm.instance_variable_get(:@metadata_path) }
     let(:logger) { double("Logger") }
 
+    before(:each) do
+      LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db
+    end
+
     context "get all" do
       it "return multiple rows" do
         write_temp_metadata(temp_metadata_path, city2_metadata)
@@ -67,7 +72,7 @@ describe LogStash::Filters::Geoip do
         expect(Time.now.to_i - past.to_i).to be < 100
         expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).not_to be_empty
         expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).to eq(md5(default_city_gz_path))
-        expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_cith_db_md5)
+        expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_city_db_md5)
         expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::FILENAME]).to eq(default_city_db_name)
       end
     end
@@ -87,7 +92,7 @@ describe LogStash::Filters::Geoip do
 
       context "with ASN database type" do
         let(:dbm) do
-          dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN", get_vendor_path)
+          dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN")
           dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
           dbm
         end
@@ -101,7 +106,7 @@ describe LogStash::Filters::Geoip do
 
       context "with invalid database type" do
         let(:dbm) do
-          dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???", get_vendor_path)
+          dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???")
           dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
           dbm
         end

x-pack/spec/filters/geoip/download_manager_spec.rb

@@ -5,13 +5,14 @@
 require_relative 'test_helper'
 require 'fileutils'
 require "filters/geoip/download_manager"
+require "filters/geoip/database_manager"
 
 describe LogStash::Filters::Geoip do
 
   describe 'DownloadManager', :aggregate_failures do
     let(:mock_metadata)  { double("database_metadata") }
     let(:download_manager) do
-      manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata, get_vendor_path)
+      manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata)
       manager
     end
     let(:logger) { double("Logger") }
@@ -21,6 +22,7 @@ describe LogStash::Filters::Geoip do
 
     before do
       stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT)
+      LogStash::Filters::Geoip::DatabaseManager.prepare_cc_db
     end
 
     context "rest client" do

x-pack/spec/filters/geoip/test_helper.rb

@@ -10,8 +10,12 @@ module GeoipHelper
     ::File.expand_path("vendor", ::File.dirname(__FILE__))
   end
 
+  def get_data_dir
+    ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
+  end
+
   def get_file_path(filename)
-    ::File.join(get_vendor_path, filename)
+    ::File.join(get_data_dir, filename)
   end
 
   def md5(file_path)
@@ -24,6 +28,7 @@ module GeoipHelper
 
   def default_city_gz_path
     get_file_path("GeoLite2-City.tgz")
+
   end
 
   def default_asn_db_path
@@ -50,7 +55,7 @@ module GeoipHelper
     get_file_path("GeoLite2-City_20200220.mmdb")
   end
 
-  def default_cith_db_md5
+  def default_city_db_md5
     md5(default_city_db_path)
   end
 
@@ -89,6 +94,7 @@ module GeoipHelper
   def get_metadata_database_name
     ::File.exist?(metadata_path) ? ::File.read(metadata_path).split(",").last[0..-2] : nil
   end
+
 end
 
 RSpec.configure do |c|