[7.x] Geoip database service (#12675) | GeoIP clean up database after new download (#12689) | fix broken test case of term of service (#12715) | change domain and endpoint of GeoIP database service (#12727) | GeoIP database add license file (#12777)

GeoIP database service license change

Fixed: #12560

logstash-core/logstash-core.gemspec

@@ -73,4 +73,11 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "elasticsearch", "~> 5"
   gem.add_runtime_dependency "manticore", '~> 0.6'
+
+  # xpack geoip database service
+  gem.add_development_dependency 'logstash-filter-geoip', '~> 7.1' # package hierarchy change
+  gem.add_dependency 'faraday' #(MIT license)
+  gem.add_dependency 'down', '~> 5.2.0' #(MIT license)
+  gem.add_dependency 'tzinfo-data' #(MIT license)
+  gem.add_dependency 'rufus-scheduler' #(MIT license)
 end

rakelib/plugins-metadata.json

@@ -133,6 +133,7 @@
   },
   "logstash-filter-geoip": {
     "default-plugins": true,
+    "core-specs": true,
     "skip-list": false
   },
   "logstash-filter-grok": {

tools/dependencies-report/src/main/resources/licenseMapping.csv

@@ -145,3 +145,4 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL
 "unf:",https://github.com/knu/ruby-unf,BSD-2-Clause
 "webhdfs:",https://github.com/kzk/webhdfs,Apache-2.0
 "xml-simple:",https://github.com/maik/xml-simple,BSD-2-Clause
+"down",https://github.com/janko/down,MIT

tools/dependencies-report/src/main/resources/notices/down-NOTICE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Janko Marohnić
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

x-pack/build.gradle

@@ -16,17 +16,34 @@ buildscript {
   }
 }
 
+configurations {
+  geolite2
+}
+
 dependencies {
   testImplementation project(':logstash-core')
   testImplementation 'org.assertj:assertj-core:3.8.0'
   testImplementation 'junit:junit:4.12'
+
+  geolite2('org.elasticsearch:geolite2-databases:20191119') {
+    transitive = false
+  }
 }
 
 test {
   exclude '/**'
 }
 
+tasks.register("unzipGeolite", Copy) {
+  from(zipTree(configurations.geolite2.singleFile)) {
+    include "GeoLite2-ASN.mmdb"
+    include "GeoLite2-City.mmdb"
+  }
+  into file("${projectDir}/spec/filters/geoip/vendor")
+}
+
 tasks.register("rubyTests", Test) {
+  dependsOn unzipGeolite
   inputs.files fileTree("${projectDir}/spec")
   inputs.files fileTree("${projectDir}/lib")
   inputs.files fileTree("${projectDir}/modules")

x-pack/lib/filters/geoip/database_manager.rb

@@ -0,0 +1,151 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require "logstash/util/loggable"
+require_relative "util"
+require_relative "database_metadata"
+require_relative "download_manager"
+require "faraday"
+require "json"
+require "zlib"
+require "stud/try"
+require "down"
+require "rufus/scheduler"
+require "date"
+
+# The mission of DatabaseManager is to ensure the plugin running an up-to-date MaxMind database and
+#   thus users are compliant with EULA.
+# DatabaseManager does a daily checking by calling an endpoint to notice a version update.
+# DatabaseMetadata records the update timestamp and md5 of the database in the metadata file
+#   to keep track of versions and the number of days disconnects to the endpoint.
+# Once a new database version release, DownloadManager downloads it, and GeoIP Filter uses it on-the-fly.
+# If the last update timestamp is 25 days ago, a warning message shows in the log;
+# if it was 30 days ago, the GeoIP Filter should shutdown in order to be compliant.
+# There are online mode and offline mode in DatabaseManager. `online` is for automatic database update
+#   while `offline` is for static database path provided by users
+
+module LogStash module Filters module Geoip class DatabaseManager
+  include LogStash::Util::Loggable
+  include LogStash::Filters::Geoip::Util
+
+  def initialize(geoip, database_path, database_type, vendor_path)
+    @vendor_path = vendor_path
+    @geoip = geoip
+    @mode = database_path.nil? ? :online : :offline
+    @database_type = database_type
+    @database_path = patch_database_path(database_path)
+
+    if @mode == :online
+      logger.info "By using `online` mode, you accepted and agreed MaxMind EULA. "\
+                  "For more details please visit https://www.maxmind.com/en/geolite2/eula"
+
+      setup
+      clean_up_database
+      execute_download_job
+
+      # check database update periodically. trigger `call` method
+      @scheduler = Rufus::Scheduler.new({:max_work_threads => 1})
+      @scheduler.every('24h', self)
+    else
+      logger.info "GeoIP plugin is in offline mode. Logstash points to static database files and will not check for update. "\
+                  "Keep in mind that if you are not using the database shipped with this plugin, "\
+                  "please go to https://www.maxmind.com/en/geolite2/eula to accept and agree the terms and conditions."
+    end
+  end
+
+  DEFAULT_DATABASE_FILENAME = %w{
+    GeoLite2-City.mmdb
+    GeoLite2-ASN.mmdb
+  }.map(&:freeze).freeze
+
+  public
+
+  def execute_download_job
+    begin
+      has_update, new_database_path = @download_manager.fetch_database
+      @database_path = new_database_path if has_update
+      @metadata.save_timestamp(@database_path)
+      has_update
+    rescue => e
+      logger.error(e.message, :cause => e.cause, :backtrace => e.backtrace)
+      check_age
+      false
+    end
+  end
+
+  # scheduler callback
+  def call(job, time)
+    logger.debug "scheduler runs database update check"
+
+    begin
+      if execute_download_job
+        @geoip.setup_filter(database_path)
+        clean_up_database
+      end
+    rescue DatabaseExpiryError => e
+      logger.error(e.message, :cause => e.cause, :backtrace => e.backtrace)
+      @geoip.terminate_filter
+    end
+  end
+
+  def close
+    @scheduler.every_jobs.each(&:unschedule) if @scheduler
+  end
+
+  def database_path
+    @database_path
+  end
+
+  protected
+  # return a valid database path or default database path
+  def patch_database_path(database_path)
+    return database_path if file_exist?(database_path)
+    return database_path if database_path = get_file_path("#{DB_PREFIX}#{@database_type}.#{DB_EXT}") and file_exist?(database_path)
+    raise "You must specify 'database => ...' in your geoip filter (I looked for '#{database_path}')"
+  end
+
+  def check_age
+    days_without_update = (Date.today - Time.at(@metadata.updated_at).to_date).to_i
+
+    case
+    when days_without_update >= 30
+      raise DatabaseExpiryError, "The MaxMind database has been used for more than 30 days. Logstash is unable to get newer version from internet. "\
+      "According to EULA, GeoIP plugin needs to stop in order to be compliant. "\
+      "Please check the network settings and allow Logstash accesses the internet to download the latest database, "\
+      "or switch to offline mode (:database => PATH_TO_YOUR_DATABASE) to use a self-managed database which you can download from https://dev.maxmind.com/geoip/geoip2/geolite2/ "
+    when days_without_update >= 25
+      logger.warn("The MaxMind database has been used for #{days_without_update} days without update. "\
+      "Logstash will stop the GeoIP plugin in #{30 - days_without_update} days. "\
+      "Please check the network settings and allow Logstash accesses the internet to download the latest database ")
+    else
+      logger.debug("The MaxMind database hasn't updated", :days_without_update => days_without_update)
+    end
+  end
+
+  # Clean up files .mmdb, .tgz which are not mentioned in metadata and not default database
+  def clean_up_database
+    if @metadata.exist?
+      protected_filenames = (@metadata.database_filenames + DEFAULT_DATABASE_FILENAME).uniq
+      existing_filenames = ::Dir.glob(get_file_path("*.{#{DB_EXT},#{GZ_EXT}}"))
+                                .map { |path| ::File.basename(path) }
+
+      (existing_filenames - protected_filenames).each do |filename|
+        ::File.delete(get_file_path(filename))
+        logger.debug("old database #{filename} is deleted")
+      end
+    end
+  end
+
+  def setup
+    @metadata = DatabaseMetadata.new(@database_type, @vendor_path)
+    @metadata.save_timestamp(@database_path) unless @metadata.exist?
+
+    @database_path = @metadata.database_path || @database_path
+
+    @download_manager = DownloadManager.new(@database_type, @metadata, @vendor_path)
+  end
+
+  class DatabaseExpiryError < StandardError
+  end
+end end end end

x-pack/lib/filters/geoip/database_metadata.rb

@@ -0,0 +1,79 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require "logstash/util/loggable"
+require_relative "util"
+require "csv"
+require "date"
+
+module LogStash module Filters module Geoip class DatabaseMetadata
+  include LogStash::Util::Loggable
+  include LogStash::Filters::Geoip::Util
+
+  def initialize(database_type, vendor_path)
+    @vendor_path = vendor_path
+    @metadata_path = get_file_path("metadata.csv")
+    @database_type = database_type
+  end
+
+  public
+
+  # csv format: database_type, update_at, gz_md5, md5, filename
+  def save_timestamp(database_path)
+    metadata = get_metadata(false)
+    metadata << [@database_type, Time.now.to_i, md5(get_gz_name(database_path)), md5(database_path),
+                 ::File.basename(database_path)]
+
+    ::CSV.open @metadata_path, 'w' do |csv|
+      metadata.each { |row| csv << row }
+    end
+
+    logger.debug("metadata updated", :metadata => metadata)
+  end
+
+  def get_all
+    file_exist?(@metadata_path)? ::CSV.read(@metadata_path, headers: false) : Array.new
+  end
+
+  # Give rows of metadata in default database type, or empty array
+  def get_metadata(match_type = true)
+    get_all.select { |row| row[Column::DATABASE_TYPE].eql?(@database_type) == match_type }
+  end
+
+  # Return database path which has valid md5
+  def database_path
+    get_metadata.map { |metadata| [metadata, get_file_path(metadata[Column::FILENAME])] }
+                .select { |metadata, path| file_exist?(path) && (md5(path) == metadata[Column::MD5]) }
+                .map { |metadata, path| path }
+                .last
+  end
+
+  def gz_md5
+    get_metadata.map { |metadata| metadata[Column::GZ_MD5] }
+                .last || ''
+  end
+
+  def updated_at
+    (get_metadata.map { |metadata| metadata[Column::UPDATE_AT] }
+                 .last || 0).to_i
+  end
+
+  # Return database related filenames in .mmdb .tgz
+  def database_filenames
+    get_all.flat_map { |metadata| [ metadata[Column::FILENAME], get_gz_name(metadata[Column::FILENAME]) ] }
+  end
+  
+  def exist?
+    file_exist?(@metadata_path)
+  end
+
+  class Column
+    DATABASE_TYPE = 0
+    UPDATE_AT     = 1
+    GZ_MD5        = 2
+    MD5           = 3
+    FILENAME      = 4
+  end
+
+end end end end

x-pack/lib/filters/geoip/download_manager.rb

@@ -0,0 +1,111 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require_relative '../../../../lib/bootstrap/util/compress'
+require "logstash/util/loggable"
+require_relative "util"
+require_relative "database_metadata"
+require "logstash-filter-geoip_jars"
+require "faraday"
+require "json"
+require "zlib"
+require "stud/try"
+require "down"
+require "fileutils"
+
+module LogStash module Filters module Geoip class DownloadManager
+  include LogStash::Util::Loggable
+  include LogStash::Filters::Geoip::Util
+
+  def initialize(database_type, metadata, vendor_path)
+    @vendor_path = vendor_path
+    @database_type = database_type
+    @metadata = metadata
+  end
+
+  GEOIP_HOST = "https://geoip.elastic.co".freeze
+  GEOIP_PATH = "/v1/database".freeze
+  GEOIP_ENDPOINT = "#{GEOIP_HOST}#{GEOIP_PATH}".freeze
+
+  public
+  # Check available update and download it. Unzip and validate the file.
+  # return [has_update, new_database_path]
+  def fetch_database
+    has_update, database_info = check_update
+
+    if has_update
+      new_database_path = unzip download_database(database_info)
+      assert_database!(new_database_path)
+      return [true, new_database_path]
+    end
+
+    [false, nil]
+  end
+
+  def database_name
+    @database_name ||= "#{DB_PREFIX}#{@database_type}"
+  end
+
+  def database_name_ext
+    @database_name_ext ||= "#{database_name}.#{DB_EXT}"
+  end
+  
+  private
+  # Call infra endpoint to get md5 of latest database and verify with metadata
+  # return [has_update, server db info]
+  def check_update
+    uuid = get_uuid
+    res = rest_client.get("#{GEOIP_ENDPOINT}?key=#{uuid}&elastic_geoip_service_tos=agree")
+    logger.debug("check update", :endpoint => GEOIP_ENDPOINT, :response => res.status)
+
+    dbs = JSON.parse(res.body)
+    target_db = dbs.select { |db| db['name'].eql?("#{database_name}.#{GZ_EXT}") }.first
+    has_update = @metadata.gz_md5 != target_db['md5_hash']
+    logger.info "new database version detected? #{has_update}"
+
+    [has_update, target_db]
+  end
+
+  def download_database(server_db)
+    Stud.try(3.times) do
+      new_database_zip_path = get_file_path("#{database_name}_#{Time.now.to_i}.#{GZ_EXT}")
+      Down.download(server_db['url'], destination: new_database_zip_path)
+      raise "the new download has wrong checksum" if md5(new_database_zip_path) != server_db['md5_hash']
+
+      logger.debug("new database downloaded in ", :path => new_database_zip_path)
+      new_database_zip_path
+    end
+  end
+
+  # extract COPYRIGHT.txt, LICENSE.txt and GeoLite2-{ASN,City}.mmdb from .tgz to temp directory
+  def unzip(zip_path)
+    new_database_path = zip_path[0...-(GZ_EXT.length)] + DB_EXT
+    temp_dir = Stud::Temporary.pathname
+
+    LogStash::Util::Tar.extract(zip_path, temp_dir)
+    logger.debug("extract database to ", :path => temp_dir)
+
+
+    FileUtils.cp(::File.join(temp_dir, database_name_ext), new_database_path)
+    FileUtils.cp_r(::Dir.glob(::File.join(temp_dir, "{COPYRIGHT,LICENSE}.txt")), @vendor_path)
+
+    new_database_path
+  end
+
+  # Make sure the path has usable database
+  def assert_database!(database_path)
+    raise "failed to load database #{database_path}" unless org.logstash.filters.geoip.GeoIPFilter.database_valid?(database_path)
+  end
+
+  def rest_client
+    @client ||= Faraday.new do |conn|
+      conn.use Faraday::Response::RaiseError
+      conn.adapter :net_http
+    end
+  end
+
+  def get_uuid
+    @uuid ||= ::File.read(::File.join(LogStash::SETTINGS.get("path.data"), "uuid"))
+  end
+end end end end

x-pack/lib/filters/geoip/util.rb

@@ -0,0 +1,33 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require "digest"
+
+
+module LogStash module Filters
+  module Geoip
+    GZ_EXT = 'tgz'.freeze
+    DB_EXT = 'mmdb'.freeze
+    DB_PREFIX = 'GeoLite2-'.freeze
+
+    module Util
+      def get_file_path(filename)
+        ::File.join(@vendor_path, filename)
+      end
+
+      def file_exist?(path)
+        !path.nil? && ::File.exist?(path) && !::File.empty?(path)
+      end
+
+      def md5(file_path)
+        file_exist?(file_path) ? Digest::MD5.hexdigest(::File.read(file_path)): ""
+      end
+
+      # replace *.mmdb to *.tgz
+      def get_gz_name(filename)
+        filename[0...-(DB_EXT.length)] + GZ_EXT
+      end
+    end
+  end
+end end

x-pack/spec/filters/geoip/database_manager_spec.rb

@@ -0,0 +1,216 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require_relative 'test_helper'
+require "filters/geoip/database_manager"
+
+describe LogStash::Filters::Geoip do
+
+  describe 'DatabaseManager', :aggregate_failures do
+    let(:mock_geoip_plugin)  { double("geoip_plugin") }
+    let(:mock_metadata)  { double("database_metadata") }
+    let(:mock_download_manager)  { double("download_manager") }
+    let(:mock_scheduler)  { double("scheduler") }
+    let(:db_manager) do
+      manager = LogStash::Filters::Geoip::DatabaseManager.new(mock_geoip_plugin, default_city_db_path, "City", get_vendor_path)
+      manager.instance_variable_set(:@metadata, mock_metadata)
+      manager.instance_variable_set(:@download_manager, mock_download_manager)
+      manager.instance_variable_set(:@scheduler, mock_scheduler)
+      manager
+    end
+    let(:logger) { double("Logger") }
+
+    context "patch database" do
+      it "use input path" do
+        path = db_manager.send(:patch_database_path, default_asn_db_path)
+        expect(path).to eq(default_asn_db_path)
+      end
+
+      it "use CC license database as default" do
+        path = db_manager.send(:patch_database_path, "")
+        expect(path).to eq(default_city_db_path)
+      end
+
+      it "failed when default database is missing" do
+        expect(db_manager).to receive(:file_exist?).and_return(false, false)
+        expect { db_manager.send(:patch_database_path, "") }.to raise_error /I looked for/
+      end
+    end
+
+    context "md5" do
+      it "return md5 if file exists" do
+        str = db_manager.send(:md5, default_city_db_path)
+        expect(str).not_to eq("")
+        expect(str).not_to be_nil
+      end
+
+      it "return empty str if file not exists" do
+        file = Stud::Temporary.file.path + "/invalid"
+        str = db_manager.send(:md5, file)
+        expect(str).to eq("")
+      end
+    end
+
+    context "check age" do
+      it "should raise error when 30 days has passed" do
+        expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i)
+        expect{ db_manager.send(:check_age) }.to raise_error /be compliant/
+      end
+
+      it "should give warning after 25 days" do
+        expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 26)).to_i)
+        expect(mock_geoip_plugin).to receive(:terminate_filter).never
+        expect(LogStash::Filters::Geoip::DatabaseManager).to receive(:logger).at_least(:once).and_return(logger)
+        expect(logger).to receive(:warn)
+        expect(logger).to receive(:info)
+
+        db_manager.send(:check_age)
+      end
+    end
+
+    context "execute download job" do
+      it "should be false if no update" do
+        original = db_manager.instance_variable_get(:@database_path)
+        expect(mock_download_manager).to receive(:fetch_database).and_return([false, nil])
+        allow(mock_metadata).to receive(:save_timestamp)
+
+        expect(db_manager.send(:execute_download_job)).to be_falsey
+        expect(db_manager.instance_variable_get(:@database_path)).to eq(original)
+      end
+
+      it "should return true if update" do
+        original = db_manager.instance_variable_get(:@database_path)
+        expect(mock_download_manager).to receive(:fetch_database).and_return([true, "NEW_PATH"])
+        allow(mock_metadata).to receive(:save_timestamp)
+
+        expect(db_manager.send(:execute_download_job)).to be_truthy
+        expect(db_manager.instance_variable_get(:@database_path)).not_to eq(original)
+      end
+
+      it "should raise error when 30 days has passed" do
+        allow(mock_download_manager).to receive(:fetch_database).and_raise("boom")
+        expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i)
+
+        expect{ db_manager.send(:execute_download_job) }.to raise_error /be compliant/
+      end
+
+
+      it "should return false when 25 days has passed" do
+        allow(mock_download_manager).to receive(:fetch_database).and_raise("boom")
+
+        expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 25)).to_i)
+
+        expect(db_manager.send(:execute_download_job)).to be_falsey
+      end
+    end
+
+    context "scheduler call" do
+      it "should call plugin termination when raise error and last update > 30 days" do
+        allow(mock_download_manager).to receive(:fetch_database).and_raise("boom")
+        expect(mock_metadata).to receive(:updated_at).and_return((Time.now - (60 * 60 * 24 * 33)).to_i)
+        expect(mock_geoip_plugin).to receive(:terminate_filter)
+        db_manager.send(:call, nil, nil)
+      end
+
+      it "should not call plugin setup when database is up to date" do
+        allow(mock_download_manager).to receive(:fetch_database).and_return([false, nil])
+        expect(mock_metadata).to receive(:save_timestamp)
+        allow(mock_geoip_plugin).to receive(:setup_filter).never
+        db_manager.send(:call, nil, nil)
+      end
+
+      it "should call scheduler when has update" do
+        allow(db_manager).to receive(:execute_download_job).and_return(true)
+        allow(mock_geoip_plugin).to receive(:setup_filter).once
+        allow(db_manager).to receive(:clean_up_database).once
+        db_manager.send(:call, nil, nil)
+      end
+    end
+
+    context "clean up database" do
+      let(:asn00) { get_file_path("GeoLite2-ASN_000000000.mmdb") }
+      let(:asn00gz) { get_file_path("GeoLite2-ASN_000000000.tgz") }
+      let(:city00) { get_file_path("GeoLite2-City_000000000.mmdb") }
+      let(:city00gz) { get_file_path("GeoLite2-City_000000000.tgz") }
+      let(:city44) { get_file_path("GeoLite2-City_4444444444.mmdb") }
+      let(:city44gz) { get_file_path("GeoLite2-City_4444444444.tgz") }
+
+      before(:each) do
+        [asn00, asn00gz, city00, city00gz, city44, city44gz].each { |file_path| ::File.delete(file_path) if ::File.exist?(file_path) }
+      end
+
+      it "should not delete when metadata file doesn't exist" do
+        expect(mock_metadata).to receive(:exist?).and_return(false)
+        allow(mock_geoip_plugin).to receive(:database_filenames).never
+
+        db_manager.send(:clean_up_database)
+      end
+
+      it "should delete file which is not in metadata" do
+        [asn00, asn00gz, city00, city00gz, city44, city44gz].each { |file_path| FileUtils.touch(file_path) }
+        expect(mock_metadata).to receive(:exist?).and_return(true)
+        expect(mock_metadata).to receive(:database_filenames).and_return(["GeoLite2-City_4444444444.mmdb"])
+
+        db_manager.send(:clean_up_database)
+        [asn00, asn00gz, city00, city00gz, city44gz].each { |file_path| expect(::File.exist?(file_path)).to be_falsey }
+        [default_city_db_path, default_asn_db_path, city44].each { |file_path| expect(::File.exist?(file_path)).to be_truthy }
+      end
+
+      it "should keep the default database" do
+        expect(mock_metadata).to receive(:exist?).and_return(true)
+        expect(mock_metadata).to receive(:database_filenames).and_return(["GeoLite2-City_4444444444.mmdb"])
+
+        db_manager.send(:clean_up_database)
+        [default_city_db_path, default_asn_db_path].each { |file_path| expect(::File.exist?(file_path)).to be_truthy }
+      end
+    end
+
+    context "setup metadata" do
+      let(:db_metadata) do
+        dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path)
+        dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
+        dbm
+      end
+
+      let(:temp_metadata_path) { db_metadata.instance_variable_get(:@metadata_path) }
+
+      before(:each) do
+        expect(::File.empty?(temp_metadata_path)).to be_truthy
+        allow(LogStash::Filters::Geoip::DatabaseMetadata).to receive(:new).and_return(db_metadata)
+      end
+
+      after(:each) do
+        ::File.delete(second_city_db_path) if ::File.exist?(second_city_db_path)
+      end
+
+      it "create metadata when file is missing" do
+        db_manager.send(:setup)
+        expect(db_manager.instance_variable_get(:@database_path)).to eql(default_city_db_path)
+        expect(db_metadata.database_path).to eql(default_city_db_path)
+        expect(::File.exist?(temp_metadata_path)).to be_truthy
+        expect(::File.empty?(temp_metadata_path)).to be_falsey
+      end
+
+      it "manager should use database path in metadata" do
+        write_temp_metadata(temp_metadata_path, city2_metadata)
+        copy_city_database(second_city_db_name)
+        expect(db_metadata).to receive(:save_timestamp).never
+
+        db_manager.send(:setup)
+        filename = db_manager.instance_variable_get(:@database_path).split('/').last
+        expect(filename).to match /#{second_city_db_name}/
+      end
+
+      it "ignore database_path in metadata if md5 does not match" do
+        write_temp_metadata(temp_metadata_path, ["City","","","INVALID_MD5",second_city_db_name])
+        copy_city_database(second_city_db_name)
+        expect(db_metadata).to receive(:save_timestamp).never
+
+        db_manager.send(:setup)
+        filename = db_manager.instance_variable_get(:@database_path).split('/').last
+        expect(filename).to match /#{default_city_db_name}/
+      end
+    end
+  end
+end

x-pack/spec/filters/geoip/database_metadata_spec.rb

@@ -0,0 +1,160 @@
+# # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# # or more contributor license agreements. Licensed under the Elastic License;
+# # you may not use this file except in compliance with the Elastic License.
+
+require_relative 'test_helper'
+require "filters/geoip/database_metadata"
+require "stud/temporary"
+
+describe LogStash::Filters::Geoip do
+
+  describe 'DatabaseMetadata', :aggregate_failures do
+    let(:dbm) do
+      dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("City", get_vendor_path)
+      dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
+      dbm
+    end
+    let(:temp_metadata_path) { dbm.instance_variable_get(:@metadata_path) }
+    let(:logger) { double("Logger") }
+
+    context "get all" do
+      it "return multiple rows" do
+        write_temp_metadata(temp_metadata_path, city2_metadata)
+
+        expect(dbm.get_all.size).to eq(3)
+      end
+    end
+
+    context "get metadata" do
+      it "return metadata" do
+        write_temp_metadata(temp_metadata_path, city2_metadata)
+
+        city = dbm.get_metadata
+        expect(city.size).to eq(2)
+
+        asn = dbm.get_metadata(false)
+        expect(asn.size).to eq(1)
+      end
+
+      it "return empty array when file is missing" do
+        metadata = dbm.get_metadata
+        expect(metadata.size).to eq(0)
+      end
+
+      it "return empty array when an empty file exist" do
+        FileUtils.touch(temp_metadata_path)
+
+        metadata = dbm.get_metadata
+        expect(metadata.size).to eq(0)
+      end
+    end
+
+    context "save timestamp" do
+      before do
+        ::File.open(default_city_gz_path, "w") { |f| f.write "make a non empty file" }
+      end
+
+      after do
+        delete_file(default_city_gz_path)
+      end
+
+      it "write the current time" do
+        dbm.save_timestamp(default_city_db_path)
+
+        metadata = dbm.get_metadata.last
+        expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::DATABASE_TYPE]).to eq("City")
+        past = metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::UPDATE_AT]
+        expect(Time.now.to_i - past.to_i).to be < 100
+        expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).not_to be_empty
+        expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::GZ_MD5]).to eq(md5(default_city_gz_path))
+        expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::MD5]).to eq(default_cith_db_md5)
+        expect(metadata[LogStash::Filters::Geoip::DatabaseMetadata::Column::FILENAME]).to eq(default_city_db_name)
+      end
+    end
+
+    context "database path" do
+      it "return the default city database path" do
+        write_temp_metadata(temp_metadata_path)
+
+        expect(dbm.database_path).to eq(default_city_db_path)
+      end
+
+      it "return the last database path with valid md5" do
+        write_temp_metadata(temp_metadata_path, city2_metadata)
+
+        expect(dbm.database_path).to eq(default_city_db_path)
+      end
+
+      context "with ASN database type" do
+        let(:dbm) do
+          dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("ASN", get_vendor_path)
+          dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
+          dbm
+        end
+
+        it "return the default asn database path" do
+          write_temp_metadata(temp_metadata_path)
+
+          expect(dbm.database_path).to eq(default_asn_db_path)
+        end
+      end
+
+      context "with invalid database type" do
+        let(:dbm) do
+          dbm = LogStash::Filters::Geoip::DatabaseMetadata.new("???", get_vendor_path)
+          dbm.instance_variable_set(:@metadata_path, Stud::Temporary.file.path)
+          dbm
+        end
+
+        it "return nil if md5 not matched" do
+          write_temp_metadata(temp_metadata_path)
+
+          expect(dbm.database_path).to be_nil
+        end
+      end
+    end
+
+    context "gz md5" do
+      it "should give the last gz md5" do
+        write_temp_metadata(temp_metadata_path, ["City","","SOME_GZ_MD5","SOME_MD5",second_city_db_name])
+        expect(dbm.gz_md5).to eq("SOME_GZ_MD5")
+      end
+
+      it "should give empty string if metadata is empty" do
+        expect(dbm.gz_md5).to eq("")
+      end
+    end
+
+    context "updated at" do
+      it "should give the last update timestamp" do
+        write_temp_metadata(temp_metadata_path, ["City","1611690807","SOME_GZ_MD5","SOME_MD5",second_city_db_name])
+        expect(dbm.updated_at).to eq(1611690807)
+      end
+
+      it "should give 0 if metadata is empty" do
+        expect(dbm.updated_at).to eq(0)
+      end
+    end
+
+    context "database filenames" do
+      it "should give filename in .mmdb .tgz" do
+        write_temp_metadata(temp_metadata_path)
+        expect(dbm.database_filenames).to match_array([default_city_db_name, default_asn_db_name,
+                                                       'GeoLite2-City.tgz', 'GeoLite2-ASN.tgz'])
+      end
+    end
+
+    context "exist" do
+      it "should be false because Stud create empty temp file" do
+        expect(dbm.exist?).to be_falsey
+      end
+
+      it "should be true if temp file has content" do
+        ::File.open(temp_metadata_path, "w") { |f| f.write("something") }
+
+        expect(dbm.exist?).to be_truthy
+      end
+    end
+
+  end
+end

x-pack/spec/filters/geoip/download_manager_spec.rb

@@ -0,0 +1,168 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require_relative 'test_helper'
+require "filters/geoip/download_manager"
+
+describe LogStash::Filters::Geoip do
+
+  describe 'DownloadManager', :aggregate_failures do
+    let(:mock_metadata)  { double("database_metadata") }
+    let(:download_manager) do
+      manager = LogStash::Filters::Geoip::DownloadManager.new( "City", mock_metadata, get_vendor_path)
+      manager
+    end
+    let(:logger) { double("Logger") }
+
+    GEOIP_STAGING_HOST = "https://geoip.elastic.dev"
+    GEOIP_STAGING_ENDPOINT = "#{GEOIP_STAGING_HOST}#{LogStash::Filters::Geoip::DownloadManager::GEOIP_PATH}"
+
+    before do
+      stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT)
+    end
+
+    context "rest client" do
+      it "can call endpoint" do
+        conn = download_manager.send(:rest_client)
+        res = conn.get("#{GEOIP_STAGING_ENDPOINT}?key=#{SecureRandom.uuid}&elastic_geoip_service_tos=agree")
+        expect(res.status).to eq(200)
+      end
+
+      it "should raise error when endpoint response 4xx" do
+        conn = download_manager.send(:rest_client)
+        expect { conn.get("#{GEOIP_STAGING_HOST}?key=#{SecureRandom.uuid}&elastic_geoip_service_tos=agree") }.to raise_error /404/
+      end
+    end
+
+    context "check update" do
+      before(:each) do
+        expect(download_manager).to receive(:get_uuid).and_return(SecureRandom.uuid)
+        mock_resp = double("geoip_endpoint",
+                           :body => ::File.read(::File.expand_path("./fixtures/normal_resp.json", ::File.dirname(__FILE__))),
+                           :status => 200)
+        allow(download_manager).to receive_message_chain("rest_client.get").and_return(mock_resp)
+      end
+
+      it "should return has_update and db info when md5 does not match" do
+        expect(mock_metadata).to receive(:gz_md5).and_return("")
+
+        has_update, info = download_manager.send(:check_update)
+        expect(has_update).to be_truthy
+        expect(info).to have_key("md5_hash")
+        expect(info).to have_key("name")
+        expect(info).to have_key("provider")
+        expect(info).to have_key("updated")
+        expect(info).to have_key("url")
+        expect(info["name"]).to include("City")
+      end
+
+      it "should return false when md5 is the same" do
+        expect(mock_metadata).to receive(:gz_md5).and_return("89d225ac546310b1e7979502ac9ad11c")
+
+        has_update, info = download_manager.send(:check_update)
+        expect(has_update).to be_falsey
+      end
+
+      it "should return true when md5 does not match" do
+        expect(mock_metadata).to receive(:gz_md5).and_return("bca2a8bad7e5e4013dc17343af52a841")
+
+        has_update, info = download_manager.send(:check_update)
+        expect(has_update).to be_truthy
+      end
+    end
+
+    context "download database" do
+      let(:db_info) do
+        {
+          "md5_hash" => md5_hash,
+          "name" => filename,
+          "provider" => "maxmind",
+          "updated" => 1609891257,
+          "url" => "https://github.com/logstash-plugins/logstash-filter-geoip/archive/master.zip"
+        }
+      end
+      let(:md5_hash) { SecureRandom.hex }
+      let(:filename) { "GeoLite2-City.tgz"}
+
+      it "should raise error if md5 does not match" do
+        allow(Down).to receive(:download)
+        expect{ download_manager.send(:download_database, db_info) }.to raise_error /wrong checksum/
+      end
+
+      it "should download file and return zip path" do
+        expect(download_manager).to receive(:md5).and_return(md5_hash)
+
+        path = download_manager.send(:download_database, db_info)
+        expect(path).to match /GeoLite2-City_\d+\.tgz/
+        expect(::File.exist?(path)).to be_truthy
+
+        delete_file(path)
+      end
+    end
+
+    context "unzip" do
+      let(:copyright_path) { get_file_path('COPYRIGHT.txt') }
+      let(:license_path) { get_file_path('LICENSE.txt') }
+      let(:readme_path) { get_file_path('README.txt') }
+
+      before do
+        file_path = ::File.expand_path("./fixtures/sample", ::File.dirname(__FILE__))
+        delete_file(file_path, copyright_path, license_path, readme_path)
+      end
+
+      it "should extract database and license related files" do
+        path = ::File.expand_path("./fixtures/sample.tgz", ::File.dirname(__FILE__))
+        unzip_db_path = download_manager.send(:unzip, path)
+
+        expect(unzip_db_path).to match /\.mmdb/
+        expect(::File.exist?(unzip_db_path)).to be_truthy
+        expect(::File.exist?(copyright_path)).to be_truthy
+        expect(::File.exist?(license_path)).to be_truthy
+        expect(::File.exist?(readme_path)).to be_falsey
+
+        delete_file(unzip_db_path, copyright_path, license_path)
+      end
+    end
+
+    context "assert database" do
+      it "should raise error if file is invalid" do
+        expect{ download_manager.send(:assert_database!, "Gemfile") }.to raise_error /failed to load database/
+      end
+
+      it "should pass validation" do
+        expect(download_manager.send(:assert_database!, default_city_db_path)).to be_nil
+      end
+    end
+
+    context "fetch database" do
+      it "should be false if no update" do
+        expect(download_manager).to receive(:check_update).and_return([false, {}])
+
+        has_update, new_database_path = download_manager.send(:fetch_database)
+
+        expect(has_update).to be_falsey
+        expect(new_database_path).to be_nil
+      end
+
+      it "should raise error" do
+        expect(download_manager).to receive(:check_update).and_return([true, {}])
+        expect(download_manager).to receive(:download_database).and_raise('boom')
+
+        expect { download_manager.send(:fetch_database) }.to raise_error
+      end
+
+      it "should be true if got update" do
+        expect(download_manager).to receive(:check_update).and_return([true, {}])
+        allow(download_manager).to receive(:download_database)
+        allow(download_manager).to receive(:unzip)
+        allow(download_manager).to receive(:assert_database!)
+
+        has_update, new_database_path = download_manager.send(:fetch_database)
+
+        expect(has_update).to be_truthy
+      end
+    end
+
+  end
+end

x-pack/spec/filters/geoip/fixtures/normal_resp.json

@@ -0,0 +1,44 @@
+[
+  {
+    "md5_hash": "bcfc39b5677554e091dbb19cd5cea4b0",
+    "name": "GeoLite2-ASN.mmdb.gz",
+    "provider": "maxmind",
+    "updated": 1615852860,
+    "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-ASN.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=ada6463b28177577f4981cbe5f29708d0196ed71cea0bf3c0bf8e9965c8f9fd3d184be852c4e84f24b2896d8043a466039e15b5581ba4fc7aa37a15c85c79999674a0966b28f53b0c5a8b1220b428d3c1e958f20a61e06758426b7308f1ba1966b04a2bf86a5a9f96b88c05753b429574829344d3043de1f7d2b93cade7b57d53ac6d3bcb4e6d11405f6f2e7ff8c25d813e3917177b9438f686f10bc4a006aadc6a7dde2343c9bc0017487684ad64f59bb2d0b7b73b3c817f24c91bd9afd2f36725937c8938def67d5cf6df3a7705bb40098548b55a6777ef2cd8e26c32efaa1bd0474f7f24d5e386d90e87d8a3c3aa63203a78004bccf2ad65cc97b26e94675"
+  },
+  {
+    "md5_hash": "be4e335eb819af148fa4e365f176923d",
+    "name": "GeoLite2-ASN.tgz",
+    "provider": "maxmind",
+    "updated": 1615939277,
+    "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-ASN.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=8d8566fdf8167d9874966c16663a76bf8a678083c753fae0397de2eaffdb9f1d19ff36dd28bb2dc3bd9230dab5256a6d08d694574b9c50cae4b8614115ef9d3d638caf29eb18cefd7a7f0154e7baaeab4c565c828a2f050bbdbb8f5a9647d67d0748960b77846674097f76ea0d721cadda9fd99379ee604eba692c9274d238a1a3d56b7c29e236182cf5e91bae63b72d1c9a1ee7c598d7c5156683aa71a9776151bec83cb99f07f75a83483d620960fd97eca4e12c3789d72ac272912df74da1d63572609883157c6d2f115f7ab1be6b3e4503e7dd501946124f1250a299338529b8abc199afe52ff9d38904603b12b674149b85d7597e57502fda05c4b65a75"
+  },
+  {
+    "md5_hash": "6cd9be41557fd4c6dd0a8609a3f96bbe",
+    "name": "GeoLite2-City.mmdb.gz",
+    "provider": "maxmind",
+    "updated": 1615420855,
+    "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-City.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=630106105d8f476a6d4e7de9fd777d8c250391ce1fbc799c7c683efeb39b319e1263948bcd326dc15f3ee0c9578f1fc95e5afe2d6b026dfac00b1fe188961df8ce3a8e5e0d71355fc0ea4d7f957af2ce8bf433210b0224d7175122ce0c1ced64dc39d2db7a979c1d173b72da58441a2358f605b92b71355cf00af4fdaa20943f21827506756b52706daaf780f173fe9f37a41fd7fc5539bbc41e79110fc4b00b37334d37179efa78c0a2ccd20ef6a5faff3baf1b5c2dfb2ef0ebb7ae4ef949f986a3cfbc8df4885476aef4ba6c06012a83418623219b48ee7ff04a41ae2ff2f421fb85fcbc04255df174647d6b9302f15441a783252c7443edfa70ef5f44068a"
+  },
+  {
+    "md5_hash": "89d225ac546310b1e7979502ac9ad11c",
+    "name": "GeoLite2-City.tgz",
+    "provider": "maxmind",
+    "updated": 1615939277,
+    "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-City.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=3f5e84337ef78e8039ed391cddbcc92b0ceb3b946d4a7f60476f0633584cd3324356c9ead4bfc19f1c8776849a26b850c7e388386c5dfa8eccc2afe7e7c21d4c7fdd093cfae5c52899d9df5ffe13db6c29a0558329c8a8aecda058f9778dd23615471023fc77cc514d372d9786cbd256e27818883c1ee4b7edee75c393c89d57e94e58c2be2f9c8ee7354864b53f66d61c917eae296e071f84776e8c358218d890333fd376753a4c0f903581480629bca86d1abf3bc65efc7da30617c4847367d0ae24ba1ce0528feba3c3c3c38ecdd9a8d820d7f1a9141e30578822564c192181a97761858b9e06cc05f7db4143c89c402cbb888dcabc1f6559f4f701b79a7c"
+  },
+  {
+    "md5_hash": "03bef5fb1fdc877304da3391052246dc",
+    "name": "GeoLite2-Country.mmdb.gz",
+    "provider": "maxmind",
+    "updated": 1615420855,
+    "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-Country.mmdb.gz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=18d3266f09a8b208573fa48ca9c30cf0041b69de4eac1656cafebcf737a9f2637b0be12f9df4dd26c07bc297a4070cd0248f8874d3d03bb3fc992f7110c1c0def845f182dcc6289d5fe4faa97daf98e3bdcd2e37405bae1f04e1b293c556c352a0c574f7a52f0f0ea92bcbfb5a74542be9e651453c79a0df1f7a84f2d48d5e704ee11df9a180f9c4c76a809c6a7edab7e36b4863556d815042b9cf43fe8bb1c60f432fcae56b1779d610e8b1388addc277b0259ac595eee34227fc9884065c7aaf44c8446c4f00849d3f8dad6eba9cc7213bac33ff166dc86c344fd14da736390615bc4d00de5ba007b0b1013f46b7e81b9827d32ae9e20f779a6580f97164f9"
+  },
+  {
+    "md5_hash": "c0e76a2e7e0f781028e849c2d389d8a1",
+    "name": "GeoLite2-Country.tgz",
+    "provider": "maxmind",
+    "updated": 1615939276,
+    "url": "https://storage.googleapis.com/elastic-paisano-staging/maxmind/GeoLite2-Country.tgz?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=elastic-paisano-staging%40elastic-apps-163815.iam.gserviceaccount.com%2F20210317%2Fhenk%2Fstorage%2Fgoog4_request&X-Goog-Date=20210317T103241Z&X-Goog-Expires=86400&X-Goog-SignedHeaders=host&X-Goog-Signature=5eaf641191c25f111afed9c569e31a5369733b3723db365b76cfbf93a7b39fd77481fe07f93fc5be2fb9ef987ef6f1c32bcb863d9d2de0e74aeece8ff568c41573c8a465e9ec5301bdc77c75b2ab369f5352f2da3f5262ae889facaf27f1685584ca49fa3bf4556ed0a92b6a4b1f1985f62378c92467d73b0c66fd1ed04cb311b903343249aed6d3ba32d7b80f0be9a08816737016038306886dcffaf141932e5fb06dfe96ff1caf8ed37f6f8128a0bdc6abf9516aeac891a791656d14f4c37b31f4c86d5dba430d92402c78d8b53dcf4ec557f0f8b6c1fb59357ae1aa7f6310289fdf16c094028570431312ea35f2c00f8cd2dcef8b98d2af5ed3ee09a7fefd"
+  }
+]

x-pack/spec/filters/geoip/test_helper.rb

@@ -0,0 +1,96 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require 'spec_helper'
+require "digest"
+
+module GeoipHelper
+  def get_vendor_path
+    ::File.expand_path("vendor", ::File.dirname(__FILE__))
+  end
+
+  def get_file_path(filename)
+    ::File.join(get_vendor_path, filename)
+  end
+
+  def md5(file_path)
+    ::File.exist?(file_path) ? Digest::MD5.hexdigest(::File.read(file_path)) : ''
+  end
+
+  def default_city_db_path
+    get_file_path("GeoLite2-City.mmdb")
+  end
+
+  def default_city_gz_path
+    get_file_path("GeoLite2-City.tgz")
+  end
+
+  def default_asn_db_path
+    get_file_path("GeoLite2-ASN.mmdb")
+  end
+
+  def metadata_path
+    get_file_path("metadata.csv")
+  end
+
+  def default_city_db_name
+    "GeoLite2-City.mmdb"
+  end
+
+  def default_asn_db_name
+    "GeoLite2-ASN.mmdb"
+  end
+
+  def second_city_db_name
+    "GeoLite2-City_20200220.mmdb"
+  end
+
+  def second_city_db_path
+    get_file_path("GeoLite2-City_20200220.mmdb")
+  end
+
+  def default_cith_db_md5
+    md5(default_city_db_path)
+  end
+
+  def DEFAULT_ASN_DB_MD5
+    md5(default_asn_db_path)
+  end
+
+
+  def write_temp_metadata(temp_file_path, row = nil)
+    now = Time.now.to_i
+    city = md5(default_city_db_path)
+    asn = md5(default_asn_db_path)
+
+    metadata = []
+    metadata << ["ASN",now,"",asn,default_asn_db_name]
+    metadata << ["City",now,"",city,default_city_db_name]
+    metadata << row if row
+    CSV.open temp_file_path, 'w' do |csv|
+      metadata.each { |row| csv << row }
+    end
+  end
+
+  def city2_metadata
+    ["City",Time.now.to_i,"",md5(default_city_db_path),second_city_db_name]
+  end
+
+  def copy_city_database(filename)
+    new_path = default_city_db_path.gsub(default_city_db_name, filename)
+    FileUtils.cp(default_city_db_path, new_path)
+  end
+
+  def delete_file(*filepaths)
+    filepaths.map { |filepath| ::File.delete(filepath) if ::File.exist?(filepath) }
+  end
+
+  def get_metadata_database_name
+    ::File.exist?(metadata_path) ? ::File.read(metadata_path).split(",").last[0..-2] : nil
+  end
+end
+
+RSpec.configure do |c|
+  c.include GeoipHelper
+end