geoip integrate air-gapped bootstrap script (#13104) (#13251)

This PR integrates Elasticsearch bootstrap script to help users keep Logstah geoip plugin run without online update check.
Add `xpack.geoip.download.endpoint` option to config geoip database service endpoint.
Users can point to `http://localhost:8080/overview.json` when using the script to bootstrap nginx docker

config/logstash.yml

@@ -302,3 +302,7 @@
 #xpack.management.elasticsearch.ssl.verification_mode: certificate
 #xpack.management.elasticsearch.sniffing: false
 #xpack.management.logstash.poll_interval: 5s
+
+# X-Pack GeoIP plugin
+# https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-manage_update
+#xpack.geoip.download.endpoint: "https://geoip.elastic.co/v1/database"

docker/data/logstash/env2yaml/env2yaml.go

@@ -121,6 +121,7 @@ func normalizeSetting(setting string) (string, error) {
 		"xpack.management.elasticsearch.ssl.truststore.password",
 		"xpack.management.elasticsearch.ssl.keystore.path",
 		"xpack.management.elasticsearch.ssl.keystore.password",
+		"xpack.geoip.download.endpoint",
 		"cloud.id",
 		"cloud.auth",
 	}

x-pack/lib/filters/geoip/database_manager.rb

@@ -183,7 +183,7 @@ module LogStash module Filters module Geoip class DatabaseManager
         database_status = :up_to_date
       end
 
-      @metric.namespace([:database, database_type.to_sym]).tap do |n|
+      metric.namespace([:database, database_type.to_sym]).tap do |n|
         n.gauge(:status, database_status)
         n.gauge(:last_updated_at, unix_time_to_iso8601(metadata[DatabaseMetadata::Column::DIRNAME]))
         n.gauge(:fail_check_in_days, days_without_update)
@@ -229,7 +229,7 @@ module LogStash module Filters module Geoip class DatabaseManager
     metadatas = @metadata.get_all
     metadatas.each do |row|
       type = row[DatabaseMetadata::Column::DATABASE_TYPE]
-      @metric.namespace([:database, type.to_sym]).tap do |n|
+      metric.namespace([:database, type.to_sym]).tap do |n|
         n.gauge(:status, @states[type].is_eula ? :up_to_date : :init)
         if @states[type].is_eula
           n.gauge(:last_updated_at, unix_time_to_iso8601(row[DatabaseMetadata::Column::DIRNAME]))
@@ -238,14 +238,14 @@ module LogStash module Filters module Geoip class DatabaseManager
       end
     end
 
-    @metric.namespace([:download_stats]).tap do |n|
+    metric.namespace([:download_stats]).tap do |n|
       check_at = metadatas.map { |row| row[DatabaseMetadata::Column::CHECK_AT].to_i }.max
       n.gauge(:last_checked_at, unix_time_to_iso8601(check_at))
     end
   end
 
   def update_download_metric(success_cnt)
-    @metric.namespace([:download_stats]).tap do |n|
+    metric.namespace([:download_stats]).tap do |n|
       n.gauge(:last_checked_at, Time.now.iso8601)
 
       if success_cnt == DB_TYPES.size
@@ -259,7 +259,7 @@ module LogStash module Filters module Geoip class DatabaseManager
   end
 
   def update_download_status(status)
-    @metric.namespace([:download_stats]).gauge(:status, status)
+    metric.namespace([:download_stats]).gauge(:status, status)
   end
 
   public
@@ -302,6 +302,11 @@ module LogStash module Filters module Geoip class DatabaseManager
     @metric = metric
   end
 
+  def metric
+    # Fallback when testing plugin and no metric collector are correctly configured.
+    @metric ||= LogStash::Instrument::NamespacedNullMetric.new
+  end
+
   class DatabaseState
     attr_reader :is_eula, :plugins, :database_path, :cc_database_path, :is_expired
     attr_writer :is_eula, :database_path, :is_expired

x-pack/lib/filters/geoip/download_manager.rb

@@ -13,6 +13,7 @@ require "zlib"
 require "stud/try"
 require "down"
 require "fileutils"
+require 'uri'
 
 module LogStash module Filters module Geoip class DownloadManager
   include LogStash::Util::Loggable
@@ -50,9 +51,8 @@ module LogStash module Filters module Geoip class DownloadManager
   # Call infra endpoint to get md5 of latest databases and verify with metadata
   # return Array of new database information [database_type, db_info]
   def check_update
-    uuid = get_uuid
-    res = rest_client.get("#{GEOIP_ENDPOINT}?key=#{uuid}&elastic_geoip_service_tos=agree")
-    logger.debug("check update", :endpoint => GEOIP_ENDPOINT, :response => res.status)
+    res = rest_client.get(service_endpoint)
+    logger.debug("check update", :endpoint => service_endpoint.to_s, :response => res.status)
 
     service_resp = JSON.parse(res.body)
 
@@ -74,7 +74,10 @@ module LogStash module Filters module Geoip class DownloadManager
       FileUtils.mkdir_p(get_dir_path(dirname))
       zip_path = get_gz_path(database_type, dirname)
 
-      Down.download(db_info['url'], destination: zip_path)
+      actual_url = download_url(db_info['url'])
+      logger.debug? && logger.debug("download #{actual_url}")
+
+      Down.download(actual_url, destination: zip_path)
       raise "the new download has wrong checksum" if md5(zip_path) != db_info['md5_hash']
 
       logger.debug("new database downloaded in ", :path => zip_path)
@@ -105,7 +108,25 @@ module LogStash module Filters module Geoip class DownloadManager
     end
   end
 
-  def get_uuid
+  def uuid
     @uuid ||= ::File.read(::File.join(LogStash::SETTINGS.get("path.data"), "uuid"))
   end
+
+  def service_endpoint
+    return @service_endpoint if @service_endpoint
+
+    uri = URI(LogStash::SETTINGS.get("xpack.geoip.download.endpoint") || GEOIP_ENDPOINT)
+    uri.query = "key=#{uuid}&elastic_geoip_service_tos=agree"
+    @service_endpoint = uri
+  end
+
+  def download_url(url)
+    uri = URI(url)
+    return url if uri.scheme
+
+    download_uri = service_endpoint.dup
+    download_uri.path = "/#{url}"
+    download_uri.to_s
+  end
+
 end end end end

x-pack/lib/filters/geoip/extension.rb

@@ -0,0 +1,20 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require "logstash/environment"
+
+module LogStash module Filters module Geoip
+  class Extension < LogStash::UniversalPlugin
+    include LogStash::Util::Loggable
+
+    def additionals_settings(settings)
+      require "logstash/runner"
+      logger.trace("Registering additional geoip settings")
+      settings.register(LogStash::Setting::NullableString.new("xpack.geoip.download.endpoint"))
+    rescue => e
+      logger.error("Cannot register new settings", :message => e.message, :backtrace => e.backtrace)
+      raise e
+    end
+  end
+end end end

x-pack/lib/x-pack/logstash_registry.rb

@@ -13,6 +13,7 @@ require "monitoring/monitoring"
 require "monitoring/inputs/metrics"
 require "monitoring/outputs/elasticsearch_monitoring"
 require "config_management/extension"
+require "filters/geoip/extension"
 require "modules/xpack_scaffold"
 require "filters/azure_event"
 
@@ -20,6 +21,7 @@ LogStash::PLUGIN_REGISTRY.add(:input, "metrics", LogStash::Inputs::Metrics)
 LogStash::PLUGIN_REGISTRY.add(:output, "elasticsearch_monitoring", LogStash::Outputs::ElasticSearchMonitoring)
 LogStash::PLUGIN_REGISTRY.add(:universal, "monitoring", LogStash::MonitoringExtension)
 LogStash::PLUGIN_REGISTRY.add(:universal, "config_management", LogStash::ConfigManagement::Extension)
+LogStash::PLUGIN_REGISTRY.add(:universal, "geoip_auto_update", LogStash::Filters::Geoip::Extension)
 
 license_levels = Hash.new
 license_levels.default = LogStash::LicenseChecker::LICENSE_TYPES

x-pack/spec/filters/geoip/download_manager_spec.rb

@@ -22,7 +22,7 @@ describe LogStash::Filters::Geoip do
     GEOIP_STAGING_ENDPOINT = "#{GEOIP_STAGING_HOST}#{LogStash::Filters::Geoip::DownloadManager::GEOIP_PATH}"
 
     before do
-      stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT)
+      allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.download.endpoint").and_return(GEOIP_STAGING_ENDPOINT)
     end
 
     context "rest client" do
@@ -40,7 +40,7 @@ describe LogStash::Filters::Geoip do
 
     context "check update" do
       before(:each) do
-        expect(download_manager).to receive(:get_uuid).and_return(SecureRandom.uuid)
+        expect(download_manager).to receive(:uuid).and_return(SecureRandom.uuid)
         mock_resp = double("geoip_endpoint",
                            :body => ::File.read(::File.expand_path("./fixtures/normal_resp.json", ::File.dirname(__FILE__))),
                            :status => 200)
@@ -184,5 +184,40 @@ describe LogStash::Filters::Geoip do
       end
     end
 
+    context "download url" do
+      before do
+        allow(download_manager).to receive(:uuid).and_return(SecureRandom.uuid)
+      end
+
+      it "should give a path with hostname when input is a filename" do
+        expect(download_manager.send(:download_url, "GeoLite2-ASN.tgz")).to match /#{GEOIP_STAGING_HOST}/
+      end
+
+      it "should give a unmodified path when input has scheme" do
+        expect(download_manager.send(:download_url, GEOIP_STAGING_ENDPOINT)).to eq(GEOIP_STAGING_ENDPOINT)
+      end
+    end
+
+    context "service endpoint" do
+      before do
+        allow(download_manager).to receive(:uuid).and_return(SecureRandom.uuid)
+      end
+
+      it "should give xpack setting" do
+        uri = download_manager.send(:service_endpoint)
+        expect(uri.to_s).to match /#{GEOIP_STAGING_ENDPOINT}/
+      end
+
+      context "empty xpack config" do
+        before do
+          allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.download.endpoint").and_return(nil)
+        end
+
+        it "should give default endpoint" do
+          uri = download_manager.send(:service_endpoint)
+          expect(uri.to_s).to match /#{LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT}/
+        end
+      end
+    end
   end
 end