From 77a6af1ba1e015ff820d674130ab21431a844f38 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Tue, 28 Sep 2021 16:23:51 +0200 Subject: [PATCH] geoip integrate air-gapped bootstrap script (#13104) (#13251) This PR integrates Elasticsearch bootstrap script to help users keep Logstah geoip plugin run without online update check. Add `xpack.geoip.download.endpoint` option to config geoip database service endpoint. Users can point to `http://localhost:8080/overview.json` when using the script to bootstrap nginx docker --- config/logstash.yml | 4 ++ docker/data/logstash/env2yaml/env2yaml.go | 1 + x-pack/lib/filters/geoip/database_manager.rb | 15 ++++--- x-pack/lib/filters/geoip/download_manager.rb | 31 ++++++++++++--- x-pack/lib/filters/geoip/extension.rb | 20 ++++++++++ x-pack/lib/x-pack/logstash_registry.rb | 2 + .../filters/geoip/download_manager_spec.rb | 39 ++++++++++++++++++- 7 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 x-pack/lib/filters/geoip/extension.rb diff --git a/config/logstash.yml b/config/logstash.yml index 24c2c266e..b9bca3435 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -302,3 +302,7 @@ #xpack.management.elasticsearch.ssl.verification_mode: certificate #xpack.management.elasticsearch.sniffing: false #xpack.management.logstash.poll_interval: 5s + +# X-Pack GeoIP plugin +# https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-manage_update +#xpack.geoip.download.endpoint: "https://geoip.elastic.co/v1/database" diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index 81029fe78..68d4ffdd2 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -121,6 +121,7 @@ func normalizeSetting(setting string) (string, error) { "xpack.management.elasticsearch.ssl.truststore.password", "xpack.management.elasticsearch.ssl.keystore.path", "xpack.management.elasticsearch.ssl.keystore.password", + "xpack.geoip.download.endpoint", "cloud.id", "cloud.auth", } diff --git a/x-pack/lib/filters/geoip/database_manager.rb b/x-pack/lib/filters/geoip/database_manager.rb index ca63fde9b..7452f63cb 100644 --- a/x-pack/lib/filters/geoip/database_manager.rb +++ b/x-pack/lib/filters/geoip/database_manager.rb @@ -183,7 +183,7 @@ module LogStash module Filters module Geoip class DatabaseManager database_status = :up_to_date end - @metric.namespace([:database, database_type.to_sym]).tap do |n| + metric.namespace([:database, database_type.to_sym]).tap do |n| n.gauge(:status, database_status) n.gauge(:last_updated_at, unix_time_to_iso8601(metadata[DatabaseMetadata::Column::DIRNAME])) n.gauge(:fail_check_in_days, days_without_update) @@ -229,7 +229,7 @@ module LogStash module Filters module Geoip class DatabaseManager metadatas = @metadata.get_all metadatas.each do |row| type = row[DatabaseMetadata::Column::DATABASE_TYPE] - @metric.namespace([:database, type.to_sym]).tap do |n| + metric.namespace([:database, type.to_sym]).tap do |n| n.gauge(:status, @states[type].is_eula ? :up_to_date : :init) if @states[type].is_eula n.gauge(:last_updated_at, unix_time_to_iso8601(row[DatabaseMetadata::Column::DIRNAME])) @@ -238,14 +238,14 @@ module LogStash module Filters module Geoip class DatabaseManager end end - @metric.namespace([:download_stats]).tap do |n| + metric.namespace([:download_stats]).tap do |n| check_at = metadatas.map { |row| row[DatabaseMetadata::Column::CHECK_AT].to_i }.max n.gauge(:last_checked_at, unix_time_to_iso8601(check_at)) end end def update_download_metric(success_cnt) - @metric.namespace([:download_stats]).tap do |n| + metric.namespace([:download_stats]).tap do |n| n.gauge(:last_checked_at, Time.now.iso8601) if success_cnt == DB_TYPES.size @@ -259,7 +259,7 @@ module LogStash module Filters module Geoip class DatabaseManager end def update_download_status(status) - @metric.namespace([:download_stats]).gauge(:status, status) + metric.namespace([:download_stats]).gauge(:status, status) end public @@ -302,6 +302,11 @@ module LogStash module Filters module Geoip class DatabaseManager @metric = metric end + def metric + # Fallback when testing plugin and no metric collector are correctly configured. + @metric ||= LogStash::Instrument::NamespacedNullMetric.new + end + class DatabaseState attr_reader :is_eula, :plugins, :database_path, :cc_database_path, :is_expired attr_writer :is_eula, :database_path, :is_expired diff --git a/x-pack/lib/filters/geoip/download_manager.rb b/x-pack/lib/filters/geoip/download_manager.rb index 3ae7c35d8..e3fa70025 100644 --- a/x-pack/lib/filters/geoip/download_manager.rb +++ b/x-pack/lib/filters/geoip/download_manager.rb @@ -13,6 +13,7 @@ require "zlib" require "stud/try" require "down" require "fileutils" +require 'uri' module LogStash module Filters module Geoip class DownloadManager include LogStash::Util::Loggable @@ -50,9 +51,8 @@ module LogStash module Filters module Geoip class DownloadManager # Call infra endpoint to get md5 of latest databases and verify with metadata # return Array of new database information [database_type, db_info] def check_update - uuid = get_uuid - res = rest_client.get("#{GEOIP_ENDPOINT}?key=#{uuid}&elastic_geoip_service_tos=agree") - logger.debug("check update", :endpoint => GEOIP_ENDPOINT, :response => res.status) + res = rest_client.get(service_endpoint) + logger.debug("check update", :endpoint => service_endpoint.to_s, :response => res.status) service_resp = JSON.parse(res.body) @@ -74,7 +74,10 @@ module LogStash module Filters module Geoip class DownloadManager FileUtils.mkdir_p(get_dir_path(dirname)) zip_path = get_gz_path(database_type, dirname) - Down.download(db_info['url'], destination: zip_path) + actual_url = download_url(db_info['url']) + logger.debug? && logger.debug("download #{actual_url}") + + Down.download(actual_url, destination: zip_path) raise "the new download has wrong checksum" if md5(zip_path) != db_info['md5_hash'] logger.debug("new database downloaded in ", :path => zip_path) @@ -105,7 +108,25 @@ module LogStash module Filters module Geoip class DownloadManager end end - def get_uuid + def uuid @uuid ||= ::File.read(::File.join(LogStash::SETTINGS.get("path.data"), "uuid")) end + + def service_endpoint + return @service_endpoint if @service_endpoint + + uri = URI(LogStash::SETTINGS.get("xpack.geoip.download.endpoint") || GEOIP_ENDPOINT) + uri.query = "key=#{uuid}&elastic_geoip_service_tos=agree" + @service_endpoint = uri + end + + def download_url(url) + uri = URI(url) + return url if uri.scheme + + download_uri = service_endpoint.dup + download_uri.path = "/#{url}" + download_uri.to_s + end + end end end end diff --git a/x-pack/lib/filters/geoip/extension.rb b/x-pack/lib/filters/geoip/extension.rb new file mode 100644 index 000000000..f2f7f5d45 --- /dev/null +++ b/x-pack/lib/filters/geoip/extension.rb @@ -0,0 +1,20 @@ +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +require "logstash/environment" + +module LogStash module Filters module Geoip + class Extension < LogStash::UniversalPlugin + include LogStash::Util::Loggable + + def additionals_settings(settings) + require "logstash/runner" + logger.trace("Registering additional geoip settings") + settings.register(LogStash::Setting::NullableString.new("xpack.geoip.download.endpoint")) + rescue => e + logger.error("Cannot register new settings", :message => e.message, :backtrace => e.backtrace) + raise e + end + end +end end end \ No newline at end of file diff --git a/x-pack/lib/x-pack/logstash_registry.rb b/x-pack/lib/x-pack/logstash_registry.rb index e52d42391..250e7fceb 100644 --- a/x-pack/lib/x-pack/logstash_registry.rb +++ b/x-pack/lib/x-pack/logstash_registry.rb @@ -13,6 +13,7 @@ require "monitoring/monitoring" require "monitoring/inputs/metrics" require "monitoring/outputs/elasticsearch_monitoring" require "config_management/extension" +require "filters/geoip/extension" require "modules/xpack_scaffold" require "filters/azure_event" @@ -20,6 +21,7 @@ LogStash::PLUGIN_REGISTRY.add(:input, "metrics", LogStash::Inputs::Metrics) LogStash::PLUGIN_REGISTRY.add(:output, "elasticsearch_monitoring", LogStash::Outputs::ElasticSearchMonitoring) LogStash::PLUGIN_REGISTRY.add(:universal, "monitoring", LogStash::MonitoringExtension) LogStash::PLUGIN_REGISTRY.add(:universal, "config_management", LogStash::ConfigManagement::Extension) +LogStash::PLUGIN_REGISTRY.add(:universal, "geoip_auto_update", LogStash::Filters::Geoip::Extension) license_levels = Hash.new license_levels.default = LogStash::LicenseChecker::LICENSE_TYPES diff --git a/x-pack/spec/filters/geoip/download_manager_spec.rb b/x-pack/spec/filters/geoip/download_manager_spec.rb index 2052dc49b..da5a4ee45 100644 --- a/x-pack/spec/filters/geoip/download_manager_spec.rb +++ b/x-pack/spec/filters/geoip/download_manager_spec.rb @@ -22,7 +22,7 @@ describe LogStash::Filters::Geoip do GEOIP_STAGING_ENDPOINT = "#{GEOIP_STAGING_HOST}#{LogStash::Filters::Geoip::DownloadManager::GEOIP_PATH}" before do - stub_const('LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT', GEOIP_STAGING_ENDPOINT) + allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.download.endpoint").and_return(GEOIP_STAGING_ENDPOINT) end context "rest client" do @@ -40,7 +40,7 @@ describe LogStash::Filters::Geoip do context "check update" do before(:each) do - expect(download_manager).to receive(:get_uuid).and_return(SecureRandom.uuid) + expect(download_manager).to receive(:uuid).and_return(SecureRandom.uuid) mock_resp = double("geoip_endpoint", :body => ::File.read(::File.expand_path("./fixtures/normal_resp.json", ::File.dirname(__FILE__))), :status => 200) @@ -184,5 +184,40 @@ describe LogStash::Filters::Geoip do end end + context "download url" do + before do + allow(download_manager).to receive(:uuid).and_return(SecureRandom.uuid) + end + + it "should give a path with hostname when input is a filename" do + expect(download_manager.send(:download_url, "GeoLite2-ASN.tgz")).to match /#{GEOIP_STAGING_HOST}/ + end + + it "should give a unmodified path when input has scheme" do + expect(download_manager.send(:download_url, GEOIP_STAGING_ENDPOINT)).to eq(GEOIP_STAGING_ENDPOINT) + end + end + + context "service endpoint" do + before do + allow(download_manager).to receive(:uuid).and_return(SecureRandom.uuid) + end + + it "should give xpack setting" do + uri = download_manager.send(:service_endpoint) + expect(uri.to_s).to match /#{GEOIP_STAGING_ENDPOINT}/ + end + + context "empty xpack config" do + before do + allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.download.endpoint").and_return(nil) + end + + it "should give default endpoint" do + uri = download_manager.send(:service_endpoint) + expect(uri.to_s).to match /#{LogStash::Filters::Geoip::DownloadManager::GEOIP_ENDPOINT}/ + end + end + end end end \ No newline at end of file