From 3c27f8ab407874cfb744ebe142177ff6f31b8c4f Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Thu, 15 Apr 2021 15:20:20 +0200 Subject: [PATCH] GeoIP database copy all files from .tgz alongside database (#12824) (#12826) This PR changes the behavior of copying license files from .tgz Originally, only two files, MaxMind LICENSE.txt and COPYRIGHT.txt, are required Now more files, README.txt and Elastic ToC, are potentially required Instead of targeting the files, this change copies all content in .tgz --- x-pack/lib/filters/geoip/download_manager.rb | 13 ++++++++++--- .../filters/geoip/download_manager_spec.rb | 18 ++++++++++++------ x-pack/spec/filters/geoip/fixtures/sample.tgz | Bin 562 -> 647 bytes 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/x-pack/lib/filters/geoip/download_manager.rb b/x-pack/lib/filters/geoip/download_manager.rb index 351a47471..62f4630a6 100644 --- a/x-pack/lib/filters/geoip/download_manager.rb +++ b/x-pack/lib/filters/geoip/download_manager.rb @@ -78,7 +78,8 @@ module LogStash module Filters module Geoip class DownloadManager end end - # extract COPYRIGHT.txt, LICENSE.txt and GeoLite2-{ASN,City}.mmdb from .tgz to temp directory + # extract all files and folders from .tgz to vendor directory + # existing files folders will be replaced def unzip(zip_path) new_database_path = zip_path[0...-(GZ_EXT.length)] + DB_EXT temp_dir = Stud::Temporary.pathname @@ -86,9 +87,15 @@ module LogStash module Filters module Geoip class DownloadManager LogStash::Util::Tar.extract(zip_path, temp_dir) logger.debug("extract database to ", :path => temp_dir) + ::Dir.each_child(temp_dir) do |file| + path = ::File.join(temp_dir, file) - FileUtils.cp(::File.join(temp_dir, database_name_ext), new_database_path) - FileUtils.cp_r(::Dir.glob(::File.join(temp_dir, "{COPYRIGHT,LICENSE}.txt")), @vendor_path) + if !::File.directory?(path) && database_name_ext.eql?(file) + FileUtils.cp(path, new_database_path) + else + FileUtils.cp_r(path, @vendor_path) + end + end new_database_path end diff --git a/x-pack/spec/filters/geoip/download_manager_spec.rb b/x-pack/spec/filters/geoip/download_manager_spec.rb index efd3ac0ed..22e092c1d 100644 --- a/x-pack/spec/filters/geoip/download_manager_spec.rb +++ b/x-pack/spec/filters/geoip/download_manager_spec.rb @@ -3,6 +3,7 @@ # you may not use this file except in compliance with the Elastic License. require_relative 'test_helper' +require 'fileutils' require "filters/geoip/download_manager" describe LogStash::Filters::Geoip do @@ -105,13 +106,17 @@ describe LogStash::Filters::Geoip do let(:copyright_path) { get_file_path('COPYRIGHT.txt') } let(:license_path) { get_file_path('LICENSE.txt') } let(:readme_path) { get_file_path('README.txt') } + let(:folder_path) { get_file_path('inner') } + let(:folder_more_path) { ::File.join(get_file_path('inner'), 'more.txt') } + let(:folder_less_path) { ::File.join(get_file_path('inner'), 'less.txt') } - before do - file_path = ::File.expand_path("./fixtures/sample", ::File.dirname(__FILE__)) + after do + file_path = ::File.expand_path("./fixtures/sample.mmdb", ::File.dirname(__FILE__)) delete_file(file_path, copyright_path, license_path, readme_path) + FileUtils.rm_r folder_path end - it "should extract database and license related files" do + it "should extract all files in tarball" do path = ::File.expand_path("./fixtures/sample.tgz", ::File.dirname(__FILE__)) unzip_db_path = download_manager.send(:unzip, path) @@ -119,9 +124,10 @@ describe LogStash::Filters::Geoip do expect(::File.exist?(unzip_db_path)).to be_truthy expect(::File.exist?(copyright_path)).to be_truthy expect(::File.exist?(license_path)).to be_truthy - expect(::File.exist?(readme_path)).to be_falsey - - delete_file(unzip_db_path, copyright_path, license_path) + expect(::File.exist?(readme_path)).to be_truthy + expect(::File.directory?(folder_path)).to be_truthy + expect(::File.exist?(folder_more_path)).to be_truthy + expect(::File.exist?(folder_less_path)).to be_truthy end end diff --git a/x-pack/spec/filters/geoip/fixtures/sample.tgz b/x-pack/spec/filters/geoip/fixtures/sample.tgz index 22bb93cb7a126085475ae6357e6f8577797dd9c8..3336704a4dae1768a583056e9065bcf6f10c01fd 100644 GIT binary patch literal 647 zcmV;20(ku&iwFRKF?V191MQeyZ__XohIidxaqa+VMB^|SLlDMs7@t5eSVRxh8Ot<<@{Lz0%U{oDvVgdPjKm5wT1**BQ7{QcglY@r2;3&!Jtkj4NN_z^c zO7IFelaM1U1+K`2S*WaS%{$*;tyW&c?izCRxN3Zqeu=8y*!tMj%o~kvH$fH5vYe5% zkX-3jX=ZU7!4+$$^5vl)8OTkqr*!!hs8wp#QW&h>+U#Q@*M=1cY{^8;@*1I&1ueo> z2osIWiX~N4(k53=!N{<9!-{2xN+d%PH-JW@S4AYD2T#6{iiUK{3U*m^lw%wG`J45;0Di8(j7-F%eJlHaA1*hDt!B{#6 zSYSn=(HKw~=)(ppRTpma_?L7&Jv+Y~_W1t?{KwSg_& literal 562 zcmV-20?qv&iwFP@FH&Ft1MOB#kJB&^-E)7%+yJTMqYbIVt?E`wrFKEP<$w?e$DSq@ zi5=OKe*Al;X}hdef|U?0Ao4D@$K#ioByYy^*<^ZqKMiP0SLZ?$MQM`2ejKH5+A$g& z)-BRFg*ahTG8o0l2%>m6jE7ecoh_nAXgmqWp?sAEn*7Ip?6z9{gy)b7JdFPcK6(VZ z0xGnJh1f2%QBXUp8cD#W+^*3XRFLgX+b=xapq*>N0ZiGO863>DM8j0$+#yzIs1KV$ z%L1e@kRhx!ZkR<76hyUu8-|8lyM~Di9_P z1?e?UY*ks~VNB$7){wRak{~)|8o<)FS%Cm)>)pdkED!sA-t_u=y7I^?548IT!=tMY zvt7v4VbHt$_n*A~@ab-L^KN;@3hz Apa1{>