diff --git a/lib/pluginmanager/bundler/logstash_uninstall.rb b/lib/pluginmanager/bundler/logstash_uninstall.rb
index 792a7def2..2ab4168e4 100644
--- a/lib/pluginmanager/bundler/logstash_uninstall.rb
+++ b/lib/pluginmanager/bundler/logstash_uninstall.rb
@@ -37,7 +37,7 @@ module Bundler
     # To be uninstalled the candidate gems need to be standalone.
     def dependants_gems(gem_name)
       builder = Dsl.new
-      builder.eval_gemfile("original gemfile", File.read(gemfile_path))
+      builder.eval_gemfile(::File.join(::File.dirname(gemfile_path), "original gemfile"), File.read(gemfile_path))
       definition = builder.to_definition(lockfile_path, {})
 
       definition.specs
@@ -66,7 +66,7 @@ module Bundler
 
       gemfile = LogStash::Gemfile.new(file).load
       gemfile.remove(gem_name)
-      builder.eval_gemfile("gemfile to changes", gemfile.generate)
+      builder.eval_gemfile(::File.join(::File.dirname(gemfile_path), "gemfile to changes"), gemfile.generate)
 
       definition = builder.to_definition(lockfile_path, {})
       definition.lock(lockfile_path)
diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/remove.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/remove.rb
index 105f711e8..93ee966bf 100644
--- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/remove.rb
+++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/remove.rb
@@ -36,14 +36,12 @@ shared_examples "logstash remove" do |logstash|
       end
     end
 
-    # Disabled because of this bug https://github.com/elastic/logstash/issues/5286
-    xcontext "when the plugin is installed" do
+    context "when the plugin is installed" do
       it "successfully removes it" do
         result = logstash.run_command_in_path("bin/logstash-plugin install logstash-filter-qatest")
         expect(logstash).to have_installed?("logstash-filter-qatest")
 
         result = logstash.run_command_in_path("bin/logstash-plugin remove logstash-filter-qatest")
-        expect(result.stdout).to match(/^Removing logstash-filter-qatest/)
         expect(logstash).not_to have_installed?("logstash-filter-qatest")
       end
     end
diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb
index 6498040e0..36b283c6a 100644
--- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb
+++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb
@@ -36,14 +36,12 @@ shared_examples "logstash uninstall" do |logstash|
       end
     end
 
-    # Disabled because of this bug https://github.com/elastic/logstash/issues/5286
-    xcontext "when the plugin is installed" do
+    context "when the plugin is installed" do
       it "successfully uninstall it" do
         result = logstash.run_command_in_path("bin/logstash-plugin install logstash-filter-qatest")
         expect(logstash).to have_installed?("logstash-filter-qatest")
 
         result = logstash.run_command_in_path("bin/logstash-plugin uninstall logstash-filter-qatest")
-        expect(result.stdout).to match(/^Uninstalling logstash-filter-qatest/)
         expect(logstash).not_to have_installed?("logstash-filter-qatest")
       end
     end