diff --git a/Makefile b/Makefile
index 5b894ccd8..76b2a4771 100644
--- a/Makefile
+++ b/Makefile
@@ -33,6 +33,7 @@ else
 TAR_OPTS=--wildcards
 endif
 
+TESTS=$(wildcard spec/support/*.rb spec/filters/*.rb spec/examples/*.rb spec/event.rb spec/jar.rb)
 default: jar
 
 # Figure out if we're using wget or curl
@@ -203,10 +204,10 @@ build/flatgems: | build vendor/bundle
 	done
 
 flatjar-test:
-	GEM_HOME= GEM_PATH= java -jar build/logstash-$(VERSION)-flatjar.jar rspec spec/support/*.rb spec/filters/*.rb spec/examples/*.rb spec/event.rb spec/jar.rb
+	cd build; GEM_HOME= GEM_PATH= java -jar logstash-$(VERSION)-flatjar.jar rspec $(TESTS)
 
 jar-test:
-	GEM_HOME= GEM_PATH= java -jar build/logstash-$(VERSION)-monolithic.jar rspec spec/support/*.rb spec/filters/*.rb spec/examples/*.rb spec/event.rb spec/jar.rb
+	cd build; GEM_HOME= GEM_PATH= java -jar logstash-$(VERSION)-monolithic.jar rspec $(TESTS)
 
 flatjar: build/logstash-$(VERSION)-flatjar.jar
 build/jar: | build build/flatgems build/monolith