diff --git a/.buildkite/scripts/benchmark/core.sh b/.buildkite/scripts/benchmark/core.sh
index 130c937a6..b778b0ccb 100755
--- a/.buildkite/scripts/benchmark/core.sh
+++ b/.buildkite/scripts/benchmark/core.sh
@@ -131,6 +131,8 @@ start_logstash() {
   cp $CONFIG_PATH/logstash.yml $LS_CONFIG_PATH/logstash.yml
   cp $CONFIG_PATH/uuid $LS_CONFIG_PATH/uuid
 
+  remove_v9_config
+
   LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx${XMX}g}
   docker run -d --name=ls --net=host --cpus=$CPU --memory=${MEM}g -e LS_JAVA_OPTS="$LS_JAVA_OPTS" \
     -e QTYPE="$QTYPE" -e WORKER="$WORKER" -e BATCH_SIZE="$BATCH_SIZE" \
@@ -142,6 +144,20 @@ start_logstash() {
     docker.elastic.co/logstash/logstash:$LS_VERSION
 }
 
+remove_v9_config() {
+  local config_path="$LS_CONFIG_PATH/logstash.yml"
+  local major_version=$(echo $LS_VERSION | cut -d. -f1)
+  if [ "$major_version" -lt 9 ]; then
+    echo "Remove v9 config 'xpack.monitoring.allow_legacy_collection' from logstash.yml"
+
+    if [[ "$(uname)" == "Darwin" ]]; then
+      sed -i '' '/xpack\.monitoring\.allow_legacy_collection/d' "$config_path"
+    else
+      sed -i '/xpack\.monitoring\.allow_legacy_collection/d' "$config_path"
+    fi
+  fi
+}
+
 start_filebeat() {
   for ((i = 0; i < FB_CNT; i++)); do
     FB_PATH="$SCRIPT_PATH/fb${i}"