Add initial buildkite pipeline for Benchmark (#16190)

skeleton pipeline for benchmark

.buildkite/scripts/benchmark/config/filebeat.yml

@@ -0,0 +1,15 @@
+http.enabled: false
+filebeat.inputs:
+- type: log
+  symlinks: true
+  paths:
+  - "/usr/share/filebeat/flog/*.log"
+logging.level: info
+output.logstash:
+  hosts:
+    - "localhost:5044"
+  ttl: 10ms
+  bulk_max_size: 2048
+# queue.mem:
+#   events: 4096
+#   flush.min_events: 2048

.buildkite/scripts/benchmark/config/logstash.yml

@@ -0,0 +1,9 @@
+api.http.host: 0.0.0.0
+pipeline.workers: ${WORKER} 
+pipeline.batch.size: ${BATCH_SIZE}
+queue.type: ${QTYPE}
+
+xpack.monitoring.enabled: true
+xpack.monitoring.elasticsearch.username: ${MONITOR_ES_USER}
+xpack.monitoring.elasticsearch.password: ${MONITOR_ES_PW}
+xpack.monitoring.elasticsearch.hosts: ["${MONITOR_ES_HOST}"]

.buildkite/scripts/benchmark/config/pipelines.yml

@@ -0,0 +1,44 @@
+- pipeline.id: main
+  config.string: |
+    input { 
+      beats {
+        port => 5044
+      }
+    } 
+    output { 
+      elasticsearch {
+        hosts => [ "${BENCHMARK_ES_HOST}" ]
+        user => "${BENCHMARK_ES_USER}"
+        password => "${BENCHMARK_ES_PW}"
+      }
+    }
+- pipeline.id: node_stats
+  config.string: |
+    input {
+      http_poller {
+        urls => {
+          NodeStats => {
+            method => get
+            url => "http://localhost:9600/_node/stats"
+          }
+        }
+        schedule => { every => "30s"}
+        codec => "json"
+      }
+    }
+    filter {
+      mutate { 
+        remove_field => [ "host", "[pipelines][.monitoring-logstash]", "event" ]
+        add_field => { "[benchmark][label]" => "${QTYPE}_w${WORKER}b${BATCH_SIZE}" }
+      }
+    }
+    output {
+      elasticsearch {
+        hosts => [ "${BENCHMARK_ES_HOST}" ]
+        user => "${BENCHMARK_ES_USER}"
+        password => "${BENCHMARK_ES_PW}"
+        data_stream_type => "metrics"
+        data_stream_dataset => "nodestats"
+        data_stream_namespace => "logstash"
+      }
+    }

.buildkite/scripts/benchmark/main.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+main() {
+    echo "hello world"
+    echo "$@"
+}
+
+main "$@"

.buildkite/scripts/benchmark/util.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+arch() { uname -m | sed -e "s|amd|x86_|" -e "s|arm|aarch|"; }
+
+# return the min value
+# usage: 
+#  g: float; h: human; d: dictionary; M: month
+#  min -g 3 2 5 1
+#  max -g 1.5 5.2 2.5 1.2 5.7
+#  max -g "null" "0"
+#  min -h 25M 13G 99K 1098M
+min() { printf "%s\n" "${@:2}" | sort "$1" | head -n1 ; }
+
+max() { min ${1}r ${@:2} ; }