[artifacts] Stabilize package testing (#134854)

* bump ansible logging verbosity

* upgrade base os

* update base images

* add specificity to docker image glob

* limit memory

* bump base vm memory

* more cpu

* decrease verbosity

* less downloads

* disable shared symlinks

* more logging

* bump concurrency to test stability

* add concurrency group

* revert

* parallelism

* Revert "parallelism"

This reverts commit 4cc00f7144.

.buildkite/scripts/steps/package_testing/test.sh

@@ -9,10 +9,10 @@ echo "--- Package Testing for $TEST_PACKAGE"
 mkdir -p target
 cd target
 if [[ "$TEST_PACKAGE" == "deb" ]]; then
-  buildkite-agent artifact download 'kibana-*.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download 'kibana-*-amd64.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.5"
 elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
-  buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download 'kibana-*-x86_64.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.6"
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
   buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
@@ -30,10 +30,14 @@ node scripts/es snapshot \
 while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
 
 function echoKibanaLogs {
-  echo '--- Kibana logs'
   if [[ "$TEST_PACKAGE" == "deb" ]] || [[ "$TEST_PACKAGE" == "rpm" ]]; then
+    echo "--- /var/log/kibana/kibana.log "
     vagrant ssh $TEST_PACKAGE -t -c 'sudo cat /var/log/kibana/kibana.log'
+
+    echo "--- Journal "
+    vagrant ssh $TEST_PACKAGE -t -c 'sudo journalctl -u kibana.service --no-pager'
   elif [[ "$TEST_PACKAGE" == "docker" ]]; then
+    echo '--- Docker logs'
     vagrant ssh $TEST_PACKAGE -t -c 'sudo docker logs kibana'
   fi
 }

test/package/Vagrantfile

@@ -1,12 +1,13 @@
 Vagrant.configure("2") do |config|
-  config.vm.synced_folder '../../target/', '/packages'
+  config.vm.synced_folder '../../target/', '/packages', SharedFoldersEnableSymlinksCreate: false
   config.vm.boot_timeout = 600
   config.vm.define "deb" do |deb|
     deb.vm.provider :virtualbox do |vb|
-      vb.memory = 2048
+      vb.memory = 4096
+      vb.cpus = 2
     end
-    deb.vm.box = 'elastic/debian-9-x86_64'
-    deb.vm.provision "ansible_local" do |ansible|
+    deb.vm.box = 'ubuntu/jammy64'
+    deb.vm.provision "ansible" do |ansible|
       ansible.playbook = "deb.yml"
     end
     deb.vm.network "private_network", ip: "192.168.56.5"
@@ -14,10 +15,11 @@ Vagrant.configure("2") do |config|
 
   config.vm.define "rpm" do |rpm|
     rpm.vm.provider :virtualbox do |vb|
-      vb.memory = 2048
+      vb.memory = 4096
+      vb.cpus = 2
     end
-    rpm.vm.box = 'elastic/centos-7-x86_64'
-    rpm.vm.provision "ansible_local" do |ansible|
+    rpm.vm.box = 'almalinux/9'
+    rpm.vm.provision "ansible" do |ansible|
       ansible.playbook = "rpm.yml"
     end
     rpm.vm.network "private_network", ip: "192.168.56.6"
@@ -25,10 +27,11 @@ Vagrant.configure("2") do |config|
 
   config.vm.define "docker" do |docker|
     docker.vm.provider :virtualbox do |vb|
-      vb.memory = 2048
+      vb.memory = 4096
+      vb.cpus = 2
     end
-    docker.vm.box = 'elastic/ubuntu-18.04-x86_64'
-    docker.vm.provision "ansible_local" do |ansible|
+    docker.vm.box = 'ubuntu/jammy64'
+    docker.vm.provision "ansible" do |ansible|
       ansible.playbook = "docker.yml"
     end
     docker.vm.network "private_network", ip: "192.168.56.7"

test/package/ansible.cfg

@@ -0,0 +1,2 @@
+[defaults]
+gathering = explicit

test/package/roles/install_kibana_deb/tasks/main.yml

@@ -26,6 +26,12 @@
     dest: /etc/kibana/kibana.yml
   register: config
 
+- name: copy node.options
+  become: yes
+  template:
+    src: templates/node.options
+    dest: /etc/kibana/node.options
+
 - name: start kibana
   become: yes
   systemd:

test/package/roles/install_kibana_docker/tasks/main.yml

@@ -1,7 +1,7 @@
 - name: find docker image
   find:
     paths: /packages/
-    patterns: kibana-*-docker-image.tar.gz
+    patterns: kibana-[0-9]*-docker-image.tar.gz
   register: kibana_docker
 
 - name: load image

test/package/roles/install_kibana_rpm/tasks/main.yml

@@ -27,15 +27,11 @@
     dest: /etc/kibana/kibana.yml
   register: config
 
-- name: open port 5601/tcp
+- name: copy node.options
   become: yes
-  command:
-    cmd: firewall-cmd --zone=public --permanent --add-port=5601/tcp
-
-- name: reload firewall
-  become: yes
-  command:
-    cmd: firewall-cmd --reload
+  template:
+    src: templates/node.options
+    dest: /etc/kibana/node.options
 
 - name: start kibana
   become: yes

test/package/templates/node.options

@@ -0,0 +1,2 @@
+--max-old-space-size=812
+--unhandled-rejections=warn