[ci] Compatibility tests for Exhaustive suite (#15641) (#15647)

This commit adds the compatibility tier for the Exhaustive tests suite.
Specifically, we introduce two new groups (running in parallel) for Linux and Windows compat tests.
Linux picks one OS per family from [^] and likewise Windows one of the three available choices from the same file.

We also support manual override, if user chooses to, by setting `LINUX_OS` or `WINDOWS_OS` as env vars in the Buildkite build prompt (in this case there is no randomization, and only one OS can be defined for Linux and Windows respectively).

For example:
```
LINUX_OS=rhel-9
WINDOWS_OS=windows=216
```

Relates:

- https://github.com/elastic/ingest-dev/issues/1722

[^1]: 4d6bd955e6/.buildkite/scripts/common/vm-images.json

(cherry picked from commit d42b938f81)

Co-authored-by: Dimitrios Liappis <dimitrios.liappis@gmail.com>

.buildkite/exhaustive_tests_pipeline.yml

@@ -1,3 +1,29 @@
 steps:
-  - label: "Test Exhaustive tests pipeline"
-    command: "echo 'Hello world'"
+  - label: "Exhaustive tests pipeline"
+    command: |
+      #!/usr/bin/env bash
+      set -eo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      echo "--- Downloading prerequisites"
+      python3 -m pip install ruamel.yaml
+      curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+      chmod a+x /usr/bin/yq
+
+      echo "--- Printing generated dynamic steps"
+      set +e
+      python3 .buildkite/scripts/exhaustive-tests/generate-steps.py >pipeline_steps.yml
+      if [[ $$? -ne 0 ]]; then
+        echo "^^^ +++"
+        echo "There was a problem rendering the pipeline steps."
+        cat pipeline_steps.yml
+        echo "Exiting now."
+        exit 1
+      else
+        set -eo pipefail
+        cat pipeline_steps.yml | yq .
+      fi
+
+      set -eo pipefail
+      echo "--- Uploading steps to buildkite"
+      cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/scripts/common/vm-images.json

@@ -0,0 +1,13 @@
+{
+    "#comment": "This file lists all custom vm images. We use it to make decisions about randomized CI jobs.",
+    "linux": {
+        "ubuntu": ["ubuntu-2204", "ubuntu-2004", "ubuntu-1804"],
+        "debian": ["debian-11", "debian-10"],
+        "rhel": ["rhel-9", "rhel-8"],
+        "oraclelinux": ["oraclelinux-8", "oraclelinux-7"],
+        "rocky": ["rocky-linux-8"],
+        "amazonlinux": ["amazonlinux-2023"],
+        "opensuse": ["opensuse-leap-15"]
+    },
+    "windows": ["windows-2022", "windows-2019", "windows-2016"]
+}

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -0,0 +1,105 @@
+import json
+import os
+import random
+import sys
+import typing
+
+from ruamel.yaml import YAML
+from ruamel.yaml.scalarstring import LiteralScalarString
+
+VM_IMAGES_FILE = ".buildkite/scripts/common/vm-images.json"
+VM_IMAGE_PREFIX = "platform-ingest-logstash-multi-jdk-"
+
+def slugify_bk_key(key: str) -> str:
+    """
+    Convert and return key to an acceptable format for Buildkite's key: field
+    Only alphanumerics, dashes and underscores are allowed.
+    """
+
+    mapping_table = str.maketrans({'.': '_', ' ': '_', '/': '_'})
+
+    return key.translate(mapping_table)
+
+def compat_linux_step(imagesuffix: str) -> dict[str, typing.Any]:
+    linux_command = LiteralScalarString("""#!/usr/bin/env bash
+set -eo pipefail
+source .buildkite/scripts/common/vm-agent.sh
+ci/unit_tests.sh""")
+
+    return compat_step(imagesuffix, command=linux_command)
+
+
+def compat_windows_step(imagesuffix: str) -> dict[str, typing.Any]:
+    windows_command = LiteralScalarString(r'''$$env:WORKSPACE=$$PWD.Path ; .\\ci\\unit_tests.bat''')
+
+    return compat_step(imagesuffix, command=windows_command)
+
+def compat_step(imagesuffix: str, command: LiteralScalarString) -> dict[str, typing.Any]:
+    step = {
+        "label": imagesuffix,
+        "key": slugify_bk_key(f"compat-linux-{imagesuffix}"),
+        "command": command,
+        "agents": {},
+    }
+
+    if "amazon" in imagesuffix.lower():
+        step["agents"] = {
+            "provider": "aws",
+            "imagePrefix": f"{VM_IMAGE_PREFIX}{imagesuffix}",
+            "instanceType": "m5.2xlarge",
+            "diskSizeGb": 200,
+       }
+    else:
+       step["agents"] = {
+            "provider": "gcp",
+            "imageProject": "elastic-images-prod",
+            "image": f"family/{VM_IMAGE_PREFIX}{imagesuffix}",
+            "machineType": "n2-standard-4",
+            "diskSizeGb": 200,
+            "diskType": "pd-ssd",
+       }
+
+    return step
+
+def randomized_linux_oses() -> typing.List[str]:
+    with open(VM_IMAGES_FILE, "r") as fp:
+        all_oses = json.load(fp)
+
+    randomized_oses = []
+    for _, family_oses in all_oses["linux"].items():
+       randomized_oses.append(random.choice(family_oses))
+    return randomized_oses
+
+def randomized_windows_os() -> str:
+    with open(VM_IMAGES_FILE, "r") as fp:
+        all_oses = json.load(fp)
+
+    return random.choice(all_oses["windows"])
+
+if __name__ == "__main__":
+    LINUX_OS_ENV_VAR_OVERRIDE = os.getenv("LINUX_OS")
+    WINDOWS_OS_ENV_VAR_OVERRIDE = os.getenv("WINDOWS_OS")
+
+    compat_linux_steps = []
+    linux_test_oses = [LINUX_OS_ENV_VAR_OVERRIDE] if LINUX_OS_ENV_VAR_OVERRIDE else randomized_linux_oses()
+    for linux_os in linux_test_oses:
+       compat_linux_steps.append(compat_linux_step(linux_os))
+
+    windows_test_os = WINDOWS_OS_ENV_VAR_OVERRIDE or randomized_windows_os()
+
+    structure = {"steps": []}
+
+    structure["steps"].append({
+            "group": "Compatibility / Linux",
+            "key": "compatibility-linux",
+            "steps": compat_linux_steps,
+    })
+
+    structure["steps"].append({
+            "group": "Compatibility / Windows",
+            "key": "compatibility-windows",
+            "steps": [compat_windows_step(imagesuffix=windows_test_os)],
+    })
+
+    print('# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json')
+    YAML().dump(structure, sys.stdout)