setting: enforce non-nullable (restore 8.15.x behavior) (#17522) (#17528)

(cherry picked from commit 712b37e1df)

Co-authored-by: Rye Biesemeyer <yaauie@users.noreply.github.com>

.buildkite/aarch64_pipeline.yml

@@ -35,48 +35,71 @@ steps:
           automatic:
             - limit: 3
 
-      - label: ":lab_coat: Integration Tests / part 1"
-        key: "integration-tests-part-1"
+      - label: ":lab_coat: Integration Tests / part 1-of-3"
+        key: "integration-tests-part-1-of-3"
         command: |
           set -euo pipefail
 
           source .buildkite/scripts/common/vm-agent.sh
-          ci/integration_tests.sh split 0
+          ci/integration_tests.sh split 0 3
         retry:
           automatic:
             - limit: 3
 
-      - label: ":lab_coat: Integration Tests / part 2"
-        key: "integration-tests-part-2"
+      - label: ":lab_coat: Integration Tests / part 2-of-3"
+        key: "integration-tests-part-2-of-3"
         command: |
           set -euo pipefail
 
           source .buildkite/scripts/common/vm-agent.sh
-          ci/integration_tests.sh split 1
+          ci/integration_tests.sh split 1 3
         retry:
           automatic:
             - limit: 3
 
-      - label: ":lab_coat: IT Persistent Queues / part 1"
-        key: "integration-tests-qa-part-1"
+      - label: ":lab_coat: Integration Tests / part 3-of-3"
+        key: "integration-tests-part-3-of-3"
+        command: |
+          set -euo pipefail
+
+          source .buildkite/scripts/common/vm-agent.sh
+          ci/integration_tests.sh split 2 3
+        retry:
+          automatic:
+            - limit: 3
+
+      - label: ":lab_coat: IT Persistent Queues / part 1-of-3"
+        key: "integration-tests-qa-part-1-of-3"
         command: |
           set -euo pipefail
 
           source .buildkite/scripts/common/vm-agent.sh
           export FEATURE_FLAG=persistent_queues
-          ci/integration_tests.sh split 0
+          ci/integration_tests.sh split 0 3
         retry:
           automatic:
             - limit: 3
 
-      - label: ":lab_coat: IT Persistent Queues / part 2"
-        key: "integration-tests-qa-part-2"
+      - label: ":lab_coat: IT Persistent Queues / part 2-of-3"
+        key: "integration-tests-qa-part-2-of-3"
         command: |
           set -euo pipefail
 
           source .buildkite/scripts/common/vm-agent.sh
           export FEATURE_FLAG=persistent_queues
-          ci/integration_tests.sh split 1
+          ci/integration_tests.sh split 1 3
+        retry:
+          automatic:
+            - limit: 3
+
+      - label: ":lab_coat: IT Persistent Queues / part 3-of-3"
+        key: "integration-tests-qa-part-3-of-3"
+        command: |
+          set -euo pipefail
+
+          source .buildkite/scripts/common/vm-agent.sh
+          export FEATURE_FLAG=persistent_queues
+          ci/integration_tests.sh split 2 3
         retry:
           automatic:
             - limit: 3

.buildkite/dra_pipeline.yml

@@ -4,8 +4,12 @@ steps:
   - label: ":pipeline: Generate steps"
     command: |
       set -euo pipefail
-
-      echo "--- Building [${WORKFLOW_TYPE}] artifacts"
+      
+      echo "--- Building [$${WORKFLOW_TYPE}] artifacts"
       python3 -m pip install pyyaml
       echo "--- Building dynamic pipeline steps"
-      python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload
+      python3 .buildkite/scripts/dra/generatesteps.py > steps.yml
+      echo "--- Printing dynamic pipeline steps"
+      cat steps.yml
+      echo "--- Uploading dynamic pipeline steps"
+      cat steps.yml | buildkite-agent pipeline upload

.buildkite/health_report_tests_pipeline.yml

@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+agents:
+  provider: gcp
+  imageProject: elastic-images-prod
+  image: family/platform-ingest-logstash-ubuntu-2204
+  machineType: "n2-standard-4"
+  diskSizeGb: 64
+
+steps:
+  - group: ":logstash: Health API integration tests"
+    key: "testing-phase"
+    steps:
+      - label: "main branch"
+        key: "integ-tests-on-main-branch"
+        command:
+          - .buildkite/scripts/health-report-tests/main.sh
+        retry:
+          automatic:
+            - limit: 3

.buildkite/jdk_availability_check_pipeline.yml

@@ -0,0 +1,14 @@
+steps:
+  - label: "JDK Availability check"
+    key: "jdk-availability-check"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
+      cpu: "4"
+      memory: "6Gi"
+      ephemeralStorage: "100Gi"
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
+      ci/check_jdk_version_availability.sh

.buildkite/pull_request_pipeline.yml

@@ -79,8 +79,8 @@ steps:
       manual:
         allowed: true
 
-  - label: ":lab_coat: Integration Tests / part 1"
-    key: "integration-tests-part-1"
+  - label: ":lab_coat: Integration Tests / part 1-of-3"
+    key: "integration-tests-part-1-of-3"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -95,10 +95,10 @@ steps:
       set -euo pipefail
 
       source .buildkite/scripts/common/container-agent.sh
-      ci/integration_tests.sh split 0
+      ci/integration_tests.sh split 0 3
 
-  - label: ":lab_coat: Integration Tests / part 2"
-    key: "integration-tests-part-2"
+  - label: ":lab_coat: Integration Tests / part 2-of-3"
+    key: "integration-tests-part-2-of-3"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -113,10 +113,28 @@ steps:
       set -euo pipefail
 
       source .buildkite/scripts/common/container-agent.sh
-      ci/integration_tests.sh split 1
+      ci/integration_tests.sh split 1 3
 
-  - label: ":lab_coat: IT Persistent Queues / part 1"
-    key: "integration-tests-qa-part-1"
+  - label: ":lab_coat: Integration Tests / part 3-of-3"
+    key: "integration-tests-part-3-of-3"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      cpu: "8"
+      memory: "16Gi"
+      ephemeralStorage: "100Gi"
+      # Run as a non-root user
+      imageUID: "1002"
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      ci/integration_tests.sh split 2 3
+
+  - label: ":lab_coat: IT Persistent Queues / part 1-of-3"
+    key: "integration-tests-qa-part-1-of-3"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -132,10 +150,10 @@ steps:
 
       source .buildkite/scripts/common/container-agent.sh
       export FEATURE_FLAG=persistent_queues
-      ci/integration_tests.sh split 0
+      ci/integration_tests.sh split 0 3
 
-  - label: ":lab_coat: IT Persistent Queues / part 2"
-    key: "integration-tests-qa-part-2"
+  - label: ":lab_coat: IT Persistent Queues / part 2-of-3"
+    key: "integration-tests-qa-part-2-of-3"
     agents:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
       cpu: "8"
@@ -151,7 +169,26 @@ steps:
 
       source .buildkite/scripts/common/container-agent.sh
       export FEATURE_FLAG=persistent_queues
-      ci/integration_tests.sh split 1
+      ci/integration_tests.sh split 1 3
+
+  - label: ":lab_coat: IT Persistent Queues / part 3-of-3"
+    key: "integration-tests-qa-part-3-of-3"
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      cpu: "8"
+      memory: "16Gi"
+      ephemeralStorage: "100Gi"
+      # Run as non root (logstash) user. UID is hardcoded in image.
+      imageUID: "1002"
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      export FEATURE_FLAG=persistent_queues
+      ci/integration_tests.sh split 2 3
 
   - label: ":lab_coat: x-pack unit tests"
     key: "x-pack-unit-tests"

.buildkite/scripts/common/trigger-pipeline-generate-steps.sh

@@ -12,7 +12,7 @@ set -eo pipefail
 # https://github.com/elastic/ingest-dev/issues/2664
 # *******************************************************
 
-ACTIVE_BRANCHES_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/branches.json"
+ACTIVE_BRANCHES_URL="https://storage.googleapis.com/artifacts-api/snapshots/branches.json"
 EXCLUDE_BRANCHES_ARRAY=()
 BRANCHES=()
 
@@ -63,7 +63,7 @@ exclude_branches_to_array
 set -u
 set +e
 # pull releaseable branches from $ACTIVE_BRANCHES_URL
-readarray -t ELIGIBLE_BRANCHES < <(curl --retry-all-errors --retry 5 --retry-delay 5 -fsSL $ACTIVE_BRANCHES_URL | jq -r '.branches[].branch')
+readarray -t ELIGIBLE_BRANCHES < <(curl --retry-all-errors --retry 5 --retry-delay 5 -fsSL $ACTIVE_BRANCHES_URL | jq -r '.branches[]')
 if [[ $? -ne 0 ]]; then
   echo "There was an error downloading or parsing the json output from [$ACTIVE_BRANCHES_URL]. Exiting."
   exit 1

.buildkite/scripts/common/vm-images.json

@@ -9,5 +9,5 @@
         "amazonlinux": ["amazonlinux-2023"],
         "opensuse": ["opensuse-leap-15"]
     },
-    "windows": ["windows-2022", "windows-2019", "windows-2016"]
+    "windows": ["windows-2025", "windows-2022", "windows-2019", "windows-2016"]
 }

.buildkite/scripts/dra/build_docker.sh

@@ -7,63 +7,42 @@ echo "####################################################################"
 source ./$(dirname "$0")/common.sh
 
 # WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+info "Building artifacts for the $WORKFLOW_TYPE workflow ..."
+
 case "$WORKFLOW_TYPE" in
     snapshot)
-        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
-        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
-            rake artifact:docker || error "artifact:docker build failed."
-            rake artifact:docker_oss || error "artifact:docker_oss build failed."
-            rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
-            rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
-        else
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
-            # Qualifier is passed from CI as optional field and specify the version postfix
-            # in case of alpha or beta releases:
-            # e.g: 8.0.0-alpha1
-            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-        fi
-        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
-        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        :  # no-op
         ;;
     staging)
-        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
-        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
-            RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
-            RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
-            RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
-            RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
-        else
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
-            # Qualifier is passed from CI as optional field and specify the version postfix
-            # in case of alpha or beta releases:
-            # e.g: 8.0.0-alpha1
-            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-        fi
-        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        export RELEASE=1
         ;;
     *)
         error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
         ;;
 esac
 
+rake artifact:docker || error "artifact:docker build failed."
+rake artifact:docker_oss || error "artifact:docker_oss build failed."
+rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
+rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+
+if [[ "$ARCH" != "aarch64" ]]; then
+  rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+fi
+
+if [[ "$WORKFLOW_TYPE" == "staging" ]] && [[ -n "$VERSION_QUALIFIER" ]]; then
+    # Qualifier is passed from CI as optional field and specify the version postfix
+    # in case of alpha or beta releases for staging builds only:
+    # e.g: 8.0.0-alpha1
+    STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER}"
+fi
+
+if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then
+    STACK_VERSION="${STACK_VERSION}-SNAPSHOT"
+fi
+
+info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+
 info "Saving tar.gz for docker images"
 save_docker_tarballs "${ARCH}" "${STACK_VERSION}"
 

.buildkite/scripts/dra/build_packages.sh

@@ -7,39 +7,35 @@ echo "####################################################################"
 source ./$(dirname "$0")/common.sh
 
 # WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+info "Building artifacts for the $WORKFLOW_TYPE workflow ..."
+
 case "$WORKFLOW_TYPE" in
     snapshot)
-        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
-        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
-            SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
-        else
-            # Qualifier is passed from CI as optional field and specify the version postfix
-            # in case of alpha or beta releases:
-            # e.g: 8.0.0-alpha1
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
-            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-        fi
-        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
-        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        :  # no-op
         ;;
     staging)
-        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
-        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
-            RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
-        else
-            # Qualifier is passed from CI as optional field and specify the version postfix
-            # in case of alpha or beta releases:
-            # e.g: 8.0.0-alpha1
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
-            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-        fi
-        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        export RELEASE=1
         ;;
     *)
         error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
         ;;
 esac
 
+SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+
+if [[ "$WORKFLOW_TYPE" == "staging" ]] && [[ -n "$VERSION_QUALIFIER" ]]; then
+    # Qualifier is passed from CI as optional field and specify the version postfix
+    # in case of alpha or beta releases for staging builds only:
+    # e.g: 8.0.0-alpha1
+    STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER}"
+fi
+
+if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then
+    STACK_VERSION="${STACK_VERSION}-SNAPSHOT"
+fi
+
+info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+
 info "Generated Artifacts"
 for file in build/logstash-*; do shasum $file;done
 

.buildkite/scripts/dra/common.sh

@@ -29,7 +29,7 @@ function save_docker_tarballs {
 # Since we are using the system jruby, we need to make sure our jvm process
 # uses at least 1g of memory, If we don't do this we can get OOM issues when
 # installing gems. See https://github.com/elastic/logstash/issues/5179
-export JRUBY_OPTS="-J-Xmx2g"
+export JRUBY_OPTS="-J-Xmx4g"
 
 # Extract the version number from the version.yml file
 # e.g.: 8.6.0

.buildkite/scripts/dra/generatesteps.py

@@ -3,6 +3,8 @@ import sys
 
 import yaml
 
+YAML_HEADER = '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n'
+
 def to_bk_key_friendly_string(key):
     """
     Convert and return key to an acceptable format for Buildkite's key: field
@@ -28,6 +30,8 @@ def package_x86_step(branch, workflow_type):
     export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
     eval "$(rbenv init -)"
     .buildkite/scripts/dra/build_packages.sh
+  artifact_paths:
+    - "**/*.hprof"
 '''
 
     return step
@@ -42,6 +46,8 @@ def package_x86_docker_step(branch, workflow_type):
     image: family/platform-ingest-logstash-ubuntu-2204
     machineType: "n2-standard-16"
     diskSizeGb: 200
+  artifact_paths:
+    - "**/*.hprof"
   command: |
     export WORKFLOW_TYPE="{workflow_type}"
     export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
@@ -61,6 +67,8 @@ def package_aarch64_docker_step(branch, workflow_type):
     imagePrefix: platform-ingest-logstash-ubuntu-2204-aarch64
     instanceType: "m6g.4xlarge"
     diskSizeGb: 200
+  artifact_paths:
+    - "**/*.hprof"
   command: |
     export WORKFLOW_TYPE="{workflow_type}"
     export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
@@ -106,6 +114,7 @@ def build_steps_to_yaml(branch, workflow_type):
 if __name__ == "__main__":
     try:
         workflow_type = os.environ["WORKFLOW_TYPE"]
+        version_qualifier = os.environ.get("VERSION_QUALIFIER", "")
     except ImportError:
         print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=<staging|snapshot>\n.Exiting.")
         exit(1)
@@ -114,18 +123,25 @@ if __name__ == "__main__":
 
     structure = {"steps": []}
 
-    # Group defining parallel steps that build and save artifacts
-    group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}")
+    if workflow_type.upper() == "SNAPSHOT" and len(version_qualifier)>0:
+        structure["steps"].append({
+            "label": f"no-op pipeline because prerelease builds (VERSION_QUALIFIER is set to [{version_qualifier}]) don't support the [{workflow_type}] workflow",
+            "command": ":",
+            "skip": "VERSION_QUALIFIER (prerelease builds) not supported with SNAPSHOT DRA",
+        })
+    else:
+        # Group defining parallel steps that build and save artifacts
+        group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}")
 
-    structure["steps"].append({
-        "group": f":Build Artifacts - {workflow_type.upper()}",
-        "key": group_key,
-        "steps": build_steps_to_yaml(branch, workflow_type),
-    })
+        structure["steps"].append({
+            "group": f":Build Artifacts - {workflow_type.upper()}",
+            "key": group_key,
+            "steps": build_steps_to_yaml(branch, workflow_type),
+        })
 
-    # Final step: pull artifacts built above and publish them via the release-manager
-    structure["steps"].extend(
-        yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
-    )
+        # Final step: pull artifacts built above and publish them via the release-manager
+        structure["steps"].extend(
+            yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
+        )
 
-    print('# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n' + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))
+    print(YAML_HEADER + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))

.buildkite/scripts/dra/publish.sh

@@ -7,7 +7,9 @@ echo "####################################################################"
 
 source ./$(dirname "$0")/common.sh
 
-PLAIN_STACK_VERSION=$STACK_VERSION
+# DRA_BRANCH can be used for manually testing packaging with PRs
+# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build
+BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
 
 # This is the branch selector that needs to be passed to the release-manager
 # It has to be the name of the branch which originates the artifacts.
@@ -15,29 +17,24 @@ RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.
 if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then
     RELEASE_BRANCH=$RELEASE_VER
 else
-    RELEASE_BRANCH=main
+    RELEASE_BRANCH="${BRANCH:="main"}"
 fi
+echo "RELEASE BRANCH: $RELEASE_BRANCH"
 
-if [ -n "$VERSION_QUALIFIER_OPT" ]; then
-  # Qualifier is passed from CI as optional field and specify the version postfix
-  # in case of alpha or beta releases:
-  # e.g: 8.0.0-alpha1
-  STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-  PLAIN_STACK_VERSION="${PLAIN_STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-fi
+VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
 
 case "$WORKFLOW_TYPE" in
     snapshot)
-        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        :
         ;;
     staging)
         ;;
     *)
-        error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
         ;;
 esac
 
-info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}"
+info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH} for version: ${STACK_VERSION} with version_qualifier: ${VERSION_QUALIFIER}"
 
 if [ "$RELEASE_VER" != "7.17" ]; then
   # Version 7.17.x doesn't generates ARM artifacts for Darwin
@@ -55,7 +52,16 @@ rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz
 info "Downloaded ARTIFACTS sha report"
 for file in build/logstash-*; do shasum $file;done
 
-mv build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv build/distributions/dependencies-${STACK_VERSION}.csv
+FINAL_VERSION=$STACK_VERSION
+if [[ -n "$VERSION_QUALIFIER" ]]; then
+  FINAL_VERSION="$FINAL_VERSION-${VERSION_QUALIFIER}"
+fi
+
+if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then
+    FINAL_VERSION="${STACK_VERSION}-SNAPSHOT"
+fi
+
+mv build/distributions/dependencies-reports/logstash-${FINAL_VERSION}.csv build/distributions/dependencies-${FINAL_VERSION}.csv
 
 # set required permissions on artifacts and directory
 chmod -R a+r build/*
@@ -73,6 +79,22 @@ release_manager_login
 # ensure the latest image has been pulled
 docker pull docker.elastic.co/infra/release-manager:latest
 
+echo "+++ :clipboard: Listing DRA artifacts for version [$STACK_VERSION], branch [$RELEASE_BRANCH], workflow [$WORKFLOW_TYPE], QUALIFIER [$VERSION_QUALIFIER]"
+docker run --rm \
+        --name release-manager \
+        -e VAULT_ROLE_ID \
+        -e VAULT_SECRET_ID \
+        --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+          cli list \
+          --project logstash \
+          --branch "${RELEASE_BRANCH}" \
+          --commit "$(git rev-parse HEAD)" \
+          --workflow "${WORKFLOW_TYPE}" \
+          --version "${STACK_VERSION}" \
+          --artifact-set main \
+          --qualifier "${VERSION_QUALIFIER}"
+
 info "Running the release manager ..."
 
 # collect the artifacts for use with the unified build
@@ -88,8 +110,9 @@ docker run --rm \
       --branch ${RELEASE_BRANCH} \
       --commit "$(git rev-parse HEAD)" \
       --workflow "${WORKFLOW_TYPE}" \
-      --version "${PLAIN_STACK_VERSION}" \
+      --version "${STACK_VERSION}" \
       --artifact-set main \
+      --qualifier "${VERSION_QUALIFIER}" \
       ${DRA_DRY_RUN} | tee rm-output.txt
 
 # extract the summary URL from a release manager output line like:

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -147,11 +147,6 @@ rake artifact:deb artifact:rpm
 set -eo pipefail
 source .buildkite/scripts/common/vm-agent-multi-jdk.sh
 source /etc/os-release
-if [[ "$$(echo $$ID_LIKE | tr '[:upper:]' '[:lower:]')" =~ (rhel|fedora) && "$${VERSION_ID%.*}" -le 7 ]]; then
-  # jruby-9.3.10.0 unavailable on centos-7 / oel-7, see https://github.com/jruby/jruby/issues/7579#issuecomment-1425885324 / https://github.com/jruby/jruby/issues/7695
-  # we only need a working jruby to run the acceptance test framework -- the packages have been prebuilt in a previous stage
-  rbenv local jruby-9.4.5.0
-fi
 ci/acceptance_tests.sh"""),
         }
         steps.append(step)

.buildkite/scripts/health-report-tests/README.md

@@ -0,0 +1,18 @@
+## Description
+This package for integration tests of the Health Report API.
+Export `LS_BRANCH` to run on a specific branch. By default, it uses the main branch.
+
+## How to run the Health Report Integration test?
+### Prerequisites
+Make sure you have python installed. Install the integration test dependencies with the following command:
+```shell
+python3 -mpip install -r .buildkite/scripts/health-report-tests/requirements.txt
+```
+
+### Run the integration tests
+```shell
+python3 .buildkite/scripts/health-report-tests/main.py
+```
+
+### Troubleshooting
+- If you get `WARNING: pip is configured with locations that require TLS/SSL,...` warning message, make sure you have python >=3.12.4 installed.

.buildkite/scripts/health-report-tests/bootstrap.py

@@ -0,0 +1,94 @@
+"""
+Health Report Integration test bootstrapper with Python script
+    - A script to resolve Logstash version if not provided
+    - Download LS docker image and spin up
+    - When tests finished, teardown the Logstash
+"""
+import os
+import subprocess
+import util
+import yaml
+
+
+class Bootstrap:
+    ELASTIC_STACK_RELEASED_VERSION_URL = "https://storage.googleapis.com/artifacts-api/releases/current/"
+
+    def __init__(self) -> None:
+        f"""
+        A constructor of the {Bootstrap}.
+        Returns:
+            Resolves Logstash branch considering provided LS_BRANCH
+            Checks out git branch
+        """
+        logstash_branch = os.environ.get("LS_BRANCH")
+        if logstash_branch is None:
+            # version is not specified, use the main branch, no need to git checkout
+            print(f"LS_BRANCH is not specified, using main branch.")
+        else:
+            # LS_BRANCH accepts major latest as a major.x or specific branch as X.Y
+            if logstash_branch.find(".x") == -1:
+                print(f"Using specified branch: {logstash_branch}")
+                util.git_check_out_branch(logstash_branch)
+            else:
+                major_version = logstash_branch.split(".")[0]
+                if major_version and major_version.isnumeric():
+                    resolved_version = self.__resolve_latest_stack_version_for(major_version)
+                    minor_version = resolved_version.split(".")[1]
+                    branch = major_version + "." + minor_version
+                    print(f"Using resolved branch: {branch}")
+                    util.git_check_out_branch(branch)
+                else:
+                    raise ValueError(f"Invalid value set to LS_BRANCH. Please set it properly (ex: 8.x or 9.0) and "
+                                     f"rerun again")
+
+    def __resolve_latest_stack_version_for(self, major_version: str) -> str:
+        resp = util.call_url_with_retry(self.ELASTIC_STACK_RELEASED_VERSION_URL + major_version)
+        release_version = resp.text.strip()
+        print(f"Resolved latest version for {major_version} is {release_version}.")
+
+        if release_version == "":
+            raise ValueError(f"Cannot resolve latest version for {major_version} major")
+        return release_version
+
+    def install_plugin(self, plugin_path: str) -> None:
+        util.run_or_raise_error(
+            ["bin/logstash-plugin", "install", plugin_path],
+            f"Failed to install {plugin_path}")
+
+    def build_logstash(self):
+        print(f"Building Logstash...")
+        util.run_or_raise_error(
+            ["./gradlew", "clean", "bootstrap", "assemble", "installDefaultGems"],
+            "Failed to build Logstash")
+        print(f"Logstash has successfully built.")
+
+    def apply_config(self, config: dict) -> None:
+        with open(os.getcwd() + "/.buildkite/scripts/health-report-tests/config/pipelines.yml", 'w') as pipelines_file:
+            yaml.dump(config, pipelines_file)
+
+    def run_logstash(self, full_start_required: bool) -> subprocess.Popen:
+        # --config.reload.automatic is to make instance active
+        # it is helpful when testing crash pipeline cases
+        config_path = os.getcwd() + "/.buildkite/scripts/health-report-tests/config"
+        process = subprocess.Popen(["bin/logstash", "--config.reload.automatic", "--path.settings", config_path,
+                                    "-w 1"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, shell=False)
+        if process.poll() is not None:
+            print(f"Logstash failed to run, check the the config and logs, then rerun.")
+            return None
+
+        # Read stdout and stderr in real-time
+        logs = []
+        for stdout_line in iter(process.stdout.readline, ""):
+            logs.append(stdout_line.strip())
+            # we don't wait for Logstash fully start as we also test slow pipeline start scenarios
+            if full_start_required is False and "Starting pipeline" in stdout_line:
+                break
+            if full_start_required is True and "Pipeline started" in stdout_line:
+                break
+            if "Logstash shut down" in stdout_line or "Logstash stopped" in stdout_line:
+                print(f"Logstash couldn't spin up.")
+                print(logs)
+                return None
+
+        print(f"Logstash is running with PID: {process.pid}.")
+        return process

.buildkite/scripts/health-report-tests/config/pipelines.yml

@@ -0,0 +1 @@
+# Intentionally left blank

.buildkite/scripts/health-report-tests/config_validator.py

@@ -0,0 +1,69 @@
+import yaml
+from typing import Any, List, Dict
+
+
+class ConfigValidator:
+    REQUIRED_KEYS = {
+        "root": ["name", "config", "conditions", "expectation"],
+        "config": ["pipeline.id", "config.string"],
+        "conditions": ["full_start_required"],
+        "expectation": ["status", "symptom", "indicators"],
+        "indicators": ["pipelines"],
+        "pipelines": ["status", "symptom", "indicators"],
+        "DYNAMIC": ["status", "symptom", "diagnosis", "impacts", "details"],
+        "details": ["status"],
+        "status": ["state"]
+    }
+
+    def __init__(self):
+        self.yaml_content = None
+
+    def __has_valid_keys(self, data: any, key_path: str, repeated: bool) -> bool:
+        if isinstance(data, str) or isinstance(data, bool):   # we reached values
+            return True
+
+        # we have two indicators section and for the next repeated ones, we go deeper
+        first_key = next(iter(data))
+        data = data[first_key] if repeated and key_path == "indicators" else data
+
+        if isinstance(data, dict):
+            # pipeline-id is a DYNAMIC
+            required = self.REQUIRED_KEYS.get("DYNAMIC" if repeated and key_path == "indicators" else key_path, [])
+            repeated = not repeated if key_path == "indicators" else repeated
+            for key in required:
+                if key not in data:
+                    print(f"Missing key '{key}' in '{key_path}'")
+                    return False
+                else:
+                    dic_keys_result = self.__has_valid_keys(data[key], key, repeated)
+                    if dic_keys_result is False:
+                        return False
+        elif isinstance(data, list):
+            for item in data:
+                list_keys_result = self.__has_valid_keys(item, key_path, repeated)
+                if list_keys_result is False:
+                    return False
+        return True
+
+    def load(self, file_path: str) -> None:
+        """Load the YAML file content into self.yaml_content."""
+        self.yaml_content: [Dict[str, Any]] = None
+        try:
+            with open(file_path, 'r') as file:
+                self.yaml_content = yaml.safe_load(file)
+        except yaml.YAMLError as exc:
+            print(f"Error in YAML file: {exc}")
+            self.yaml_content = None
+
+    def is_valid(self) -> bool:
+        """Validate the entire YAML structure."""
+        if self.yaml_content is None:
+            print(f"YAML content is empty.")
+            return False
+
+        if not isinstance(self.yaml_content, dict):
+            print(f"YAML structure is not as expected, it should start with a Dict.")
+            return False
+
+        result = self.__has_valid_keys(self.yaml_content, "root", False)
+        return True if result is True else False

.buildkite/scripts/health-report-tests/logstash_health_report.py

@@ -0,0 +1,16 @@
+"""
+A class to provide information about Logstash node stats.
+"""
+
+import util
+
+
+class LogstashHealthReport:
+    LOGSTASH_HEALTH_REPORT_URL = "http://localhost:9600/_health_report"
+
+    def __init__(self):
+        pass
+
+    def get(self):
+        response = util.call_url_with_retry(self.LOGSTASH_HEALTH_REPORT_URL)
+        return response.json()

.buildkite/scripts/health-report-tests/main.py

@@ -0,0 +1,87 @@
+"""
+ Main entry point of the LS health report API integration test suites
+"""
+import glob
+import os
+import time
+import traceback
+import yaml
+from bootstrap import Bootstrap
+from scenario_executor import ScenarioExecutor
+from config_validator import ConfigValidator
+
+
+class BootstrapContextManager:
+
+    def __init__(self):
+        pass
+
+    def __enter__(self):
+        print(f"Starting Logstash Health Report Integration test.")
+        self.bootstrap = Bootstrap()
+        self.bootstrap.build_logstash()
+
+        plugin_path = os.getcwd() + "/qa/support/logstash-integration-failure_injector/logstash-integration" \
+                                    "-failure_injector-*.gem"
+        matching_files = glob.glob(plugin_path)
+        if len(matching_files) == 0:
+            raise ValueError(f"Could not find logstash-integration-failure_injector plugin.")
+
+        self.bootstrap.install_plugin(matching_files[0])
+        print(f"logstash-integration-failure_injector successfully installed.")
+        return self.bootstrap
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        if exc_type is not None:
+            print(traceback.format_exception(exc_type, exc_value, exc_traceback))
+
+
+def main():
+    with BootstrapContextManager() as bootstrap:
+        scenario_executor = ScenarioExecutor()
+        config_validator = ConfigValidator()
+
+        working_dir = os.getcwd()
+        scenario_files_path = working_dir + "/.buildkite/scripts/health-report-tests/tests/*.yaml"
+        scenario_files = glob.glob(scenario_files_path)
+
+        for scenario_file in scenario_files:
+            print(f"Validating {scenario_file} scenario file.")
+            config_validator.load(scenario_file)
+            if config_validator.is_valid() is False:
+                print(f"{scenario_file} scenario file is not valid.")
+                return
+            else:
+                print(f"Validation succeeded.")
+
+        has_failed_scenario = False
+        for scenario_file in scenario_files:
+            with open(scenario_file, 'r') as file:
+                # scenario_content: Dict[str, Any] = None
+                scenario_content = yaml.safe_load(file)
+                print(f"Testing `{scenario_content.get('name')}` scenario.")
+                scenario_name = scenario_content['name']
+
+                is_full_start_required = next(sub.get('full_start_required') for sub in
+                                              scenario_content.get('conditions') if 'full_start_required' in sub)
+                config = scenario_content['config']
+                if config is not None:
+                    bootstrap.apply_config(config)
+                    expectations = scenario_content.get("expectation")
+                    process = bootstrap.run_logstash(is_full_start_required)
+                    if process is not None:
+                        try:
+                            scenario_executor.on(scenario_name, expectations)
+                        except Exception as e:
+                            print(e)
+                            has_failed_scenario = True
+                        process.terminate()
+                        time.sleep(5)   # leave some window to terminate the process
+
+        if has_failed_scenario:
+            # intentionally fail due to visibility
+            raise Exception("Some of scenarios failed, check the log for details.")
+
+
+if __name__ == "__main__":
+    main()

.buildkite/scripts/health-report-tests/main.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -euo pipefail
+
+export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$PATH"
+export JAVA_HOME="/opt/buildkite-agent/.java"
+eval "$(rbenv init -)"
+eval "$(pyenv init -)"
+
+echo "--- Installing pip"
+sudo apt-get install python3-pip -y
+
+echo "--- Installing dependencies"
+python3 -mpip install -r .buildkite/scripts/health-report-tests/requirements.txt
+
+echo "--- Running tests"
+python3 .buildkite/scripts/health-report-tests/main.py

.buildkite/scripts/health-report-tests/requirements.txt

@@ -0,0 +1,2 @@
+requests==2.32.3
+pyyaml==6.0.2

.buildkite/scripts/health-report-tests/scenario_executor.py

@@ -0,0 +1,65 @@
+"""
+A class to execute the given scenario for Logstash Health Report integration test
+"""
+import time
+from logstash_health_report import LogstashHealthReport
+
+
+class ScenarioExecutor:
+    logstash_health_report_api = LogstashHealthReport()
+
+    def __init__(self):
+        pass
+
+    def __has_intersection(self, expects, results):
+        # we expect expects to be existing in results
+        for expect in expects:
+            for result in results:
+                if result.get('help_url') and "health-report-pipeline-status.html#" not in result.get('help_url'):
+                    return False
+                if not all(key in result and result[key] == value for key, value in expect.items()):
+                    return False
+        return True
+
+    def __get_difference(self, differences: list, expectations: dict, reports: dict) -> dict:
+        for key in expectations.keys():
+
+            if type(expectations.get(key)) != type(reports.get(key)):
+                differences.append(f"Scenario expectation and Health API report structure differs for {key}.")
+                return differences
+
+            if isinstance(expectations.get(key), str):
+                if expectations.get(key) != reports.get(key):
+                    differences.append({key: {"expected": expectations.get(key), "got": reports.get(key)}})
+                continue
+            elif isinstance(expectations.get(key), dict):
+                self.__get_difference(differences, expectations.get(key), reports.get(key))
+            elif isinstance(expectations.get(key), list):
+                if not self.__has_intersection(expectations.get(key), reports.get(key)):
+                    differences.append({key: {"expected": expectations.get(key), "got": reports.get(key)}})
+        return differences
+
+    def __is_expected(self, expectations: dict) -> None:
+        reports = self.logstash_health_report_api.get()
+        differences = self.__get_difference([], expectations, reports)
+        if differences:
+            print("Differences found in 'expectation' section between YAML content and stats:")
+            for diff in differences:
+                print(f"Difference: {diff}")
+            return False
+        else:
+            return True
+
+    def on(self, scenario_name: str, expectations: dict) -> None:
+        # retriable check the expectations
+        attempts = 5
+        while self.__is_expected(expectations) is False:
+            attempts = attempts - 1
+            if attempts == 0:
+                break
+            time.sleep(1)
+
+        if attempts == 0:
+            raise Exception(f"{scenario_name} failed.")
+        else:
+            print(f"Scenario `{scenario_name}` expectaion meets the health report stats.")

.buildkite/scripts/health-report-tests/tests/abnormal-termination.yaml

@@ -0,0 +1,31 @@
+name: "Abnormally terminated pipeline"
+config:
+  - pipeline.id: abnormally-terminated-pp
+    config.string: |
+      input { heartbeat { interval => 1 } }
+      filter { failure_injector { crash_at => filter } }
+      output { stdout {} }
+    pipeline.workers: 1
+    pipeline.batch.size: 1
+conditions:
+  - full_start_required: true
+expectation:
+  status: "red"
+  symptom: "1 indicator is unhealthy (`pipelines`)"
+  indicators:
+    pipelines:
+      status: "red"
+      symptom: "1 indicator is unhealthy (`abnormally-terminated-pp`)"
+      indicators:
+        abnormally-terminated-pp:
+          status: "red"
+          symptom: "The pipeline is unhealthy; 1 area is impacted and 1 diagnosis is available"
+          diagnosis:
+            - cause: "pipeline is not running, likely because it has encountered an error"
+            - action: "view logs to determine the cause of abnormal pipeline shutdown"
+          impacts:
+            - description: "the pipeline is not currently processing"
+            - impact_areas: ["pipeline_execution"]
+          details:
+            status:
+              state: "TERMINATED"

.buildkite/scripts/health-report-tests/tests/normal-termination.yaml

@@ -0,0 +1,29 @@
+name: "Successfully terminated pipeline"
+config:
+  - pipeline.id: normally-terminated-pp
+    config.string: |
+      input { generator { count => 1 } }
+      output { stdout {} }
+    pipeline.workers: 1
+    pipeline.batch.size: 1
+conditions:
+  - full_start_required: true
+expectation:
+  status: "yellow"
+  symptom: "1 indicator is concerning (`pipelines`)"
+  indicators:
+    pipelines:
+      status: "yellow"
+      symptom: "1 indicator is concerning (`normally-terminated-pp`)"
+      indicators:
+        normally-terminated-pp:
+          status: "yellow"
+          symptom: "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available"
+          diagnosis:
+            - cause: "pipeline has finished running because its inputs have been closed and events have been processed"
+            - action: "if you expect this pipeline to run indefinitely, you will need to configure its inputs to continue receiving or fetching events"
+          impacts:
+            - impact_areas: ["pipeline_execution"]
+          details:
+            status:
+              state: "FINISHED"

.buildkite/scripts/health-report-tests/tests/slow-start.yaml

@@ -0,0 +1,30 @@
+name: "Slow start pipeline"
+config:
+  - pipeline.id: slow-start-pp
+    config.string: |
+      input { heartbeat {} }
+      filter { failure_injector { degrade_at => [register] } }
+      output { stdout {} }
+    pipeline.workers: 1
+    pipeline.batch.size: 1
+conditions:
+  - full_start_required: false
+expectation:
+  status: "yellow"
+  symptom: "1 indicator is concerning (`pipelines`)"
+  indicators:
+    pipelines:
+      status: "yellow"
+      symptom: "1 indicator is concerning (`slow-start-pp`)"
+      indicators:
+        slow-start-pp:
+          status: "yellow"
+          symptom: "The pipeline is concerning; 1 area is impacted and 1 diagnosis is available"
+          diagnosis:
+            - cause: "pipeline is loading"
+            - action: "if pipeline does not come up quickly, you may need to check the logs to see if it is stalled"
+          impacts:
+            - impact_areas: ["pipeline_execution"]
+          details:
+            status:
+              state: "LOADING"

.buildkite/scripts/health-report-tests/util.py

@@ -0,0 +1,36 @@
+import os
+import requests
+import subprocess
+from requests.adapters import HTTPAdapter, Retry
+
+
+def call_url_with_retry(url: str, max_retries: int = 5, delay: int = 1) -> requests.Response:
+    f"""
+    Calls the given {url} with maximum of {max_retries} retries with {delay} delay.
+    """
+    schema = "https://" if "https://" in url else "http://"
+    session = requests.Session()
+    # retry on most common failures such as connection timeout(408), etc...
+    retries = Retry(total=max_retries, backoff_factor=delay, status_forcelist=[408, 502, 503, 504])
+    session.mount(schema, HTTPAdapter(max_retries=retries))
+    return session.get(url)
+
+
+def git_check_out_branch(branch_name: str) -> None:
+    f"""
+    Checks out specified branch or fails with error if checkout operation fails.
+    """
+    run_or_raise_error(["git", "checkout", branch_name],
+                       "Error occurred while checking out the " + branch_name + " branch")
+
+
+def run_or_raise_error(commands: list, error_message):
+    f"""
+    Executes the {list} commands and raises an {Exception} if opration fails.
+    """
+    result = subprocess.run(commands, env=os.environ.copy(), universal_newlines=True, stdout=subprocess.PIPE)
+    if result.returncode != 0:
+        full_error_message = (error_message + ", output: " + result.stdout.decode('utf-8')) \
+            if result.stdout else error_message
+        raise Exception(f"{full_error_message}")
+

.buildkite/scripts/jdk-matrix-tests/generate-steps.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
 import os
 import sys
 import typing
+from functools import partial
 
 from ruamel.yaml import YAML
 from ruamel.yaml.scalarstring import LiteralScalarString
@@ -177,17 +178,15 @@ class LinuxJobs(Jobs):
       super().__init__(os=os, jdk=jdk, group_key=group_key, agent=agent)
 
     def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]:
-        return [
-            self.init_annotation,
-            self.java_unit_test,
-            self.ruby_unit_test,
-            self.integration_tests_part_1,
-            self.integration_tests_part_2,
-            self.pq_integration_tests_part_1,
-            self.pq_integration_tests_part_2,
-            self.x_pack_unit_tests,
-            self.x_pack_integration,
-        ]
+        jobs=list()
+        jobs.append(self.init_annotation)
+        jobs.append(self.java_unit_test)
+        jobs.append(self.ruby_unit_test)
+        jobs.extend(self.integration_test_parts(3))
+        jobs.extend(self.pq_integration_test_parts(3))
+        jobs.append(self.x_pack_unit_tests)
+        jobs.append(self.x_pack_integration)
+        return jobs
 
     def prepare_shell(self) -> str:
         jdk_dir = f"/opt/buildkite-agent/.java/{self.jdk}"
@@ -259,17 +258,14 @@ ci/unit_tests.sh ruby
             retry=copy.deepcopy(ENABLED_RETRIES),
         )
 
-    def integration_tests_part_1(self) -> JobRetValues:
-        return self.integration_tests(part=1)
+    def integration_test_parts(self, parts) -> list[partial[JobRetValues]]:
+        return [partial(self.integration_tests, part=idx+1, parts=parts) for idx in range(parts)]
 
-    def integration_tests_part_2(self) -> JobRetValues:
-        return self.integration_tests(part=2)
-
-    def integration_tests(self, part: int) -> JobRetValues:
-        step_name_human = f"Integration Tests - {part}"
-        step_key = f"{self.group_key}-integration-tests-{part}"
+    def integration_tests(self, part: int, parts: int) -> JobRetValues:
+        step_name_human = f"Integration Tests - {part}/{parts}"
+        step_key = f"{self.group_key}-integration-tests-{part}-of-{parts}"
         test_command = f"""
-ci/integration_tests.sh split {part-1}
+ci/integration_tests.sh split {part-1} {parts}
         """
 
         return JobRetValues(
@@ -281,18 +277,15 @@ ci/integration_tests.sh split {part-1}
             retry=copy.deepcopy(ENABLED_RETRIES),
         )
 
-    def pq_integration_tests_part_1(self) -> JobRetValues:
-        return self.pq_integration_tests(part=1)
+    def pq_integration_test_parts(self, parts) -> list[partial[JobRetValues]]:
+        return [partial(self.pq_integration_tests, part=idx+1, parts=parts) for idx in range(parts)]
 
-    def pq_integration_tests_part_2(self) -> JobRetValues:
-        return self.pq_integration_tests(part=2)
-
-    def pq_integration_tests(self, part: int) -> JobRetValues:
-        step_name_human = f"IT Persistent Queues - {part}"
-        step_key = f"{self.group_key}-it-persistent-queues-{part}"
+    def pq_integration_tests(self, part: int, parts: int) -> JobRetValues:
+        step_name_human = f"IT Persistent Queues - {part}/{parts}"
+        step_key = f"{self.group_key}-it-persistent-queues-{part}-of-{parts}"
         test_command = f"""
 export FEATURE_FLAG=persistent_queues
-ci/integration_tests.sh split {part-1}
+ci/integration_tests.sh split {part-1} {parts}
         """
 
         return JobRetValues(

.buildkite/windows_jdk_matrix_pipeline.yml

@@ -15,6 +15,8 @@ steps:
         multiple: true
         default: "${DEFAULT_MATRIX_OS}"
         options:
+          - label: "Windows 2025"
+            value: "windows-2025"
           - label: "Windows 2022"
             value: "windows-2022"
           - label: "Windows 2019"

.ruby-version

@@ -1 +1 @@
-jruby-9.3.10.0
+jruby-9.4.9.0

Gemfile.template

@@ -13,7 +13,7 @@ gem "ruby-maven-libs", "~> 3", ">= 3.9.6.1"
 gem "logstash-output-elasticsearch", ">= 11.14.0"
 gem "polyglot", require: false
 gem "treetop", require: false
-gem "faraday", "~> 1", :require => false # due elasticsearch-transport (elastic-transport) depending faraday '~> 1'
+gem "minitar", "~> 1", :group => :build
 gem "childprocess", "~> 4", :group => :build
 gem "fpm", "~> 1", ">= 1.14.1", :group => :build # compound due to bugfix https://github.com/jordansissel/fpm/pull/1856
 gem "gems", "~> 1", :group => :build
@@ -26,6 +26,8 @@ gem "stud", "~> 0.0.22", :group => :build
 gem "fileutils", "~> 1.7"
 
 gem "rubocop", :group => :development
+# rubocop-ast 1.43.0 carries a dep on `prism` which requires native c extensions
+gem 'rubocop-ast', '= 1.42.0', :group => :development
 gem "belzebuth", :group => :development
 gem "benchmark-ips", :group => :development
 gem "ci_reporter_rspec", "~> 1", :group => :development
@@ -39,5 +41,9 @@ gem "simplecov", "~> 0.22.0", :group => :development
 gem "simplecov-json", require: false, :group => :development
 gem "jar-dependencies", "= 0.4.1" # Gem::LoadError with jar-dependencies 0.4.2
 gem "murmurhash3", "= 0.1.6" # Pins until version 0.1.7-java is released
+gem "date", "= 3.3.3"
 gem "thwait"
 gem "bigdecimal", "~> 3.1"
+gem "psych", "5.2.2"
+gem "cgi", "0.3.7" # Pins until a new jruby version with updated cgi is released
+gem "uri", "0.12.3" # Pins until a new jruby version with updated cgi is released

bin/pqcheck.bat

@@ -16,7 +16,7 @@ for %%i in ("%LS_HOME%\logstash-core\lib\jars\*.jar") do (
 	call :concat "%%i"
 )
 
-"%JAVACMD%" "%JAVA_OPTS%" -cp "%CLASSPATH%" org.logstash.ackedqueue.PqCheck %*
+"%JAVACMD%" %JAVA_OPTS% org.logstash.ackedqueue.PqCheck %*
 
 :concat
 IF not defined CLASSPATH (

bin/pqrepair.bat

@@ -16,7 +16,7 @@ for %%i in ("%LS_HOME%\logstash-core\lib\jars\*.jar") do (
 	call :concat "%%i"
 )
 
-"%JAVACMD%" %JAVA_OPTS% -cp "%CLASSPATH%" org.logstash.ackedqueue.PqRepair %*
+"%JAVACMD%" %JAVA_OPTS% org.logstash.ackedqueue.PqRepair %*
 
 :concat
 IF not defined CLASSPATH (

bin/setup.bat

@@ -42,7 +42,7 @@ if defined LS_JAVA_HOME (
 )
 
 if not exist "%JAVACMD%" (
-  echo could not find java; set JAVA_HOME or ensure java is in PATH 1>&2
+  echo could not find java; set LS_JAVA_HOME or ensure java is in PATH 1>&2
   exit /b 1
 )
 

build.gradle

@@ -101,6 +101,7 @@ allprojects {
       "--add-opens=java.base/java.lang=ALL-UNNAMED",
       "--add-opens=java.base/java.util=ALL-UNNAMED"
     ]
+    maxHeapSize = "2g"
     //https://stackoverflow.com/questions/3963708/gradle-how-to-display-test-results-in-the-console-in-real-time
     testLogging {
       // set options for log level LIFECYCLE
@@ -145,7 +146,6 @@ subprojects {
 }
 
 version = versionMap['logstash-core']
-String artifactVersionsApi = "https://artifacts-api.elastic.co/v1/versions"
 
 tasks.register("configureArchitecture") {
     String arch = System.properties['os.arch']
@@ -171,33 +171,28 @@ tasks.register("configureArtifactInfo") {
     description "Set the url to download stack artifacts for select stack version"
 
     doLast {
-        def versionQualifier = System.getenv('VERSION_QUALIFIER')
-        if (versionQualifier) {
-            version = "$version-$versionQualifier"
-        }
+        def splitVersion = version.split('\\.')
+        int major = splitVersion[0].toInteger()
+        int minor = splitVersion[1].toInteger()
+        String branch = "${major}.${minor}"
+        String fallbackMajorX = "${major}.x"
+        boolean isFallBackPreviousMajor = minor - 1 < 0
+        String fallbackBranch = isFallBackPreviousMajor ? "${major-1}.x" : "${major}.${minor-1}"
+        def qualifiedVersion = ""
 
-        boolean isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
-        String apiResponse = artifactVersionsApi.toURL().text
-
-        def dlVersions = new JsonSlurper().parseText(apiResponse)
-        String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
-        if (qualifiedVersion == null) {
-            if (!isReleaseBuild) {
-                project.ext.set("useProjectSpecificArtifactSnapshotUrl", true)
-                project.ext.set("stackArtifactSuffix", "${version}-SNAPSHOT")
-                return
+        for (b in [branch, fallbackMajorX, fallbackBranch]) {
+            def url = "https://storage.googleapis.com/artifacts-api/snapshots/${b}.json"
+            try {
+                def snapshotInfo = new JsonSlurper().parseText(url.toURL().text)
+                qualifiedVersion = snapshotInfo.version
+                println "ArtifactInfo version: ${qualifiedVersion}"
+                break
+            } catch (Exception e) {
+                println "Failed to fetch branch ${branch} from ${url}: ${e.message}"
             }
-            throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}")
         }
-        // find latest reference to last build
-        String buildsListApi = "${artifactVersionsApi}/${qualifiedVersion}/builds/"
-        apiResponse = buildsListApi.toURL().text
-        def dlBuilds = new JsonSlurper().parseText(apiResponse)
-        def stackBuildVersion = dlBuilds["builds"][0]
 
-        project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}")
-        project.ext.set("stackArtifactSuffix", qualifiedVersion)
-        project.ext.set("useProjectSpecificArtifactSnapshotUrl", false)
+        project.ext.set("artifactApiVersion", qualifiedVersion)
     }
 }
 
@@ -440,23 +435,13 @@ tasks.register("downloadFilebeat") {
 
     doLast {
         download {
-            String beatVersion = project.ext.get("stackArtifactSuffix")
-            String downloadedFilebeatName = "filebeat-${beatVersion}-${project.ext.get("beatsArchitecture")}"
+            String beatsVersion = project.ext.get("artifactApiVersion")
+            String downloadedFilebeatName = "filebeat-${beatsVersion}-${project.ext.get("beatsArchitecture")}"
             project.ext.set("unpackedFilebeatName", downloadedFilebeatName)
 
-            if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
-                def res = SnapshotArtifactURLs.packageUrls("beats", beatVersion, downloadedFilebeatName)
-                project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: res.packageUrl)
-                project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
-            } else {
-                // find url of build artifact
-                String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
-                String apiResponse = artifactApiUrl.toURL().text
-                def buildUrls = new JsonSlurper().parseText(apiResponse)
-
-                project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
-                project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
-            }
+            def res = SnapshotArtifactURLs.packageUrls("beats", beatsVersion, downloadedFilebeatName)
+            project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: res.packageUrl)
+            project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
 
             src project.ext.filebeatSnapshotUrl
             onlyIfNewer true
@@ -492,20 +477,12 @@ tasks.register("checkEsSHA") {
     description "Download ES version remote's fingerprint file"
 
     doLast {
-        String esVersion = project.ext.get("stackArtifactSuffix")
+        String esVersion = project.ext.get("artifactApiVersion")
         String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
         String remoteSHA
 
-        if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
-            def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
-            remoteSHA = res.packageShaUrl
-        } else {
-            // find url of build artifact
-            String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
-            String apiResponse = artifactApiUrl.toURL().text
-            def buildUrls = new JsonSlurper().parseText(apiResponse)
-            remoteSHA = buildUrls.package.sha_url.toURL().text
-        }
+        def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
+        remoteSHA = res.packageShaUrl
 
         def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
         if (localESArchive.exists()) {
@@ -539,25 +516,14 @@ tasks.register("downloadEs") {
 
     doLast {
         download {
-            String esVersion = project.ext.get("stackArtifactSuffix")
+            String esVersion = project.ext.get("artifactApiVersion")
             String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
 
             project.ext.set("unpackedElasticsearchName", "elasticsearch-${esVersion}")
 
-            if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
-                def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
-                project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: res.packageUrl)
-                project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
-            } else {
-                // find url of build artifact
-                String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
-                String apiResponse = artifactApiUrl.toURL().text
-
-                def buildUrls = new JsonSlurper().parseText(apiResponse)
-
-                project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
-                project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
-            }
+            def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
+            project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: res.packageUrl)
+            project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
 
             src project.ext.elasticsearchSnapshotURL
             onlyIfNewer true
@@ -740,46 +706,66 @@ class JDKDetails {
         return createElasticCatalogDownloadUrl()
     }
 
-    private String createElasticCatalogDownloadUrl() {
-        // Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK
-
-        // arch x86_64 never used, only aarch64 if macos
+    // throws an error iff local version in versions.yml doesn't match the latest from JVM catalog.
+    void checkLocalVersionMatchingLatest() {
+        // retrieve the metadata from remote
         def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${major}_${osName}"
-
-        // Append the cpu's arch only if Mac on aarch64, all the other OSes doesn't have CPU extension
-        if (arch == "aarch64") {
-            url += "_${arch}"
-        }
-        println "Retrieving JDK from catalog..."
         def catalogMetadataUrl = URI.create(url).toURL()
         def catalogConnection = catalogMetadataUrl.openConnection()
         catalogConnection.requestMethod = 'GET'
         assert catalogConnection.responseCode == 200
 
         def metadataRetrieved = catalogConnection.content.text
-        println "Retrieved!"
-
         def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved)
-        return catalogMetadata.url
-    }
 
-    private String createAdoptDownloadUrl() {
-        String releaseName = major > 8 ?
-                "jdk-${revision}+${build}" :
-                "jdk${revision}u${build}"
-        String vendorOsName = vendorOsName(osName)
-        switch (vendor) {
-            case "adoptium":
-                return "https://api.adoptium.net/v3/binary/version/${releaseName}/${vendorOsName}/${arch}/jdk/hotspot/normal/adoptium"
-            default:
-                throw RuntimeException("Can't handle vendor: ${vendor}")
+        if (catalogMetadata.version != revision || catalogMetadata.revision != build) {
+            throw new GradleException("Found new jdk version. Please update version.yml to ${catalogMetadata.version} build ${catalogMetadata.revision}")
         }
     }
 
-    private String vendorOsName(String osName) {
-        if (osName == "darwin")
-            return "mac"
-        return osName
+    private String createElasticCatalogDownloadUrl() {
+        // Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK
+
+        // arch x86_64 is default, aarch64 if macos or linux
+        def url = "https://jvm-catalog.elastic.co/jdk/adoptiumjdk-${revision}+${build}-${osName}"
+
+        // Append the cpu's arch only if not x86_64, which is the default
+        if (arch == "aarch64") {
+            url += "-${arch}"
+        }
+        println "Retrieving JDK from catalog..."
+        def catalogMetadataUrl = URI.create(url).toURL()
+        def catalogConnection = catalogMetadataUrl.openConnection()
+        catalogConnection.requestMethod = 'GET'
+        if (catalogConnection.responseCode != 200) {
+            println "Can't find adoptiumjdk ${revision} for ${osName} on Elastic JVM catalog"
+            throw new GradleException("JVM not present on catalog")
+        }
+
+        def metadataRetrieved = catalogConnection.content.text
+        println "Retrieved!"
+
+        def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved)
+        validateMetadata(catalogMetadata)
+
+        return catalogMetadata.url
+    }
+
+    //Verify that the artifact metadata correspond to the request, if not throws an error
+    private void validateMetadata(Map metadata) {
+        if (metadata.version != revision) {
+            throw new GradleException("Expected to retrieve a JDK for version ${revision} but received: ${metadata.version}")
+        }
+        if (!isSameArchitecture(metadata.architecture)) {
+            throw new GradleException("Expected to retrieve a JDK for architecture ${arch} but received: ${metadata.architecture}")
+        }
+    }
+
+    private boolean isSameArchitecture(String metadataArch) {
+        if (arch == 'x64') {
+            return metadataArch == 'x86_64'
+        }
+        return metadataArch == arch
     }
 
     private String parseJdkArchitecture(String jdkArch) {
@@ -791,16 +777,22 @@ class JDKDetails {
                 return "aarch64"
                 break
             default:
-                throw RuntimeException("Can't handle CPU architechture: ${jdkArch}")
+                throw new GradleException("Can't handle CPU architechture: ${jdkArch}")
         }
     }
 }
 
 tasks.register("lint") {
-    // Calls rake's 'lint' task
+    description = "Lint Ruby source files. Use -PrubySource=file1.rb,file2.rb to specify files"
     dependsOn installDevelopmentGems
     doLast {
-        rake(projectDir, buildDir, 'lint:report')
+        if (project.hasProperty("rubySource")) {
+            // Split the comma-separated files and pass them as separate arguments
+            def files = project.property("rubySource").split(",")
+            rake(projectDir, buildDir, "lint:report", *files)
+        } else {
+            rake(projectDir, buildDir, "lint:report")
+        }
     }
 }
 
@@ -840,6 +832,15 @@ tasks.register("downloadJdk", Download) {
     }
 }
 
+tasks.register("checkNewJdkVersion") {
+    def versionYml = new Yaml().load(new File("$projectDir/versions.yml").text)
+
+    // use Linux x86_64 as canary platform
+    def jdkDetails = new JDKDetails(versionYml, "linux", "x86_64")
+    // throws Gradle exception if local and remote doesn't match
+    jdkDetails.checkLocalVersionMatchingLatest()
+}
+
 tasks.register("deleteLocalJdk", Delete) {
     // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin]
     String osName = selectOsType()

catalog-info.yaml

@@ -32,6 +32,8 @@ spec:
     - resource:logstash-linux-jdk-matrix-pipeline
     - resource:logstash-windows-jdk-matrix-pipeline
     - resource:logstash-benchmark-pipeline
+    - resource:logstash-health-report-tests-pipeline
+    - resource:logstash-jdk-availability-check-pipeline
 
 # ***********************************
 # Declare serverless IT pipeline
@@ -642,4 +644,112 @@ spec:
 
 # *******************************
 # SECTION END: Benchmark pipeline
+# *******************************
+
+# ***********************************
+# Declare Health Report Tests pipeline
+# ***********************************
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: logstash-health-report-tests-pipeline
+  description: Buildkite pipeline for the Logstash Health Report Tests
+  links:
+    - title: ':logstash Logstash Health Report Tests (Daily, Auto) pipeline'
+      url: https://buildkite.com/elastic/logstash-health-report-tests-pipeline
+spec:
+  type: buildkite-pipeline
+  owner: group:logstash
+  system: platform-ingest
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: logstash-health-report-tests-pipeline
+      description: ':logstash: Logstash Health Report tests :pipeline:'
+    spec:
+      repository: elastic/logstash
+      pipeline_file: ".buildkite/health_report_tests_pipeline.yml"
+      maximum_timeout_in_minutes: 60
+      provider_settings:
+        trigger_mode: none # don't trigger jobs from github activity
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+        SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        logstash:
+          access_level: MANAGE_BUILD_AND_READ
+        ingest-eng-prod:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+      schedules:
+        Daily Health Report tests on main branch:
+          branch: main
+          cronline: 30 20 * * *
+          message: Daily trigger of Health Report Tests Pipeline
+
+# *******************************
+# SECTION END: Health Report Tests pipeline
+# *******************************
+
+# ***********************************
+# Declare JDK check pipeline
+# ***********************************
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: logstash-jdk-availability-check-pipeline
+  description: ":logstash: check availability of new JDK version"
+spec:
+  type: buildkite-pipeline
+  owner: group:logstash
+  system: platform-ingest
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: logstash-jdk-availability-check-pipeline
+    spec:
+      repository: elastic/logstash
+      pipeline_file: ".buildkite/jdk_availability_check_pipeline.yml"
+      maximum_timeout_in_minutes: 10
+      provider_settings:
+        trigger_mode: none # don't trigger jobs from github activity
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+        SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true'
+      teams:
+        logstash:
+          access_level: MANAGE_BUILD_AND_READ
+        ingest-eng-prod:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+      schedules:
+        Weekly JDK availability check (main):
+          branch: main
+          cronline: 0 2 * * 1  # every Monday@2AM UTC
+          message: Weekly trigger of JDK update availability pipeline per branch
+          env:
+            PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline'
+        Weekly JDK availability check (8.x):
+          branch: 8.x
+          cronline: 0 2 * * 1  # every Monday@2AM UTC
+          message: Weekly trigger of JDK update availability pipeline per branch
+          env:
+            PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline'
+
+# *******************************
+# SECTION END: JDK check pipeline
 # *******************************

ci/acceptance_tests.sh

@@ -19,7 +19,7 @@ function get_package_type {
 # uses at least 1g of memory, If we don't do this we can get OOM issues when
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
-export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
 export OSS=true
 
 if [ -n "$BUILD_JAVA_HOME" ]; then

ci/check_jdk_version_availability.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
+
+echo "Checking local JDK version against latest remote from JVM catalog"
+./gradlew checkNewJdkVersion

ci/docker_acceptance_tests.sh

@@ -6,7 +6,7 @@ set -x
 # uses at least 1g of memory, If we don't do this we can get OOM issues when
 # installing gems. See https://github.com/elastic/logstash/issues/5179
 export JRUBY_OPTS="-J-Xmx1g"
-export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
+export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
 
 if [ -n "$BUILD_JAVA_HOME" ]; then
   GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"

ci/integration_tests.sh

@@ -19,24 +19,15 @@ if [[ $1 = "setup" ]]; then
  exit 0
 
 elif [[ $1 == "split" ]]; then
-    cd qa/integration
-    glob1=(specs/*spec.rb)
-    glob2=(specs/**/*spec.rb)
-    all_specs=("${glob1[@]}" "${glob2[@]}")
+  # Source shared function for splitting integration tests
+  source "$(dirname "${BASH_SOURCE[0]}")/partition-files.lib.sh"
 
-    specs0=${all_specs[@]::$((${#all_specs[@]} / 2 ))}
-    specs1=${all_specs[@]:$((${#all_specs[@]} / 2 ))}
-    cd ../..
-    if [[ $2 == 0 ]]; then
-       echo "Running the first half of integration specs: $specs0"
-       ./gradlew runIntegrationTests -PrubyIntegrationSpecs="$specs0" --console=plain
-    elif [[ $2 == 1 ]]; then
-       echo "Running the second half of integration specs: $specs1"
-       ./gradlew runIntegrationTests -PrubyIntegrationSpecs="$specs1" --console=plain
-    else
-       echo "Error, must specify 0 or 1 after the split. For example ci/integration_tests.sh split 0"
-       exit 1
-    fi
+  index="${2:?index}"
+  count="${3:-2}"
+  specs=($(cd qa/integration; partition_files "${index}" "${count}" < <(find specs -name '*_spec.rb') ))
+
+  echo "Running integration tests partition[${index}] of ${count}: ${specs[*]}"
+  ./gradlew runIntegrationTests -PrubyIntegrationSpecs="${specs[*]}" --console=plain
 
 elif [[ !  -z  $@  ]]; then
     echo "Running integration tests 'rspec $@'"

ci/partition-files.lib.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# partition_files returns a consistent partition of the filenames given on stdin
+# Usage: partition_files <partition_index> <partition_count=2> < <(ls files)
+# partition_index: the zero-based index of the partition to select `[0,partition_count)`
+# partition_count: the number of partitions `[2,#files]`
+partition_files() (
+    set -e
+
+    local files
+    # ensure files is consistently sorted and distinct
+    IFS=$'\n' read -ra files -d '' <<<"$(cat - | sort | uniq)" || true
+
+    local partition_index="${1:?}"
+    local partition_count="${2:?}"
+
+    _error () { >&2 echo "ERROR: ${1:-UNSPECIFIED}"; exit 1; }
+
+    # safeguard against nonsense invocations
+    if (( ${#files[@]} < 2 )); then
+      _error "#files(${#files[@]}) must be at least 2 in order to partition"
+    elif ( ! [[ "${partition_count}" =~ ^[0-9]+$ ]] ) || (( partition_count < 2 )) || (( partition_count > ${#files[@]})); then
+      _error "partition_count(${partition_count}) must be a number that is at least 2 and not greater than #files(${#files[@]})"
+    elif ( ! [[ "${partition_index}" =~ ^[0-9]+$ ]] ) || (( partition_index < 0 )) || (( partition_index >= $partition_count )) ; then
+      _error "partition_index(${partition_index}) must be a number that is greater 0 and less than partition_count(${partition_count})"
+    fi
+
+    # round-robbin emit those in our selected partition
+    for index in "${!files[@]}"; do
+      partition="$(( index % partition_count ))"
+      if (( partition == partition_index )); then
+        echo "${files[$index]}"
+      fi
+    done
+)
+
+if [[ "$0" == "${BASH_SOURCE[0]}" ]]; then
+  if [[ "$1" == "test" ]]; then
+    status=0
+
+    SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+    file_list="$( cd "${SCRIPT_DIR}"; find . -type f )"
+
+    # for any legal partitioning into N partitions, we ensure that
+    # the combined output of `partition_files I N` where `I` is all numbers in
+    # the range `[0,N)` produces no repeats and no omissions, even if the
+    # input list is not consistently ordered.
+    for n in $(seq 2 $(wc -l <<<"${file_list}")); do
+      result=""
+      for i in $(seq 0 $(( n - 1 ))); do
+        for file in $(partition_files $i $n <<<"$( shuf <<<"${file_list}" )"); do
+          result+="${file}"$'\n'
+        done
+      done
+
+      repeated="$( uniq --repeated <<<"$( sort <<<"${result}" )" )"
+      if (( $(printf "${repeated}" | wc -l) > 0 )); then
+        status=1
+        echo "[n=${n}]FAIL(repeated):"$'\n'"${repeated}"
+      fi
+
+      missing=$( comm -23 <(sort <<<"${file_list}") <( sort <<<"${result}" ) )
+      if (( $(printf "${missing}" | wc -l) > 0 )); then
+        status=1
+        echo "[n=${n}]FAIL(omitted):"$'\n'"${missing}"
+      fi
+    done
+
+    if (( status > 0 )); then
+      echo "There were failures. The input list was:"
+      echo "${file_list}"
+    fi
+
+    exit "${status}"
+  else
+    partition_files $@
+  fi
+fi

config/jvm.options

@@ -34,7 +34,7 @@
 ## basic
 
 # set the I/O temp directory
-#-Djava.io.tmpdir=$HOME
+#-Djava.io.tmpdir=${HOME}
 
 # set to headless, just in case
 -Djava.awt.headless=true

config/log4j2.properties

@@ -154,7 +154,7 @@ appender.deprecation_rolling.policies.size.size = 100MB
 appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
 appender.deprecation_rolling.strategy.max = 30
 
-logger.deprecation.name = org.logstash.deprecation, deprecation
+logger.deprecation.name = org.logstash.deprecation
 logger.deprecation.level = WARN
 logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_plain_rolling
 logger.deprecation.additivity = false

config/logstash.yml

@@ -332,7 +332,7 @@
 #
 # Determine where to allocate memory buffers, for plugins that leverage them.
 # Default to direct, optionally can be switched to heap to select Java heap space.
-# pipeline.buffer.type: direct
+# pipeline.buffer.type: heap
 #
 # ------------ X-Pack Settings (not applicable for OSS build)--------------
 #

docker/Makefile

@@ -31,7 +31,7 @@ build-from-local-full-artifacts: dockerfile env2yaml
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
 	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
-	docker build --network=host -t $(IMAGE_TAG)-full:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-full data/logstash || \
+	docker build --progress=plain --network=host -t $(IMAGE_TAG)-full:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-full data/logstash || \
 	  (docker kill $(HTTPD); false); \
 	docker tag $(IMAGE_TAG)-full:$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG);
 	docker kill $(HTTPD)
@@ -41,7 +41,7 @@ build-from-local-oss-artifacts: dockerfile env2yaml
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
 	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
-	docker build --network=host -t $(IMAGE_TAG)-oss:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-oss data/logstash || \
+	docker build --progress=plain --network=host -t $(IMAGE_TAG)-oss:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-oss data/logstash || \
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
 
@@ -50,7 +50,7 @@ build-from-local-ubi8-artifacts: dockerfile env2yaml
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
 	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
-	docker build --network=host -t $(IMAGE_TAG)-ubi8:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-ubi8 data/logstash || \
+	docker build --progress=plain --network=host -t $(IMAGE_TAG)-ubi8:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-ubi8 data/logstash || \
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
 
@@ -59,7 +59,7 @@ build-from-local-wolfi-artifacts: dockerfile
 	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
 	           python:3 bash -c 'cd /mnt && python3 -m http.server'
 	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
-	docker build --network=host -t $(IMAGE_TAG)-wolfi:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-wolfi data/logstash || \
+	docker build --progress=plain --network=host -t $(IMAGE_TAG)-wolfi:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-wolfi data/logstash || \
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
 

docker/templates/Dockerfile.erb

@@ -15,7 +15,7 @@ RUN go build
 <%# Start image_flavor 'ironbank' %>
 ARG BASE_REGISTRY=registry1.dso.mil
 ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9
-ARG BASE_TAG=9.3
+ARG BASE_TAG=9.5
 ARG LOGSTASH_VERSION=<%= elastic_version %>
 ARG GOLANG_VERSION=1.21.8
 
@@ -106,7 +106,7 @@ FROM <%= base_image %>
 
 RUN for iter in {1..10}; do \
 <% if image_flavor == 'wolfi' %>
-  <%= package_manager %> add --no-cache curl bash && \
+  <%= package_manager %> add --no-cache curl bash openssl && \
 <% else -%>
   <% if image_flavor == 'full' || image_flavor == 'oss' -%>
     export DEBIAN_FRONTEND=noninteractive && \
@@ -191,7 +191,7 @@ COPY --from=builder-env2yaml /tmp/go/src/env2yaml/env2yaml /usr/local/bin/env2ya
 <% else -%>
 COPY env2yaml/env2yaml-amd64 env2yaml/env2yaml-arm64 env2yaml/
 # Copy over the appropriate env2yaml artifact
-RUN env2yamlarch="$(<%= arch_command %>)"; \
+RUN set -eux; env2yamlarch="$(<%= arch_command %>)"; \
   case "${env2yamlarch}" in \
     'x86_64'|'amd64') \
       env2yamlarch=amd64; \

docker/templates/hardening_manifest.yaml.erb

@@ -14,7 +14,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.3"
+  BASE_TAG: "9.5"
   LOGSTASH_VERSION: "<%= elastic_version %>"
   GOLANG_VERSION: "1.21.8"
 

docs/static/arcsight-module.asciidoc

@@ -6,6 +6,8 @@
 <titleabbrev>ArcSight Module</titleabbrev>
 ++++
 
+deprecated[8.16.0]
+
 NOTE: The Logstash ArcSight module is an
 https://www.elastic.co/products/x-pack[{xpack}] feature under the Basic License
 and is therefore free to use. Please contact

docs/static/azure-module.asciidoc

@@ -7,7 +7,7 @@ experimental[]
 <titleabbrev>Azure Module (deprecated)</titleabbrev>
 ++++
 
-deprecated[7.8.0, "We recommend using the Azure modules in {filebeat-ref}/filebeat-module-azure.html[{Filebeat}] and {metricbeat-ref}/metricbeat-module-azure.html[{metricbeat}], which are compliant with the {ecs-ref}/index.html[Elastic Common Schema (ECS)]"]
+deprecated[7.8.0, "Replaced by the https://www.elastic.co/guide/en/integrations/current/azure-events.html[Azure Logs integration]."]
 
 The https://azure.microsoft.com/en-us/overview/what-is-azure/[Microsoft Azure]
 module in Logstash helps you easily integrate your Azure activity logs and SQL

docs/static/dead-letter-queues.asciidoc

@@ -21,9 +21,10 @@ loss in this situation, you can <<configuring-dlq,configure Logstash>> to write
 unsuccessful events to a dead letter queue instead of dropping them.
 
 NOTE: The dead letter queue is currently supported only for the
-<<plugins-outputs-elasticsearch,{es} output>>. The dead letter queue is used for
-documents with response codes of 400 or 404, both of which indicate an event
+<<plugins-outputs-elasticsearch,{es} output>> and <<conditionals, conditional statements evaluation>>.
+The dead letter queue is used for documents with response codes of 400 or 404, both of which indicate an event
 that cannot be retried.
+It's also used when a conditional evaluation encounter an error.
 
 Each event written to the dead letter queue includes the original event,
 metadata that describes the reason the event could not be processed, information
@@ -57,7 +58,12 @@ status code per entry to indicate why the action could not be performed.
 If the DLQ is configured, individual indexing failures are routed there.
 
 Even if you regularly process events, events remain in the dead letter queue.
-The dead letter queue requires <<dlq-clear,manual intervention>> to clear it. 
+The dead letter queue requires <<dlq-clear,manual intervention>> to clear it.
+
+[[conditionals-dlq]]
+==== Conditional statements and the dead letter queue
+When a conditional statement reaches an error in processing an event, such as comparing string and integer values,
+the event, as it is at the time of evaluation, is inserted into the dead letter queue.
 
 [[configuring-dlq]]
 ==== Configuring {ls} to use dead letter queues

docs/static/ls-to-cloud.asciidoc

@@ -7,7 +7,7 @@ When you configure the Elasticsearch output plugin to use <<plugins-outputs-elas
 Examples:
 
 * `output {elasticsearch { cloud_id => "<cloud id>" cloud_auth => "<cloud auth>" } }`
-* `output {elasticsearch { cloud_id => "<cloud id>" api_key => "<api key>" } }``
+* `output {elasticsearch { cloud_id => "<cloud id>" api_key => "<api key>" } }`
 
 {ess-leadin-short}
 

docs/static/monitoring/monitoring-apis.asciidoc

@@ -2,13 +2,13 @@
 [[monitoring]]
 == APIs for monitoring {ls}
 
-{ls} provides monitoring APIs for retrieving runtime metrics
-about {ls}:
+{ls} provides monitoring APIs for retrieving runtime information about {ls}:
 
 * <<node-info-api>>
 * <<plugins-api>>
 * <<node-stats-api>>
 * <<hot-threads-api>>
+* <<logstash-health-report-api>>
 
 
 You can use the root resource to retrieve general information about the Logstash instance, including
@@ -714,6 +714,11 @@ Example response:
       },
       "queue" : {
         "type" : "memory"
+      },
+      "pipeline": {
+        "workers": 4,
+        "batch_size": 125,
+        "batch_delay": 50
       }
     },
     "test2" : {
@@ -795,6 +800,11 @@ Example response:
       },
       "queue" : {
         "type" : "memory"
+      },
+      "pipeline": {
+        "workers": 4,
+        "batch_size": 125,
+        "batch_delay": 50
       }
     }
   }
@@ -957,6 +967,11 @@ Example response:
         "events_count": 0,
         "queue_size_in_bytes": 3885,
         "max_queue_size_in_bytes": 1073741824
+      },
+      "pipeline": {
+        "workers": 4,
+        "batch_size": 125,
+        "batch_delay": 50
       }
     }
   }
@@ -1184,3 +1199,155 @@ Example of a human-readable response:
 	 org.jruby.internal.runtime.NativeThread.join(NativeThread.java:75)
 
 --------------------------------------------------
+
+
+[[logstash-health-report-api]]
+=== Health report API
+
+An API that reports the health status of Logstash.
+
+[source,js]
+--------------------------------------------------
+curl -XGET 'localhost:9600/_health_report?pretty'
+--------------------------------------------------
+
+==== Description
+
+The health API returns a report with the health status of Logstash and the pipelines that are running inside of it.
+The report contains a list of indicators that compose Logstash functionality.
+
+Each indicator has a health status of: `green`, `unknown`, `yellow`, or `red`.
+The indicator will provide an explanation and metadata describing the reason for its current health status.
+
+The top-level status is controlled by the worst indicator status.
+
+In the event that an indicator's status is non-green, a list of impacts may be present in the indicator result which detail the functionalities that are negatively affected by the health issue.
+Each impact carries with it a severity level, an area of the system that is affected, and a simple description of the impact on the system.
+
+Some health indicators can determine the root cause of a health problem and prescribe a set of steps that can be performed in order to improve the health of the system.
+The root cause and remediation steps are encapsulated in a `diagnosis`.
+A diagnosis contains a cause detailing a root cause analysis, an action containing a brief description of the steps to take to fix the problem, and the URL for detailed troubleshooting help.
+
+NOTE: The health indicators perform root cause analysis of non-green health statuses.
+      This can be computationally expensive when called frequently.
+
+==== Response body
+
+`status`::
+(Optional, string) Health status of {ls}, based on the aggregated status of all indicators. Statuses are:
+
+`green`:::
+{ls} is healthy.
+
+`unknown`:::
+The health of {ls} could not be determined.
+
+`yellow`:::
+The functionality of {ls} is in a degraded state and may need remediation to avoid the health becoming `red`.
+
+`red`:::
+{ls} is experiencing an outage or certain features are unavailable for use.
+
+`indicators`::
+(object) Information about the health of the {ls} indicators.
+
++
+.Properties of `indicators`
+[%collapsible%open]
+====
+`<indicator>`::
+(object) Contains health results for an indicator.
++
+.Properties of `<indicator>`
+[%collapsible%open]
+=======
+`status`::
+(string) Health status of the indicator. Statuses are:
+
+`green`:::
+The indicator is healthy.
+
+`unknown`:::
+The health of the indicator could not be determined.
+
+`yellow`:::
+The functionality of an indicator is in a degraded state and may need remediation to avoid the health becoming `red`.
+
+`red`:::
+The indicator is experiencing an outage or certain features are unavailable for use.
+
+`symptom`::
+(string) A message providing information about the current health status.
+
+`details`::
+(Optional, object) An object that contains additional information about the indicator that has lead to the current health status result.
+Each indicator has <<logstash-health-api-response-details, a unique set of details>>.
+
+`impacts`::
+(Optional, array) If a non-healthy status is returned, indicators may include a list of impacts that this health status will have on {ls}.
++
+.Properties of `impacts`
+[%collapsible%open]
+========
+`severity`::
+(integer) How important this impact is to the functionality of {ls}.
+A value of 1 is the highest severity, with larger values indicating lower severity.
+
+`description`::
+(string) A description of the impact on {ls}.
+
+`impact_areas`::
+(array of strings) The areas {ls} functionality that this impact affects.
+Possible values are:
++
+--
+* `pipeline_execution`
+--
+
+========
+
+`diagnosis`::
+(Optional, array) If a non-healthy status is returned, indicators may include a list of diagnosis that encapsulate the cause of the health issue and an action to take in order to remediate the problem.
++
+.Properties of `diagnosis`
+[%collapsible%open]
+========
+`cause`::
+(string) A description of a root cause of this health problem.
+
+`action`::
+(string) A brief description the steps that should be taken to remediate the problem.
+A more detailed step-by-step guide to remediate the problem is provided by the `help_url` field.
+
+`help_url`::
+(string) A link to the troubleshooting guide that'll fix the health problem.
+========
+=======
+====
+
+[role="child_attributes"]
+[[logstash-health-api-response-details]]
+==== Indicator Details
+
+Each health indicator in the health API returns a set of details that further explains the state of the system.
+The details have contents and a structure that is unique to each indicator.
+
+[[logstash-health-api-response-details-pipeline]]
+===== Pipeline Indicator Details
+
+`pipelines/indicators/<pipeline_id>/details`::
+(object) Information about the specified pipeline.
++
+.Properties of `pipelines/indicators/<pipeline_id>/details`
+[%collapsible%open]
+====
+`status`::
+(object) Details related to the pipeline's current status and run-state.
++
+.Properties of `status`
+[%collapsible%open]
+========
+`state`::
+(string) The current state of the pipeline, including whether it is `loading`, `running`, `finished`, or `terminated`.
+========
+====

docs/static/monitoring/monitoring-ea-dashboards.asciidoc

@@ -71,8 +71,8 @@ details.
 * Under **Logs**, modify the log paths to match your {ls} environment.
 
 . Configure the integration to collect metrics.
-* Make sure that **Metrics (Technical Preview)** is turned on, and **Metrics (Stack Monitoring)** is turned off.
-* Under **Metrics (Technical Preview)**, make sure the {ls} URL setting
+* Make sure that **Metrics (Elastic Agent)** is turned on (default), and **Metrics (Stack Monitoring)** is turned off.
+* Under **Metrics (Elastic Agent)**, make sure the {ls} URL setting
 points to your {ls} instance URLs. +
 By default, the integration collects {ls}
 monitoring metrics from `https://localhost:9600`. If that host and port number are not

docs/static/monitoring/monitoring-ea-serverless.asciidoc

@@ -42,7 +42,7 @@ For more info, check out the {serverless-docs}/observability/what-is-observabili
 
 **Configure the integration to collect metrics**
 
-* Make sure that  **Metrics (Stack Monitoring)** is OFF, and that **Metrics (Technical Preview)** is ON.
+* Make sure that  **Metrics (Stack Monitoring)** is OFF, and that **Metrics (Elastic Agent)** is ON.
 * Set the {ls} URL to point to your {ls} instance. +
 By default, the integration collects {ls}
 monitoring metrics from `https://localhost:9600`. If that host and port number are not

docs/static/monitoring/monitoring-ea.asciidoc

@@ -81,8 +81,8 @@ details about it.
 to match your {ls} environment.
 
 . Configure the integration to collect metrics
-* Make sure that **Metrics (Stack Monitoring)** is turned on, and **Metrics (Technical Preview)** is turned off, if you
-want to collect metrics from your {ls} instance
+* Make sure that **Metrics (Stack Monitoring)** is turned on, and **Metrics (Elastic Agent)** is turned off, if you
+want to collect metrics from your {ls} instance.
 * Under **Metrics (Stack Monitoring)**, make sure the hosts setting
 points to your {ls} host URLs. By default, the integration collects {ls}
 monitoring metrics from `localhost:9600`. If that host and port number are not

docs/static/monitoring/monitoring-view.asciidoc

@@ -1,9 +1,9 @@
 After you have confirmed enrollment and data is coming in,  click **View assets** to access dashboards related to the {ls} integration.
 
 For traditional Stack Monitoring UI, the dashboards marked **[Logs {ls}]** are used to visualize the logs
-produced by your {ls} instances, with those marked **[Metrics {ls}]** for the technical preview metrics
+produced by your {ls} instances, with those marked **[Metrics {ls}]** for metrics
 dashboards.
-These are populated with data only if you selected the **Metrics (Technical Preview)** checkbox.
+These are populated with data only if you selected the **Metrics (Elastic Agent)** checkbox.
 
 --
 [role="screenshot"]

docs/static/persistent-queues.asciidoc

@@ -126,8 +126,8 @@ These tables show examples of overhead by event type and how that affects the mu
 | JSON document size (bytes) | Serialized {ls} event size (bytes) | Overhead (bytes) | Overhead (%) | Multiplication Factor
 | 947 | 1133 | 186 | 20% | 1.20
 | 2707 | 3206 | 499 | 18% | 1.18
-| 6751 | 7388 | 637 | 9% | 1.9
-| 58901 | 59693 | 792 | 1% | 1.1
+| 6751 | 7388 | 637 | 9% | 1.09
+| 58901 | 59693 | 792 | 1% | 1.01
 |=======================================================================
 
 *Example*

docs/static/plugin-manager.asciidoc

@@ -112,6 +112,22 @@ bin/logstash-plugin update logstash-input-github <2>
 <1> updates all installed plugins
 <2> updates only the plugin you specify
 
+[discrete]
+[[updating-major]]
+==== Major version plugin updates
+
+To avoid introducing breaking changes, the plugin manager updates only plugins for which newer _minor_ or _patch_ versions exist by default.
+If you wish to also include breaking changes, specify `--level=major`.
+
+[source,shell]
+----------------------------------
+bin/logstash-plugin update --level=major <1>
+bin/logstash-plugin update --level=major logstash-input-github <2>
+----------------------------------
+<1> updates all installed plugins to latest, including major versions with breaking changes
+<2> updates only the plugin you specify to latest, including major versions with breaking changes
+
+
 [discrete]
 [[removing-plugins]]
 === Removing plugins

docs/static/releasenotes.asciidoc

@@ -3,6 +3,22 @@
 
 This section summarizes the changes in the following releases:
 
+* <<logstash-8-17-4,Logstash 8.17.4>>
+* <<logstash-8-17-3,Logstash 8.17.3>>
+* <<logstash-8-17-2,Logstash 8.17.2>>
+* <<logstash-8-17-1,Logstash 8.17.1>>
+* <<logstash-8-17-0,Logstash 8.17.0>>
+* <<logstash-8-16-6,Logstash 8.16.6>>
+* <<logstash-8-16-5,Logstash 8.16.5>>
+* <<logstash-8-16-4,Logstash 8.16.4>>
+* <<logstash-8-16-3,Logstash 8.16.3>>
+* <<logstash-8-16-2,Logstash 8.16.2>>
+* <<logstash-8-16-1,Logstash 8.16.1>>
+* <<logstash-8-16-0,Logstash 8.16.0>>
+* <<logstash-8-15-5,Logstash 8.15.5>>
+* <<logstash-8-15-4,Logstash 8.15.4>>
+* <<logstash-8-15-3,Logstash 8.15.3>>
+* <<logstash-8-15-2,Logstash 8.15.2>>
 * <<logstash-8-15-1,Logstash 8.15.1>>
 * <<logstash-8-15-0,Logstash 8.15.0>>
 * <<logstash-8-14-3,Logstash 8.14.3>>
@@ -66,9 +82,803 @@ This section summarizes the changes in the following releases:
 * <<logstash-8-0-0-alpha1,Logstash 8.0.0-alpha1>>
 
 
+[[logstash-8-17-4]]
+=== Logstash 8.17.4 Release Notes
+
+[[known-issues-8-17-4]]
+==== Known issues
+
+** The https://github.com/logstash-plugins/logstash-input-http_poller[http_poller input] plugin will terminate during pipeline startup due to an issue with an underlying library. Please upgrade to logstash-input-http_poller 5.6.1 using {ls}'s plugin manager with `bin/logstash-plugin update logstash-input-http_poller`.
+** The https://github.com/logstash-plugins/logstash-input-elasticsearch[elasticsearch input] and https://github.com/logstash-plugins/logstash-filter-elasticsearch[elasticsearch filter] plugins will terminate during pipeline startup due to an upgrade of the underlying elasticsearch ruby client from 7.x to 8.x. Please upgrade to logstash-input-elasticsearch 4.21.2 and logstash-filter-elasticsearch 3.17.1 using {ls}'s plugin manager with `bin/logstash-plugin update logstash-input-elasticsearch logstash-filter-elasticsearch` or downgrade to Logstash 8.17.3.
+
+[[notable-8-17-4]]
+==== Notable issues fixed
+
+* Fix pqcheck and pqrepair on Windows. https://github.com/elastic/logstash/pull/17210[#17210]
+* Avoid possible integer overflow in string tokenization. https://github.com/elastic/logstash/pull/17353[#17353]
+* Fixed an issue where the /_node/stats API displayed empty pipeline metrics when Logstash monitoring was configured to use legacy collectors. https://github.com/elastic/logstash/pull/17185[#17185]
+
+[[plugins-8-17-4]]
+==== Plugins
+
+*Syslog Input - 3.7.1*
+
+* Fix issue where the priority field was not being set correctly when grok failed https://github.com/logstash-plugins/logstash-input-syslog/pull/78[#76]
+
+*Jdbc Integration - 5.5.3*
+
+* [DOC] Rework inline comment to a callout in preparation for upcoming MD conversion https://github.com/logstash-plugins/logstash-integration-jdbc/pull/181[#181]
+
+
+[[logstash-8-17-3]]
+=== Logstash 8.17.3 Release Notes
+
+[[notable-8-17-3]]
+==== Notable issues fixed
+
+* Improves performance of the Persistent Queue, especially in the case of large events, by moving deserialization out of the exclusive access lock. https://github.com/elastic/logstash/pull/17050[#17050]
+* Improve error logging when Centralized Pipeline Management cannot find a configured pipeline. https://github.com/elastic/logstash/pull/17052[#17052]
+* Update logstash-keystore to allow spaces in values when `stdin` is used to set values https://github.com/elastic/logstash/pull/17039[#17039]
+
+[[plugins-8-17-3]]
+==== Plugins
+
+*Beats Input - 6.9.3*
+
+* Upgrade netty to 4.1.118 https://github.com/logstash-plugins/logstash-input-beats/pull/514[#514]
+
+*Http Input - 3.10.2*
+
+* Upgrade netty to 4.1.118 https://github.com/logstash-plugins/logstash-input-http/pull/194[#194]
+
+*Tcp Input - 6.4.6*
+
+* Upgrade netty to 4.1.118 https://github.com/logstash-plugins/logstash-input-tcp/pull/233[#233]
+
+
+[[logstash-8-17-2]]
+=== Logstash 8.17.2 Release Notes
+
+[[notable-8-17-2]]
+==== Notable issues fixed
+
+* The plugin manager's `update` command now correctly updates only _minor_ versions of plugins by default to avoid breaking changes.
+If you wish to also include breaking changes, you must specify `--level=major` https://github.com/elastic/logstash/pull/16974[#16974]
+* The plugin manager no longer has issues installing plugins with embedded jars or depending on snakeyaml https://github.com/elastic/logstash/pull/16924[#16924]
+* The plugin manager now correctly supports authenticated proxies by transmitting username and password from proxy environment URI https://github.com/elastic/logstash/pull/16958[#16958]
+* The logstash-keystore now correctly accepts spaces in values when added via stdin https://github.com/elastic/logstash/pull/17041[#17041]
+* The buffered-tokenizer, which is used by many plugins to split streams of bytes by a delimiter, now properly resumes at the next delimiter after encountering a buffer-full condition https://github.com/elastic/logstash/pull/17022[#17022]
+
+
+[[dependencies-8-17-2]]
+==== Updates to dependencies
+
+* Update JDK to 21.0.6+7 https://github.com/elastic/logstash/pull/16989[#16989]
+
+[[plugins-8-17-2]]
+==== Plugins
+
+*Elastic_integration Filter - 8.17.1*
+
+* Provides a guidance in logs when plugin version mismatches with connected Elasticsearch `major.minor` version https://github.com/elastic/logstash-filter-elastic_integration/pull/255[#255]
+* Embeds Ingest Node components from Elasticsearch 8.17
+* Compatible with Logstash 8.15+
+
+*Elasticsearch Filter - 3.17.0*
+
+* Added support for custom headers https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/190[#190]
+
+*Beats Input - 6.9.2*
+
+* Name netty threads according to their purpose and the plugin id https://github.com/logstash-plugins/logstash-input-beats/pull/511[#511]
+
+*Elasticsearch Input - 4.21.1*
+
+* Fix: prevent plugin crash when hits contain illegal structure https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/183[#183]
+* When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit.
+
+* Add support for custom headers https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/217[#217]
+
+*Http Input - 3.10.1*
+
+* Properly naming netty threads https://github.com/logstash-plugins/logstash-input-http/pull/191[#191]
+
+* Add improved proactive rate-limiting, rejecting new requests when queue has been actively blocking for more than 10 seconds https://github.com/logstash-plugins/logstash-input-http/pull/179[#179]
+
+*Tcp Input - 6.4.5*
+
+* Name netty threads with plugin id and their purpose https://github.com/logstash-plugins/logstash-input-tcp/pull/229[#229]
+
+*Snmp Integration - 4.0.6*
+
+* [DOC] Fix typo in snmptrap migration section https://github.com/logstash-plugins/logstash-integration-snmp/pull/74[#74]
+
+*Elasticsearch Output - 11.22.12*
+
+* Properly handle http code 413 (Payload Too Large) https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1199[#1199]
+* Remove irrelevant log warning about elastic stack version https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1202[#1202]
+
+
+[[logstash-8-17-1]]
+=== Logstash 8.17.1 Release Notes
+
+[[notable-8.17.1]]
+==== Notable issues fixed
+
+* Reset internal size counter in BufferedTokenizer during flush https://github.com/elastic/logstash/pull/16760[#16760].
+  Fixes <<known-issue-8-16-1-json_lines,"input buffer full" error>> that could appear with versions 8.16.0, 8.16.1, and 8.17.0.
+* Avoid lock contention when ecs_compatibility is explicitly specified https://github.com/elastic/logstash/pull/16786[#16786]
+* Ensure that the Jackson read constraints defaults (Maximum Number value length, Maximum String value length, and Maximum Nesting depth) are applied at runtime if they are absent from jvm.options https://github.com/elastic/logstash/pull/16832[#16832]
+* Fix environment variables `${VAR}` were not interpreted in jvm.options https://github.com/elastic/logstash/pull/16834[#16834]
+* Show pipeline metrics (workers, batch_size, batch_delay) in the Node Stats API https://github.com/elastic/logstash/pull/16839[#16839]
+
+[[dependencies-8.17.1]]
+==== Updates to dependencies
+
+* Update Iron Bank base image to ubi9/9.5 https://github.com/elastic/logstash/pull/16825[#16825]
+
+[[plugins-8.17.1]]
+==== Plugins
+
+*Elastic_integration Filter - 8.17.0*
+
+* Aligns with stack major and minor versions https://github.com/elastic/logstash-filter-elastic_integration/pull/212[#212]
+    * Embeds Ingest Node components from Elasticsearch 8.17
+    * Compatible with Logstash 8.15+
+
+*Elasticsearch Filter - 3.16.2*
+
+* Add `x-elastic-product-origin` header to Elasticsearch requests https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/185[#185]
+
+*Azure_event_hubs Input - 1.5.1*
+
+* Updated multiple Java dependencies https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/99[#99]
+
+*Elasticsearch Input - 4.20.5*
+
+* Add `x-elastic-product-origin` header to Elasticsearch requests https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/211[#211]
+
+*Elastic_enterprise_search Integration - 3.0.1*
+
+* Add deprecation log for App Search and Workplace Search. Both products are removed from Elastic Stack in version 9 https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/22[#22]
+
+*Jdbc Integration - 5.5.2*
+
+* The input plugin's prior behaviour of opening a new database connection for each scheduled run (removed in `v5.4.1`) is restored, ensuring that infrequently-run schedules do not hold open connections to their databases indefinitely, _without_ reintroducing the leak https://github.com/logstash-plugins/logstash-integration-jdbc/pull/130[#130]
+
+*Kafka Integration - 11.5.4*
+
+* Update kafka client to 3.8.1 and transitive dependencies https://github.com/logstash-plugins/logstash-integration-kafka/pull/188[#188]
+* Removed `jar-dependencies` dependency https://github.com/logstash-plugins/logstash-integration-kafka/pull/187[#187]
+
+*Logstash Integration - 1.0.4*
+
+* Fixes a buffer-over-limit exception in the downstream input plugin by emitting event-oriented chunks in the upstream output plugin https://github.com/logstash-plugins/logstash-integration-logstash/pull/25[#25]
+
+*Snmp Integration - 4.0.5*
+
+* Fix typo resulting in "uninitialized constant" exception for invalid column name https://github.com/logstash-plugins/logstash-integration-snmp/pull/73[#73]
+
+*Elasticsearch Output - 11.22.10*
+
+* Add `x-elastic-product-origin` header to Elasticsearch requests https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1194[#1194]
+
+
+[[logstash-8-17-0]]
+=== Logstash 8.17.0 Release Notes
+
+[[known-issues-8-17-0]]
+==== Known issues
+
+[[known-issue-8-17-0-jvm]]
+===== JVM version changes needed when upgrading {ls} from 8.12.0 (or earlier)
+
+If the `jvm.options` file was modified and not overwritten with the newest version, you may see a "deserialize invocation error" message, causing the pipeline to crash.
+Users are affected if the Persistent Queue (PQ) is enabled, and the pipeline is processing messages larger than 20MB.
+
+**Solution:** Apply the default change contained in the newer 'jvm.options' file, as seen in this https://github.com/elastic/logstash/blob/v8.17.0/config/jvm.options#L74-L90[example].
+
+[[known-issue-8-17-0-json_lines]]
+===== "Input buffer full" error with {ls} 8.16.0, 8.16.1, or 8.17.0
+
+If you are using `json_lines` codec 3.2.0 (or later) with {ls} 8.16.0, 8.16.1, or 8.17.0, you may see an error similar to this one, crashing the pipelines:
+```
+unable to process event. {:message=>"input buffer full", :class=>"Java::JavaLang::IllegalStateException", :backtrace=>["org.logstash.common.BufferedTokenizerExt.extract(BufferedTokenizerExt.java:83)", "usr.share.logstash.vendor.bundle.jruby.$3_dot_1_dot_0.gems.logstash_minus_codec_minus_json_lines_minus_3_dot_2_dot_2.lib.logstash.codecs.json_lines.RUBY$method$decode$0(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-codec-json_lines-3.2.2/lib/logstash/codecs/json_lines.rb:69)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:165)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:185)",
+```
+The issue was fixed in https://github.com/elastic/logstash/pull/16760.
+
+This problem is most likely to be seen when you are using the <<plugins-integrations-logstash,{ls} integration>> plugin to ship data between two {ls} instances, but may appear in other situations, too.
+
+**Workaround for {ls}-to-{ls} communication**
+
+The {ls}-to-{ls} issue can be mitigated by:
+
+* Downgrading the _receiving_ {ls} to `8.16.2`, or any {ls} in the `8.15` series,  **_AND/OR_**
+* Upgrading the <<plugins-integrations-logstash,{ls} integration>> plugin of the _sending_ {ls} to version `1.0.4`.
+
+**Workaround for other `json_lines` codec situations**
+
+Other `json_lines` codec issues can be mitigated by:
+
+* Downgrading {ls} to `8.16.2`, or any {ls} in the `8.15` series.
+
+[[notable-8-17-0]]
+==== Notable fixes and improvements
+
+* Add warning that `allow_superuser` will default to `false` in 9.0.0 https://github.com/elastic/logstash/pull/16555[#16555]
+* Update deprecation warning to mention ArcSight module will be removed in 9.0.0 https://github.com/elastic/logstash/pull/16648[#16648]
+* Update deprecation warning for http.* settings to mention removal in 9.0.0 https://github.com/elastic/logstash/pull/16538[#16538]
+
+[[core-8-17-0]]
+==== Changes to Logstash core
+
+* Make max inflight warning global to all pipelines https://github.com/elastic/logstash/pull/16601[#16601]
+* Correctly guide user to use LS_JAVA_HOME instead of JAVA_HOME to configure Java on Windows https://github.com/elastic/logstash/pull/16636[#16636]
+* Ensure jackson configurations are applied if found in "jvm.options" https://github.com/elastic/logstash/pull/16757[#16757]
+* Set `platform = 'java'` in custom java plugins' gemspecs https://github.com/elastic/logstash/pull/16628[#16628]
+* Fix offline installation of java plugins containing "-java" in their name https://github.com/elastic/logstash/pull/16637[#16637]
+
+[[dependencies-8.17.0]]
+==== Updates to dependencies
+
+* Pin jar-dependencies gem to 0.4.1 to avoid clashing with version bundled with JRuby https://github.com/elastic/logstash/pull/16750[#16750]
+* Update JDK to 21.0.5+11 https://github.com/elastic/logstash/pull/16631[#16631]
+
+[[docs-8.17.0]]
+==== Documentation enhancements
+
+* Troubleshooting update for JDK bug handling cgroups v1 https://github.com/elastic/logstash/pull/16731[#16731]
+
+==== Plugins
+
+*Http_client Mixin - 7.5.0*
+
+* Adds new mixin configuration option `with_obsolete` to mark `ssl` options as obsolete https://github.com/logstash-plugins/logstash-mixin-http_client/pull/46[#46]
+
+[[logstash-8-16-6]]
+=== Logstash 8.16.6 Release Notes
+
+[[notable-8-16-6]]
+==== Notable issues fixed
+
+* Fix pqcheck and pqrepair on Windows. https://github.com/elastic/logstash/pull/17210[#17210]
+* Avoid possible integer overflow in string tokenization. https://github.com/elastic/logstash/pull/17353[#17353]
+
+[[plugins-8-16-6]]
+==== Plugins
+
+*Syslog Input - 3.7.1*
+
+* Fix issue where the priority field was not being set correctly when grok failed https://github.com/logstash-plugins/logstash-input-syslog/pull/78[#76]
+
+*Jdbc Integration - 5.5.3*
+
+* [DOC] Rework inline comment to a callout in preparation for upcoming MD conversion https://github.com/logstash-plugins/logstash-integration-jdbc/pull/181[#181]
+
+[[logstash-8-16-5]]
+=== Logstash 8.16.5 Release Notes
+
+[[notable-8-16-5]]
+==== Notable issues fixed
+
+* Improves performance of the Persistent Queue, especially in the case of large events, by moving deserialization out of the exclusive access lock. https://github.com/elastic/logstash/pull/17050[#17050]
+* Improve error logging when Centralized Pipeline Management cannot find a configured pipeline. https://github.com/elastic/logstash/pull/17052[#17052]
+
+[[plugins-8-16-5]]
+==== Plugins
+
+*Beats Input - 6.9.3*
+
+* Upgrade netty to 4.1.118 https://github.com/logstash-plugins/logstash-input-beats/pull/514[#514]
+
+*Http Input - 3.10.2*
+
+* Upgrade netty to 4.1.118 https://github.com/logstash-plugins/logstash-input-http/pull/194[#194]
+
+*Tcp Input - 6.4.6*
+
+* Upgrade netty to 4.1.118 https://github.com/logstash-plugins/logstash-input-tcp/pull/233[#233]
+
+
+[[logstash-8-16-4]]
+=== Logstash 8.16.4 Release Notes
+
+[[notable-8-16-4]]
+==== Notable issues fixed
+
+* The plugin manager's `update` command now correctly updates only _minor_ versions of plugins by default to avoid breaking changes.
+If you wish to also include breaking changes, you must specify `--level=major` https://github.com/elastic/logstash/pull/16975[#16975]
+* The plugin manager no longer has issues installing plugins with embedded jars or depending on snakeyaml https://github.com/elastic/logstash/pull/16925[#16925]
+* The plugin manager now correctly supports authenticated proxies by transmitting username and password from proxy environment URI https://github.com/elastic/logstash/pull/16957[#16957]
+* The buffered-tokenizer, which is used by many plugins to split streams of bytes by a delimiter, now properly resumes at the next delimiter after encountering a buffer-full condition https://github.com/elastic/logstash/pull/17021[#17021]
+
+[[dependencies-8-16-4]]
+==== Updates to dependencies
+
+* Update JDK to 21.0.6+7 https://github.com/elastic/logstash/pull/16990[#16990]
+
+[[plugins-8-16-4]]
+==== Plugins
+
+*Elastic_integration Filter - 8.16.1*
+
+* Provides a guidance in logs when plugin version mismatches with connected Elasticsearch `major.minor` version https://github.com/elastic/logstash-filter-elastic_integration/pull/253[#253]
+* Embeds Ingest Node components from Elasticsearch 8.16
+* Compatible with Logstash 8.15+
+
+*Elasticsearch Filter - 3.17.0*
+
+* Added support for custom headers https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/190[#190]
+
+*Beats Input - 6.9.2*
+
+* Name netty threads according to their purpose and the plugin id https://github.com/logstash-plugins/logstash-input-beats/pull/511[#511]
+
+*Elasticsearch Input - 4.21.1*
+
+* Fix: prevent plugin crash when hits contain illegal structure https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/183[#183]
+* When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit.
+
+* Add support for custom headers https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/217[#217]
+
+*Http Input - 3.10.1*
+
+* Properly naming netty threads https://github.com/logstash-plugins/logstash-input-http/pull/191[#191]
+
+* Add improved proactive rate-limiting, rejecting new requests when queue has been actively blocking for more than 10 seconds https://github.com/logstash-plugins/logstash-input-http/pull/179[#179]
+
+*Tcp Input - 6.4.5*
+
+* Name netty threads with plugin id and their purpose https://github.com/logstash-plugins/logstash-input-tcp/pull/229[#229]
+
+*Snmp Integration - 4.0.6*
+
+* [DOC] Fix typo in snmptrap migration section https://github.com/logstash-plugins/logstash-integration-snmp/pull/74[#74]
+
+*Elasticsearch Output - 11.22.12*
+
+* Properly handle http code 413 (Payload Too Large) https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1199[#1199]
+* Remove irrelevant log warning about elastic stack version https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1202[#1202]
+
+
+[[logstash-8-16-3]]
+=== Logstash 8.16.3 Release Notes
+
+[[notable-8.16.3]]
+==== Notable issues fixed
+
+* Avoid lock contention when ecs_compatibility is explicitly specified https://github.com/elastic/logstash/pull/16786[#16786]
+* Ensure that the Jackson read constraints defaults (Maximum Number value length, Maximum String value length, and Maximum Nesting depth) are applied at runtime if they are absent from jvm.options https://github.com/elastic/logstash/pull/16832[#16832]
+
+[[dependencies-8.16.3]]
+==== Updates to dependencies
+
+* Update Iron Bank base image to ubi9/9.5 https://github.com/elastic/logstash/pull/16825[#16825]
+
+[[plugins-8.16.3]]
+==== Plugins
+
+*Elastic_integration Filter - 8.16.0*
+
+* Aligns with stack major and minor versions https://github.com/elastic/logstash-filter-elastic_integration/pull/210[#210]
+  * Embeds Ingest Node components from Elasticsearch 8.16
+  * Compatible with Logstash 8.15+
+
+*Azure_event_hubs Input - 1.5.1*
+
+* Updated multiple Java dependencies https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/99[#99]
+
+*Elastic_enterprise_search Integration - 3.0.1*
+
+* Add deprecation log for App Search and Workplace Search. https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/22[#22]
+
+*Jdbc Integration - 5.5.2*
+
+* The input plugin's prior behaviour of opening a new database connection for each scheduled run (removed in `v5.4.1`) is restored, ensuring that infrequently-run schedules do not hold open connections to their databases indefinitely, _without_ reintroducing the leak https://github.com/logstash-plugins/logstash-integration-jdbc/pull/130[#130]
+
+*Kafka Integration - 11.5.4*
+
+* Update kafka client to 3.8.1 and transitive dependencies https://github.com/logstash-plugins/logstash-integration-kafka/pull/188[#188]
+* Removed `jar-dependencies` dependency https://github.com/logstash-plugins/logstash-integration-kafka/pull/187[#187]
+
+*Snmp Integration - 4.0.5*
+
+* Fix typo resulting in "uninitialized constant" exception for invalid column name https://github.com/logstash-plugins/logstash-integration-snmp/pull/73[#73]
+
+[[logstash-8-16-2]]
+=== Logstash 8.16.2 Release Notes
+
+[[notable-8-16-2]]
+==== Notable issues fixed
+
+* Reset internal size counter in BufferedTokenizer during flush https://github.com/elastic/logstash/pull/16771[#16771].
+  Fixes <<known-issue-8-16-1-json_lines,"input buffer full" error>> that could appear with versions 8.16.0 and 8.16.1.
+* Ensure overrides to jackson settings are applied during startup https://github.com/elastic/logstash/pull/16758[#16758].
+
+[[dependencies-8-16-2]]
+==== Updates to dependencies
+
+* Pin `jar-dependencies` to `0.4.1` and `date` to `3.3.3` to avoid clashes between what's bundled with JRuby and newer versions in Rubygems https://github.com/elastic/logstash/pull/16749[#16749] https://github.com/elastic/logstash/pull/16779[#16779]
+
+==== Plugins
+
+*Elastic_integration Filter - 0.1.17*
+
+* Add `x-elastic-product-origin` header to Elasticsearch requests https://github.com/elastic/logstash-filter-elastic_integration/pull/197[#197]
+
+*Elasticsearch Filter - 3.16.2*
+
+* Add `x-elastic-product-origin` header to Elasticsearch requests https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/185[#185]
+
+*Elasticsearch Input - 4.20.5*
+
+* Add `x-elastic-product-origin` header to Elasticsearch requests https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/211[#211]
+
+*Jdbc Integration - 5.5.1*
+
+* Document `statement_retry_attempts` and `statement_retry_attempts_wait_time` options https://github.com/logstash-plugins/logstash-integration-jdbc/pull/177[#177]
+
+*Kafka Integration - 11.5.3*
+
+* Update kafka client to 3.7.1 and transitive dependencies https://github.com/logstash-plugins/logstash-integration-kafka/pull/186[#186]
+
+*Logstash Integration - 1.0.4*
+
+* Align output plugin with documentation by producing event-oriented ndjson-compatible payloads instead of JSON array of events https://github.com/logstash-plugins/logstash-integration-logstash/pull/25[#25]
+
+*Elasticsearch Output - 11.22.10*
+
+* Add `x-elastic-product-origin` header to Elasticsearch requests https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1195[#1195]
+
+[[logstash-8-16-1]]
+=== Logstash 8.16.1 Release Notes
+
+[[known-issues-8-16-1]]
+==== Known issue
+
+[[known-issue-8-16-1-json_lines]]
+===== "Input buffer full" error with {ls} 8.16.0, 8.16.1, or 8.17.0
+
+If you are using `json_lines` codec 3.2.0 (or later) with {ls} 8.16.0, 8.16.1, or 8.17.0, you may see an error similar to this one, crashing the pipelines:
+```
+unable to process event. {:message=>"input buffer full", :class=>"Java::JavaLang::IllegalStateException", :backtrace=>["org.logstash.common.BufferedTokenizerExt.extract(BufferedTokenizerExt.java:83)", "usr.share.logstash.vendor.bundle.jruby.$3_dot_1_dot_0.gems.logstash_minus_codec_minus_json_lines_minus_3_dot_2_dot_2.lib.logstash.codecs.json_lines.RUBY$method$decode$0(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-codec-json_lines-3.2.2/lib/logstash/codecs/json_lines.rb:69)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:165)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:185)",
+```
+The issue was fixed in https://github.com/elastic/logstash/pull/16760.
+
+This problem is most likely to be seen when you are using the <<plugins-integrations-logstash,{ls} integration>> plugin to ship data between two {ls} instances, but may appear in other situations, too. 
+
+**Workaround for {ls}-to-{ls} communication** 
+
+The {ls}-to-{ls} issue can be mitigated by:
+
+* Downgrading the _receiving_ {ls} to `8.16.2`, or any {ls} in the `8.15` series,  **_AND/OR_**
+* Upgrading the {ls} integration filter of the _sending_ {ls} to version `1.0.4`.
+
+**Workaround for other `json_lines` codec situations**
+
+Other `json_lines` codec issues can be mitigated by:
+
+* Downgrading {ls} to `8.16.2`, or any {ls} in the `8.15` series.
+
+[[notable-8-16-1]]
+==== Notable issues fixed
+
+* PipelineBusV2 deadlock proofing: We fixed an issue that could cause a deadlock when the pipeline-to-pipeline feature was in use, causing pipelines (and consequently) {ls} to never terminate https://github.com/elastic/logstash/pull/16680[#16680]
+
+==== Plugins
+
+*Elastic_integration Filter - 0.1.16*
+
+* Reflect the Elasticsearch GeoIP changes into the plugin and sync with Elasticsearch 8.16 branch https://github.com/elastic/logstash-filter-elastic_integration/pull/170[#170]
+
+*Xml Filter - 4.2.1*
+
+* patch rexml to improve performance of multi-threaded xml parsing https://github.com/logstash-plugins/logstash-filter-xml/pull/84[#84]
+
+*Beats Input - 6.9.1*
+
+* Upgrade netty to 4.1.115 https://github.com/logstash-plugins/logstash-input-beats/pull/507[#507]
+
+*Http Input - 3.9.2*
+
+* Upgrade netty to 4.1.115 https://github.com/logstash-plugins/logstash-input-http/pull/183[#183]
+
+*Tcp Input - 6.4.4*
+
+* Upgrade netty to 4.1.115 https://github.com/logstash-plugins/logstash-input-tcp/pull/227[#227]
+
+*Http Output - 5.7.1*
+
+* Added new development `rackup` dependency to fix tests 
+
+
+[[logstash-8-16-0]]
+=== Logstash 8.16.0 Release Notes
+
+[[known-issues-8-16-0]]
+==== Known issues
+
+[[known-issue-8-16-0-shutdown-failure]]
+===== {ls} may fail to shut down under some circumstances
+{ls} may fail to shut down when you are using <<pipeline-to-pipeline>>.
+Check out issue https://github.com/elastic/logstash/issues/16657[#16657] for details.
+
+Workaround: Add `-Dlogstash.pipelinebus.implementation=v1` to `config/jvm.options`.
+This change reverts the `PipelineBus` to `v1`, a version that does not exhibit this issue, but may impact performance in pipeline-to-pipeline scenarios.
+
+[[known-issue-8-16-0-json_lines]]
+===== "Input buffer full" error with {ls} 8.16.0, 8.16.1, or 8.17.0
+
+If you are using `json_lines` codec 3.2.0 (or later) with {ls} 8.16.0, 8.16.1, or 8.17.0, you may see an error similar to this one, crashing the pipelines:
+```
+unable to process event. {:message=>"input buffer full", :class=>"Java::JavaLang::IllegalStateException", :backtrace=>["org.logstash.common.BufferedTokenizerExt.extract(BufferedTokenizerExt.java:83)", "usr.share.logstash.vendor.bundle.jruby.$3_dot_1_dot_0.gems.logstash_minus_codec_minus_json_lines_minus_3_dot_2_dot_2.lib.logstash.codecs.json_lines.RUBY$method$decode$0(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/logstash-codec-json_lines-3.2.2/lib/logstash/codecs/json_lines.rb:69)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:165)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:185)",
+```
+The issue was fixed in https://github.com/elastic/logstash/pull/16760.
+
+This problem is most likely to be seen when you are using the <<plugins-integrations-logstash,{ls} integration>> plugin to ship data between two {ls} instances, but may appear in other situations, too. 
+
+**Workaround for {ls}-to-{ls} communication** 
+
+The {ls}-to-{ls} issue can be mitigated by:
+
+* Downgrading the _receiving_ {ls} to `8.16.2`, or any {ls} in the `8.15` series,  **_AND/OR_**
+* Upgrading the {ls} integration filter  of the _sending_ {ls} to version `1.0.4`.
+
+**Workaround for other `json_lines` codec situations**
+
+Other `json_lines` codec issues can be mitigated by:
+
+* Downgrading {ls} to `8.16.2`, or any {ls} in the `8.15` series.
+
+
+[[health-api-8-16-0]]
+==== Announcing the new {ls} Health Report API
+
+The new Health Report API (`GET /_health_report`) is available starting with {ls} `8.16.0`.
+This API uses indicators capable of detecting the degraded status of pipelines and
+providing actionable insights https://github.com/elastic/logstash/pull/16520[#16520], https://github.com/elastic/logstash/pull/16532[#16532].
+
+**Upgrading from earlier versions.** If your existing automation relies on liveliness scripts that expect the {ls} API status to be unavailable or to return a hardcoded `green` status, you can set a property to preserve pre-8.16.0. 
+To maintain existing behavior for API responses, add the `-Dlogstash.forceApiStatus=green` property to your `config/jvm.options` file.
+
+This setting prevents the new Health API status from affecting the top-level `status` field of existing {ls} API responses, forcing other APIs to return the previous hard-coded `green` value. https://github.com/elastic/logstash/pull/16535[#16535]
+
+Check out the <<logstash-health-report-api>> docs more for info.
+
+[[featured-8-16-0]]
+==== New features and enhancements
+
+* {ls} now gracefully handles `if` conditionals in pipeline definitions that can't be evaluated (https://github.com/elastic/logstash/pull/16322[#16322]), either by dropping
+the event or by sending it to the pipeline's DLQ if enabled. https://github.com/elastic/logstash/pull/16423[#16423]
+
+[[core-8-16-0]]
+==== Other changes to Logstash core
+
+* Added deprecation logs for modules `netflow`, `fb_apache` and `azure`. https://github.com/elastic/logstash/pull/16548[#16548]
+
+* Added deprecation logs for users that doesn't explicitly select a value for `pipeline.buffer.type` forcing them to proactively make a choice before version `9.0` when this setting will default to heap. https://github.com/elastic/logstash/pull/16498[#16498]
+
+* The flag `--event_api.tags.illegal` was deprecated and will be removed in version 9. This flag remains available throughout all version 8.x releases. Users who rely on this flag to allow non strings assignment to `tags` field should update their pipeline. https://github.com/elastic/logstash/pull/16507[#16507]
+
+[[dependencies-8.16.0]]
+==== Updates to dependencies
+
+* Updated JRuby to 9.4.9.0 https://github.com/elastic/logstash/pull/16638[#16638]
+
+
+[[plugins-8-16-0]]
+==== Plugins
+
+*Cef Codec - 6.2.8*
+
+* [DOC] Added missing documentation of the `raw_data_field` option https://github.com/logstash-plugins/logstash-codec-cef/pull/105[#105]
+
+*Json_lines Codec - 3.2.2*
+
+* Raised the default value of the `decode_size_limit_bytes` option to 512 MB https://github.com/logstash-plugins/logstash-codec-json_lines/pull/46[#46]
+
+* Added the `decode_size_limit_bytes` option to limit the maximum size of JSON lines that can be parsed. https://github.com/logstash-plugins/logstash-codec-json_lines/pull/43[#43]
+
+*Elastic_integration Filter - 0.1.15*
+
+* Use Elasticsearch code from its `8.16` branch and adapt to changes in Elasticsearch GeoIP processor https://github.com/elastic/logstash-filter-elastic_integration/pull/170[#170]
+
+*Geoip Filter - 7.3.1*
+
+* Fixed a pipeline crash when looking up a database with customised fields https://github.com/logstash-plugins/logstash-filter-geoip/pull/225[#225]
+
+*Azure_event_hubs Input - 1.5.0*
+
+* Updated Azure Event Hub client library to version `3.3.0` https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/96[#96]
+
+*Beats Input - 6.9.0*
+
+* Improved plugin's shutdown process and fixed a crash when a connection is terminated while processing messages https://github.com/logstash-plugins/logstash-input-beats/pull/500[#500]
+
+*Http Input - 3.9.1*
+
+* Fixed an issue where the value of `ssl_enabled` during `run` wasn't correctly logged https://github.com/logstash-plugins/logstash-input-http/pull/180[#180]
+
+* Separated Netty boss and worker groups to improve the graceful shutdown https://github.com/logstash-plugins/logstash-input-http/pull/178[#178]
+
+*Tcp Input - 6.4.3*
+
+* Updated dependencies for TCP input https://github.com/logstash-plugins/logstash-input-tcp/pull/224[#224]
+
+*Jdbc Integration - 5.5.0*
+
+* Added support for SQL `DATE` columns to jdbc static and streaming filters https://github.com/logstash-plugins/logstash-integration-jdbc/pull/171[#171]
+
+*Rabbitmq Integration - 7.4.0*
+
+* Removed obsolete `verify_ssl` and `debug` options https://github.com/logstash-plugins/logstash-integration-rabbitmq/pull/60[#60]
+
+[[logstash-8-15-5]]
+=== Logstash 8.15.5 Release Notes
+
+[[notable-8-15-5]]
+==== Notable issues fixed
+
+* PipelineBusV2 deadlock proofing: We fixed an issue that could cause a deadlock when the pipeline-to-pipeline feature was in use, causing pipelines (and consequently) {ls} to never terminate https://github.com/elastic/logstash/pull/16681[#16681]
+* We reverted a change in BufferedTokenizer (https://github.com/elastic/logstash/pull/16482[#16482]) that improved handling of large messages but introduced a double encoding bug https://github.com/elastic/logstash/pull/16687[#16687].
+
+==== Plugins
+
+*Elastic_integration Filter - 0.1.16*
+
+* Reflect the Elasticsearch GeoIP changes into the plugin and sync with Elasticsearch 8.16 branch https://github.com/elastic/logstash-filter-elastic_integration/pull/170[#170]
+
+*Xml Filter - 4.2.1*
+
+* patch rexml to improve performance of multi-threaded xml parsing https://github.com/logstash-plugins/logstash-filter-xml/pull/84[#84]
+
+*Tcp Input - 6.4.4*
+
+* update netty to 4.1.115 https://github.com/logstash-plugins/logstash-input-tcp/pull/227[#227]
+
+*Http Output - 5.7.1*
+
+* Added new development `rackup` dependency to fix tests 
+
+[[logstash-8-15-4]]
+=== Logstash 8.15.4 Release Notes
+
+[[known-issues-8-15-4]]
+==== Known issue
+
+**{ls} may fail to shut down under some circumstances when you are using <<pipeline-to-pipeline>>.**
+Check out issue https://github.com/elastic/logstash/issues/16657[#16657] for details.
+
+Workaround: Add `-Dlogstash.pipelinebus.implementation=v1` to `config/jvm.options`.
+This change reverts the `PipelineBus` to `v1`, a version that does not exhibit this issue, but may impact performance in pipeline-to-pipeline scenarios.
+
+[[notable-8-15-4]]
+==== Notable issues fixed
+
+* Fixed an issue where Logstash could not consume lines correctly when a codec with a delimiter is in use and the input buffer becomes full https://github.com/elastic/logstash/pull/16482[#16482]
+
+[[dependencies-8-15-4]]
+==== Updates to dependencies
+
+* Updated JRuby to 9.4.9.0 https://github.com/elastic/logstash/pull/16638[#16638]
+
+[[plugins-8-15-4]]
+==== Plugins
+
+*Cef Codec - 6.2.8*
+
+* [DOC] Added `raw_data_field` to docs https://github.com/logstash-plugins/logstash-codec-cef/pull/105[#105]
+
+*Elastic_integration Filter - 0.1.15*
+
+* Fixed the connection failure where SSL verification mode is disabled over SSL connection https://github.com/elastic/logstash-filter-elastic_integration/pull/165[#165]
+
+*Geoip Filter - 7.3.1*
+
+* Fixed issue causing pipelines to crash during lookup when a database has custom fields https://github.com/logstash-plugins/logstash-filter-geoip/pull/225[#225]
+
+*Tcp Input - 6.4.3*
+
+* Updated dependencies https://github.com/logstash-plugins/logstash-input-tcp/pull/224[#224]
+
+
+[[logstash-8-15-3]]
+=== Logstash 8.15.3 Release Notes
+
+[[known-issues-8-15-3]]
+==== Known issue
+
+**{ls} may fail to shut down under some circumstances when you are using <<pipeline-to-pipeline>>.**
+Check out issue https://github.com/elastic/logstash/issues/16657[#16657] for details.
+
+Workaround: Add `-Dlogstash.pipelinebus.implementation=v1` to `config/jvm.options`.
+This change reverts the `PipelineBus` to `v1`, a version that does not exhibit this issue, but may impact performance in pipeline-to-pipeline scenarios.
+
+
+[[notable-8.15.3]]
+==== Notable issues fixed
+
+* Improved the pipeline bootstrap error logs to include the cause's backtrace, giving a hint where the issue occurred https://github.com/elastic/logstash/pull/16495[#16495]
+
+* Fixed Logstash core compatibility issues with `logstash-input-azure_event_hubs` versions `1.4.8` and earlier https://github.com/elastic/logstash/pull/16485[#16485]
+
+==== Plugins
+
+*Elastic_integration Filter - 0.1.14*
+
+* Enabled the use of org.elasticsearch.ingest.common.Processors in Ingest Pipelines, resolving an issue where some integrations would fail to load https://github.com/elastic/logstash-filter-elastic_integration/pull/162[#162]
+
+*Azure_event_hubs Input - 1.4.9*
+
+* Fixed issue with `getHostContext` method accessibility, causing plugin not to be able to run  https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/93[#93]
+
+* Fixed connection placeholder replacements errors with Logstash `8.15.1` and `8.15.2` https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/92[#92]
+
+*Kafka Integration - 11.5.2*
+
+* Updated avro to 1.11.4 and confluent kafka to 7.4.7 https://github.com/logstash-plugins/logstash-integration-kafka/pull/184[#184]
+
+
+[[logstash-8-15-2]]
+=== Logstash 8.15.2 Release Notes
+
+[[known-issues-8-15-2]]
+==== Known issue
+
+**{ls} may fail to shut down under some circumstances when you are using <<pipeline-to-pipeline>>.**
+Check out issue https://github.com/elastic/logstash/issues/16657[#16657] for details.
+
+Workaround: Add `-Dlogstash.pipelinebus.implementation=v1` to `config/jvm.options`.
+This change reverts the `PipelineBus` to `v1`, a version that does not exhibit this issue, but may impact performance in pipeline-to-pipeline scenarios.
+
+
+[[notable-8.15.2]]
+==== Notable issues fixed
+
+* Fixed a https://github.com/elastic/logstash/issues/16437[regression] from {ls} 8.15.1 in which {ls} removes all quotes from docker env variables, possibly causing {ls} not to start https://github.com/elastic/logstash/pull/16456[#16456]
+
+==== Plugins
+
+*Beats Input - 6.8.4*
+
+* Fix to populate the `@metadata` fields even if the source's metadata value is `nil` https://github.com/logstash-plugins/logstash-input-beats/pull/502[#502]
+
+*Dead_letter_queue Input - 2.0.1*
+
+* Fix NullPointerException when the plugin closes https://github.com/logstash-plugins/logstash-input-dead_letter_queue/pull/53[#53]
+
+*Elastic_serverless_forwarder Input - 0.1.5*
+
+* [DOC] Fix attributes to accurately set and clear default codec values https://github.com/logstash-plugins/logstash-input-elastic_serverless_forwarder/pull/8[#8]
+
+*Logstash Integration - 1.0.3*
+
+* [DOC] Fix attributes to accurately set and clear default codec values https://github.com/logstash-plugins/logstash-integration-logstash/pull/23[#23]
+
+*Elasticsearch Output - 11.22.9*
+
+* Vendor ECS template for Elasticsearch 9.x in built gem https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1188[#1188]
+* Added ECS template for Elasticsearch 9.x https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1187[#1187]
+
+
 [[logstash-8-15-1]]
 === Logstash 8.15.1 Release Notes
 
+[[known-issues-8-15-1]]
+==== Known issues
+
+* **{ls} may fail to start under some circumstances.** Single and double quotes are stripped from a pipeline configuration if the configuration includes environment or keystore variable references.
+If this situation occurs, {ls} may fail to start or some plugins may use a malformed configuration.
+Check out issue https://github.com/elastic/logstash/issues/16437[#16437] for details.
++
+Workaround: Downgrade to {ls} 8.15.0, or temporarily avoid using environment and keystore variable references.
+
+* **{ls} may fail to shut down under some circumstances when you are using <<pipeline-to-pipeline>>.**
+Check out issue https://github.com/elastic/logstash/issues/16657[#16657] for details.
++
+Workaround: Add `-Dlogstash.pipelinebus.implementation=v1` to `config/jvm.options`.
+This change reverts the `PipelineBus` to `v1`, a version that does not exhibit this issue, but may impact performance in pipeline-to-pipeline scenarios.
+
 [[notable-8.15.1]]
 ==== Performance improvements and notable issues fixed
 
@@ -102,6 +912,15 @@ This section summarizes the changes in the following releases:
 [[logstash-8-15-0]]
 === Logstash 8.15.0 Release Notes
 
+[[known-issues-8-15-0]]
+==== Known issue
+
+**{ls} may fail to shut down under some circumstances when you are using <<pipeline-to-pipeline>>.**
+Check out issue https://github.com/elastic/logstash/issues/16657[#16657] for details.
+
+Workaround: Add `-Dlogstash.pipelinebus.implementation=v1` to `config/jvm.options`.
+This change reverts the `PipelineBus` to `v1`, a version that does not exhibit this issue, but may impact performance in pipeline-to-pipeline scenarios.
+
 [[snmp-ga-8.15.0]]
 ==== Announcing the new {ls} SNMP integration plugin
 
@@ -224,6 +1043,14 @@ This new image flavor builds on top of a smaller and more secure base image, and
 [[logstash-8-14-3]]
 === Logstash 8.14.3 Release Notes
 
+[[known-issues-8-14-3]]
+==== Known issue
+
+**{ls} performance regression in JSON encoding**
+{ls} `8.14.1` fixed a bug in the JSON encoding of strings containing non-unicode data https://github.com/elastic/logstash/issues/15833[#15833].
+The fix introduced a performance regression that has since been solved with https://github.com/elastic/logstash/pull/16313[#16313] and included in {ls} `8.15.0`.
+There is no workaround for this issue, please upgrade to {ls} 8.15.0 or later.
+
 [[notable-8.14.3]]
 ==== Enhancements and notable issues fixed
 
@@ -2475,4 +3302,4 @@ We have added another flag to the Benchmark CLI to allow passing a data file wit
 This feature allows users to run the Benchmark CLI in a custom test case with a custom config and a custom dataset. https://github.com/elastic/logstash/pull/12437[#12437]
 
 ==== Plugin releases
-Plugins align with release 7.14.0
+Plugins align with release 7.14.0

docs/static/security/api-keys.asciidoc

@@ -249,8 +249,7 @@ POST /_security/api_key
   "name": "logstash_host001", <1>
   "role_descriptors": {
     "logstash_monitoring": { <2>
-      "cluster": ["monitor"],
-      "index": ["read"]
+      "cluster": ["monitor", "manage_logstash_pipelines"]
     }
   }
 }

docs/static/security/es-security.asciidoc

@@ -78,7 +78,7 @@ As always, there's a definite argument for consistency across deployments.
 [[es-sec-plugin]]
 ==== Configure the elasticsearch output
 
-Use the <<plugins-outputs-elasticsearch,`elasticsearch output`'s>> <<plugins-outputs-elasticsearch-cacert,`cacert` option>> to point to the certificate's location. 
+Use the <<plugins-outputs-elasticsearch,`elasticsearch output`'s>> <<plugins-outputs-elasticsearch-ssl_certificate_authorities,`ssl_certificate_authorities` option>> to point to the certificate's location. 
 
 **Example**
 
@@ -87,7 +87,7 @@ Use the <<plugins-outputs-elasticsearch,`elasticsearch output`'s>> <<plugins-out
 output {
   elasticsearch {
     hosts => ["https://...] <1>
-    cacert => '/etc/logstash/config/certs/ca.crt' <2>
+    ssl_certificate_authorities => ['/etc/logstash/config/certs/ca.crt'] <2>
   }
 }
 -------

docs/static/security/grant-access.asciidoc

@@ -1,6 +1,6 @@
 [discrete]
 [[ls-user-access]]
-=== Granting access to the Logstash indices
+=== Granting access to the indices Logstash creates
 
 To access the indices Logstash creates, users need the `read` and
 `view_index_metadata` privileges:
@@ -13,14 +13,20 @@ privileges  for the Logstash indices. You can create roles from the
 ---------------------------------------------------------------
 POST _security/role/logstash_reader
 {
-  "cluster": ["manage_logstash_pipelines"]
+  "cluster": ["manage_logstash_pipelines"],
+  "indices": [
+    {
+      "names": [ "logstash-*" ],
+      "privileges": ["read","view_index_metadata"]
+    }
+  ]
 }
 ---------------------------------------------------------------
 
 . Assign your Logstash users the `logstash_reader` role. If the Logstash user
 will be using
 {logstash-ref}/logstash-centralized-pipeline-management.html[centralized pipeline management],
-also assign the `logstash_admin` role. You can create and manage users from the
+also assign the `logstash_system` role. You can create and manage users from the
 **Management > Users** UI in {kib} or through the `user` API:
 +
 [source, sh]
@@ -28,9 +34,9 @@ also assign the `logstash_admin` role. You can create and manage users from the
 POST _security/user/logstash_user
 {
   "password" : "x-pack-test-password",
-  "roles" : [ "logstash_reader", "logstash_admin"], <1>
+  "roles" : [ "logstash_reader", "logstash_system"], <1>
   "full_name" : "Kibana User for Logstash"
 }
 ---------------------------------------------------------------
-<1> `logstash_admin` is a built-in role that provides access to system
-indices for managing configurations.
+<1> `logstash_system` is a built-in role that provides the necessary permissions to
+check the availability of the supported features of {es} cluster.

docs/static/security/pki-auth.asciidoc

@@ -11,12 +11,12 @@ client-certificate for authentication, you configure the `keystore` and
 output {
   elasticsearch {
     ...
-    keystore => /path/to/keystore.jks
-    keystore_password => realpassword
-    truststore =>  /path/to/truststore.jks <1>
-    truststore_password =>  realpassword
+    ssl_keystore_path => /path/to/keystore.jks
+    ssl_keystore_password => realpassword
+    ssl_truststore_path =>  /path/to/truststore.jks <1>
+    ssl_truststore_password =>  realpassword
   }
 }
 --------------------------------------------------
 <1> If you use a separate truststore, the truststore path and password are
-also required.
+also required.

docs/static/security/tls-encryption.asciidoc

@@ -3,21 +3,23 @@
 === Configuring Logstash to use TLS/SSL encryption
 
 If TLS encryption is enabled on an on premise {es} cluster, you need to
-configure the `ssl` and `cacert` options in your Logstash `.conf` file:
+configure the `ssl_enabled` and `ssl_certificate_authorities` options in your Logstash `.conf` file:
+
+NOTE: See https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html[elasticsearch output plugin documentation] for a full list of options
 
 [source,js]
 --------------------------------------------------
 output {
   elasticsearch {
     ...
-    ssl => true
-    cacert => '/path/to/cert.pem' <1>
+    ssl_enabled => true
+    ssl_certificate_authorities => '/path/to/cert.pem' <1>
   }
 }
 --------------------------------------------------
 <1> The path to the local `.pem` file that contains the Certificate
     Authority's certificate.
-    
+
 NOTE: Hosted {ess} simplifies security. This configuration step is not necessary for hosted Elasticsearch Service on Elastic Cloud.
 {ess-leadin-short} 
-    
+    

docs/static/settings/configuration-management-settings.asciidoc

@@ -54,9 +54,10 @@ section in your Logstash configuration, or a different one. Defaults to
 If your {es} cluster is protected with basic authentication, these settings
 provide the username and password that the Logstash instance uses to
 authenticate for accessing the configuration data. The username you specify here
-should have the built-in `logstash_admin` role and the customized `logstash_writer` role, which provides access to system
-indices for managing configurations. Starting with Elasticsearch version 7.10.0, the
-`logstash_admin` role inherits the `manage_logstash_pipelines` cluster privilege for centralized pipeline management.
+should have the built-in `logstash_admin` and `logstash_system` roles.
+These roles provide access to system indices for managing configurations.
+
+NOTE: Starting with Elasticsearch version 7.10.0, the `logstash_admin` role inherits the `manage_logstash_pipelines` cluster privilege for centralized pipeline management.
 If a user has created their own roles and granted them access to the .logstash index, those roles will continue to work in 7.x but will need to be updated for 8.0.
 
 `xpack.management.elasticsearch.proxy`::
@@ -143,8 +144,8 @@ If you're using {es} in {ecloud}, you can set your auth credentials here.
 This setting is an alternative to both `xpack.management.elasticsearch.username`
 and `xpack.management.elasticsearch.password`. If `cloud_auth` is configured,
 those settings should not be used.
-The credentials you specify here should be for a user with the `logstash_admin` role, which
-provides access to system indices for managing configurations.
+The credentials you specify here should be for a user with the `logstash_admin` and `logstash_system` roles, which
+provide access to system indices for managing configurations.
 
 `xpack.management.elasticsearch.api_key`::
 

docs/static/troubleshoot/health-pipeline-flow-worker-utilization.asciidoc

@@ -0,0 +1,44 @@
+[[health-report-pipeline-flow-worker-utilization]]
+=== Health Report Pipeline Flow: Worker Utilization
+
+The Pipeline indicator has a `flow:worker_utilization` probe that is capable of producing one of several diagnoses about blockages in the pipeline.
+
+A pipeline is considered "blocked" when its workers are fully-utilized, because if they are consistently spending 100% of their time processing events, they are unable to pick up new events from the queue.
+This can cause back-pressure to cascade to upstream services, which can result in data loss or duplicate processing depending on upstream configuration.
+
+The issue typically stems from one or more causes:
+
+* a downstream resource being blocked,
+* a plugin consuming more resources than expected, and/or
+* insufficient resources being allocated to the pipeline.
+
+To address the issue, observe the <<plugin-flow-rates>> from the <<node-stats-api>>, and identify which plugins have the highest `worker_utilization`.
+This will tell you which plugins are spending the most of the pipeline's worker resources.
+
+* If the offending plugin connects to a downstream service or another pipeline that is exerting back-pressure, the issue needs to be addressed in the downstream service or pipeline.
+* If the offending plugin connects to a downstream service with high network latency, throughput for the pipeline may be improved by <<tuning-logstash-settings, allocating more worker resources to the pipeline>>.
+* If the offending plugin is a computation-heavy filter such as `grok` or `kv`, its configuration may need to be tuned to eliminate wasted computation.
+
+[[health-report-pipeline-flow-worker-utilization-diagnosis-blocked-5m]]
+==== [[blocked-5m]]Blocked Pipeline (5 minutes)
+
+A pipeline that has been completely blocked for five minutes or more represents a critical blockage to the flow of events through your pipeline that needs to be addressed immediately to avoid or limit data loss.
+See above for troubleshooting steps.
+
+[[health-report-pipeline-flow-worker-utilization-diagnosis-nearly-blocked-5m]]
+==== [[nearly-blocked-5m]]Nearly Blocked Pipeline (5 minutes)
+
+A pipeline that has been nearly blocked for five minutes or more may be creating intermittent blockage to the flow of events through your pipeline, which can result in the risk of data loss.
+See above for troubleshooting steps.
+
+[[health-report-pipeline-flow-worker-utilization-diagnosis-blocked-1m]]
+==== [[blocked-1m]]Blocked Pipeline (1 minute)
+
+A pipeline that has been completely blocked for one minute or more represents a high-risk or upcoming blockage to the flow of events through your pipeline that likely needs to be addressed soon to avoid or limit data loss.
+See above for troubleshooting steps.
+
+[[health-report-pipeline-flow-worker-utilization-diagnosis-nearly-blocked-1m]]
+==== [[nearly-blocked-1m]]Nearly Blocked Pipeline (1 minute)
+
+A pipeline that has been nearly blocked for one minute or more may be creating intermittent blockage to the flow of events through your pipeline, which can result in the risk of data loss.
+See above for troubleshooting steps.

docs/static/troubleshoot/health-pipeline-status.asciidoc

@@ -0,0 +1,37 @@
+[[health-report-pipeline-status]]
+=== Health Report Pipeline Status
+
+The Pipeline indicator has a `status` probe that is capable of producing one of several diagnoses about the pipeline's lifecycle, indicating whether the pipeline is currently running.
+
+[[health-report-pipeline-status-diagnosis-loading]]
+==== [[loading]]Loading Pipeline
+
+A pipeline that is loading is not yet processing data, and is considered a temporarily-degraded pipeline state.
+Some plugins perform actions or pre-validation that can delay the starting of the pipeline, such as when a plugin pre-establishes a connection to an external service before allowing the pipeline to start.
+When these plugins take significant time to start up, the whole pipeline can remain in a loading state for an extended time.
+
+If your pipeline does not come up in a reasonable amount of time, consider checking the Logstash logs to see if the plugin shows evidence of being caught in a retry loop.
+
+[[health-report-pipeline-status-diagnosis-finished]]
+==== [[finished]]Finished Pipeline
+
+A logstash pipeline whose input plugins have all completed will be shut down once events have finished processing.
+
+Many plugins can be configured to run indefinitely, either by listening for new inbound events or by polling for events on a schedule.
+A finished pipeline will not produce or process any more events until it is restarted, which will occur if the pipeline's definition is changed and pipeline reloads are enabled.
+If you wish to keep your pipeline runing, consider configuring its input to run on a schedule or otherwise listen for new events.
+
+[[health-report-pipeline-status-diagnosis-terminated]]
+==== [[terminated]]Terminated Pipeline
+
+When a Logstash pipeline's filter or output plugins crash, the entire pipeline is terminated and intervention is required.
+
+A terminated pipeline will not produce or process any more events until it is restarted, which will occur if the pipeline's definition is changed and pipeline reloads are enabled.
+Check the logs to determine the cause of the crash, and report the issue to the plugin maintainers.
+
+[[health-report-pipeline-status-diagnosis-unknown]]
+==== [[unknown]]Unknown Pipeline
+
+When a Logstash pipeline either cannot be created or has recently been deleted the health report doesn't know enough to produce a meaningful status.
+
+Check the logs to determine if the pipeline crashed during creation, and report the issue to the plugin maintainers.

docs/static/troubleshoot/troubleshooting.asciidoc

@@ -28,3 +28,5 @@ include::ts-logstash.asciidoc[]
 include::ts-plugins-general.asciidoc[]
 include::ts-plugins.asciidoc[]
 include::ts-other-issues.asciidoc[]
+include::health-pipeline-status.asciidoc[]
+include::health-pipeline-flow-worker-utilization.asciidoc[]

docs/static/troubleshoot/ts-logstash.asciidoc

@@ -106,6 +106,76 @@ This issue affects some OpenJDK-derived JVM versions (Adoptium, OpenJDK, and Azu
 -Djdk.io.File.enableADS=true
 -----
 
+[[ts-container-cgroup]]
+===== Container exits with 'An unexpected error occurred!' message
+
+{ls} running in a container may not start due to a https://bugs.openjdk.org/browse/JDK-8343191[bug in the JDK]. 
+
+*Sample error*
+
+[source,sh]
+-----
+[FATAL] 2024-11-11 11:11:11.465 [LogStash::Runner] runner - An unexpected error occurred! {:error=>#<Java::JavaLang::NullPointerException: >, :backtrace=>[
+        "java.util.Objects.requireNonNull(java/util/Objects.java:233)",
+        "sun.nio.fs.UnixFileSystem.getPath(sun/nio/fs/UnixFileSystem.java:296)",
+        "java.nio.file.Path.of(java/nio/file/Path.java:148)",
+        "java.nio.file.Paths.get(java/nio/file/Paths.java:69)",
+        "jdk.internal.platform.CgroupUtil.lambda$readStringValue$1(jdk/internal/platform/CgroupUtil.java:67)",
+        "java.security.AccessController.doPrivileged(java/security/AccessController.java:571)",
+        "jdk.internal.platform.CgroupUtil.readStringValue(jdk/internal/platform/CgroupUtil.java:69)",
+        "jdk.internal.platform.CgroupSubsystemController.getStringValue(jdk/internal/platform/CgroupSubsystemController.java:65)",
+        "jdk.internal.platform.cgroupv1.CgroupV1Subsystem.getCpuSetCpus(jdk/internal/platform/cgroupv1/CgroupV1Subsystem.java:275)",
+        "jdk.internal.platform.CgroupMetrics.getCpuSetCpus(jdk/internal/platform/CgroupMetrics.java:100)",
+        "com.sun.management.internal.OperatingSystemImpl.isCpuSetSameAsHostCpuSet(com/sun/management/internal/OperatingSystemImpl.java:277)",
+        "com.sun.management.internal.OperatingSystemImpl$ContainerCpuTicks.getContainerCpuLoad(com/sun/management/internal/OperatingSystemImpl.java:96)",
+        "com.sun.management.internal.OperatingSystemImpl.getProcessCpuLoad(com/sun/management/internal/OperatingSystemImpl.java:271)",
+        "org.logstash.instrument.monitors.ProcessMonitor$Report.<init>(org/logstash/instrument/monitors/ProcessMonitor.java:63)",
+        "org.logstash.instrument.monitors.ProcessMonitor.detect(org/logstash/instrument/monitors/ProcessMonitor.java:136)",
+        "org.logstash.instrument.reports.ProcessReport.generate(org/logstash/instrument/reports/ProcessReport.java:35)",
+        "jdk.internal.reflect.DirectMethodHandleAccessor.invoke(jdk/internal/reflect/DirectMethodHandleAccessor.java:103)",
+        "java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:580)",
+        "org.jruby.javasupport.JavaMethod.invokeDirectWithExceptionHandling(org/jruby/javasupport/JavaMethod.java:300)",
+        "org.jruby.javasupport.JavaMethod.invokeStaticDirect(org/jruby/javasupport/JavaMethod.java:222)",
+        "RUBY.collect_process_metrics(/usr/share/logstash/logstash-core/lib/logstash/instrument/periodic_poller/jvm.rb:102)",
+        "RUBY.collect(/usr/share/logstash/logstash-core/lib/logstash/instrument/periodic_poller/jvm.rb:73)",
+        "RUBY.start(/usr/share/logstash/logstash-core/lib/logstash/instrument/periodic_poller/base.rb:72)",
+        "org.jruby.RubySymbol$SymbolProcBody.yieldSpecific(org/jruby/RubySymbol.java:1541)",
+        "org.jruby.RubySymbol$SymbolProcBody.doYield(org/jruby/RubySymbol.java:1534)",
+        "org.jruby.RubyArray.collectArray(org/jruby/RubyArray.java:2770)",
+        "org.jruby.RubyArray.map(org/jruby/RubyArray.java:2803)",
+        "org.jruby.RubyArray$INVOKER$i$0$0$map.call(org/jruby/RubyArray$INVOKER$i$0$0$map.gen)",
+        "RUBY.start(/usr/share/logstash/logstash-core/lib/logstash/instrument/periodic_pollers.rb:41)",
+        "RUBY.configure_metrics_collectors(/usr/share/logstash/logstash-core/lib/logstash/agent.rb:477)",
+        "RUBY.initialize(/usr/share/logstash/logstash-core/lib/logstash/agent.rb:88)",
+        "org.jruby.RubyClass.new(org/jruby/RubyClass.java:949)",
+        "org.jruby.RubyClass$INVOKER$i$newInstance.call(org/jruby/RubyClass$INVOKER$i$newInstance.gen)",
+        "RUBY.create_agent(/usr/share/logstash/logstash-core/lib/logstash/runner.rb:552)",
+        "RUBY.execute(/usr/share/logstash/logstash-core/lib/logstash/runner.rb:434)",
+        "RUBY.run(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/clamp-1.0.1/lib/clamp/command.rb:68)",
+        "RUBY.run(/usr/share/logstash/logstash-core/lib/logstash/runner.rb:293)",
+        "RUBY.run(/usr/share/logstash/vendor/bundle/jruby/3.1.0/gems/clamp-1.0.1/lib/clamp/command.rb:133)",
+        "usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:89)",
+        "usr.share.logstash.lib.bootstrap.environment.run(usr/share/logstash/lib/bootstrap//usr/share/logstash/lib/bootstrap/environment.rb)",
+        "java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:733)",
+        "org.jruby.Ruby.runScript(org/jruby/Ruby.java:1245)",
+        "org.jruby.Ruby.runNormally(org/jruby/Ruby.java:1157)",
+        "org.jruby.Ruby.runFromMain(org/jruby/Ruby.java:983)",
+        "org.logstash.Logstash.run(org/logstash/Logstash.java:163)",
+        "org.logstash.Logstash.main(org/logstash/Logstash.java:73)"
+    ]
+}
+[FATAL] 2024-11-11 11:11:11.516 [LogStash::Runner] Logstash - Logstash stopped processing because of an error: (SystemExit) exit
+    org.jruby.exceptions.SystemExit: (SystemExit) exit
+    at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java: 921) ~[jruby.jar:?]
+    at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java: 880) ~[jruby.jar:?]
+    at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb: 90) ~[?:?]
+-----
+
+This error can happen when cgroups v2 is not enabled, such as when running on a Red Had version 8 operating system.
+
+*Work around*
+
+Follow your operating system's instructions for enabling cgroups v2.
 
 [[ts-pqs]]
 ==== Troubleshooting persistent queues
@@ -265,4 +335,4 @@ adding to the field name a `_1` suffix:
       "data":"{\"name\": [}"
    }
 }
------
+-----

lib/bootstrap/bundler.rb

@@ -15,6 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# work around https://github.com/jruby/jruby/issues/8579
+require_relative './patches/jar_dependencies'
+
 module LogStash
   module Bundler
     extend self
@@ -264,6 +267,7 @@ module LogStash
       elsif options[:update]
         arguments << "update"
         arguments << expand_logstash_mixin_dependencies(options[:update])
+        arguments << "--#{options[:level] || 'minor'}"
         arguments << "--local" if options[:local]
         arguments << "--conservative" if options[:conservative]
       elsif options[:clean]

lib/bootstrap/patches/jar_dependencies.rb

@@ -21,7 +21,7 @@ def require_jar(*args)
   return nil unless Jars.require?
   result = Jars.require_jar(*args)
   if result.is_a? String
-    # JAR_DEBUG=1 will now show theses
+    # JARS_VERBOSE=true will show these
     Jars.debug { "--- jar coordinate #{args[0..-2].join(':')} already loaded with version #{result} - omit version #{args[-1]}" }
     Jars.debug { "    try to load from #{caller.join("\n\t")}" }
     return false
@@ -29,3 +29,29 @@ def require_jar(*args)
   Jars.debug { "    register #{args.inspect} - #{result == true}" }
   result
 end
+
+# work around https://github.com/jruby/jruby/issues/8579
+# the ruby maven 3.9.3 + maven-libs 3.9.9 gems will output unnecessary text we need to trim down during `load_from_maven`
+# remove everything from "--" until the end of the line
+# the `[...-5]` is just to remove the color changing characters from the end of the string that exist before "--"
+require 'jars/installer'
+
+class ::Jars::Installer
+  def self.load_from_maven(file)
+    Jars.debug { "[load_from_maven] called with arguments: #{file.inspect}" }
+    result = []
+    ::File.read(file).each_line do |line|
+      if line.match?(/ --/)
+        Jars.debug { "[load_from_maven] line: #{line.inspect}" }
+        fixed_line = line.strip.gsub(/ --.+?$/, "")[0...-5]
+        Jars.debug { "[load_from_maven] fixed_line: #{fixed_line.inspect}" }
+        dep = ::Jars::Installer::Dependency.new(fixed_line)
+      else
+        dep = ::Jars::Installer::Dependency.new(line)
+      end
+      result << dep if dep && dep.scope == :runtime
+    end
+    Jars.debug { "[load_from_maven] returned: #{result.inspect}" }
+    result
+  end
+end

lib/pluginmanager/pack_installer/pack.rb

@@ -39,7 +39,7 @@ module LogStash module PluginManager module PackInstaller
     class GemInformation
       EXTENSION = ".gem"
       SPLIT_CHAR = "-"
-      JAVA_PLATFORM_RE = /-java/
+      JAVA_PLATFORM_RE = /-java$/
       DEPENDENCIES_DIR_RE = /dependencies/
 
       attr_reader :file, :name, :version, :platform

lib/pluginmanager/proxy_support.rb

@@ -55,8 +55,8 @@ def apply_env_proxy_settings(settings)
   scheme = settings[:protocol].downcase
   java.lang.System.setProperty("#{scheme}.proxyHost", settings[:host])
   java.lang.System.setProperty("#{scheme}.proxyPort", settings[:port].to_s)
-  java.lang.System.setProperty("#{scheme}.proxyUsername", settings[:username].to_s)
-  java.lang.System.setProperty("#{scheme}.proxyPassword", settings[:password].to_s)
+  java.lang.System.setProperty("#{scheme}.proxyUser", settings[:username].to_s)
+  java.lang.System.setProperty("#{scheme}.proxyPass", settings[:password].to_s)
 end
 
 def extract_proxy_values_from_uri(proxy_uri)

lib/pluginmanager/update.rb

@@ -24,7 +24,13 @@ class LogStash::PluginManager::Update < LogStash::PluginManager::Command
   # These are local gems used by LS and needs to be filtered out of other plugin gems
   NON_PLUGIN_LOCAL_GEMS = ["logstash-core", "logstash-core-plugin-api"]
 
+  SUPPORTED_LEVELS = %w(major minor patch)
+
   parameter "[PLUGIN] ...", "Plugin name(s) to upgrade to latest version", :attribute_name => :plugins_arg
+  option "--level", "LEVEL", "restrict updates to given semantic version level (one of #{SUPPORTED_LEVELS})", :default => "minor" do |given_level|
+    fail("unsupported level `#{given_level}`; expected one of #{SUPPORTED_LEVELS}") unless SUPPORTED_LEVELS.include?(given_level)
+    given_level
+  end
   option "--[no-]verify", :flag, "verify plugin validity before installation", :default => true
   option "--local", :flag, "force local-only plugin update. see bin/logstash-plugin package|unpack", :default => false
   option "--[no-]conservative", :flag, "do a conservative update of plugin's dependencies", :default => true
@@ -82,6 +88,7 @@ class LogStash::PluginManager::Update < LogStash::PluginManager::Command
     # Bundler cannot update and clean gems in one operation so we have to call the CLI twice.
     Bundler.settings.temporary(:frozen => false) do # Unfreeze the bundle when updating gems
       output = LogStash::Bundler.invoke! update: plugins,
+                                         level: level,
                                          rubygems_source: gemfile.gemset.sources,
                                          local: local?,
                                          conservative: conservative?

logstash-core-plugin-api/logstash-core-plugin-api.gemspec

@@ -12,7 +12,13 @@ if File.exist?(project_versions_yaml_path)
   # we ignore the copy in git and we overwrite an existing file
   # each time we build the logstash-core gem
   original_lines = IO.readlines(project_versions_yaml_path)
-  original_lines << ""
+  # introduce the version qualifier (e.g. beta1, rc1) into the copied yml so it's displayed by Logstash
+  unless ENV['VERSION_QUALIFIER'].to_s.strip.empty?
+    logstash_version_line = original_lines.find {|line| line.match(/^logstash:/) }
+    logstash_version_line.chomp!
+    logstash_version_line << "-#{ENV['VERSION_QUALIFIER']}\n"
+  end
+  original_lines << "\n"
   original_lines << "# This is a copy the project level versions.yml into this gem's root and it is created when the gemspec is evaluated."
   gem_versions_yaml_path = File.expand_path("./versions-gem-copy.yml", File.dirname(__FILE__))
   File.open(gem_versions_yaml_path, 'w') do |new_file|

logstash-core/benchmarks/build.gradle

@@ -50,7 +50,7 @@ jar {
 }
 
 ext {
-  jmh = 1.22
+  jmh = 1.37
 }
 
 dependencies {
@@ -79,17 +79,15 @@ tasks.register("jmh", JavaExec) {
 
  dependsOn=[':logstash-core-benchmarks:clean', ':logstash-core-benchmarks:shadowJar']
 
-  main = "-jar"
+  mainClass = "-jar"
 
   def include = project.properties.get('include', '')
 
   doFirst {
     args = [
             "-Djava.io.tmpdir=${buildDir.absolutePath}",
-            "-XX:+UseConcMarkSweepGC", "-XX:CMSInitiatingOccupancyFraction=75",
-            "-XX:+UseCMSInitiatingOccupancyOnly", "-XX:+DisableExplicitGC",
             "-XX:+HeapDumpOnOutOfMemoryError", "-Xms2g", "-Xmx2g",
-            shadowJar.archivePath,
+            shadowJar.archiveFile.get().asFile,
             include
     ]
   }

logstash-core/benchmarks/src/main/java/org/logstash/benchmark/BufferedTokenizerExtBenchmark.java

@@ -0,0 +1,83 @@
+package org.logstash.benchmark;
+
+import org.jruby.RubyArray;
+import org.jruby.RubyString;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.builtin.IRubyObject;
+import org.logstash.RubyUtil;
+import org.logstash.common.BufferedTokenizerExt;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.logstash.RubyUtil.RUBY;
+
+@Warmup(iterations = 3, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 10, time = 100, timeUnit = TimeUnit.MILLISECONDS)
+@Fork(1)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Thread)
+public class BufferedTokenizerExtBenchmark {
+
+    private BufferedTokenizerExt sut;
+    private ThreadContext context;
+    private RubyString singleTokenPerFragment;
+    private RubyString multipleTokensPerFragment;
+    private RubyString multipleTokensSpreadMultipleFragments_1;
+    private RubyString multipleTokensSpreadMultipleFragments_2;
+    private RubyString multipleTokensSpreadMultipleFragments_3;
+
+    @Setup(Level.Invocation)
+    public void setUp() {
+        sut = new BufferedTokenizerExt(RubyUtil.RUBY, RubyUtil.BUFFERED_TOKENIZER);
+        context = RUBY.getCurrentContext();
+        IRubyObject[] args = {};
+        sut.init(context, args);
+        singleTokenPerFragment = RubyUtil.RUBY.newString("a".repeat(512) + "\n");
+
+        multipleTokensPerFragment = RubyUtil.RUBY.newString("a".repeat(512) + "\n" + "b".repeat(512) + "\n" + "c".repeat(512) + "\n");
+
+        multipleTokensSpreadMultipleFragments_1 = RubyUtil.RUBY.newString("a".repeat(512) + "\n" + "b".repeat(512) + "\n" + "c".repeat(256));
+        multipleTokensSpreadMultipleFragments_2 = RubyUtil.RUBY.newString("c".repeat(256) + "\n" + "d".repeat(512) + "\n" + "e".repeat(256));
+        multipleTokensSpreadMultipleFragments_3 = RubyUtil.RUBY.newString("f".repeat(256) + "\n" + "g".repeat(512) + "\n" + "h".repeat(512) + "\n");
+    }
+
+    @SuppressWarnings("unchecked")
+    @Benchmark
+    public final void onlyOneTokenPerFragment(Blackhole blackhole) {
+        RubyArray<RubyString> tokens = (RubyArray<RubyString>) sut.extract(context, singleTokenPerFragment);
+        blackhole.consume(tokens);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Benchmark
+    public final void multipleTokenPerFragment(Blackhole blackhole) {
+        RubyArray<RubyString> tokens = (RubyArray<RubyString>) sut.extract(context, multipleTokensPerFragment);
+        blackhole.consume(tokens);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Benchmark
+    public final void multipleTokensCrossingMultipleFragments(Blackhole blackhole) {
+        RubyArray<RubyString> tokens = (RubyArray<RubyString>) sut.extract(context, multipleTokensSpreadMultipleFragments_1);
+        blackhole.consume(tokens);
+
+        tokens = (RubyArray<RubyString>) sut.extract(context, multipleTokensSpreadMultipleFragments_2);
+        blackhole.consume(tokens);
+
+        tokens = (RubyArray<RubyString>) sut.extract(context, multipleTokensSpreadMultipleFragments_3);
+        blackhole.consume(tokens);
+    }
+}

logstash-core/build.gradle

@@ -57,6 +57,7 @@ def versionMap = (Map) (new Yaml()).load(new File("$projectDir/../versions.yml")
 
 description = """Logstash Core Java"""
 
+String logstashCoreVersion = versionMap['logstash-core']
 String jacksonVersion = versionMap['jackson']
 String jacksonDatabindVersion = versionMap['jackson-databind']
 String jrubyVersion = versionMap['jruby']['version']
@@ -123,6 +124,9 @@ tasks.register("javaTests", Test) {
     exclude '/org/logstash/plugins/factory/PluginFactoryExtTest.class'
     exclude '/org/logstash/execution/ObservedExecutionTest.class'
 
+    // 10GB is needed by the BufferedTokenizerExtWithSizeLimitTest.givenTooLongInputExtractDoesntOverflow test
+    maxHeapSize = "10g"
+
     jacoco {
         enabled = true
         destinationFile = layout.buildDirectory.file('jacoco/test.exec').get().asFile
@@ -183,6 +187,23 @@ artifacts {
     }
 }
 
+task generateVersionInfoResources(type: DefaultTask) {
+    ext.outDir = layout.buildDirectory.dir("generated-resources/version-info").get()
+
+    inputs.property("version-info:logstash-core", logstashCoreVersion)
+    outputs.dir(ext.outDir)
+
+    doLast {
+        mkdir outDir;
+        def resourceFile = outDir.file('version-info.properties').asFile
+        resourceFile.text = "logstash-core: ${logstashCoreVersion}"
+    }
+}
+sourceSets {
+    main { output.dir(generateVersionInfoResources.outputs.files) }
+}
+processResources.dependsOn generateVersionInfoResources
+
 configurations {
     provided
 }

logstash-core/lib/logstash/agent.rb

@@ -26,6 +26,7 @@ require "logstash/pipeline_action"
 require "logstash/state_resolver"
 require "logstash/pipelines_registry"
 require "logstash/persisted_queue_config_validator"
+require "logstash/pipeline_resource_usage_validator"
 require "stud/trap"
 require "uri"
 require "socket"
@@ -40,6 +41,8 @@ class LogStash::Agent
   attr_reader :metric, :name, :settings, :dispatcher, :ephemeral_id, :pipeline_bus
   attr_accessor :logger
 
+  attr_reader :health_observer
+
   # initialize method for LogStash::Agent
   # @param params [Hash] potential parameters are:
   #   :name [String] - identifier for the agent
@@ -51,6 +54,9 @@ class LogStash::Agent
     @auto_reload = setting("config.reload.automatic")
     @ephemeral_id = SecureRandom.uuid
 
+    java_import("org.logstash.health.HealthObserver")
+    @health_observer ||= HealthObserver.new
+
     # Mutex to synchronize in the exclusive method
     # Initial usage for the Ruby pipeline initialization which is not thread safe
     @webserver_control_lock = Mutex.new
@@ -97,6 +103,7 @@ class LogStash::Agent
     initialize_geoip_database_metrics(metric)
 
     @pq_config_validator = LogStash::PersistedQueueConfigValidator.new
+    @pipeline_resource_usage_validator = LogStash::PipelineResourceUsageValidator.new(Java::java.lang.Runtime.getRuntime().maxMemory)
 
     @dispatcher = LogStash::EventDispatcher.new(self)
     LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, dispatcher)
@@ -151,6 +158,31 @@ class LogStash::Agent
     transition_to_stopped
   end
 
+  include org.logstash.health.PipelineIndicator::PipelineDetailsProvider
+  def pipeline_details(pipeline_id)
+    logger.trace("fetching pipeline details for `#{pipeline_id}`")
+    pipeline_id = pipeline_id.to_sym
+
+    java_import org.logstash.health.PipelineIndicator
+
+    pipeline_state = @pipelines_registry.states.get(pipeline_id)
+    if pipeline_state.nil?
+      return PipelineIndicator::Details.new(PipelineIndicator::Status::UNKNOWN)
+    end
+
+    pipeline_state.synchronize do |sync_state|
+      status = case
+               when sync_state.loading?    then PipelineIndicator::Status::LOADING
+               when sync_state.crashed?    then PipelineIndicator::Status::TERMINATED
+               when sync_state.running?    then PipelineIndicator::Status::RUNNING
+               when sync_state.finished?   then PipelineIndicator::Status::FINISHED
+               else                             PipelineIndicator::Status::UNKNOWN
+               end
+
+      PipelineIndicator::Details.new(status, sync_state.pipeline&.to_java.collectWorkerUtilizationFlowObservation)
+    end
+  end
+
   def auto_reload?
     @auto_reload
   end
@@ -191,13 +223,15 @@ class LogStash::Agent
 
     converge_result = resolve_actions_and_converge_state(results.response)
     update_metrics(converge_result)
-
-    logger.info(
-        "Pipelines running",
-        :count => running_pipelines.size,
-        :running_pipelines => running_pipelines.keys,
-        :non_running_pipelines => non_running_pipelines.keys
-    ) if converge_result.success? && converge_result.total > 0
+    if converge_result.success? && converge_result.total > 0
+      logger.info(
+          "Pipelines running",
+          :count => running_pipelines.size,
+          :running_pipelines => running_pipelines.keys,
+          :non_running_pipelines => non_running_pipelines.keys
+      )
+      @pipeline_resource_usage_validator.check(@pipelines_registry)
+    end
 
     dispatch_events(converge_result)
 
@@ -395,7 +429,13 @@ class LogStash::Agent
             )
           end
         rescue SystemExit, Exception => e
-          logger.error("Failed to execute action", :action => action, :exception => e.class.name, :message => e.message, :backtrace => e.backtrace)
+          error_details = { :action => action, :exception => e.class.name, :message => e.message, :backtrace => e.backtrace }
+          cause = e.cause
+          if cause && e != cause
+            error_details[:cause] = { :exception => cause.class, :message => cause.message }
+            error_details[:cause][:backtrace] = cause.backtrace if cause.backtrace
+          end
+          logger.error('Failed to execute action', error_details)
           converge_result.add(action, LogStash::ConvergeResult::FailedAction.from_exception(e))
         end
       end

logstash-core/lib/logstash/api/command_factory.rb

@@ -18,6 +18,7 @@
 require "logstash/api/service"
 require "logstash/api/commands/system/basicinfo_command"
 require "logstash/api/commands/system/plugins_command"
+require "logstash/api/commands/health_report"
 require "logstash/api/commands/stats"
 require "logstash/api/commands/node"
 require "logstash/api/commands/default_metadata"
@@ -34,6 +35,7 @@ module LogStash
           :plugins_command => ::LogStash::Api::Commands::System::Plugins,
           :stats => ::LogStash::Api::Commands::Stats,
           :node => ::LogStash::Api::Commands::Node,
+          :health_report => ::LogStash::Api::Commands::HealthReport,
           :default_metadata => ::LogStash::Api::Commands::DefaultMetadata
         }
       end

logstash-core/lib/logstash/api/commands/default_metadata.rb

@@ -22,20 +22,14 @@ module LogStash
     module Commands
       class DefaultMetadata < Commands::Base
         def all
-          res = {:host => host,
-             :version => version,
-             :http_address => http_address,
-             :id => service.agent.id,
-             :name => service.agent.name,
-             :ephemeral_id => service.agent.ephemeral_id,
-             :status => "green",  # This is hard-coded to mirror x-pack behavior
-             :snapshot => ::BUILD_INFO["build_snapshot"],
+          res = base_info.merge({
+             :status => service.agent.health_observer.status,
              :pipeline => {
                :workers => LogStash::SETTINGS.get("pipeline.workers"),
                :batch_size => LogStash::SETTINGS.get("pipeline.batch.size"),
                :batch_delay => LogStash::SETTINGS.get("pipeline.batch.delay"),
              },
-            }
+            })
           monitoring = {}
           if enabled_xpack_monitoring?
             monitoring = monitoring.merge({
@@ -49,12 +43,24 @@ module LogStash
           res.merge(monitoring.empty? ? {} : {:monitoring => monitoring})
         end
 
+        def base_info
+          {
+            :host => host,
+            :version => version,
+            :http_address => http_address,
+            :id => service.agent.id,
+            :name => service.agent.name,
+            :ephemeral_id => service.agent.ephemeral_id,
+            :snapshot => ::BUILD_INFO["build_snapshot"],
+          }
+        end
+
         def host
           @@host ||= Socket.gethostname
         end
 
         def version
-          LOGSTASH_CORE_VERSION
+          LOGSTASH_VERSION
         end
 
         def http_address

logstash-core/lib/logstash/api/commands/health_report.rb

@@ -0,0 +1,31 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "logstash/api/commands/base"
+
+module LogStash
+  module Api
+    module Commands
+      class HealthReport < Commands::Base
+
+        def all(selected_fields = [])
+          service.agent.health_observer.report
+        end
+      end
+    end
+  end
+end

logstash-core/lib/logstash/api/commands/stats.rb

@@ -183,7 +183,12 @@ module LogStash
                 :outputs => plugin_stats(stats, :outputs)
               },
               :reloads => stats[:reloads],
-              :queue => stats[:queue]
+              :queue => stats[:queue],
+              :pipeline => {
+                :workers => stats.dig(:config, :workers),
+                :batch_size => stats.dig(:config, :batch_size),
+                :batch_delay => stats.dig(:config, :batch_delay),
+              }
             }
             ret[:dead_letter_queue] = stats[:dlq] if stats.include?(:dlq)
 

logstash-core/lib/logstash/api/modules/health_report.rb

@@ -0,0 +1,49 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module LogStash
+  module Api
+    module Modules
+      class HealthReport < ::LogStash::Api::Modules::Base
+
+        get "/" do
+          payload = health_report.all.then do |health_report_pojo|
+            # The app_helper needs a ruby-hash.
+            # Manually creating a map of properties works around the issue.
+            base_metadata.merge({
+              status:     health_report_pojo.status,
+              symptom:    health_report_pojo.symptom,
+              indicators: health_report_pojo.indicators,
+            })
+          end
+
+          respond_with(payload, {exclude_default_metadata: true})
+        end
+
+        private
+
+        def health_report
+          @health_report ||= factory.build(:health_report)
+        end
+
+        def base_metadata
+          @factory.build(:default_metadata).base_info
+        end
+      end
+    end
+  end
+end

logstash-core/lib/logstash/api/rack_app.rb

@@ -18,6 +18,7 @@
 require "rack"
 require "sinatra/base"
 require "logstash/api/modules/base"
+require "logstash/api/modules/health_report"
 require "logstash/api/modules/node"
 require "logstash/api/modules/node_stats"
 require "logstash/api/modules/plugins"
@@ -123,6 +124,7 @@ module LogStash
 
       def self.rack_namespaces(agent)
         {
+          "/_health_report" => LogStash::Api::Modules::HealthReport,
           "/_node" => LogStash::Api::Modules::Node,
           "/_stats" => LogStash::Api::Modules::Stats,
           "/_node/stats" => LogStash::Api::Modules::NodeStats,

logstash-core/lib/logstash/config/modules_common.rb

@@ -67,6 +67,9 @@ module LogStash module Config
         raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-unavailable", **i18n_opts)
       end
 
+      specified_and_available_names
+        .each { |mn| deprecation_logger.deprecated("The #{mn} module has been deprecated and will be removed in version 9.") }
+
       specified_and_available_names.each do |module_name|
         connect_fail_args = {}
         begin

logstash-core/lib/logstash/elasticsearch_client.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 require "elasticsearch"
-require "elasticsearch/transport/transport/http/manticore"
+require "elastic/transport/transport/http/manticore"
 require 'logstash/util/manticore_ssl_config_helper'
 require 'logstash/util/password'
 
@@ -24,7 +24,7 @@ module LogStash class ElasticsearchClient
   include LogStash::Util::Loggable
 
   class Response
-    # duplicated here from Elasticsearch::Transport::Transport::Response
+    # duplicated here from Elastic::Transport::Transport::Response
     # to create a normalised response across different client IMPL
     attr_reader :status, :body, :headers
 
@@ -65,8 +65,13 @@ module LogStash class ElasticsearchClient
     def can_connect?
       begin
         head(SecureRandom.hex(32).prepend('_'))
-      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+      rescue Elastic::Transport::Transport::Errors::BadRequest
         true
+      rescue Elastic::Transport::Transport::Errors::Unauthorized
+        true
+      rescue Exception => e
+        return true if e.message.include?('Connection refused')
+        raise e
       rescue Manticore::SocketException
         false
       end
@@ -116,7 +121,7 @@ module LogStash class ElasticsearchClient
 
     def client_args
       {
-        :transport_class => Elasticsearch::Transport::Transport::HTTP::Manticore,
+        :transport_class => Elastic::Transport::Transport::HTTP::Manticore,
         :hosts => [*unpack_hosts],
         # :logger => @logger, # silence the client logging
       }

logstash-core/lib/logstash/environment.rb

@@ -35,10 +35,10 @@ module LogStash
 
   [
            Setting::Boolean.new("allow_superuser", true),
-            Setting::String.new("node.name", Socket.gethostname),
-    Setting::NullableString.new("path.config", nil, false),
+            Setting::SettingString.new("node.name", Socket.gethostname),
+    Setting::SettingNullableString.new("path.config", nil, false),
  Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
-    Setting::NullableString.new("config.string", nil, false),
+    Setting::SettingNullableString.new("config.string", nil, false),
            Setting::Modules.new("modules.cli", LogStash::Util::ModulesSettingArray, []),
            Setting::Modules.new("modules", LogStash::Util::ModulesSettingArray, []),
                     Setting.new("modules_list", Array, []),
@@ -50,10 +50,10 @@ module LogStash
            Setting::Boolean.new("config.reload.automatic", false),
          Setting::TimeValue.new("config.reload.interval", "3s"), # in seconds
            Setting::Boolean.new("config.support_escapes", false),
-            Setting::String.new("config.field_reference.escape_style", "none", true, %w(none percent ampersand)),
-            Setting::String.new("event_api.tags.illegal", "rename", true, %w(rename warn)),
+            Setting::SettingString.new("config.field_reference.escape_style", "none", true, %w(none percent ampersand)),
+            Setting::SettingString.new("event_api.tags.illegal", "rename", true, %w(rename warn)),
            Setting::Boolean.new("metric.collect", true),
-            Setting::String.new("pipeline.id", "main"),
+            Setting::SettingString.new("pipeline.id", "main"),
            Setting::Boolean.new("pipeline.system", false),
    Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
    Setting::PositiveInteger.new("pipeline.batch.size", 125),
@@ -65,32 +65,32 @@ module LogStash
    Setting::CoercibleString.new("pipeline.ordered", "auto", true, ["auto", "true", "false"]),
    Setting::CoercibleString.new("pipeline.ecs_compatibility", "v8", true, %w(disabled v1 v8)),
                     Setting.new("path.plugins", Array, []),
-    Setting::NullableString.new("interactive", nil, false),
+    Setting::SettingNullableString.new("interactive", nil, false),
            Setting::Boolean.new("config.debug", false),
-            Setting::String.new("log.level", "info", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
+            Setting::SettingString.new("log.level", "info", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
            Setting::Boolean.new("version", false),
            Setting::Boolean.new("help", false),
            Setting::Boolean.new("enable-local-plugin-development", false),
-            Setting::String.new("log.format", "plain", true, ["json", "plain"]),
+            Setting::SettingString.new("log.format", "plain", true, ["json", "plain"]),
            Setting::Boolean.new("log.format.json.fix_duplicate_message_fields", false),
-           Setting::Boolean.new("api.enabled", true).with_deprecated_alias("http.enabled"),
-            Setting::String.new("api.http.host", "127.0.0.1").with_deprecated_alias("http.host"),
-         Setting::PortRange.new("api.http.port", 9600..9700).with_deprecated_alias("http.port"),
-            Setting::String.new("api.environment", "production").with_deprecated_alias("http.environment"),
-            Setting::String.new("api.auth.type", "none", true, %w(none basic)),
-            Setting::String.new("api.auth.basic.username", nil, false).nullable,
+           Setting::Boolean.new("api.enabled", true).with_deprecated_alias("http.enabled", "9"),
+            Setting::SettingString.new("api.http.host", "127.0.0.1").with_deprecated_alias("http.host", "9"),
+         Setting::PortRange.new("api.http.port", 9600..9700).with_deprecated_alias("http.port", "9"),
+            Setting::SettingString.new("api.environment", "production").with_deprecated_alias("http.environment", "9"),
+            Setting::SettingString.new("api.auth.type", "none", true, %w(none basic)),
+            Setting::SettingString.new("api.auth.basic.username", nil, false).nullable,
           Setting::Password.new("api.auth.basic.password", nil, false).nullable,
-            Setting::String.new("api.auth.basic.password_policy.mode", "WARN", true, %w[WARN ERROR]),
+            Setting::SettingString.new("api.auth.basic.password_policy.mode", "WARN", true, %w[WARN ERROR]),
            Setting::Numeric.new("api.auth.basic.password_policy.length.minimum", 8),
-            Setting::String.new("api.auth.basic.password_policy.include.upper", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
-            Setting::String.new("api.auth.basic.password_policy.include.lower", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
-            Setting::String.new("api.auth.basic.password_policy.include.digit", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
-            Setting::String.new("api.auth.basic.password_policy.include.symbol", "OPTIONAL", true, %w[REQUIRED OPTIONAL]),
+            Setting::SettingString.new("api.auth.basic.password_policy.include.upper", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
+            Setting::SettingString.new("api.auth.basic.password_policy.include.lower", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
+            Setting::SettingString.new("api.auth.basic.password_policy.include.digit", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
+            Setting::SettingString.new("api.auth.basic.password_policy.include.symbol", "OPTIONAL", true, %w[REQUIRED OPTIONAL]),
            Setting::Boolean.new("api.ssl.enabled", false),
   Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable,
           Setting::Password.new("api.ssl.keystore.password", nil, false).nullable,
        Setting::StringArray.new("api.ssl.supported_protocols", nil, true, %w[TLSv1 TLSv1.1 TLSv1.2 TLSv1.3]),
-            Setting::String.new("queue.type", "memory", true, ["persisted", "memory"]),
+            Setting::SettingString.new("queue.type", "memory", true, ["persisted", "memory"]),
            Setting::Boolean.new("queue.drain", false),
              Setting::Bytes.new("queue.page_capacity", "64mb"),
              Setting::Bytes.new("queue.max_bytes", "1024mb"),
@@ -102,16 +102,16 @@ module LogStash
            Setting::Boolean.new("dead_letter_queue.enable", false),
              Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"),
            Setting::Numeric.new("dead_letter_queue.flush_interval", 5000),
-            Setting::String.new("dead_letter_queue.storage_policy", "drop_newer", true, ["drop_newer", "drop_older"]),
-    Setting::NullableString.new("dead_letter_queue.retain.age"), # example 5d
+            Setting::SettingString.new("dead_letter_queue.storage_policy", "drop_newer", true, ["drop_newer", "drop_older"]),
+    Setting::SettingNullableString.new("dead_letter_queue.retain.age"), # example 5d
          Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
          Setting::TimeValue.new("slowlog.threshold.info", "-1"),
          Setting::TimeValue.new("slowlog.threshold.debug", "-1"),
          Setting::TimeValue.new("slowlog.threshold.trace", "-1"),
-            Setting::String.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"),
-            Setting::String.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on
-    Setting::NullableString.new("monitoring.cluster_uuid"),
-            Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"])
+            Setting::SettingString.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"),
+            Setting::SettingString.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on
+    Setting::SettingNullableString.new("monitoring.cluster_uuid"),
+            Setting::SettingString.new("pipeline.buffer.type", nil, false, ["direct", "heap"])
   # post_process
   ].each {|setting| SETTINGS.register(setting) }
 

logstash-core/lib/logstash/java_pipeline.rb

@@ -39,7 +39,6 @@ module LogStash; class JavaPipeline < AbstractPipeline
     :started_at,
     :thread
 
-  MAX_INFLIGHT_WARN_THRESHOLD = 10_000
   SECOND = 1
   MEMORY = "memory".freeze
 
@@ -65,6 +64,7 @@ module LogStash; class JavaPipeline < AbstractPipeline
     @flushing = java.util.concurrent.atomic.AtomicBoolean.new(false)
     @flushRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
     @shutdownRequested = java.util.concurrent.atomic.AtomicBoolean.new(false)
+    @crash_detected = Concurrent::AtomicBoolean.new(false)
     @outputs_registered = Concurrent::AtomicBoolean.new(false)
 
     # @finished_execution signals that the pipeline thread has finished its execution
@@ -87,6 +87,10 @@ module LogStash; class JavaPipeline < AbstractPipeline
     @finished_execution.true?
   end
 
+  def finished_run?
+    @finished_run.true?
+  end
+
   def ready?
     @ready.value
   end
@@ -229,6 +233,10 @@ module LogStash; class JavaPipeline < AbstractPipeline
     @running.false?
   end
 
+  def crashed?
+    @crash_detected.true?
+  end
+
   # register_plugins calls #register_plugin on the plugins list and upon exception will call Plugin#do_close on all registered plugins
   # @param plugins [Array[Plugin]] the list of plugins to register
   def register_plugins(plugins)
@@ -275,10 +283,6 @@ module LogStash; class JavaPipeline < AbstractPipeline
         "pipeline.sources" => pipeline_source_details)
       @logger.info("Starting pipeline", pipeline_log_params)
 
-      if max_inflight > MAX_INFLIGHT_WARN_THRESHOLD
-        @logger.warn("CAUTION: Recommended inflight events max exceeded! Logstash will run with up to #{max_inflight} events in memory in your current configuration. If your message sizes are large this may cause instability with the default heap size. Please consider setting a non-standard heap size, changing the batch size (currently #{batch_size}), or changing the number of pipeline workers (currently #{pipeline_workers})", default_logging_keys)
-      end
-
       filter_queue_client.set_batch_dimensions(batch_size, batch_delay)
 
       # First launch WorkerLoop initialization in separate threads which concurrently
@@ -305,6 +309,7 @@ module LogStash; class JavaPipeline < AbstractPipeline
           rescue => e
             # WorkerLoop.run() catches all Java Exception class and re-throws as IllegalStateException with the
             # original exception as the cause
+            @crash_detected.make_true
             @logger.error(
               "Pipeline worker error, the pipeline will be stopped",
               default_logging_keys(:error => e.cause.message, :exception => e.cause.class, :backtrace => e.cause.backtrace)
@@ -319,6 +324,7 @@ module LogStash; class JavaPipeline < AbstractPipeline
       begin
         start_inputs
       rescue => e
+        @crash_detected.make_true
         # if there is any exception in starting inputs, make sure we shutdown workers.
         # exception will already by logged in start_inputs
         shutdown_workers
@@ -628,7 +634,7 @@ module LogStash; class JavaPipeline < AbstractPipeline
     case settings.get("pipeline.ordered")
     when "auto"
       if settings.set?("pipeline.workers") && settings.get("pipeline.workers") == 1
-        @logger.warn("'pipeline.ordered' is enabled and is likely less efficient, consider disabling if preserving event order is not necessary")
+        @logger.warn("'pipeline.ordered' is enabled and is likely less efficient, consider disabling if preserving event order is not necessary") unless settings.get("pipeline.system")
         return true
       end
     when "true"

logstash-core/lib/logstash/modules/logstash_config.rb

@@ -69,13 +69,13 @@ module LogStash module Modules class LogStashConfig
     # validate the values and replace them in the template.
     case default
       when String
-        get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
+        get_setting(LogStash::Setting::SettingNullableString.new(name, default.to_s))
       when Numeric
         get_setting(LogStash::Setting::Numeric.new(name, default))
       when true, false
         get_setting(LogStash::Setting::Boolean.new(name, default))
       else
-        get_setting(LogStash::Setting::NullableString.new(name, default.to_s))
+        get_setting(LogStash::Setting::SettingNullableString.new(name, default.to_s))
       end
   end
 

logstash-core/lib/logstash/patches/clamp.rb

@@ -79,8 +79,13 @@ module Clamp
           new_flag = opts[:new_flag]
           new_value = opts.fetch(:new_value, value)
           passthrough = opts.fetch(:passthrough, false)
+          obsoleted_version = opts[:obsoleted_version]
 
-          LogStash::DeprecationMessage.instance << "DEPRECATION WARNING: The flag #{option.switches} has been deprecated, please use \"--#{new_flag}=#{new_value}\" instead."
+          dmsg = "DEPRECATION WARNING: The flag #{option.switches} has been deprecated"
+          dmsg += obsoleted_version.nil? ? " and may be removed in a future release" : " and will be removed in version #{obsoleted_version}"
+          dmsg += new_flag.nil? ? ".": ", please use \"--#{new_flag}=#{new_value}\" instead."
+
+          LogStash::DeprecationMessage.instance << dmsg
 
           if passthrough
             LogStash::SETTINGS.set(option.attribute_name, value)

logstash-core/lib/logstash/pipeline_action/create.rb

@@ -46,13 +46,21 @@ module LogStash module PipelineAction
     # The execute assume that the thread safety access of the pipeline
     # is managed by the caller.
     def execute(agent, pipelines_registry)
+      attach_health_indicator(agent)
       new_pipeline = LogStash::JavaPipeline.new(@pipeline_config, @metric, agent)
       success = pipelines_registry.create_pipeline(pipeline_id, new_pipeline) do
         new_pipeline.start # block until the pipeline is correctly started or crashed
       end
+
       LogStash::ConvergeResult::ActionResult.create(self, success)
     end
 
+    def attach_health_indicator(agent)
+      health_observer = agent.health_observer
+      health_observer.detach_pipeline_indicator(pipeline_id) # just in case ...
+      health_observer.attach_pipeline_indicator(pipeline_id, agent)
+    end
+
     def to_s
       "PipelineAction::Create<#{pipeline_id}>"
     end

logstash-core/lib/logstash/pipeline_action/delete.rb

@@ -27,10 +27,15 @@ module LogStash module PipelineAction
 
     def execute(agent, pipelines_registry)
       success = pipelines_registry.delete_pipeline(@pipeline_id)
+      detach_health_indicator(agent) if success
 
       LogStash::ConvergeResult::ActionResult.create(self, success)
     end
 
+    def detach_health_indicator(agent)
+      agent.health_observer.detach_pipeline_indicator(pipeline_id)
+    end
+
     def to_s
       "PipelineAction::Delete<#{pipeline_id}>"
     end

logstash-core/lib/logstash/pipeline_action/stop_and_delete.rb

@@ -31,10 +31,15 @@ module LogStash module PipelineAction
       end
 
       success = pipelines_registry.delete_pipeline(@pipeline_id)
+      detach_health_indicator(agent) if success
 
       LogStash::ConvergeResult::ActionResult.create(self, success)
     end
 
+    def detach_health_indicator(agent)
+      agent.health_observer.detach_pipeline_indicator(pipeline_id)
+    end
+
     def to_s
       "PipelineAction::StopAndDelete<#{pipeline_id}>"
     end

logstash-core/lib/logstash/pipeline_resource_usage_validator.rb

@@ -0,0 +1,54 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module LogStash
+  class PipelineResourceUsageValidator
+    include LogStash::Util::Loggable
+
+    WARN_HEAP_THRESHOLD = 10 # 10%
+
+    def initialize(max_heap_size)
+      @max_heap_size = max_heap_size
+    end
+
+    def check(pipelines_registry)
+      return if pipelines_registry.size == 0
+
+      percentage_of_heap = compute_percentage(pipelines_registry)
+
+      if percentage_of_heap >= WARN_HEAP_THRESHOLD
+        logger.warn("For a baseline of 2KB events, the maximum heap memory consumed across #{pipelines_registry.size} pipelines may reach up to #{percentage_of_heap}% of the entire heap (more if the events are bigger). The recommended percentage is less than #{WARN_HEAP_THRESHOLD}%. Consider reducing the number of pipelines, or the batch size and worker count per pipeline.")
+      else
+        logger.debug("For a baseline of 2KB events, the maximum heap memory consumed across #{pipelines_registry.size} pipelines may reach up to #{percentage_of_heap}% of the entire heap (more if the events are bigger).")
+      end
+    end
+
+    def compute_percentage(pipelines_registry)
+      max_event_count = sum_event_count(pipelines_registry)
+      estimated_heap_usage = max_event_count * 2.0 * 1024 # assume 2KB per event
+      percentage_of_heap = ((estimated_heap_usage / @max_heap_size) * 100).round(2)
+    end
+
+    def sum_event_count(pipelines_registry)
+      pipelines_registry.loaded_pipelines.inject(0) do |sum, (pipeline_id, pipeline)|
+        batch_size = pipeline.settings.get("pipeline.batch.size")
+        pipeline_workers = pipeline.settings.get("pipeline.workers")
+        sum + (batch_size * pipeline_workers)
+      end
+    end
+  end
+end

logstash-core/lib/logstash/pipelines_registry.rb

@@ -28,6 +28,7 @@ module LogStash
       @lock = Monitor.new
     end
 
+    # a terminated pipeline has either crashed OR finished normally
     def terminated?
       @lock.synchronize do
         # a loading pipeline is never considered terminated
@@ -35,6 +36,20 @@ module LogStash
       end
     end
 
+    # a finished pipeline finished _normally_ without exception
+    def finished?
+      @lock.synchronize do
+        # a loading pipeline is never considered terminated
+        @loading.false? && @pipeline.finished_run?
+      end
+    end
+
+    def crashed?
+      @lock.synchronize do
+        @pipeline&.crashed?
+      end
+    end
+
     def running?
       @lock.synchronize do
         # not terminated and not loading
@@ -104,6 +119,7 @@ module LogStash
       end
     end
 
+
     def empty?
       @lock.synchronize do
         @states.empty?

logstash-core/lib/logstash/plugins/ecs_compatibility_support.rb

@@ -9,10 +9,11 @@ module LogStash
 
       def ecs_compatibility
         @_ecs_compatibility || LogStash::Util.synchronize(self) do
-          @_ecs_compatibility ||= begin
-            # use config_init-set value if present
-            break @ecs_compatibility unless @ecs_compatibility.nil?
+          # use config_init-set value if present
+          @_ecs_compatibility ||= @ecs_compatibility
 
+          # load default from settings
+          @_ecs_compatibility ||= begin
             pipeline = execution_context.pipeline
             pipeline_settings = pipeline && pipeline.settings
             pipeline_settings ||= LogStash::SETTINGS

logstash-core/lib/logstash/runner.rb

@@ -92,10 +92,6 @@ class LogStash::Runner < Clamp::StrictCommand
          :default => LogStash::SETTINGS.get_default("config.field_reference.escape_style"),
          :attribute_name => "config.field_reference.escape_style"
 
-  option ["--event_api.tags.illegal"], "STRING",
-         I18n.t("logstash.runner.flag.event_api.tags.illegal"),
-         :default => LogStash::SETTINGS.get_default("event_api.tags.illegal"),
-         :attribute_name => "event_api.tags.illegal"
 
   # Module settings
   option ["--modules"], "MODULES",
@@ -257,15 +253,24 @@ class LogStash::Runner < Clamp::StrictCommand
 
   deprecated_option ["--http.enabled"], :flag,
     I18n.t("logstash.runner.flag.http_enabled"),
-    :new_flag => "api.enabled", :passthrough => true # use settings to disambiguate
+    :new_flag => "api.enabled", :passthrough => true, # use settings to disambiguate
+    :obsoleted_version => "9"
 
   deprecated_option ["--http.host"], "HTTP_HOST",
     I18n.t("logstash.runner.flag.http_host"),
-    :new_flag => "api.http.host", :passthrough => true # use settings to disambiguate
+    :new_flag => "api.http.host", :passthrough => true, # use settings to disambiguate
+    :obsoleted_version => "9"
 
   deprecated_option ["--http.port"], "HTTP_PORT",
     I18n.t("logstash.runner.flag.http_port"),
-    :new_flag => "api.http.port", :passthrough => true # use settings to disambiguate
+    :new_flag => "api.http.port", :passthrough => true, # use settings to disambiguate
+    :obsoleted_version => "9"
+
+  deprecated_option ["--event_api.tags.illegal"], "STRING",
+         I18n.t("logstash.runner.flag.event_api.tags.illegal"),
+         :default => LogStash::SETTINGS.get_default("event_api.tags.illegal"),
+         :attribute_name => "event_api.tags.illegal", :passthrough => true,
+         :obsoleted_version => "9"
 
   # We configure the registry and load any plugin that can register hooks
   # with logstash, this needs to be done before any operation.
@@ -310,9 +315,17 @@ class LogStash::Runner < Clamp::StrictCommand
     if setting("config.debug") && !logger.debug?
       logger.warn("--config.debug was specified, but log.level was not set to \'debug\'! No config info will be logged.")
     end
-    if setting("pipeline.buffer.type") != nil
-      configure_pipeline_buffer_type
+    if setting("pipeline.buffer.type") == nil
+      deprecation_logger.deprecated(
+        "'pipeline.buffer.type' setting is not explicitly defined."\
+        "Before moving to 9.x set it to 'heap' and tune heap size upward, or set it to 'direct' to maintain existing behavior."
+      )
+
+      # set to direct to keep backward ecs_compatibility
+      buffer_type_setting = @settings.get_setting("pipeline.buffer.type")
+      buffer_type_setting.set("direct")
     end
+    configure_pipeline_buffer_type
 
     while (msg = LogStash::DeprecationMessage.instance.shift)
       deprecation_logger.deprecated msg
@@ -340,8 +353,8 @@ class LogStash::Runner < Clamp::StrictCommand
     # Add local modules to the registry before everything else
     LogStash::Modules::Util.register_local_modules(LogStash::Environment::LOGSTASH_HOME)
 
-    # Set up the Jackson defaults
-    LogStash::Util::Jackson.set_jackson_defaults(logger)
+    # Verify the Jackson defaults
+    LogStash::Util::Jackson.verify_jackson_overrides
 
     @dispatcher = LogStash::EventDispatcher.new(self)
     LogStash::PLUGIN_REGISTRY.hooks.register_emitter(self.class, @dispatcher)
@@ -480,8 +493,15 @@ class LogStash::Runner < Clamp::StrictCommand
 
   def running_as_superuser
     if Process.euid() == 0
+      unless @settings.set?("allow_superuser")
+        deprecation_logger.deprecated("Starting from version 9.0, " +
+          "running with superuser privileges is not permitted unless you explicitly set 'allow_superuser' to true, " +
+          "thereby acknowledging the possible security risks")
+      end
+
       if setting("allow_superuser")
-        deprecation_logger.deprecated("NOTICE: Running Logstash as superuser is not recommended and won't be allowed in the future. Set 'allow_superuser' to 'false' to avoid startup errors in future releases.")
+        logger.warn("NOTICE: Running Logstash as a superuser is strongly discouraged as it poses a security risk. " +
+                      "Set 'allow_superuser' to false for better security.")
       else
         raise(RuntimeError, "Logstash cannot be run as superuser.")
       end

logstash-core/lib/logstash/settings.rb

@@ -86,7 +86,10 @@ module LogStash
     end
 
     def register(setting)
-      return setting.map { |s| register(s) } if setting.kind_of?(Array)
+      # Method #with_deprecated_alias returns collection containing couple of other settings.
+      # In case the method is implemented in Ruby returns an Array while for the Java implementation
+      # return a List, so the following type checking before going deep by one layer.
+      return setting.map { |s| register(s) } if setting.kind_of?(Array) || setting.kind_of?(java.util.List)
 
       if @settings.key?(setting.name)
         raise ArgumentError.new("Setting \"#{setting.name}\" has already been registered as #{setting.inspect}")
@@ -151,7 +154,7 @@ module LogStash
     def to_hash
       hash = {}
       @settings.each do |name, setting|
-        next if setting.kind_of? Setting::DeprecatedAlias
+        next if (setting.kind_of? Setting::DeprecatedAlias) || (setting.kind_of? Java::org.logstash.settings.DeprecatedAlias)
         hash[name] = setting.value
       end
       hash
@@ -244,54 +247,73 @@ module LogStash
   class Setting
     include LogStash::Settings::LOGGABLE_PROXY
 
-    attr_reader :name, :default
+    attr_reader :wrapped_setting
 
     def initialize(name, klass, default = nil, strict = true, &validator_proc)
-      @name = name
       unless klass.is_a?(Class)
-        raise ArgumentError.new("Setting \"#{@name}\" must be initialized with a class (received #{klass})")
+        raise ArgumentError.new("Setting \"#{name}\" must be initialized with a class (received #{klass})")
       end
+      setting_builder = Java::org.logstash.settings.BaseSetting.create(name)
+                            .defaultValue(default)
+                            .strict(strict)
+      if validator_proc
+        setting_builder = setting_builder.validator(validator_proc)
+      end
+
+      @wrapped_setting = setting_builder.build()
+
       @klass = klass
       @validator_proc = validator_proc
-      @value = nil
-      @value_is_set = false
-      @strict = strict
 
-      validate(default) if @strict
-      @default = default
+      validate(default) if strict?
+    end
+
+    def default
+      @wrapped_setting.default
+    end
+
+    def name
+      @wrapped_setting.name
+    end
+
+    def initialize_copy(original)
+      @wrapped_setting = original.wrapped_setting.clone
+    end
+
+    # To be used only internally
+    def update_wrapper(wrapped_setting)
+      @wrapped_setting = wrapped_setting
     end
 
     def value
-      @value_is_set ? @value : default
+      @wrapped_setting.value()
     end
 
     def set?
-      @value_is_set
+      @wrapped_setting.set?
     end
 
     def strict?
-      @strict
+      @wrapped_setting.strict?
     end
 
     def set(value)
-      validate(value) if @strict
-      @value = value
-      @value_is_set = true
-      @value
+      validate(value) if strict?
+      @wrapped_setting.set(value)
+      @wrapped_setting.value
     end
 
     def reset
-      @value = nil
-      @value_is_set = false
+      @wrapped_setting.reset
     end
 
     def to_hash
       {
-        "name" => @name,
+        "name" => @wrapped_setting.name,
         "klass" => @klass,
-        "value" => @value,
-        "value_is_set" => @value_is_set,
-        "default" => @default,
+        "value" => @wrapped_setting.value,
+        "value_is_set" => @wrapped_setting.set?,
+        "default" => @wrapped_setting.default,
         # Proc#== will only return true if it's the same obj
         # so no there's no point in comparing it
         # also thereś no use case atm to return the proc
@@ -301,7 +323,7 @@ module LogStash
     end
 
     def inspect
-      "<#{self.class.name}(#{name}): #{value.inspect}" + (@value_is_set ? '' : ' (DEFAULT)') + ">"
+      "<#{self.class.name}(#{name}): #{value.inspect}" + (@wrapped_setting.set? ? '' : ' (DEFAULT)') + ">"
     end
 
     def ==(other)
@@ -312,8 +334,8 @@ module LogStash
       validate(value)
     end
 
-    def with_deprecated_alias(deprecated_alias_name)
-      SettingWithDeprecatedAlias.wrap(self, deprecated_alias_name)
+    def with_deprecated_alias(deprecated_alias_name, obsoleted_version=nil)
+      SettingWithDeprecatedAlias.wrap(self, deprecated_alias_name, obsoleted_version)
     end
 
     ##
@@ -323,58 +345,65 @@ module LogStash
     end
 
     def format(output)
-      effective_value = self.value
-      default_value = self.default
-      setting_name = self.name
+      @wrapped_setting.format(output)
+    end
 
-      if default_value == value # print setting and its default value
-        output << "#{setting_name}: #{effective_value.inspect}" unless effective_value.nil?
-      elsif default_value.nil? # print setting and warn it has been set
-        output << "*#{setting_name}: #{effective_value.inspect}"
-      elsif effective_value.nil? # default setting not set by user
-        output << "#{setting_name}: #{default_value.inspect}"
-      else # print setting, warn it has been set, and show default value
-        output << "*#{setting_name}: #{effective_value.inspect} (default: #{default_value.inspect})"
-      end
+    def clone(*args)
+      copy = self.dup
+      copy.update_wrapper(@wrapped_setting.clone())
+      copy
     end
 
     protected
     def validate(input)
       if !input.is_a?(@klass)
-        raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{input} (#{input.class})")
+        raise ArgumentError.new("Setting \"#{@wrapped_setting.name}\" must be a #{@klass}. Received: #{input} (#{input.class})")
       end
 
       if @validator_proc && !@validator_proc.call(input)
-        raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{input}")
+        raise ArgumentError.new("Failed to validate setting \"#{@wrapped_setting.name}\" with value: #{input}")
       end
     end
 
     class Coercible < Setting
       def initialize(name, klass, default = nil, strict = true, &validator_proc)
-        @name = name
         unless klass.is_a?(Class)
-          raise ArgumentError.new("Setting \"#{@name}\" must be initialized with a class (received #{klass})")
+          raise ArgumentError.new("Setting \"#{name}\" must be initialized with a class (received #{klass})")
         end
+
         @klass = klass
         @validator_proc = validator_proc
-        @value = nil
-        @value_is_set = false
+
+        # needed to have the name method accessible when invoking validate
+        @wrapped_setting = Java::org.logstash.settings.BaseSetting.create(name)
+                                      .defaultValue(default)
+                                      .strict(strict)
+                                      .build()
 
         if strict
           coerced_default = coerce(default)
           validate(coerced_default)
-          @default = coerced_default
+          updated_default = coerced_default
         else
-          @default = default
+          updated_default = default
         end
+
+        # default value must be coerced to the right type before being set
+        setting_builder = Java::org.logstash.settings.BaseSetting.create(name)
+                              .defaultValue(updated_default)
+                              .strict(strict)
+        if validator_proc
+          setting_builder = setting_builder.validator(validator_proc)
+        end
+
+        @wrapped_setting = setting_builder.build()
       end
 
       def set(value)
         coerced_value = coerce(value)
         validate(coerced_value)
-        @value = coerce(coerced_value)
-        @value_is_set = true
-        @value
+        @wrapped_setting.set(coerced_value)
+        coerced_value
       end
 
       def coerce(value)
@@ -383,22 +412,7 @@ module LogStash
     end
     ### Specific settings #####
 
-    class Boolean < Coercible
-      def initialize(name, default, strict = true, &validator_proc)
-        super(name, Object, default, strict, &validator_proc)
-      end
-
-      def coerce(value)
-        case value
-        when TrueClass, "true"
-          true
-        when FalseClass, "false"
-          false
-        else
-          raise ArgumentError.new("could not coerce #{value} into a boolean")
-        end
-      end
-    end
+    java_import org.logstash.settings.Boolean
 
     class Numeric < Coercible
       def initialize(name, default = nil, strict = true)
@@ -509,27 +523,10 @@ module LogStash
         @validator_class.validate(value)
       end
     end
+    
+    java_import org.logstash.settings.SettingString
 
-    class String < Setting
-      def initialize(name, default = nil, strict = true, possible_strings = [])
-        @possible_strings = possible_strings
-        super(name, ::String, default, strict)
-      end
-
-      def validate(value)
-        super(value)
-        unless @possible_strings.empty? || @possible_strings.include?(value)
-          raise ArgumentError.new("Invalid value \"#{name}: #{value}\". Options are: #{@possible_strings.inspect}")
-        end
-      end
-    end
-
-    class NullableString < String
-      def validate(value)
-        return if value.nil?
-        super(value)
-      end
-    end
+    java_import org.logstash.settings.SettingNullableString
 
     class Password < Coercible
       def initialize(name, default = nil, strict = true)
@@ -733,15 +730,15 @@ module LogStash
       protected
       def validate(input)
         if !input.is_a?(@klass)
-          raise ArgumentError.new("Setting \"#{@name}\" must be a #{@klass}. Received: #{input} (#{input.class})")
+          raise ArgumentError.new("Setting \"#{@wrapped_setting.name}\" must be a #{@klass}. Received: #{input} (#{input.class})")
         end
 
         unless input.all? {|el| el.kind_of?(@element_class) }
-          raise ArgumentError.new("Values of setting \"#{@name}\" must be #{@element_class}. Received: #{input.map(&:class)}")
+          raise ArgumentError.new("Values of setting \"#{@wrapped_setting.name}\" must be #{@element_class}. Received: #{input.map(&:class)}")
         end
 
         if @validator_proc && !@validator_proc.call(input)
-          raise ArgumentError.new("Failed to validate setting \"#{@name}\" with value: #{input}")
+          raise ArgumentError.new("Failed to validate setting \"#{@wrapped_setting.name}\" with value: #{input}")
         end
       end
     end
@@ -782,7 +779,7 @@ module LogStash
         return unless invalid_value.any?
 
         raise ArgumentError,
-          "Failed to validate the setting \"#{@name}\" value(s): #{invalid_value.inspect}. Valid options are: #{@possible_strings.inspect}"
+          "Failed to validate the setting \"#{@wrapped_setting.name}\" value(s): #{invalid_value.inspect}. Valid options are: #{@possible_strings.inspect}"
       end
     end
 
@@ -792,9 +789,9 @@ module LogStash
       end
 
       def set(value)
-        @value = coerce(value)
-        @value_is_set = true
-        @value
+        coerced_value = coerce(value)
+        @wrapped_setting.set(coerced_value)
+        coerced_value
       end
 
       def coerce(value)
@@ -810,6 +807,8 @@ module LogStash
       end
     end
 
+    java_import org.logstash.settings.NullableSetting
+
     # @see Setting#nullable
     # @api internal
     class Nullable < SimpleDelegator
@@ -833,14 +832,14 @@ module LogStash
     class DeprecatedAlias < SimpleDelegator
       # include LogStash::Util::Loggable
       alias_method :wrapped, :__getobj__
-      attr_reader :canonical_proxy
+      attr_reader :canonical_proxy, :obsoleted_version
 
-      def initialize(canonical_proxy, alias_name)
+      def initialize(canonical_proxy, alias_name, obsoleted_version)
         @canonical_proxy = canonical_proxy
+        @obsoleted_version = obsoleted_version
 
         clone = @canonical_proxy.canonical_setting.clone
-        clone.instance_variable_set(:@name, alias_name)
-        clone.instance_variable_set(:@default, nil)
+        clone.update_wrapper(clone.wrapped_setting.deprecate(alias_name))
 
         super(clone)
       end
@@ -869,9 +868,15 @@ module LogStash
       private
 
       def do_log_setter_deprecation
-        deprecation_logger.deprecated(I18n.t("logstash.settings.deprecation.set",
-                                             :deprecated_alias => name,
-                                             :canonical_name => canonical_proxy.name))
+        deprecation_logger.deprecated(
+          I18n.t("logstash.settings.deprecation.set",
+                 :deprecated_alias => name,
+                 :canonical_name => canonical_proxy.name,
+                 :obsoleted_sentences =>
+                   @obsoleted_version.nil? ?
+                     I18n.t("logstash.settings.deprecation.obsoleted_future") :
+                     I18n.t("logstash.settings.deprecation.obsoleted_version", :obsoleted_version => @obsoleted_version))
+        )
       end
     end
 
@@ -888,10 +893,11 @@ module LogStash
       # including the canonical setting and a deprecated alias.
       # @param canonical_setting [Setting]: the setting to wrap
       # @param deprecated_alias_name [String]: the name for the deprecated alias
+      # @param obsoleted_version [String]: the version of Logstash that deprecated alias will be removed
       #
       # @return [SettingWithDeprecatedAlias,DeprecatedSetting]
-      def self.wrap(canonical_setting, deprecated_alias_name)
-        setting_proxy = new(canonical_setting, deprecated_alias_name)
+      def self.wrap(canonical_setting, deprecated_alias_name, obsoleted_version=nil)
+        setting_proxy = new(canonical_setting, deprecated_alias_name, obsoleted_version)
 
         [setting_proxy, setting_proxy.deprecated_alias]
       end
@@ -899,10 +905,10 @@ module LogStash
       attr_reader :deprecated_alias
       alias_method :canonical_setting, :__getobj__
 
-      def initialize(canonical_setting, deprecated_alias_name)
+      def initialize(canonical_setting, deprecated_alias_name, obsoleted_version)
         super(canonical_setting)
 
-        @deprecated_alias = DeprecatedAlias.new(self, deprecated_alias_name)
+        @deprecated_alias = DeprecatedAlias.new(self, deprecated_alias_name, obsoleted_version)
       end
 
       def set(value)