From ba5af17681327e0a32c99bb7d6128a3d8a862de0 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 16 Apr 2024 17:24:39 +0100 Subject: [PATCH 01/32] update Gemfile lock --- Gemfile.jruby-3.1.lock.release | 1012 ++++++++++++++++++++++++++++++++ 1 file changed, 1012 insertions(+) create mode 100644 Gemfile.jruby-3.1.lock.release diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release new file mode 100644 index 000000000..19a2e9d17 --- /dev/null +++ b/Gemfile.jruby-3.1.lock.release @@ -0,0 +1,1012 @@ +PATH + remote: logstash-core-plugin-api + specs: + logstash-core-plugin-api (2.1.16-java) + logstash-core (= 8.13.3) + +PATH + remote: logstash-core + specs: + logstash-core (8.13.3-java) + clamp (~> 1) + concurrent-ruby (~> 1, < 1.1.10) + down (~> 5.2.0) + elasticsearch (~> 7) + filesize (~> 0.2) + gems (~> 1) + i18n (~> 1) + jrjackson (= 0.4.18) + jruby-openssl (~> 0.14.1) + manticore (~> 0.6) + minitar (~> 0.8) + pry (~> 0.12) + puma (~> 6.3, >= 6.4.2) + rack (~> 2) + ruby-maven-libs (~> 3, >= 3.8.9) + rubyzip (~> 1) + sinatra (~> 2) + stud (~> 0.0.19) + thread_safe (~> 0.3.6) + thwait + treetop (~> 1) + tzinfo-data + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.6) + public_suffix (>= 2.0.2, < 6.0) + amazing_print (1.6.0) + arr-pm (0.0.12) + ast (2.4.2) + atomic (1.1.101-java) + avl_tree (1.2.1) + atomic (~> 1.1) + avro (1.10.2) + multi_json (~> 1) + aws-eventstream (1.3.0) + aws-partitions (1.899.0) + aws-sdk-cloudfront (1.87.0) + aws-sdk-core (~> 3, >= 3.191.0) + aws-sigv4 (~> 1.1) + aws-sdk-cloudwatch (1.86.0) + aws-sdk-core (~> 3, >= 3.191.0) + aws-sigv4 (~> 1.1) + aws-sdk-core (3.191.6) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.651.0) + aws-sigv4 (~> 1.8) + jmespath (~> 1, >= 1.6.1) + aws-sdk-kms (1.78.0) + aws-sdk-core (~> 3, >= 3.191.0) + aws-sigv4 (~> 1.1) + aws-sdk-resourcegroups (1.58.0) + aws-sdk-core (~> 3, >= 3.191.0) + aws-sigv4 (~> 1.1) + aws-sdk-s3 (1.146.1) + aws-sdk-core (~> 3, >= 3.191.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.8) + aws-sdk-sns (1.72.0) + aws-sdk-core (~> 3, >= 3.191.0) + aws-sigv4 (~> 1.1) + aws-sdk-sqs (1.70.0) + aws-sdk-core (~> 3, >= 3.191.0) + aws-sigv4 (~> 1.1) + aws-sigv4 (1.8.0) + aws-eventstream (~> 1, >= 1.0.2) + back_pressure (1.0.0) + backports (3.25.0) + base64 (0.2.0) + belzebuth (0.2.3) + childprocess + benchmark-ips (2.13.0) + bigdecimal (3.1.7-java) + bindata (2.5.0) + buftok (0.2.0) + builder (3.2.4) + cabin (0.9.0) + childprocess (4.1.0) + ci_reporter (2.1.0) + builder (>= 2.1.2) + rexml + ci_reporter_rspec (1.0.0) + ci_reporter (~> 2.0) + rspec (>= 2.14, < 4) + clamp (1.0.1) + coderay (1.1.3) + concurrent-ruby (1.1.9) + crack (0.4.6) + bigdecimal + rexml + dalli (3.2.8) + date (3.3.4-java) + diff-lcs (1.5.1) + docile (1.4.0) + domain_name (0.6.20240107) + dotenv (2.8.1) + down (5.2.4) + addressable (~> 2.8) + e2mmap (0.1.0) + edn (1.1.1) + elastic-enterprise-search (8.9.0) + elastic-transport (~> 8.1) + jwt (>= 1.5, < 3.0) + elastic-transport (8.3.2) + faraday (< 3) + multi_json + elasticsearch (7.17.10) + elasticsearch-api (= 7.17.10) + elasticsearch-transport (= 7.17.10) + elasticsearch-api (7.17.10) + multi_json + elasticsearch-transport (7.17.10) + faraday (>= 1, < 3) + multi_json + equalizer (0.0.11) + et-orbi (1.2.11) + tzinfo + faraday (1.10.3) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.4) + multipart-post (~> 2) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + ffi (1.16.3-java) + filesize (0.2.0) + fivemat (1.3.7) + flores (0.0.8) + fpm (1.15.1) + arr-pm (~> 0.0.11) + backports (>= 2.6.2) + cabin (>= 0.6.0) + clamp (~> 1.0.0) + pleaserun (~> 0.0.29) + rexml + stud + fugit (1.10.1) + et-orbi (~> 1, >= 1.2.7) + raabro (~> 1.4) + gelfd2 (0.4.1) + gem_publisher (1.5.0) + gems (1.2.0) + gene_pool (1.5.0) + concurrent-ruby (>= 1.0) + hashdiff (1.1.0) + hitimes (1.3.1-java) + http (3.3.0) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 2.0) + http_parser.rb (~> 0.6.0) + http-cookie (1.0.5) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.6.0-java) + i18n (1.14.4) + concurrent-ruby (~> 1.0) + insist (1.0.0) + jar-dependencies (0.4.1) + jls-grok (0.11.5) + cabin (>= 0.6.0) + jls-lumberjack (0.0.26) + concurrent-ruby + jmespath (1.6.2) + jrjackson (0.4.18-java) + jruby-jms (1.3.0-java) + gene_pool + semantic_logger + jruby-openssl (0.14.3-java) + jruby-stdin-channel (0.2.0-java) + json (2.7.2-java) + json-schema (2.8.1) + addressable (>= 2.4) + jwt (2.8.1) + base64 + kramdown (2.4.0) + rexml + language_server-protocol (3.17.0.3) + logstash-codec-avro (3.4.1-java) + avro (~> 1.10.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + logstash-codec-cef (6.2.7-java) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-codec-collectd (3.1.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + logstash-codec-dots (3.0.6) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-codec-edn (3.1.0) + edn + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + logstash-codec-edn_lines (3.1.0) + edn + logstash-codec-line + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + logstash-codec-es_bulk (3.1.0) + logstash-codec-line + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + logstash-codec-fluent (3.4.2-java) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + msgpack (~> 1.1) + logstash-codec-graphite (3.0.6) + logstash-codec-line + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-event_support (~> 1.0) + logstash-codec-json (3.1.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0, >= 1.0.1) + logstash-mixin-validator_support (~> 1.0) + logstash-codec-json_lines (3.1.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0, >= 1.0.1) + logstash-mixin-validator_support (~> 1.0) + logstash-codec-line (3.1.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-codec-msgpack (3.1.0-java) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + msgpack (~> 1.1) + logstash-codec-multiline (3.1.1) + concurrent-ruby + jls-grok (~> 0.11.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-patterns-core + logstash-codec-netflow (4.3.2) + bindata (>= 1.5.0) + logstash-core-plugin-api (~> 2.0) + logstash-mixin-event_support (~> 1.0) + logstash-codec-plain (3.1.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-codec-rubydebug (3.1.0) + amazing_print (~> 1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-devutils (2.6.2-java) + fivemat + gem_publisher + kramdown (~> 2) + logstash-codec-plain + logstash-core (>= 6.3) + minitar + rake + rspec (~> 3.0) + rspec-wait + stud (>= 0.0.20) + logstash-filter-aggregate (2.10.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-anonymize (3.0.7) + logstash-core-plugin-api (>= 1.60, <= 2.99) + murmurhash3 (= 0.1.6) + logstash-filter-cidr (3.1.3-java) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-clone (4.2.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.1) + logstash-filter-csv (3.1.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-validator_support (~> 1.0) + logstash-filter-date (3.1.15) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-de_dot (1.0.4) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-dissect (1.2.5) + jar-dependencies + logstash-core-plugin-api (>= 2.1.1, <= 2.99) + logstash-filter-dns (3.2.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + lru_redux (~> 1.1.0) + logstash-filter-drop (3.0.5) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-elastic_integration (0.1.8-java) + logstash-core (>= 8.7.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-elasticsearch (3.16.1) + elasticsearch (>= 7.14.9) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ca_trusted_fingerprint_support (~> 1.0) + logstash-mixin-normalize_config_support (~> 1.0) + manticore (>= 0.7.1) + logstash-filter-fingerprint (3.4.4) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.2) + murmurhash3 (= 0.1.6) + logstash-filter-geoip (7.2.13-java) + logstash-core (>= 7.14.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.2) + logstash-filter-grok (4.4.3) + jls-grok (~> 0.11.3) + logstash-core (>= 5.6.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.0) + logstash-patterns-core (>= 4.3.0, < 5) + stud (~> 0.0.22) + logstash-filter-http (1.5.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.2) + logstash-mixin-http_client (>= 7.3.0, < 8.0.0) + logstash-mixin-validator_support (~> 1.0) + logstash-filter-json (3.2.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-validator_support (~> 1.0) + logstash-filter-kv (4.7.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-validator_support (~> 1.0) + logstash-filter-memcached (1.2.0) + dalli (~> 3) + logstash-core-plugin-api (~> 2.0) + logstash-filter-metrics (4.0.7) + logstash-core-plugin-api (>= 1.60, <= 2.99) + metriks + thread_safe + logstash-filter-mutate (3.5.8) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-prune (3.0.4) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-ruby (3.1.8) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-sleep (3.0.7) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-split (3.1.8) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-syslog_pri (3.2.1) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-filter-throttle (4.0.4) + atomic + logstash-core-plugin-api (>= 1.60, <= 2.99) + thread_safe + logstash-filter-translate (3.4.2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-deprecation_logger_support (~> 1.0) + logstash-mixin-ecs_compatibility_support (~> 1.2) + logstash-mixin-scheduler (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + psych (>= 5.1.0) + logstash-filter-truncate (1.0.6) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-urldecode (3.0.6) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-useragent (3.3.5-java) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-filter-uuid (3.0.5) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-xml (4.2.0) + logstash-core (>= 8.4.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + nokogiri (>= 1.13.8) + xml-simple + logstash-input-azure_event_hubs (1.4.5) + logstash-codec-json + logstash-codec-plain + logstash-core-plugin-api (~> 2.0) + stud (>= 0.0.22) + logstash-input-beats (6.8.2-java) + concurrent-ruby (~> 1.0) + jar-dependencies (~> 0.3, >= 0.3.4) + logstash-codec-multiline (>= 2.0.5) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-normalize_config_support (~> 1.0) + logstash-mixin-plugin_factory_support (~> 1.0) + thread_safe (~> 0.3.5) + logstash-input-couchdb_changes (3.1.6) + json + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + stud (>= 0.0.22) + logstash-input-dead_letter_queue (2.0.0) + logstash-codec-plain + logstash-core (>= 8.4.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-input-elastic_serverless_forwarder (0.1.4-java) + logstash-codec-json_lines + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-input-http (>= 3.7.2) + logstash-mixin-ecs_compatibility_support (~> 1.2) + logstash-mixin-normalize_config_support (~> 1.0) + logstash-mixin-plugin_factory_support + logstash-input-elasticsearch (4.20.2) + elasticsearch (>= 7.17.9) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ca_trusted_fingerprint_support (~> 1.0) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-normalize_config_support (~> 1.0) + logstash-mixin-scheduler (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + manticore (>= 0.7.1) + tzinfo + tzinfo-data + logstash-input-exec (3.6.0) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-scheduler (~> 1.0) + stud (~> 0.0.22) + logstash-input-file (4.4.6) + addressable + concurrent-ruby (~> 1.0) + logstash-codec-multiline (~> 3.0) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-input-ganglia (3.1.4) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + stud (~> 0.0.22) + logstash-input-gelf (3.3.2) + gelfd2 (= 0.4.1) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + stud (>= 0.0.22, < 0.1.0) + logstash-input-generator (3.1.0) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-input-graphite (3.0.6) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-input-tcp + logstash-input-heartbeat (3.1.1) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-deprecation_logger_support (~> 1.0) + logstash-mixin-ecs_compatibility_support (~> 1.2) + logstash-mixin-event_support (~> 1.0) + stud + logstash-input-http (3.8.0-java) + jar-dependencies (~> 0.3, >= 0.3.4) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.2) + logstash-mixin-normalize_config_support (~> 1.0) + logstash-input-http_poller (5.5.1) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0, >= 1.0.1) + logstash-mixin-http_client (>= 7.3.0, < 8.0.0) + logstash-mixin-scheduler (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + logstash-input-jms (3.2.2-java) + jruby-jms (>= 1.2.0) + logstash-codec-json (~> 3.0) + logstash-codec-plain (~> 3.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + semantic_logger (< 4.0.0) + logstash-input-pipe (3.1.0) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + stud (~> 0.0.22) + logstash-input-redis (3.7.0) + logstash-codec-json + logstash-core-plugin-api (>= 1.60, <= 2.99) + redis (>= 4.0.1, < 5) + logstash-input-snmp (1.3.3) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + stud (>= 0.0.22, < 0.1.0) + logstash-input-snmptrap (3.1.0) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + snmp + logstash-input-stdin (3.4.0) + jruby-stdin-channel + logstash-codec-line + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.2) + logstash-input-syslog (3.7.0) + concurrent-ruby + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-filter-date + logstash-filter-grok (>= 4.4.1) + logstash-mixin-ecs_compatibility_support (~> 1.2) + stud (>= 0.0.22, < 0.1.0) + logstash-input-tcp (6.4.1-java) + jruby-openssl (>= 0.12.2) + logstash-codec-json + logstash-codec-json_lines + logstash-codec-line + logstash-codec-multiline + logstash-codec-plain + logstash-core (>= 8.1.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.2) + logstash-mixin-normalize_config_support (~> 1.0) + logstash-input-twitter (4.1.1) + http-form_data (~> 2) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + public_suffix (> 4, < 6) + stud (>= 0.0.22, < 0.1) + twitter (= 6.2.0) + logstash-input-udp (3.5.0) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.2) + stud (~> 0.0.22) + logstash-input-unix (3.1.2) + logstash-codec-line + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-integration-aws (7.1.6-java) + aws-sdk-cloudfront + aws-sdk-cloudwatch + aws-sdk-core (~> 3) + aws-sdk-resourcegroups + aws-sdk-s3 + aws-sdk-sns + aws-sdk-sqs + concurrent-ruby + logstash-codec-json + logstash-codec-plain + logstash-core-plugin-api (>= 2.1.12, <= 2.99) + rexml + rufus-scheduler (>= 3.0.9) + stud (~> 0.0.22) + logstash-integration-elastic_enterprise_search (3.0.0) + elastic-enterprise-search (>= 7.16, < 9) + logstash-codec-plain + logstash-core-plugin-api (~> 2.0) + logstash-mixin-deprecation_logger_support (~> 1.0) + manticore (~> 0.8) + logstash-integration-jdbc (5.4.9) + logstash-codec-plain + logstash-core (>= 6.5.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ecs_compatibility_support (~> 1.3) + logstash-mixin-event_support (~> 1.0) + logstash-mixin-scheduler (~> 1.0) + logstash-mixin-validator_support (~> 1.0) + lru_redux + sequel (>= 5.74.0) + tzinfo + tzinfo-data + logstash-integration-kafka (11.3.4-java) + logstash-codec-json + logstash-codec-plain + logstash-core (>= 8.3.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-deprecation_logger_support (~> 1.0) + manticore (>= 0.5.4, < 1.0.0) + stud (>= 0.0.22, < 0.1.0) + logstash-integration-logstash (1.0.2-java) + logstash-codec-json_lines (~> 3.1) + logstash-core-plugin-api (>= 2.1.12, <= 2.99) + logstash-input-http (>= 3.7.0) + logstash-mixin-http_client (~> 7.3) + logstash-mixin-plugin_factory_support (~> 1.0) + logstash-mixin-validator_support (~> 1.1) + stud + logstash-integration-rabbitmq (7.3.3-java) + back_pressure (~> 1.0) + logstash-codec-json + logstash-core (>= 6.5.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + march_hare (~> 4.0) + stud (~> 0.0.22) + logstash-mixin-ca_trusted_fingerprint_support (1.0.1-java) + logstash-core (>= 6.8.0) + logstash-mixin-deprecation_logger_support (1.0.0-java) + logstash-core (>= 5.0.0) + logstash-mixin-ecs_compatibility_support (1.3.0-java) + logstash-core (>= 6.0.0) + logstash-mixin-event_support (1.0.1-java) + logstash-core (>= 6.8) + logstash-mixin-http_client (7.3.0) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-normalize_config_support (~> 1.0) + manticore (>= 0.8.0, < 1.0.0) + logstash-mixin-normalize_config_support (1.0.0-java) + logstash-core (>= 6.8.0) + logstash-mixin-plugin_factory_support (1.0.0-java) + logstash-core (>= 7.13.0) + logstash-mixin-scheduler (1.0.1-java) + logstash-core (>= 7.16) + rufus-scheduler (>= 3.0.9) + logstash-mixin-validator_support (1.1.1-java) + logstash-core (>= 6.8) + logstash-output-csv (3.0.10) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-output-file + logstash-output-elasticsearch (11.22.5-java) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-ca_trusted_fingerprint_support (~> 1.0) + logstash-mixin-deprecation_logger_support (~> 1.0) + logstash-mixin-ecs_compatibility_support (~> 1.0) + logstash-mixin-normalize_config_support (~> 1.0) + manticore (>= 0.8.0, < 1.0.0) + stud (~> 0.0, >= 0.0.17) + logstash-output-email (4.1.3) + logstash-core-plugin-api (>= 1.60, <= 2.99) + mail (~> 2.8) + mustache (>= 0.99.8) + logstash-output-file (4.3.0) + logstash-codec-json_lines + logstash-codec-line + logstash-core-plugin-api (>= 2.0.0, < 2.99) + logstash-output-graphite (3.1.6) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-output-http (5.6.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-http_client (>= 7.3.0, < 8.0.0) + logstash-output-lumberjack (3.1.9) + jls-lumberjack (>= 0.0.26) + logstash-core-plugin-api (>= 1.60, <= 2.99) + stud + logstash-output-nagios (3.0.6) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-output-null (3.0.5) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-output-pipe (3.0.6) + logstash-codec-plain + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-output-redis (5.0.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + redis (~> 4) + stud + logstash-output-stdout (3.1.4) + logstash-codec-rubydebug + logstash-core-plugin-api (>= 1.60.1, < 2.99) + logstash-output-tcp (6.2.0) + jruby-openssl (>= 0.12.2) + logstash-codec-json + logstash-core (>= 8.1.0) + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-mixin-normalize_config_support (~> 1.0) + stud + logstash-output-udp (3.2.0) + logstash-codec-json + logstash-core-plugin-api (>= 1.60, <= 2.99) + logstash-output-webhdfs (3.1.0-java) + logstash-core-plugin-api (>= 1.60, <= 2.99) + webhdfs + logstash-patterns-core (4.3.4) + logstash-core-plugin-api (>= 1.60, <= 2.99) + lru_redux (1.1.0) + mail (2.8.1) + mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp + manticore (0.9.1-java) + openssl_pkcs8_pure + march_hare (4.6.0-java) + memoizable (0.4.2) + thread_safe (~> 0.3, >= 0.3.1) + method_source (1.0.0) + metriks (0.9.9.8) + atomic (~> 1.0) + avl_tree (~> 1.2.0) + hitimes (~> 1.1) + mini_mime (1.1.5) + minitar (0.9) + msgpack (1.7.2-java) + multi_json (1.15.0) + multipart-post (2.4.0) + murmurhash3 (0.1.6-java) + mustache (0.99.8) + mustermann (2.0.2) + ruby2_keywords (~> 0.0.1) + naught (1.1.0) + net-imap (0.4.10) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.4.0.1) + net-protocol + nio4r (2.7.1-java) + nokogiri (1.16.3-java) + racc (~> 1.4) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) + openssl_pkcs8_pure (0.0.0.2) + paquet (0.2.1) + parallel (1.24.0) + parser (3.3.0.5) + ast (~> 2.4.1) + racc + pleaserun (0.0.32) + cabin (> 0) + clamp + dotenv + insist + mustache (= 0.99.8) + stud + polyglot (0.3.5) + pry (0.14.2-java) + coderay (~> 1.1) + method_source (~> 1.0) + spoon (~> 0.0) + psych (5.1.2-java) + jar-dependencies (>= 0.1.7) + public_suffix (5.0.5) + puma (6.4.2-java) + nio4r (~> 2.0) + raabro (1.4.0) + racc (1.7.3-java) + rack (2.2.9) + rack-protection (2.2.4) + rack + rack-test (2.1.0) + rack (>= 1.3) + rainbow (3.1.1) + rake (13.1.0) + redis (4.8.1) + regexp_parser (2.9.0) + rexml (3.2.6) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) + rspec-wait (0.0.9) + rspec (>= 3, < 4) + rubocop (1.62.1) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.31.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.31.2) + parser (>= 3.3.0.4) + ruby-maven-libs (3.8.9) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + rubyzip (1.3.0) + rufus-scheduler (3.9.1) + fugit (~> 1.1, >= 1.1.6) + sawyer (0.9.2) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) + semantic_logger (3.4.1) + concurrent-ruby (~> 1.0) + sequel (5.78.0) + bigdecimal + simple_oauth (0.3.1) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov-json (0.2.3) + json + simplecov + simplecov_json_formatter (0.1.4) + sinatra (2.2.4) + mustermann (~> 2.0) + rack (~> 2.2) + rack-protection (= 2.2.4) + tilt (~> 2.0) + snmp (1.3.2) + spoon (0.0.6) + ffi + stud (0.0.23) + thread_safe (0.3.6-java) + thwait (0.2.0) + e2mmap + tilt (2.3.0) + timeout (0.4.1) + treetop (1.6.12) + polyglot (~> 0.3) + twitter (6.2.0) + addressable (~> 2.3) + buftok (~> 0.2.0) + equalizer (~> 0.0.11) + http (~> 3.0) + http-form_data (~> 2.0) + http_parser.rb (~> 0.6.0) + memoizable (~> 0.4.0) + multipart-post (~> 2.0) + naught (~> 1.0) + simple_oauth (~> 0.3.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2024.1) + tzinfo (>= 1.0.0) + unicode-display_width (2.5.0) + webhdfs (0.11.0) + addressable + webmock (3.23.0) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) + xml-simple (1.1.9) + rexml + +PLATFORMS + java + +DEPENDENCIES + belzebuth + benchmark-ips + bigdecimal (~> 3.1) + childprocess (~> 4) + ci_reporter_rspec (~> 1) + faraday (~> 1) + flores (~> 0.0.8) + fpm (~> 1, >= 1.14.1) + gems (~> 1) + jar-dependencies (= 0.4.1) + json-schema (~> 2) + logstash-codec-avro + logstash-codec-cef + logstash-codec-collectd + logstash-codec-dots + logstash-codec-edn + logstash-codec-edn_lines + logstash-codec-es_bulk + logstash-codec-fluent + logstash-codec-graphite + logstash-codec-json + logstash-codec-json_lines + logstash-codec-line + logstash-codec-msgpack + logstash-codec-multiline + logstash-codec-netflow + logstash-codec-plain + logstash-codec-rubydebug + logstash-core! + logstash-core-plugin-api! + logstash-devutils (~> 2.6.0) + logstash-filter-aggregate + logstash-filter-anonymize + logstash-filter-cidr + logstash-filter-clone + logstash-filter-csv + logstash-filter-date + logstash-filter-de_dot + logstash-filter-dissect + logstash-filter-dns + logstash-filter-drop + logstash-filter-elastic_integration + logstash-filter-elasticsearch + logstash-filter-fingerprint + logstash-filter-geoip + logstash-filter-grok + logstash-filter-http + logstash-filter-json + logstash-filter-kv + logstash-filter-memcached + logstash-filter-metrics + logstash-filter-mutate + logstash-filter-prune + logstash-filter-ruby + logstash-filter-sleep + logstash-filter-split + logstash-filter-syslog_pri + logstash-filter-throttle + logstash-filter-translate + logstash-filter-truncate + logstash-filter-urldecode + logstash-filter-useragent + logstash-filter-uuid + logstash-filter-xml + logstash-input-azure_event_hubs + logstash-input-beats + logstash-input-couchdb_changes + logstash-input-dead_letter_queue + logstash-input-elastic_serverless_forwarder + logstash-input-elasticsearch + logstash-input-exec + logstash-input-file + logstash-input-ganglia + logstash-input-gelf + logstash-input-generator + logstash-input-graphite + logstash-input-heartbeat + logstash-input-http + logstash-input-http_poller + logstash-input-jms + logstash-input-pipe + logstash-input-redis + logstash-input-snmp + logstash-input-snmptrap + logstash-input-stdin + logstash-input-syslog + logstash-input-tcp + logstash-input-twitter + logstash-input-udp + logstash-input-unix + logstash-integration-aws + logstash-integration-elastic_enterprise_search + logstash-integration-jdbc + logstash-integration-kafka + logstash-integration-logstash + logstash-integration-rabbitmq + logstash-output-csv + logstash-output-elasticsearch (>= 11.14.0) + logstash-output-email + logstash-output-file + logstash-output-graphite + logstash-output-http + logstash-output-lumberjack + logstash-output-nagios + logstash-output-null + logstash-output-pipe + logstash-output-redis + logstash-output-stdout + logstash-output-tcp + logstash-output-udp + logstash-output-webhdfs + murmurhash3 (= 0.1.6) + octokit (~> 4.25) + paquet (~> 0.2) + pleaserun (~> 0.0.28) + polyglot + rack-test + rake (~> 13) + rspec (~> 3.5) + rubocop + ruby-maven-libs (~> 3, >= 3.8.9) + ruby-progressbar (~> 1) + rubyzip (~> 1) + simplecov (~> 0.22.0) + simplecov-json + stud (~> 0.0.22) + thwait + treetop + webmock (~> 3) + +BUNDLED WITH + 2.3.26 From 18583787b3cc1095a002f4a8e1f4d9436e712c54 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:16:02 +0100 Subject: [PATCH 02/32] bump lock file for 8.14 (#16087) * Update minor plugin versions in gemfile lock * Update Gemfile.jruby-3.1.lock.release --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> --- Gemfile.jruby-3.1.lock.release | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index 19a2e9d17..6b297807f 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -2,12 +2,12 @@ PATH remote: logstash-core-plugin-api specs: logstash-core-plugin-api (2.1.16-java) - logstash-core (= 8.13.3) + logstash-core (= 8.14.0) PATH remote: logstash-core specs: - logstash-core (8.13.3-java) + logstash-core (8.14.0-java) clamp (~> 1) concurrent-ruby (~> 1, < 1.1.10) down (~> 5.2.0) @@ -45,11 +45,11 @@ GEM avro (1.10.2) multi_json (~> 1) aws-eventstream (1.3.0) - aws-partitions (1.899.0) - aws-sdk-cloudfront (1.87.0) + aws-partitions (1.913.0) + aws-sdk-cloudfront (1.88.0) aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) - aws-sdk-cloudwatch (1.86.0) + aws-sdk-cloudwatch (1.88.0) aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) aws-sdk-core (3.191.6) @@ -57,10 +57,10 @@ GEM aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.78.0) + aws-sdk-kms (1.79.0) aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) - aws-sdk-resourcegroups (1.58.0) + aws-sdk-resourcegroups (1.59.0) aws-sdk-core (~> 3, >= 3.191.0) aws-sigv4 (~> 1.1) aws-sdk-s3 (1.146.1) @@ -193,7 +193,7 @@ GEM jruby-jms (1.3.0-java) gene_pool semantic_logger - jruby-openssl (0.14.3-java) + jruby-openssl (0.14.4-java) jruby-stdin-channel (0.2.0-java) json (2.7.2-java) json-schema (2.8.1) @@ -602,7 +602,7 @@ GEM sequel (>= 5.74.0) tzinfo tzinfo-data - logstash-integration-kafka (11.3.4-java) + logstash-integration-kafka (11.4.1-java) logstash-codec-json logstash-codec-plain logstash-core (>= 8.3.0) @@ -717,7 +717,7 @@ GEM march_hare (4.6.0-java) memoizable (0.4.2) thread_safe (~> 0.3, >= 0.3.1) - method_source (1.0.0) + method_source (1.1.0) metriks (0.9.9.8) atomic (~> 1.0) avl_tree (~> 1.2.0) @@ -739,10 +739,10 @@ GEM net-protocol net-protocol (0.2.2) timeout - net-smtp (0.4.0.1) + net-smtp (0.5.0) net-protocol nio4r (2.7.1-java) - nokogiri (1.16.3-java) + nokogiri (1.16.4-java) racc (~> 1.4) octokit (4.25.1) faraday (>= 1, < 3) @@ -778,7 +778,7 @@ GEM rack-test (2.1.0) rack (>= 1.3) rainbow (3.1.1) - rake (13.1.0) + rake (13.2.1) redis (4.8.1) regexp_parser (2.9.0) rexml (3.2.6) @@ -797,7 +797,7 @@ GEM rspec-support (3.13.1) rspec-wait (0.0.9) rspec (>= 3, < 4) - rubocop (1.62.1) + rubocop (1.63.2) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -821,7 +821,7 @@ GEM faraday (>= 0.17.3, < 3) semantic_logger (3.4.1) concurrent-ruby (~> 1.0) - sequel (5.78.0) + sequel (5.79.0) bigdecimal simple_oauth (0.3.1) simplecov (0.22.0) From ceddff3ab3092070ac3b6b8408ec456b4df1fbd0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 18:01:16 +0200 Subject: [PATCH 03/32] Provide opt-in flag to avoid fields name clash when log format is json (#15969) (#16094) Adds log.format.json.fix_duplicate_message_fields feature flag to rename the clashing fields when json logging format (log.format) is selected. In case two message fields clashes on structured log message, then the second is renamed attaching _1 suffix to the field name. By default the feature is disabled and requires user to explicitly enable the behaviour. Co-authored-by: Rob Bavey (cherry picked from commit 830733d75856c3c4d3086facf46bbb2eb12c61c3) Co-authored-by: Andrea Selva --- config/logstash.yml | 2 + docker/data/logstash/env2yaml/env2yaml.go | 1 + .../running-logstash-command-line.asciidoc | 3 + docs/static/settings-file.asciidoc | 4 ++ docs/static/troubleshoot/ts-logstash.asciidoc | 62 +++++++++++++++++++ logstash-core/lib/logstash/environment.rb | 2 + logstash-core/lib/logstash/runner.rb | 5 ++ logstash-core/locales/en.yml | 2 + .../log/CustomLogEventSerializer.java | 14 ++++- .../org/logstash/log/CustomLogEventTests.java | 32 ++++++++++ 10 files changed, 126 insertions(+), 1 deletion(-) diff --git a/config/logstash.yml b/config/logstash.yml index afa378a17..62b5912c4 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -314,6 +314,8 @@ # * json # # log.format: plain +# log.format.json.fix_duplicate_message_fields: false +# # path.logs: # # ------------ Other Settings -------------- diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index e89997854..7bcaa33d1 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -71,6 +71,7 @@ var validSettings = []string{ "http.port", // DEPRECATED: prefer `api.http.port` "log.level", "log.format", + "log.format.json.fix_duplicate_message_fields", "modules", "metric.collect", "path.logs", diff --git a/docs/static/running-logstash-command-line.asciidoc b/docs/static/running-logstash-command-line.asciidoc index 5eba5c596..646ea60ac 100644 --- a/docs/static/running-logstash-command-line.asciidoc +++ b/docs/static/running-logstash-command-line.asciidoc @@ -230,6 +230,9 @@ With this command, Logstash concatenates three config files, `/tmp/one`, `/tmp/t Specify if Logstash should write its own logs in JSON form (one event per line) or in plain text (using Ruby's Object#inspect). The default is "plain". +*`--log.format.json.fix_duplicate_message_fields ENABLED`*:: + Avoid `message` field collision using JSON log format. Possible values are `false` (default) and `true`. + *`--path.settings SETTINGS_DIR`*:: Set the directory containing the `logstash.yml` <> as well as the log4j logging configuration. This can also be set through the LS_SETTINGS_DIR environment variable. diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index 32856d7eb..82f8ddaab 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -336,6 +336,10 @@ The log level. Valid options are: | The log format. Set to `json` to log in JSON format, or `plain` to use `Object#.inspect`. | `plain` +| `log.format.json.fix_duplicate_message_fields` +| When the log format is `json` avoid collision of field names in log lines. +| `false` + | `path.logs` | The directory where Logstash will write its log to. | `LOGSTASH_HOME/logs` diff --git a/docs/static/troubleshoot/ts-logstash.asciidoc b/docs/static/troubleshoot/ts-logstash.asciidoc index 42288c4d3..219639466 100644 --- a/docs/static/troubleshoot/ts-logstash.asciidoc +++ b/docs/static/troubleshoot/ts-logstash.asciidoc @@ -204,3 +204,65 @@ As the logging library used in Logstash is synchronous, heavy logging can affect *Solution* Reset the logging level to `info`. + +[[ts-pipeline-logging-json-duplicated-message-field]] +==== Logging in json format can write duplicate `message` fields + +*Symptoms* + +When log format is `json` and certain log events (for example errors from JSON codec plugin) +contains two instances of the `message` field. + +Without setting this flag, json log would contain objects like: + +[source,json] +----- +{ + "level":"WARN", + "loggerName":"logstash.codecs.jsonlines", + "timeMillis":1712937761955, + "thread":"[main] "log.format", :default => LogStash::SETTINGS.get_default("log.format") + option ["--log.format.json.fix_duplicate_message_fields"], "FORMAT_JSON_STRICT", + I18n.t("logstash.runner.flag.log_format_json_fix_duplicate_message_fields"), + :attribute_name => "log.format.json.fix_duplicate_message_fields", + :default => LogStash::SETTINGS.get_default("log.format.json.fix_duplicate_message_fields") + option ["--path.settings"], "SETTINGS_DIR", I18n.t("logstash.runner.flag.path_settings"), :attribute_name => "path.settings", diff --git a/logstash-core/locales/en.yml b/logstash-core/locales/en.yml index 78193863a..bd9feedff 100644 --- a/logstash-core/locales/en.yml +++ b/logstash-core/locales/en.yml @@ -423,6 +423,8 @@ en: log_format: |+ Specify if Logstash should write its own logs in JSON form (one event per line) or in plain text (using Ruby's Object#inspect) + log_format_json_fix_duplicate_message_fields: |+ + Enable to avoid duplication of message fields in JSON form. debug: |+ Set the log level to debug. DEPRECATED: use --log.level=debug instead. diff --git a/logstash-core/src/main/java/org/logstash/log/CustomLogEventSerializer.java b/logstash-core/src/main/java/org/logstash/log/CustomLogEventSerializer.java index b52c14a6f..8d91429b6 100644 --- a/logstash-core/src/main/java/org/logstash/log/CustomLogEventSerializer.java +++ b/logstash-core/src/main/java/org/logstash/log/CustomLogEventSerializer.java @@ -69,7 +69,9 @@ public class CustomLogEventSerializer extends JsonSerializer { } for (final Map.Entry entry : message.getParams().entrySet()) { - final String paramName = entry.getKey().toString(); + // Given that message params is a map and the generator just started a new object, containing + // only one 'message' field, it could clash only on this field; fixit post-fixing it with '_1' + final String paramName = renameParamNameIfClashingWithMessage(entry); final Object paramValue = entry.getValue(); try { @@ -94,6 +96,16 @@ public class CustomLogEventSerializer extends JsonSerializer { } } + private static String renameParamNameIfClashingWithMessage(Map.Entry entry) { + final String paramName = entry.getKey().toString(); + if ("message".equals(paramName)) { + if ("true".equalsIgnoreCase(System.getProperty("ls.log.format.json.fix_duplicate_message_fields"))) { + return "message_1"; + } + } + return paramName; + } + private boolean isValueSafeToWrite(Object value) { return value == null || value instanceof String || diff --git a/logstash-core/src/test/java/org/logstash/log/CustomLogEventTests.java b/logstash-core/src/test/java/org/logstash/log/CustomLogEventTests.java index c340fb921..7b320e4ee 100644 --- a/logstash-core/src/test/java/org/logstash/log/CustomLogEventTests.java +++ b/logstash-core/src/test/java/org/logstash/log/CustomLogEventTests.java @@ -59,9 +59,11 @@ import org.logstash.RubyUtil; import static junit.framework.TestCase.assertFalse; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertNotNull; +import static org.junit.Assert.assertTrue; public class CustomLogEventTests { private static final String CONFIG = "log4j2-test1.xml"; + public static final String STRICT_JSON_PROPERTY_NAME = "ls.log.format.json.fix_duplicate_message_fields"; @ClassRule public static LoggerContextRule CTX = new LoggerContextRule(CONFIG); @@ -174,4 +176,34 @@ public class CustomLogEventTests { assertEquals(1, logEventMapValue.get("first")); assertEquals(2, logEventMapValue.get("second")); } + + @Test + @SuppressWarnings("unchecked") + public void testJSONLayoutWhenParamsContainsAnotherMessageField() throws JsonProcessingException { + String prevSetting = System.getProperty(STRICT_JSON_PROPERTY_NAME); + System.setProperty(STRICT_JSON_PROPERTY_NAME, Boolean.TRUE.toString()); + + ListAppender appender = CTX.getListAppender("JSONEventLogger").clear(); + Logger logger = LogManager.getLogger("JSONEventLogger"); + + Map paramsWithAnotherMessageField = Collections.singletonMap("message", "something to say"); + logger.error("here is a map: {}", paramsWithAnotherMessageField); + + List messages = appender.getMessages(); + assertEquals(1, messages.size()); + + Map loggedMessage = ObjectMappers.JSON_MAPPER.readValue(messages.get(0), Map.class); + assertEquals(5, loggedMessage.size()); + + Map actualLogEvent = (Map) loggedMessage.get("logEvent"); + assertEquals("here is a map: {}", actualLogEvent.get("message")); + assertEquals("something to say", actualLogEvent.get("message_1")); + + // tear down + if (prevSetting == null) { + System.clearProperty(STRICT_JSON_PROPERTY_NAME); + } else { + System.setProperty(STRICT_JSON_PROPERTY_NAME, prevSetting); + } + } } From c2195bc1ee1a971a6aa18b6bee33721301d47ba5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 10:00:03 +0100 Subject: [PATCH 04/32] upgrade jruby to 9.4.7.0 (#16125) (#16126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 4350855e7b127d95b6b29a0ffea19a529fc5111d) Co-authored-by: João Duarte --- rubyUtils.gradle | 5 +---- versions.yml | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/rubyUtils.gradle b/rubyUtils.gradle index e16fa703c..52875b13d 100644 --- a/rubyUtils.gradle +++ b/rubyUtils.gradle @@ -25,7 +25,7 @@ buildscript { dependencies { classpath "org.yaml:snakeyaml:${snakeYamlVersion}" classpath "de.undercouch:gradle-download-task:4.0.4" - classpath "org.jruby:jruby-core:9.4.6.0" + classpath "org.jruby:jruby-core:9.4.7.0" } } @@ -207,8 +207,6 @@ def versionsPath = project.hasProperty("LOGSTASH_CORE_PATH") ? LOGSTASH_CORE_PAT versionMap = (Map) (new Yaml()).load(new File("${versionsPath}").text) String jRubyURL -String previousJRubyVersion = "9.3.9.0" -String previousJRubyURL = "https://repo1.maven.org/maven2/org/jruby/jruby-dist/${previousJRubyVersion}/jruby-dist-${previousJRubyVersion}-bin.tar.gz" String jRubyVersion String jRubySha1 Boolean doChecksum @@ -224,7 +222,6 @@ if (versionMap["jruby-runtime-override"]) { doChecksum = true } def jrubyTarPath = "${projectDir}/vendor/_/jruby-dist-${jRubyVersion}-bin.tar.gz" -def previousJrubyTarPath = "${projectDir}/vendor/_/jruby-dist-${previousJRubyVersion}-bin.tar.gz" def customJRubyDir = project.hasProperty("custom.jruby.path") ? project.property("custom.jruby.path") : "" def customJRubyVersion = customJRubyDir == "" ? "" : Files.readAllLines(Paths.get(customJRubyDir, "VERSION")).get(0).trim() diff --git a/versions.yml b/versions.yml index 96954fe67..431bff8ba 100644 --- a/versions.yml +++ b/versions.yml @@ -13,8 +13,8 @@ bundled_jdk: # jruby must reference a *released* version of jruby which can be downloaded from the official download url # *and* for which jars artifacts are published for compile-time jruby: - version: 9.4.6.0 - sha1: 871d520c9f2494ca56138200c6b2a95c54d0a639 + version: 9.4.7.0 + sha1: ea3b99edcb48ed436173977551a113759541c18c # jruby-runtime-override, if specified, will override the jruby version installed in vendor/jruby #jruby-runtime-override: # url: https://oss.sonatype.org/content/repositories/snapshots/org/jruby/jruby-dist/9.3.0.0-SNAPSHOT/jruby-dist-9.3.0.0-20210723.214927-259-bin.tar.gz From ed535a822a840618c8e5c8eecba4281756603c3e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 23:39:49 +0100 Subject: [PATCH 05/32] Release notes for 8.13.3 (#16108) * Update release notes for 8.13.3 Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/static/releasenotes.asciidoc | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index 13f728353..d3a23366f 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,7 @@ This section summarizes the changes in the following releases: +* <> * <> * <> * <> @@ -58,6 +59,30 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-13-3]] +=== Logstash 8.13.3 Release Notes + +No user-facing changes in Logstash core. + +==== Plugins + +*Beats Input - 6.8.3* + +* Updated netty to 4.1.109 https://github.com/logstash-plugins/logstash-input-beats/pull/495[#495] + +*Http Input - 3.8.1* + +* Updated netty to 4.1.109 https://github.com/logstash-plugins/logstash-input-http/pull/173[#173] + +*Tcp Input - 6.4.2* + +* Updated netty to 4.1.109 https://github.com/logstash-plugins/logstash-input-tcp/pull/220[#220] + +*Multiline Codec - 3.1.2* + +* Fixed a race condition in periodic runner that blocks clean up process and pipeline shutdown https://github.com/logstash-plugins/logstash-codec-multiline/pull/72[#72] + + [[logstash-8-13-2]] === Logstash 8.13.2 Release Notes @@ -2154,4 +2179,4 @@ We have added another flag to the Benchmark CLI to allow passing a data file wit This feature allows users to run the Benchmark CLI in a custom test case with a custom config and a custom dataset. https://github.com/elastic/logstash/pull/12437[#12437] ==== Plugin releases -Plugins align with release 7.14.0 +Plugins align with release 7.14.0 \ No newline at end of file From 1f122085286fcab740e3643729b2c486e03899ea Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 15:34:38 +0100 Subject: [PATCH 06/32] bump lock file for 8.14 (#16129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: João Duarte --- Gemfile.jruby-3.1.lock.release | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index 6b297807f..71df9b89e 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -193,7 +193,7 @@ GEM jruby-jms (1.3.0-java) gene_pool semantic_logger - jruby-openssl (0.14.4-java) + jruby-openssl (0.14.5-java) jruby-stdin-channel (0.2.0-java) json (2.7.2-java) json-schema (2.8.1) @@ -264,7 +264,7 @@ GEM logstash-mixin-event_support (~> 1.0) logstash-mixin-validator_support (~> 1.0) msgpack (~> 1.1) - logstash-codec-multiline (3.1.1) + logstash-codec-multiline (3.1.2) concurrent-ruby jls-grok (~> 0.11.1) logstash-core-plugin-api (>= 1.60, <= 2.99) @@ -406,7 +406,7 @@ GEM logstash-codec-plain logstash-core-plugin-api (~> 2.0) stud (>= 0.0.22) - logstash-input-beats (6.8.2-java) + logstash-input-beats (6.8.3-java) concurrent-ruby (~> 1.0) jar-dependencies (~> 0.3, >= 0.3.4) logstash-codec-multiline (>= 2.0.5) @@ -481,7 +481,7 @@ GEM logstash-mixin-ecs_compatibility_support (~> 1.2) logstash-mixin-event_support (~> 1.0) stud - logstash-input-http (3.8.0-java) + logstash-input-http (3.8.1-java) jar-dependencies (~> 0.3, >= 0.3.4) logstash-codec-plain logstash-core-plugin-api (>= 1.60, <= 2.99) @@ -540,7 +540,7 @@ GEM logstash-filter-grok (>= 4.4.1) logstash-mixin-ecs_compatibility_support (~> 1.2) stud (>= 0.0.22, < 0.1.0) - logstash-input-tcp (6.4.1-java) + logstash-input-tcp (6.4.2-java) jruby-openssl (>= 0.12.2) logstash-codec-json logstash-codec-json_lines @@ -650,7 +650,7 @@ GEM logstash-output-csv (3.0.10) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-output-file - logstash-output-elasticsearch (11.22.5-java) + logstash-output-elasticsearch (11.22.6-java) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-mixin-ca_trusted_fingerprint_support (~> 1.0) logstash-mixin-deprecation_logger_support (~> 1.0) @@ -750,7 +750,7 @@ GEM openssl_pkcs8_pure (0.0.0.2) paquet (0.2.1) parallel (1.24.0) - parser (3.3.0.5) + parser (3.3.1.0) ast (~> 2.4.1) racc pleaserun (0.0.32) @@ -795,9 +795,9 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-support (3.13.1) - rspec-wait (0.0.9) - rspec (>= 3, < 4) - rubocop (1.63.2) + rspec-wait (0.0.10) + rspec (>= 3.0) + rubocop (1.63.4) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -808,8 +808,8 @@ GEM rubocop-ast (>= 1.31.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.31.2) - parser (>= 3.3.0.4) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) ruby-maven-libs (3.8.9) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) From b1773f8c026864f413736294b5170eeb03185d9b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 21:50:23 +0100 Subject: [PATCH 07/32] force ruby-maven-libs constraint to >= 3.9.6.1 (#16130) (#16131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 973c2ba3aaef223dd1fac5f102bcf06829e8c713) Co-authored-by: João Duarte --- Gemfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.template b/Gemfile.template index 49471791c..bb6aa79ed 100644 --- a/Gemfile.template +++ b/Gemfile.template @@ -9,7 +9,7 @@ gem "paquet", "~> 0.2" gem "pleaserun", "~>0.0.28", require: false gem "rake", "~> 13", require: false gem "ruby-progressbar", "~> 1", require: false -gem "ruby-maven-libs", "~> 3", ">= 3.8.9" +gem "ruby-maven-libs", "~> 3", ">= 3.9.6.1" gem "logstash-output-elasticsearch", ">= 11.14.0" gem "polyglot", require: false gem "treetop", require: false From e673e0d36ff5e3eb2e5d5929708ae9c6fd3ac3d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 22:23:22 +0100 Subject: [PATCH 08/32] bump lock file for 8.14 (#16132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update patch plugin versions in gemfile lock * Update Gemfile.jruby-3.1.lock.release --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: João Duarte --- Gemfile.jruby-3.1.lock.release | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index 71df9b89e..b48d5268b 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -810,7 +810,7 @@ GEM unicode-display_width (>= 2.4.0, < 3.0) rubocop-ast (1.31.3) parser (>= 3.3.1.0) - ruby-maven-libs (3.8.9) + ruby-maven-libs (3.9.6.1) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyzip (1.3.0) @@ -998,7 +998,7 @@ DEPENDENCIES rake (~> 13) rspec (~> 3.5) rubocop - ruby-maven-libs (~> 3, >= 3.8.9) + ruby-maven-libs (~> 3, >= 3.9.6.1) ruby-progressbar (~> 1) rubyzip (~> 1) simplecov (~> 0.22.0) From 581ac90ec6e55ae5afab9c2a77a4b8284e1d9508 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 18:06:55 -0400 Subject: [PATCH 09/32] Release notes for 8.13.4 (#16144) (#16151) Refined release notes for 8.13.4 --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: andsel (cherry picked from commit 49a65324a8a65d24a63fdc85cc7117fb7df6e222) --- docs/static/releasenotes.asciidoc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index d3a23366f..2d7f5692d 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,7 @@ This section summarizes the changes in the following releases: +* <> * <> * <> * <> @@ -59,6 +60,18 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-13-4]] +=== Logstash 8.13.4 Release Notes + +No user-facing changes in Logstash core. + +==== Plugins + +*Elasticsearch Output - 11.22.6* + +* [DOC] Logstash output.elasticsearch index can be alias or datastream https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1179[#1179] + + [[logstash-8-13-3]] === Logstash 8.13.3 Release Notes From 1401565c48f6a9e3e50be11553168602430f61cc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 May 2024 14:18:55 +0100 Subject: [PATCH 10/32] bump lock file for 8.14 (#16154) * Update minor plugin versions in gemfile lock * Update Gemfile.jruby-3.1.lock.release --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> --- Gemfile.jruby-3.1.lock.release | 54 +++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index b48d5268b..b141f7192 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -45,33 +45,33 @@ GEM avro (1.10.2) multi_json (~> 1) aws-eventstream (1.3.0) - aws-partitions (1.913.0) - aws-sdk-cloudfront (1.88.0) - aws-sdk-core (~> 3, >= 3.191.0) + aws-partitions (1.925.0) + aws-sdk-cloudfront (1.89.0) + aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-cloudwatch (1.88.0) - aws-sdk-core (~> 3, >= 3.191.0) + aws-sdk-cloudwatch (1.90.0) + aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-core (3.191.6) + aws-sdk-core (3.194.2) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.79.0) - aws-sdk-core (~> 3, >= 3.191.0) + aws-sdk-kms (1.80.0) + aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-resourcegroups (1.59.0) - aws-sdk-core (~> 3, >= 3.191.0) + aws-sdk-resourcegroups (1.60.0) + aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-s3 (1.146.1) - aws-sdk-core (~> 3, >= 3.191.0) + aws-sdk-s3 (1.149.1) + aws-sdk-core (~> 3, >= 3.194.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.8) - aws-sdk-sns (1.72.0) - aws-sdk-core (~> 3, >= 3.191.0) + aws-sdk-sns (1.74.0) + aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) - aws-sdk-sqs (1.70.0) - aws-sdk-core (~> 3, >= 3.191.0) + aws-sdk-sqs (1.73.0) + aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) aws-sigv4 (1.8.0) aws-eventstream (~> 1, >= 1.0.2) @@ -81,7 +81,7 @@ GEM belzebuth (0.2.3) childprocess benchmark-ips (2.13.0) - bigdecimal (3.1.7-java) + bigdecimal (3.1.8-java) bindata (2.5.0) buftok (0.2.0) builder (3.2.4) @@ -161,8 +161,8 @@ GEM pleaserun (~> 0.0.29) rexml stud - fugit (1.10.1) - et-orbi (~> 1, >= 1.2.7) + fugit (1.11.0) + et-orbi (~> 1, >= 1.2.11) raabro (~> 1.4) gelfd2 (0.4.1) gem_publisher (1.5.0) @@ -180,7 +180,7 @@ GEM domain_name (~> 0.5) http-form_data (2.3.0) http_parser.rb (0.6.0-java) - i18n (1.14.4) + i18n (1.14.5) concurrent-ruby (~> 1.0) insist (1.0.0) jar-dependencies (0.4.1) @@ -590,7 +590,7 @@ GEM logstash-core-plugin-api (~> 2.0) logstash-mixin-deprecation_logger_support (~> 1.0) manticore (~> 0.8) - logstash-integration-jdbc (5.4.9) + logstash-integration-jdbc (5.4.10) logstash-codec-plain logstash-core (>= 6.5.0) logstash-core-plugin-api (>= 1.60, <= 2.99) @@ -726,13 +726,13 @@ GEM minitar (0.9) msgpack (1.7.2-java) multi_json (1.15.0) - multipart-post (2.4.0) + multipart-post (2.4.1) murmurhash3 (0.1.6-java) mustache (0.99.8) mustermann (2.0.2) ruby2_keywords (~> 0.0.1) naught (1.1.0) - net-imap (0.4.10) + net-imap (0.4.11) date net-protocol net-pop (0.1.2) @@ -741,7 +741,7 @@ GEM timeout net-smtp (0.5.0) net-protocol - nio4r (2.7.1-java) + nio4r (2.7.3-java) nokogiri (1.16.4-java) racc (~> 1.4) octokit (4.25.1) @@ -791,13 +791,13 @@ GEM rspec-expectations (3.13.0) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-mocks (3.13.0) + rspec-mocks (3.13.1) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-support (3.13.1) rspec-wait (0.0.10) rspec (>= 3.0) - rubocop (1.63.4) + rubocop (1.63.5) json (~> 2.3) language_server-protocol (>= 3.17.0) parallel (~> 1.10) @@ -821,7 +821,7 @@ GEM faraday (>= 0.17.3, < 3) semantic_logger (3.4.1) concurrent-ruby (~> 1.0) - sequel (5.79.0) + sequel (5.80.0) bigdecimal simple_oauth (0.3.1) simplecov (0.22.0) From 304b8b25e62a2be218a8f6573086c7e45e11ca5b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 14:24:08 -0400 Subject: [PATCH 11/32] [DOC] Remove reference to puppet LS module (#12356) (#16165) As the module is not maintained since 2018 and it was community supported, I would like to remove it from the documentation. (cherry picked from commit 53d9480176a5bd038aef075e750d840e28f38ed4) Co-authored-by: Luca Belluccini --- docs/static/config-management.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/static/config-management.asciidoc b/docs/static/config-management.asciidoc index 2960f651c..17052e344 100644 --- a/docs/static/config-management.asciidoc +++ b/docs/static/config-management.asciidoc @@ -6,8 +6,7 @@ manage updates to your configuration over time. The topics in this section describe Logstash configuration management features only. For information about other config management tools, such as Puppet and -Chef, see the documentation for those projects. Also take a look at the -https://forge.puppet.com/elastic/logstash[Logstash Puppet module documentation]. +Chef, see the documentation for those projects. :edit_url!: include::management/centralized-pipelines.asciidoc[] From 7277a369a18a53b461a8da34c97ed767c6fc150c Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Tue, 21 May 2024 06:50:35 -0700 Subject: [PATCH 12/32] Handle non-unicode payload in Logstash. (#16072) (#16168) * A logic to handle non-unicode payload in Logstash. Co-authored-by: Ry Biesemeyer * Upgrade jrjackson to 0.4.20 * Code review: simplify the logic with a standard String#encode interface with replace option. Co-authored-by: Ry Biesemeyer --------- Co-authored-by: Ry Biesemeyer Co-authored-by: Ry Biesemeyer (cherry picked from commit 979d30d7016db6f0df8150f794a74d3785199fb2) --- logstash-core/lib/logstash/json.rb | 22 +++++- .../lib/logstash/util/unicode_normalizer.rb | 38 ++++++++++ .../lib/logstash/util/unicode_trimmer.rb | 17 +++++ logstash-core/spec/logstash/json_spec.rb | 74 +++++++++++++++++++ versions.yml | 2 +- 5 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 logstash-core/lib/logstash/util/unicode_normalizer.rb diff --git a/logstash-core/lib/logstash/json.rb b/logstash-core/lib/logstash/json.rb index 905c9cc12..6ffe70e53 100644 --- a/logstash-core/lib/logstash/json.rb +++ b/logstash-core/lib/logstash/json.rb @@ -16,6 +16,7 @@ # under the License. require "logstash/environment" +require "logstash/util/unicode_normalizer" require "jrjackson" module LogStash @@ -32,16 +33,35 @@ module LogStash end def jruby_dump(o, options = {}) + encoding_normalized_data = normalize_encoding(o.dup).freeze + # TODO [guyboertje] remove these comments in 5.0 # test for enumerable here to work around an omission in JrJackson::Json.dump to # also look for Java::JavaUtil::ArrayList, see TODO submit issue # o.is_a?(Enumerable) ? JrJackson::Raw.generate(o) : JrJackson::Json.dump(o) - JrJackson::Base.generate(o, options) + JrJackson::Base.generate(encoding_normalized_data, options) rescue => e raise LogStash::Json::GeneratorError.new(e.message) end alias_method :load, "jruby_load".to_sym alias_method :dump, "jruby_dump".to_sym + + private + def normalize_encoding(data) + case data + when String + LogStash::UnicodeNormalizer.normalize_string_encoding(data) + when Array + data.map { |item| normalize_encoding(item) } + when Hash + # origin key might change when normalizing, so requires transformation + data.to_hash # if coming from jruby objects such as UnmodifiableMap + .transform_keys { |key| normalize_encoding(key) } + .transform_values { |value| normalize_encoding(value) } + else + data # use as it is + end + end end end diff --git a/logstash-core/lib/logstash/util/unicode_normalizer.rb b/logstash-core/lib/logstash/util/unicode_normalizer.rb new file mode 100644 index 000000000..7397a3eb3 --- /dev/null +++ b/logstash-core/lib/logstash/util/unicode_normalizer.rb @@ -0,0 +1,38 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module LogStash + + # A class to normalize the invalid unicode data + class UnicodeNormalizer + + include LogStash::Util::Loggable + + # Tries to normalize input string to UTF-8 when + # input string encoding is not UTF-8, + # and replaces invalid unicode bytes with replacement characters ('uFFFD') + # string_data - The String data to be normalized. + # Returns the normalized string data. + def self.normalize_string_encoding(string_data) + # when given BINARY-flagged string, assume it is UTF-8 so that + # subsequent cleanup retains valid UTF-8 sequences + source_encoding = string_data.encoding + source_encoding = Encoding::UTF_8 if source_encoding == Encoding::BINARY + string_data.encode(Encoding::UTF_8, source_encoding, invalid: :replace, undef: :replace).scrub + end + end +end diff --git a/logstash-core/lib/logstash/util/unicode_trimmer.rb b/logstash-core/lib/logstash/util/unicode_trimmer.rb index ae973d1ff..68468e67e 100644 --- a/logstash-core/lib/logstash/util/unicode_trimmer.rb +++ b/logstash-core/lib/logstash/util/unicode_trimmer.rb @@ -1,3 +1,20 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # encoding: utf-8 module LogStash::Util::UnicodeTrimmer diff --git a/logstash-core/spec/logstash/json_spec.rb b/logstash-core/spec/logstash/json_spec.rb index 8cd4ffb04..a53e8375f 100644 --- a/logstash-core/spec/logstash/json_spec.rb +++ b/logstash-core/spec/logstash/json_spec.rb @@ -118,4 +118,78 @@ describe "LogStash::Json" do o = LogStash::Json.load(" ") expect(o).to be_nil end + + context "Unicode edge-cases" do + matcher :be_utf8 do + match(:notify_expectation_failures => true) do |actual| + aggregate_failures do + expect(actual).to have_attributes(:encoding => Encoding::UTF_8, :valid_encoding? => true) + expect(actual.bytes).to eq(@expected_bytes) unless @expected_bytes.nil? + end + end + chain :with_bytes do |expected_bytes| + @expected_bytes = expected_bytes + end + end + + let(:result) { LogStash::Json::dump(input) } + + context "with valid non-unicode encoding" do + let(:input) { "Th\xEFs \xCCs W\xCFnd\xD8w\x8A".b.force_encoding(Encoding::WINDOWS_1252).freeze } + it 'transcodes to equivalent UTF-8 code-points' do + aggregate_failures do + expect(result).to be_utf8.with_bytes("\u{22}Th\u{EF}s \u{CC}s W\u{CF}nd\u{D8}w\u{160}\u{22}".bytes) + end + end + end + + context "with unicode that has invalid sequences" do + let(:input) { "Thïs is a not-quite-v\xCEalid uni\xF0\x9D\x84code string 💖ok".b.force_encoding(Encoding::UTF_8).freeze } + it 'replaces each invalid sequence with the xFFFD replacement character' do + expect(result).to be_utf8.with_bytes("\x22Thïs is a not-quite-v\u{FFFD}alid uni\u{FFFD}code string 💖ok\x22".bytes) + end + end + + context 'with valid unicode' do + let(:input) { "valid \u{A7}\u{a9c5}\u{18a5}\u{1f984} unicode".encode('UTF-8').freeze } + it 'keeps the unicode in-tact' do + expect(result).to be_utf8.with_bytes(('"' + input + '"').bytes) + end + end + + context 'with binary-flagged input' do + + context 'that contains only lower-ascii' do + let(:input) { "hello, world. This is a test including newline(\x0A) literal-backslash(\x5C) double-quote(\x22)".b.force_encoding(Encoding::BINARY).freeze} + it 'does not munge the bytes' do + expect(result).to be_utf8.with_bytes("\x22hello, world. This is a test including newline(\x5Cn) literal-backslash(\x5C\x5C) double-quote(\x5C\x22)\x22".bytes) + end + end + + context 'that contains bytes outside lower-ascii' do + let(:input) { "Thïs is a not-quite-v\xCEalid uni\xF0\x9D\x84code string 💖ok".b.force_encoding(Encoding::BINARY).freeze } + it 'replaces each invalid sequence with the xFFFD replacement character' do + expect(result).to be_utf8.with_bytes("\x22Thïs is a not-quite-v\u{FFFD}alid uni\u{FFFD}code string 💖ok\x22".bytes) + end + end + + end + + context 'with hash data structure' do + let(:input) {{"Th\xEFs key and".b.force_encoding(Encoding::WINDOWS_1252).freeze => + {"Thïs key also".b.force_encoding(Encoding::UTF_8).freeze => "not-quite-v\xCEalid uni\xF0\x9D\x84code string 💖ok".b.force_encoding(Encoding::UTF_8).freeze}}} + it 'normalizes and replaces each invalid key-value with the xFFFD replacement character' do + expect(result).to be_utf8.with_bytes("{\"Th\u{EF}s key and\":{\"Thïs key also\":\"not-quite-v\u{FFFD}alid uni\u{FFFD}code string 💖ok\"}}".bytes) + end + end + + context 'with array data structure' do + let(:input) {["Th\xEFs entry and".b.force_encoding(Encoding::WINDOWS_1252).freeze, + "Thïs entry also".b.force_encoding(Encoding::UTF_8).freeze, + "not-quite-v\xCEalid uni\xF0\x9D\x84code strings 💖ok".b.force_encoding(Encoding::UTF_8).freeze]} + it 'normalizes and replaces each invalid array values with the xFFFD replacement character' do + expect(result).to be_utf8.with_bytes("[\"Th\u{EF}s entry and\",\"Thïs entry also\",\"not-quite-v\u{FFFD}alid uni\u{FFFD}code strings 💖ok\"]".bytes) + end + end + end end diff --git a/versions.yml b/versions.yml index 431bff8ba..1eeef4e62 100644 --- a/versions.yml +++ b/versions.yml @@ -24,6 +24,6 @@ jruby: # Note: this file is copied to the root of logstash-core because its gemspec needs it when # bundler evaluates the gemspec via bin/logstash # Ensure Jackson version here is kept in sync with version used by jrjackson gem -jrjackson: 0.4.18 +jrjackson: 0.4.20 jackson: 2.15.3 jackson-databind: 2.15.3 From 9cdb8b839f24b52a281f6b9011fc4dc1237f32eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 10:55:53 -0700 Subject: [PATCH 13/32] PQ: avoid blocking writer when precisely full (#16176) (#16178) * pq: avoid blocking writer when queue is precisely full A PQ is considered full (and therefore needs to block before releasing the writer) when its persisted size on disk _exceeds_ its `queue.max_bytes` capacity. This removes an edge-case preemptive block when the persisted size after writing an event _meets_ its `queue.max_bytes` precisely AND its current head page has insufficient room to also accept a hypothetical future event. Fixes: elastic/logstash#16172 * docs: PQ `queue.max_bytes` cannot be less than `queue.page_capacity` (cherry picked from commit ea930861ef414718096b12f481547704581da90c) Co-authored-by: Ry Biesemeyer --- docs/static/persistent-queues.asciidoc | 2 +- .../java/org/logstash/ackedqueue/Queue.java | 20 ++++------- .../org/logstash/ackedqueue/QueueTest.java | 35 ++++++++++++++++++- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/docs/static/persistent-queues.asciidoc b/docs/static/persistent-queues.asciidoc index 51630f942..eb6676f33 100644 --- a/docs/static/persistent-queues.asciidoc +++ b/docs/static/persistent-queues.asciidoc @@ -61,7 +61,7 @@ The total of `queue.max_bytes` for _all_ queues should be lower than the capacity of your disk. + TIP: If you are using persistent queues to protect against data loss, but don't -require much buffering, you can set `queue.max_bytes` to a smaller value. +require much buffering, you can set `queue.max_bytes` to a smaller value as long as it is not less than the value of `queue.page_capacity`. A smaller value produces smaller queues and improves queue performance. `queue.checkpoint.acks`:: Sets the number of acked events before forcing a checkpoint. diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java b/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java index 0a7e4a04c..cb3c4e297 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java @@ -506,18 +506,13 @@ public final class Queue implements Closeable { } /** - *

Checks if the Queue is full, with "full" defined as either of:

- *

Assuming a maximum size of the queue larger than 0 is defined:

+ *

Checks if the Queue is full, with "full" defined as either of: *

    - *
  • The sum of the size of all allocated pages is more than the allowed maximum Queue - * size
    • - *
  • The sum of the size of all allocated pages equal to the allowed maximum Queue size - * and the current head page has no remaining capacity.
    • - *
- *

or assuming a max unread count larger than 0, is defined "full" is also defined as:

- *
    - *
  • The current number of unread events exceeds or is equal to the configured maximum - * number of allowed unread events.
    • + *
  • {@code maxUnread} is non-zero and is met or exceeded by the total + * number of unread events on all pages ({@code unreadCount})
    • + *
  • {@code maxBytes} is non-zero and is exceeded by the sum of the sizes + * of all allocated tail pages plus the portion of the current head page + * containing events
    • *
* @return True iff the queue is full */ @@ -535,8 +530,7 @@ public final class Queue implements Closeable { return false; } - final long persistedByteSize = getPersistedByteSize(); - return ((persistedByteSize > this.maxBytes) || (persistedByteSize == this.maxBytes && !this.headPage.hasSpace(1))); + return getPersistedByteSize() > this.maxBytes; } private boolean isMaxUnreadReached() { diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/QueueTest.java b/logstash-core/src/test/java/org/logstash/ackedqueue/QueueTest.java index 25bca9fc7..d33781187 100644 --- a/logstash-core/src/test/java/org/logstash/ackedqueue/QueueTest.java +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/QueueTest.java @@ -519,9 +519,12 @@ public class QueueTest { public void reachMaxSizeTest() throws IOException, InterruptedException { Queueable element = new StringElement("0123456789"); // 10 bytes + int pageSize = computeCapacityForMmapPageIO(element, 10); + int queueMaxSize = (pageSize * 10) - 1; // 100th will overflow max capacity while still on 10th page + // allow 10 elements per page but only 100 events in total Settings settings = TestSettings.persistedQueueSettings( - computeCapacityForMmapPageIO(element, 10), computeCapacityForMmapPageIO(element, 100), dataPath + pageSize, queueMaxSize, dataPath ); try (Queue q = new Queue(settings)) { q.open(); @@ -542,6 +545,36 @@ public class QueueTest { } } + @Test(timeout = 50_000) + public void preciselyMaxSizeTest() throws IOException, InterruptedException { + Queueable element = new StringElement("0123456789"); // 10 bytes + + int pageSize = computeCapacityForMmapPageIO(element, 10); + int queueMaxSize = (pageSize * 10); // 100th will precisely fit max capacity + + // allow 10 elements per page but only 100 events in total + Settings settings = TestSettings.persistedQueueSettings( + pageSize, queueMaxSize, dataPath + ); + try (Queue q = new Queue(settings)) { + q.open(); + + int elementCount = 100; // should be able to write 100 events before getting full + for (int i = 0; i < elementCount; i++) { + q.write(element); + } + + assertThat(q.isFull(), is(false)); + + // we expect this next write call to block so let's wrap it in a Future + executor.submit(() -> q.write(element)); + while (!q.isFull()) { + Thread.sleep(10); + } + assertThat(q.isFull(), is(true)); + } + } + @Test(timeout = 50_000) public void ackingMakesQueueNotFullAgainTest() throws IOException, InterruptedException, ExecutionException { From 9878e2737a5516394bf19f9dca0e6a67d16dd45d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Fri, 31 May 2024 11:28:03 +0100 Subject: [PATCH 14/32] Upgrade nokogiri to 1.16.5 (#16188) --- Gemfile.jruby-3.1.lock.release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index b141f7192..ac5e5b9a9 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -742,7 +742,7 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.3-java) - nokogiri (1.16.4-java) + nokogiri (1.16.5-java) racc (~> 1.4) octokit (4.25.1) faraday (>= 1, < 3) From 78fb3792822170c393561fe271c51c62c20b3051 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 19:11:05 -0400 Subject: [PATCH 15/32] Release notes for 8.14.0 (#16155) Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/static/releasenotes.asciidoc | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index 2d7f5692d..e119ddc3f 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,7 @@ This section summarizes the changes in the following releases: +* <> * <> * <> * <> @@ -60,6 +61,44 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-14-0]] +=== Logstash 8.14.0 Release Notes +[[featured-8-14-0]] +==== Announcing the new SNMP integration plugin (Technical Preview) + +The new {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[`logstash-integration-snmp`] plugin is now available in _Technical Preview_, and can be installed on {ls} 8.13. + +experimental[] + +The `logstash-integration-snmp` plugin combines our +classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0. +Current 1.x versions of the `input-snmp` plugin are bundled with {ls} by default, and will soon be replaced by the 4.0.0+ version contained in this new integration. + +If you want to try the new `integration-snmp` plugin while it is in Technical Preview, run `bin/logstash-plugin install logstash-integration-snmp`. + +IMPORTANT: Before you install the new integration, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between current stand-alone plugins and the new versions included in `integration-snmp`. + +[[notable-8.14.0]] +==== Enhancements and notable issues fixed + +* Fixed a bug that created duplicated `io.netty.allocator.maxOrder` system property when using environment variable `LS_JAVA_OPTS` in Docker https://github.com/elastic/logstash/pull/16079[#16079] + +==== Plugins + +*Jdbc Integration - 5.4.10* + +* [DOC] Added database-specific considerations https://github.com/logstash-plugins/logstash-integration-jdbc/pull/167[#167] + +*Kafka Integration - 11.4.1* + +* Added `message_headers` option to set headers of record for Kafka output https://github.com/logstash-plugins/logstash-integration-kafka/pull/162[#162] + +[[dependencies-8.14.0]] +==== Updates to dependencies + +* Update JRuby to 9.4.7.0 https://github.com/elastic/logstash/pull/16125[#16125] + + [[logstash-8-13-4]] === Logstash 8.13.4 Release Notes From 7afb42c13adb00709fbce7f5fd37953015633b03 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Wed, 5 Jun 2024 09:15:37 -0400 Subject: [PATCH 16/32] Bump version to 8.14.1 (#16196) --- Gemfile.jruby-3.1.lock.release | 4 ++-- versions.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index ac5e5b9a9..df8d4c94c 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -2,12 +2,12 @@ PATH remote: logstash-core-plugin-api specs: logstash-core-plugin-api (2.1.16-java) - logstash-core (= 8.14.0) + logstash-core (= 8.14.1) PATH remote: logstash-core specs: - logstash-core (8.14.0-java) + logstash-core (8.14.1-java) clamp (~> 1) concurrent-ruby (~> 1, < 1.1.10) down (~> 5.2.0) diff --git a/versions.yml b/versions.yml index 1eeef4e62..b78c7446f 100644 --- a/versions.yml +++ b/versions.yml @@ -1,7 +1,7 @@ --- # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var -logstash: 8.14.0 -logstash-core: 8.14.0 +logstash: 8.14.1 +logstash-core: 8.14.1 logstash-core-plugin-api: 2.1.16 bundled_jdk: From 550e935835e640152d5e962196f5b0dbb5f018f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 20:29:56 +0100 Subject: [PATCH 17/32] Revert PR #16050 (#16203) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR was created to skip resolving environment variable references in comments present in the “config.string” pipelines defined in the pipelines.yml file. However it introduced a bug that no longer resolves env var references in values of settings like pipeline.batch.size or queue.max_bytes. For now we’ll revert this PR and create a fix that handles both problems. (cherry picked from commit efa83787a565064fbf2f4327741455f8863aefe2) Co-authored-by: Andrea Selva --- .../lib/logstash/config/source/multi_local.rb | 2 +- .../config/source/multi_local_spec.rb | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/logstash-core/lib/logstash/config/source/multi_local.rb b/logstash-core/lib/logstash/config/source/multi_local.rb index 2a523e850..14697835c 100644 --- a/logstash-core/lib/logstash/config/source/multi_local.rb +++ b/logstash-core/lib/logstash/config/source/multi_local.rb @@ -30,7 +30,7 @@ module LogStash module Config module Source end def pipeline_configs - pipelines = retrieve_yaml_pipelines + pipelines = deep_replace(retrieve_yaml_pipelines) pipelines_settings = pipelines.map do |pipeline_settings| clone = @original_settings.clone clone.merge_pipeline_settings(pipeline_settings) diff --git a/logstash-core/spec/logstash/config/source/multi_local_spec.rb b/logstash-core/spec/logstash/config/source/multi_local_spec.rb index 2d7747da5..a8e9b4a37 100644 --- a/logstash-core/spec/logstash/config/source/multi_local_spec.rb +++ b/logstash-core/spec/logstash/config/source/multi_local_spec.rb @@ -146,16 +146,17 @@ describe LogStash::Config::Source::MultiLocal do describe "#pipeline_configs" do - let(:config_string) { - "input { - udp { - port => 5555 # intentional comment contains \"${UDP_DEV_PORT}\" variable, shouldn't break functionalities - host => \"127.0.0.1\" - } - # another intentional comment contains \"${UDP_PROD_HOST}\" variable, shouldn't break functionalities - } - output {}" - } +# let(:config_string) { +# "input { +# udp { +# port => 5555 # intentional comment contains \"${UDP_DEV_PORT}\" variable, shouldn't break functionalities +# host => \"127.0.0.1\" +# } +# # another intentional comment contains \"${UDP_PROD_HOST}\" variable, shouldn't break functionalities +# } +# output {}" +# } + let(:config_string) { "input {} output {}" } let(:retrieved_pipelines) do [ { "pipeline.id" => "main", "config.string" => config_string }, From 224421f3e95e5440fb27387e3f0093d79a5fbe55 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 18:01:22 +0200 Subject: [PATCH 18/32] Pin rexml gem version to 3.2.6 (#16209) (#16210) This commit pinned the `rexml` gem version to `3.2.6` (cherry picked from commit 23221caddb8e34beea3d7aa4737bf5b2b396214f) Co-authored-by: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> --- Gemfile.template | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile.template b/Gemfile.template index bb6aa79ed..f7141767f 100644 --- a/Gemfile.template +++ b/Gemfile.template @@ -25,6 +25,7 @@ gem "rubocop", :group => :development gem "belzebuth", :group => :development gem "benchmark-ips", :group => :development gem "ci_reporter_rspec", "~> 1", :group => :development +gem "rexml", "3.2.6", :group => :development gem "flores", "~> 0.0.8", :group => :development gem "json-schema", "~> 2", :group => :development gem "logstash-devutils", "~> 2.6.0", :group => :development From 8fa13707fafb5a76bcbecc2f416b0ab8f18c6c2b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:52:24 -0700 Subject: [PATCH 19/32] Update patch plugin versions in gemfile lock (#16211) Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> --- Gemfile.jruby-3.1.lock.release | 41 ++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index df8d4c94c..9c8c3c2f5 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -15,7 +15,7 @@ PATH filesize (~> 0.2) gems (~> 1) i18n (~> 1) - jrjackson (= 0.4.18) + jrjackson (= 0.4.20) jruby-openssl (~> 0.14.1) manticore (~> 0.6) minitar (~> 0.8) @@ -115,12 +115,13 @@ GEM elastic-transport (8.3.2) faraday (< 3) multi_json - elasticsearch (7.17.10) - elasticsearch-api (= 7.17.10) - elasticsearch-transport (= 7.17.10) - elasticsearch-api (7.17.10) + elasticsearch (7.17.11) + elasticsearch-api (= 7.17.11) + elasticsearch-transport (= 7.17.11) + elasticsearch-api (7.17.11) multi_json - elasticsearch-transport (7.17.10) + elasticsearch-transport (7.17.11) + base64 faraday (>= 1, < 3) multi_json equalizer (0.0.11) @@ -176,7 +177,7 @@ GEM http-cookie (~> 1.0) http-form_data (~> 2.0) http_parser.rb (~> 0.6.0) - http-cookie (1.0.5) + http-cookie (1.0.6) domain_name (~> 0.5) http-form_data (2.3.0) http_parser.rb (0.6.0-java) @@ -189,11 +190,11 @@ GEM jls-lumberjack (0.0.26) concurrent-ruby jmespath (1.6.2) - jrjackson (0.4.18-java) + jrjackson (0.4.20-java) jruby-jms (1.3.0-java) gene_pool semantic_logger - jruby-openssl (0.14.5-java) + jruby-openssl (0.14.6-java) jruby-stdin-channel (0.2.0-java) json (2.7.2-java) json-schema (2.8.1) @@ -319,7 +320,7 @@ GEM lru_redux (~> 1.1.0) logstash-filter-drop (3.0.5) logstash-core-plugin-api (>= 1.60, <= 2.99) - logstash-filter-elastic_integration (0.1.8-java) + logstash-filter-elastic_integration (0.1.9-java) logstash-core (>= 8.7.0) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-filter-elasticsearch (3.16.1) @@ -401,7 +402,7 @@ GEM logstash-core-plugin-api (>= 1.60, <= 2.99) nokogiri (>= 1.13.8) xml-simple - logstash-input-azure_event_hubs (1.4.5) + logstash-input-azure_event_hubs (1.4.7) logstash-codec-json logstash-codec-plain logstash-core-plugin-api (~> 2.0) @@ -590,7 +591,7 @@ GEM logstash-core-plugin-api (~> 2.0) logstash-mixin-deprecation_logger_support (~> 1.0) manticore (~> 0.8) - logstash-integration-jdbc (5.4.10) + logstash-integration-jdbc (5.4.11) logstash-codec-plain logstash-core (>= 6.5.0) logstash-core-plugin-api (>= 1.60, <= 2.99) @@ -602,7 +603,7 @@ GEM sequel (>= 5.74.0) tzinfo tzinfo-data - logstash-integration-kafka (11.4.1-java) + logstash-integration-kafka (11.4.2-java) logstash-codec-json logstash-codec-plain logstash-core (>= 8.3.0) @@ -668,7 +669,7 @@ GEM logstash-core-plugin-api (>= 2.0.0, < 2.99) logstash-output-graphite (3.1.6) logstash-core-plugin-api (>= 1.60, <= 2.99) - logstash-output-http (5.6.0) + logstash-output-http (5.6.1) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-mixin-http_client (>= 7.3.0, < 8.0.0) logstash-output-lumberjack (3.1.9) @@ -691,7 +692,7 @@ GEM logstash-output-stdout (3.1.4) logstash-codec-rubydebug logstash-core-plugin-api (>= 1.60.1, < 2.99) - logstash-output-tcp (6.2.0) + logstash-output-tcp (6.2.1) jruby-openssl (>= 0.12.2) logstash-codec-json logstash-core (>= 8.1.0) @@ -732,7 +733,7 @@ GEM mustermann (2.0.2) ruby2_keywords (~> 0.0.1) naught (1.1.0) - net-imap (0.4.11) + net-imap (0.4.12) date net-protocol net-pop (0.1.2) @@ -750,7 +751,7 @@ GEM openssl_pkcs8_pure (0.0.0.2) paquet (0.2.1) parallel (1.24.0) - parser (3.3.1.0) + parser (3.3.2.0) ast (~> 2.4.1) racc pleaserun (0.0.32) @@ -780,7 +781,7 @@ GEM rainbow (3.1.1) rake (13.2.1) redis (4.8.1) - regexp_parser (2.9.0) + regexp_parser (2.9.2) rexml (3.2.6) rspec (3.13.0) rspec-core (~> 3.13.0) @@ -867,7 +868,7 @@ GEM unicode-display_width (2.5.0) webhdfs (0.11.0) addressable - webmock (3.23.0) + webmock (3.23.1) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) @@ -876,6 +877,7 @@ GEM PLATFORMS java + universal-java-11 DEPENDENCIES belzebuth @@ -996,6 +998,7 @@ DEPENDENCIES polyglot rack-test rake (~> 13) + rexml (= 3.2.6) rspec (~> 3.5) rubocop ruby-maven-libs (~> 3, >= 3.9.6.1) From f9d6b42a7eeae05b5e36236fb56235b12cbd1618 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:13:59 -0700 Subject: [PATCH 20/32] Release notes for 8.14.1 (#16212) * Update release notes for 8.14.1 * Snip generated context * Manually fill release notes for Elastic Integration filter * Reword release notes from core to be user-centric --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: Ry Biesemeyer --- docs/static/releasenotes.asciidoc | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index e119ddc3f..c877c5bce 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,7 @@ This section summarizes the changes in the following releases: +* <> * <> * <> * <> @@ -61,6 +62,45 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-14-1]] +=== Logstash 8.14.1 Release Notes + +* Fixes JSON serialization of payloads that are not UTF-8, eliminating an issue where the Elasticsearch Output could get stuck in a retry loop (#16072) https://github.com/elastic/logstash/pull/16168[#16168] +* Fixes Persistent Queue bug in which a PQ configured with `queue.max_bytes` equal to its `queue.page_capacity` could become permanently blocked when _precisely_ full https://github.com/elastic/logstash/pull/16178[#16178] +* Fixes a regression in multi-local pipeline loader that caused variable-references in a configured `pipelines.yml` to not be replaced by their values in the environment and/or keystore https://github.com/elastic/logstash/pull/16201[#16201] + + +==== Plugins + +*Elastic_integration Filter - 0.1.9* + + - [DOC] Removes Tech Preview label and adds link to extending integrations topic in LSR https://github.com/elastic/logstash-filter-elastic_integration/pull/142[#142] + + +*Azure_event_hubs Input - 1.4.7* + +* [DOCS] Clarify examples for single and multiple event hubs https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/90[#90] + +* [DOCS] Add outbound port requirements for Event Hub https://github.com/logstash-plugins/logstash-input-azure_event_hubs/pull/88[#88] + +*Jdbc Integration - 5.4.11* + +* Fixes an issue in which any one instance of a JDBC input plugin using `jdbc_default_timezone` changes the behaviour of plugin instances that do _not_ use `jdbc_default_timezone`, ensuring that timezone offsets remain consistent for each instance of the plugin _as configured_ https://github.com/logstash-plugins/logstash-integration-jdbc/pull/151[#151] +* Fixes an exception that could occur while reloading `jdbc_static` databases when the underlying connection to the remote has been broken https://github.com/logstash-plugins/logstash-integration-jdbc/pull/165[#165] + +*Kafka Integration - 11.4.2* + +* Add default client_id of logstash to kafka output https://github.com/logstash-plugins/logstash-integration-kafka/pull/169[#169] + +*Http Output - 5.6.1* + +* Added body logging for non 2xx responses https://github.com/logstash-plugins/logstash-output-http/pull/142[#142] + +*Tcp Output - 6.2.1* + +* Document correct default plugin codec https://github.com/logstash-plugins/logstash-output-tcp/pull/54[#54] + + [[logstash-8-14-0]] === Logstash 8.14.0 Release Notes [[featured-8-14-0]] From 8949dc77b6e9a4dc61428e053efdc5590bdcd08a Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:45:47 +0200 Subject: [PATCH 21/32] Bump 8.14.2 (#16216) Bump 8.14.2 --- Gemfile.jruby-3.1.lock.release | 4 ++-- versions.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index 9c8c3c2f5..fd4fc606b 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -2,12 +2,12 @@ PATH remote: logstash-core-plugin-api specs: logstash-core-plugin-api (2.1.16-java) - logstash-core (= 8.14.1) + logstash-core (= 8.14.2) PATH remote: logstash-core specs: - logstash-core (8.14.1-java) + logstash-core (8.14.2-java) clamp (~> 1) concurrent-ruby (~> 1, < 1.1.10) down (~> 5.2.0) diff --git a/versions.yml b/versions.yml index b78c7446f..a69b0f71d 100644 --- a/versions.yml +++ b/versions.yml @@ -1,7 +1,7 @@ --- # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var -logstash: 8.14.1 -logstash-core: 8.14.1 +logstash: 8.14.2 +logstash-core: 8.14.2 logstash-core-plugin-api: 2.1.16 bundled_jdk: From 0ba5330b5f8176c34bea807e04cf6b02fbf7a679 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:47:52 -0700 Subject: [PATCH 22/32] Geoip database management cache invalidation (#16222) (#16223) * geoip: failing specs demonstrating elastic/logstash#16221 * geoip: invalidate cached db state when receiving updates/expiries (cherry picked from commit 801f0f441ea336277913c505cca9ebb3944b9c61) Co-authored-by: Ry Biesemeyer --- x-pack/lib/filters/geoip/database_manager.rb | 20 +++++++---- .../filters/geoip/database_manager_spec.rb | 35 +++++++++++++++---- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/x-pack/lib/filters/geoip/database_manager.rb b/x-pack/lib/filters/geoip/database_manager.rb index 9e8f7605f..1bdd9dc7c 100644 --- a/x-pack/lib/filters/geoip/database_manager.rb +++ b/x-pack/lib/filters/geoip/database_manager.rb @@ -136,17 +136,23 @@ module LogStash module Filters module Geoip class DatabaseManager end def update!(database_type, updated_db_info) - new_database_path = updated_db_info.path - notify_plugins(database_type, :update, new_database_path) do |db_type, ids| - logger.info("geoip filter plugin will use database #{new_database_path}", - :database_type => db_type, :pipeline_ids => ids) unless ids.empty? + @trigger_lock.synchronize do + new_database_path = updated_db_info.path + @states[database_type].database_path = new_database_path + notify_plugins(database_type, :update, new_database_path) do |db_type, ids| + logger.info("geoip filter plugin will use database #{new_database_path}", + :database_type => db_type, :pipeline_ids => ids) unless ids.empty? + end end end def expire!(database_type) - notify_plugins(database_type, :expire) do |db_type, ids| - logger.warn("geoip filter plugin will stop filtering and will tag all events with the '_geoip_expired_database' tag.", - :database_type => db_type, :pipeline_ids => ids) + @trigger_lock.synchronize do + @states[database_type].database_path = nil + notify_plugins(database_type, :expire) do |db_type, ids| + logger.warn("geoip filter plugin will stop filtering and will tag all events with the '_geoip_expired_database' tag.", + :database_type => db_type, :pipeline_ids => ids) + end end end diff --git a/x-pack/spec/filters/geoip/database_manager_spec.rb b/x-pack/spec/filters/geoip/database_manager_spec.rb index f30c584eb..67b2e0d94 100644 --- a/x-pack/spec/filters/geoip/database_manager_spec.rb +++ b/x-pack/spec/filters/geoip/database_manager_spec.rb @@ -8,11 +8,14 @@ require "filters/geoip/database_manager" describe LogStash::Filters::Geoip do describe 'DatabaseManager', :aggregate_failures do let(:pipeline_id) { SecureRandom.hex(16) } - let(:mock_geoip_plugin) do - double("LogStash::Filters::Geoip").tap do |c| - allow(c).to receive(:execution_context).and_return(double("EC", pipeline_id: pipeline_id)) - allow(c).to receive(:update_filter).with(anything) - end + let(:mock_geoip_plugin) { mock_geoip_plugin_factory.call } + let(:mock_geoip_plugin_factory) do + ->() { + double("LogStash::Filters::Geoip").tap do |c| + allow(c).to receive(:execution_context).and_return(double("EC", pipeline_id: pipeline_id)) + allow(c).to receive(:update_filter).with(anything) + end + } end let(:eula_database_infos) { Hash.new { LogStash::GeoipDatabaseManagement::DbInfo::PENDING } } @@ -119,10 +122,19 @@ describe LogStash::Filters::Geoip do shared_examples "subscribed to expire notifications" do context "when the manager expires the db" do - it "notifies the plugin" do + before(:each) do db_manager.eula_subscription("City").notify(LogStash::GeoipDatabaseManagement::DbInfo::EXPIRED) + end + it "notifies the plugin" do expect(mock_geoip_plugin).to have_received(:update_filter).with(:expire) end + context "subsequent subscriptions" do + it "are given the nil path" do + plugin2 = mock_geoip_plugin_factory.call + path2 = db_manager.subscribe_database_path("City", nil, plugin2) + expect(path2).to be_nil + end + end end context "when the manager expires a different DB" do it 'does not notify the plugin' do @@ -135,10 +147,19 @@ describe LogStash::Filters::Geoip do shared_examples "subscribed to update notifications" do context "when the manager updates the db" do let(:updated_db_path) { "/this/that/another.mmdb" } - it "notifies the plugin" do + before(:each) do db_manager.eula_subscription("City").notify(LogStash::GeoipDatabaseManagement::DbInfo.new(path: updated_db_path)) + end + it "notifies the plugin" do expect(mock_geoip_plugin).to have_received(:update_filter).with(:update, updated_db_path) end + context "subsequent subscriptions" do + it "are given the updated path" do + plugin2 = mock_geoip_plugin_factory.call + path2 = db_manager.subscribe_database_path("City", nil, plugin2) + expect(path2).to eq updated_db_path + end + end end context "when the manager updates a different DB" do let(:updated_db_path) { "/this/that/another.mmdb" } From a42c71a90497005a99d971633419cd5221f0090f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 20 Jun 2024 11:46:32 -0700 Subject: [PATCH 23/32] Avoid to log file not found errors when DLQ segments are removed concurrently between writer and reader. (#16204) (#16249) * Rework the logic to delete DLQ eldest segments to be more resilient on file not found errors and avoid to log warn messages that there isn't any action the user can do to solve. * Fixed test case, when path point to a file that doesn't exist, rely always on path name comparator. Reworked the code to simplify, not needing anymore the tri-state variable (cherry picked from commit 321e407e537c16ad32b69be844ebfc1f0aea02d5) Co-authored-by: Andrea Selva --- .../common/io/DeadLetterQueueReader.java | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/logstash-core/src/main/java/org/logstash/common/io/DeadLetterQueueReader.java b/logstash-core/src/main/java/org/logstash/common/io/DeadLetterQueueReader.java index e76da789b..b445406ed 100644 --- a/logstash-core/src/main/java/org/logstash/common/io/DeadLetterQueueReader.java +++ b/logstash-core/src/main/java/org/logstash/common/io/DeadLetterQueueReader.java @@ -374,24 +374,32 @@ public final class DeadLetterQueueReader implements Closeable { } private int compareByFileTimestamp(Path p1, Path p2) { - FileTime timestamp1; - // if one of the getLastModifiedTime raise an error, consider them equals - // and fallback to the other comparator - try { - timestamp1 = Files.getLastModifiedTime(p1); - } catch (IOException ex) { - logger.warn("Error reading file's timestamp for {}", p1, ex); + final Optional timestampResult1 = readLastModifiedTime(p1); + final Optional timestampResult2 = readLastModifiedTime(p2); + + // if one of the readLastModifiedTime encountered a file not found error or generic IO error, + // consider them equals and fallback to the other comparator + if (!timestampResult1.isPresent() || !timestampResult2.isPresent()) { return 0; } - FileTime timestamp2; + return timestampResult1.get().compareTo(timestampResult2.get()); + } + + /** + * Builder method. Read the timestamp if file on path p is present. + * When the file + * */ + private static Optional readLastModifiedTime(Path p) { try { - timestamp2 = Files.getLastModifiedTime(p2); + return Optional.of(Files.getLastModifiedTime(p)); + } catch (NoSuchFileException fileNotFoundEx) { + logger.debug("File {} doesn't exist", p); + return Optional.empty(); } catch (IOException ex) { - logger.warn("Error reading file's timestamp for {}", p2, ex); - return 0; + logger.warn("Error reading file's timestamp for {}", p, ex); + return Optional.empty(); } - return timestamp1.compareTo(timestamp2); } /** @@ -407,6 +415,9 @@ public final class DeadLetterQueueReader implements Closeable { Files.delete(segment); logger.debug("Deleted segment {}", segment); return Optional.of(eventsInSegment); + } catch (NoSuchFileException fileNotFoundEx) { + logger.debug("Expected file segment {} was already removed by a writer", segment); + return Optional.empty(); } catch (IOException ex) { logger.warn("Problem occurred in cleaning the segment {} after a repositioning", segment, ex); return Optional.empty(); From a5b7d2bfadf2405948f87bcba33bdd93894e650b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 12:21:13 -0700 Subject: [PATCH 24/32] json: remove unnecessary dup/freeze in serialization (#16213) (#16253) (cherry picked from commit 92909cb1c49ebff9868657515188b491892c6ccf) Co-authored-by: Ry Biesemeyer --- logstash-core/lib/logstash/json.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-core/lib/logstash/json.rb b/logstash-core/lib/logstash/json.rb index 6ffe70e53..9a441d4ec 100644 --- a/logstash-core/lib/logstash/json.rb +++ b/logstash-core/lib/logstash/json.rb @@ -33,7 +33,7 @@ module LogStash end def jruby_dump(o, options = {}) - encoding_normalized_data = normalize_encoding(o.dup).freeze + encoding_normalized_data = normalize_encoding(o) # TODO [guyboertje] remove these comments in 5.0 # test for enumerable here to work around an omission in JrJackson::Json.dump to From ff37802a77a905a63395afda2fea78198fd3a205 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:46:55 -0700 Subject: [PATCH 25/32] Unicode pipeline and plugin ids (#15971) (#16257) * fix: restore support for unicode pipeline- and plugin-id's JRuby's `Ruby#newSymbol(String)` throws an exception when provided a `String` that contains characters outside of lower-ASCII because JRuby internals expect "the incoming String to be one of our mangled ISO-8859-1 strings" as noted in a comment on jruby/jruby#6217. Instead, we use `Ruby#newString(String)` to create a new `RubyString` (which works properly), and then rely on `RubyString#intern` to get our `RubySymbol`. This fixes a regression introduced in the 8.7 series in which pipeline id's are consistently represented as ruby symbols in the metrics store, and ensures similar issue does not exist when specifying a plugin id that contains characters above the lower-ASCII plane. * fix: use properly-encoded RubySymbol in PipelineConfig We cannot rely on `RubySymbol#toString` to produce a properly-encoded `String` whe the string contains characters above the lower-ASCII plane because the result is effectively a binary ruby-internal marshal of the bytes that only holds when the symbol contains lower-ASCII. Instead, we can use the internally-memoizing `RubySymbol#name` to get a properly-encoded `RubyString`, and `RubyString#asJavaString()` to get a properly-encoded java-`String`. * fix: properly serialize unicode pipeline names in API output Jackson's JSON serializer leaks the JRuby-internal byte structure of Symbols, which only aligns with the byte-structure of the symbol's actual string when that string is wholly-comprised of lower-ASCII characters. By pre-converting Symbols to Strings, we ensure that the result is readable and useful. * spec: bypass monitoring specs for unicode pipeline ids when PQ enabled (cherry picked from commit 0ec16ca39825d3e01fca4120fe2ef88775af5798) Co-authored-by: Ry Biesemeyer --- logstash-core/build.gradle | 1 + logstash-core/lib/logstash/api/app_helpers.rb | 12 +- .../logstash/config/ir/ConfigCompiler.java | 2 +- .../logstash/config/ir/PipelineConfig.java | 2 +- .../compiler/AbstractOutputDelegatorExt.java | 2 +- .../ir/compiler/JavaFilterDelegatorExt.java | 2 +- .../ir/compiler/JavaInputDelegatorExt.java | 2 +- .../execution/AbstractPipelineExt.java | 12 +- .../ext/JRubyWrappedWriteClientExt.java | 2 +- .../logstash/plugins/CounterMetricImpl.java | 2 +- .../plugins/NamespacedMetricImpl.java | 4 +- .../org/logstash/plugins/RootMetricImpl.java | 4 +- .../plugins/factory/PluginFactoryExt.java | 2 +- .../factory/PluginMetricsFactoryExt.java | 2 +- .../config/ir/PipelineConfigTest.java | 2 +- .../ir/compiler/OutputDelegatorTest.java | 2 +- .../ir/compiler/PluginDelegatorTestCase.java | 8 +- .../instrument/metrics/MetricExtFactory.java | 2 +- .../org/logstash/plugins/MetricTestCase.java | 6 +- .../plugins/NamespacedMetricImplTest.java | 32 +- qa/integration/services/logstash_service.rb | 18 +- qa/integration/specs/monitoring_api_spec.rb | 321 ++++++++++++------ 22 files changed, 284 insertions(+), 158 deletions(-) diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index fe758d3a3..aa2dccea2 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -140,6 +140,7 @@ jacocoTestReport { javaTests.finalizedBy(jacocoTestReport) tasks.register("rubyTests", Test) { + dependsOn compileTestJava inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/spec") systemProperty 'logstash.root.dir', projectDir.parent diff --git a/logstash-core/lib/logstash/api/app_helpers.rb b/logstash-core/lib/logstash/api/app_helpers.rb index 953043296..6df4cc706 100644 --- a/logstash-core/lib/logstash/api/app_helpers.rb +++ b/logstash-core/lib/logstash/api/app_helpers.rb @@ -47,7 +47,7 @@ module LogStash::Api::AppHelpers end content_type "application/json" - LogStash::Json.dump(data, {:pretty => pretty?}) + LogStash::Json.dump(stringify_symbols(data), {:pretty => pretty?}) else content_type "text/plain" data.to_s @@ -81,6 +81,16 @@ module LogStash::Api::AppHelpers params.has_key?("pretty") end + # Recursively stringify symbols in the provided data structure + def stringify_symbols(data) + case data + when Hash then Hash[data.each_pair.map { |k,v| [stringify_symbols(k), stringify_symbols(v)] }] + when Array then data.map { |v| stringify_symbols(v) } + when Symbol then data.to_s.encode('UTF-8') + else data + end + end + def generate_error_hash(error) { :path => request.path, diff --git a/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java b/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java index e1c6e4e61..f722d8404 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/ConfigCompiler.java @@ -105,7 +105,7 @@ public final class ConfigCompiler { } private static Statement readStatementFromRubyHash(RubyHash hash, String key) { - IRubyObject inputValue = hash.fastARef(RubyUtil.RUBY.newSymbol(key)); + IRubyObject inputValue = hash.fastARef(RubyUtil.RUBY.newString(key).intern()); return inputValue.toJava(Statement.class); } diff --git a/logstash-core/src/main/java/org/logstash/config/ir/PipelineConfig.java b/logstash-core/src/main/java/org/logstash/config/ir/PipelineConfig.java index 8767064c5..cc2508757 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/PipelineConfig.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/PipelineConfig.java @@ -73,7 +73,7 @@ public final class PipelineConfig { RubyArray.newArray(RUBY, uncastedConfigParts); this.source = source; - this.pipelineId = pipelineId.toString(); + this.pipelineId = pipelineId.name(RUBY.getCurrentContext()).asJavaString(); SourceWithMetadata[] castedConfigParts = (SourceWithMetadata[]) configParts.toJava(SourceWithMetadata[].class); List confParts = Arrays.asList(castedConfigParts); confParts.sort(Comparator.comparing(SourceWithMetadata::getProtocol) diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java index bb2063bad..88728f531 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java @@ -128,7 +128,7 @@ public abstract class AbstractOutputDelegatorExt extends RubyObject { final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); this.id = RubyString.newString(context.runtime, id); synchronized (metric) { - namespacedMetric = metric.namespace(context, context.runtime.newSymbol(id)); + namespacedMetric = metric.namespace(context, context.runtime.newString(id).intern()); metricEvents = namespacedMetric.namespace(context, MetricKeys.EVENTS_KEY); namespacedMetric.gauge(context, MetricKeys.NAME_KEY, configName(context)); eventMetricOut = LongCounter.fromRubyBase(metricEvents, MetricKeys.OUT_KEY); diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/JavaFilterDelegatorExt.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/JavaFilterDelegatorExt.java index 85c40dc4f..1431e681d 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/JavaFilterDelegatorExt.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/JavaFilterDelegatorExt.java @@ -65,7 +65,7 @@ public class JavaFilterDelegatorExt extends AbstractFilterDelegatorExt { new JavaFilterDelegatorExt(RubyUtil.RUBY, RubyUtil.JAVA_FILTER_DELEGATOR_CLASS); instance.configName = RubyUtil.RUBY.newString(configName); AbstractNamespacedMetricExt scopedMetric = - metric.namespace(RubyUtil.RUBY.getCurrentContext(), RubyUtil.RUBY.newSymbol(filter.getId())); + metric.namespace(RubyUtil.RUBY.getCurrentContext(), RubyUtil.RUBY.newString(filter.getId()).intern()); instance.initMetrics(id, scopedMetric); instance.filter = filter; instance.initializeFilterMatchListener(pluginArgs); diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/JavaInputDelegatorExt.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/JavaInputDelegatorExt.java index 28a52b0b2..9e7bf0d13 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/JavaInputDelegatorExt.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/JavaInputDelegatorExt.java @@ -60,7 +60,7 @@ public class JavaInputDelegatorExt extends RubyObject { final Map pluginArgs) { final JavaInputDelegatorExt instance = new JavaInputDelegatorExt(RubyUtil.RUBY, RubyUtil.JAVA_INPUT_DELEGATOR_CLASS); - AbstractNamespacedMetricExt scopedMetric = metric.namespace(RubyUtil.RUBY.getCurrentContext(), RubyUtil.RUBY.newSymbol(input.getId())); + AbstractNamespacedMetricExt scopedMetric = metric.namespace(RubyUtil.RUBY.getCurrentContext(), RubyUtil.RUBY.newString(input.getId()).intern()); scopedMetric.gauge(RubyUtil.RUBY.getCurrentContext(), MetricKeys.NAME_KEY, RubyUtil.RUBY.newString(input.getName())); instance.setMetric(RubyUtil.RUBY.getCurrentContext(), scopedMetric); instance.input = input; diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index d75c4e60b..82cd216e6 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -651,20 +651,20 @@ public class AbstractPipelineExt extends RubyBasicObject { private void initializePluginThroughputFlowMetric(final ThreadContext context, final UptimeMetric uptime, final String id) { final Metric uptimeInPreciseSeconds = uptime.withUnitsPrecise(SECONDS); - final RubySymbol[] eventsNamespace = buildNamespace(PLUGINS_KEY, INPUTS_KEY, RubyUtil.RUBY.newSymbol(id), EVENTS_KEY); + final RubySymbol[] eventsNamespace = buildNamespace(PLUGINS_KEY, INPUTS_KEY, RubyUtil.RUBY.newString(id).intern(), EVENTS_KEY); final LongCounter eventsOut = initOrGetCounterMetric(context, eventsNamespace, OUT_KEY); final FlowMetric throughputFlow = createFlowMetric(PLUGIN_THROUGHPUT_KEY, eventsOut, uptimeInPreciseSeconds); this.flowMetrics.add(throughputFlow); - final RubySymbol[] flowNamespace = buildNamespace(PLUGINS_KEY, INPUTS_KEY, RubyUtil.RUBY.newSymbol(id), FLOW_KEY); + final RubySymbol[] flowNamespace = buildNamespace(PLUGINS_KEY, INPUTS_KEY, RubyUtil.RUBY.newString(id).intern(), FLOW_KEY); storeMetric(context, flowNamespace, throughputFlow); } private void initializePluginWorkerFlowMetrics(final ThreadContext context, final int workerCount, final UptimeMetric uptime, final RubySymbol key, final String id) { final Metric uptimeInPreciseMillis = uptime.withUnitsPrecise(MILLISECONDS); - final RubySymbol[] eventsNamespace = buildNamespace(PLUGINS_KEY, key, RubyUtil.RUBY.newSymbol(id), EVENTS_KEY); + final RubySymbol[] eventsNamespace = buildNamespace(PLUGINS_KEY, key, RubyUtil.RUBY.newString(id).intern(), EVENTS_KEY); final TimerMetric durationInMillis = initOrGetTimerMetric(context, eventsNamespace, DURATION_IN_MILLIS_KEY); final LongCounter counterEvents = initOrGetCounterMetric(context, eventsNamespace, IN_KEY); final FlowMetric workerCostPerEvent = createFlowMetric(WORKER_MILLIS_PER_EVENT_KEY, durationInMillis, counterEvents); @@ -675,7 +675,7 @@ public class AbstractPipelineExt extends RubyBasicObject { final FlowMetric workerUtilization = createFlowMetric(WORKER_UTILIZATION_KEY, percentScaledDurationInMillis, availableWorkerTimeInMillis); this.flowMetrics.add(workerUtilization); - final RubySymbol[] flowNamespace = buildNamespace(PLUGINS_KEY, key, RubyUtil.RUBY.newSymbol(id), FLOW_KEY); + final RubySymbol[] flowNamespace = buildNamespace(PLUGINS_KEY, key, RubyUtil.RUBY.newString(id).intern(), FLOW_KEY); storeMetric(context, flowNamespace, workerCostPerEvent); storeMetric(context, flowNamespace, workerUtilization); } @@ -685,7 +685,7 @@ public class AbstractPipelineExt extends RubyBasicObject { final Metric metric) { final IRubyObject collector = this.metric.collector(context); final IRubyObject fullNamespace = pipelineNamespacedPath(subPipelineNamespacePath); - final IRubyObject metricKey = context.runtime.newSymbol(metric.getName()); + final IRubyObject metricKey = context.runtime.newString(metric.getName()).intern(); final IRubyObject wasRegistered = collector.callMethod(context, "register?", new IRubyObject[]{fullNamespace, metricKey, JavaUtil.convertJavaToUsableRubyObject(context.runtime, metric)}); if (!wasRegistered.toJava(Boolean.class)) { @@ -746,7 +746,7 @@ public class AbstractPipelineExt extends RubyBasicObject { RubyUtil.RUBY.getCurrentContext(), new IRubyObject[]{ inputQueueClient(), pipelineId().convertToString().intern(), - metric(), RubyUtil.RUBY.newSymbol(inputName) + metric(), RubyUtil.RUBY.newString(inputName).intern() } ); } diff --git a/logstash-core/src/main/java/org/logstash/ext/JRubyWrappedWriteClientExt.java b/logstash-core/src/main/java/org/logstash/ext/JRubyWrappedWriteClientExt.java index ace6fcafc..919ebf257 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JRubyWrappedWriteClientExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JRubyWrappedWriteClientExt.java @@ -77,7 +77,7 @@ public final class JRubyWrappedWriteClientExt extends RubyObject implements Queu final IRubyObject pluginId) { this.writeClient = queueWriteClientExt; - final RubySymbol pipelineIdSym = getRuntime().newSymbol(pipelineId); + final RubySymbol pipelineIdSym = getRuntime().newString(pipelineId).intern(); final RubySymbol pluginIdSym = pluginId.asString().intern(); // Synchronize on the metric since setting up new fields on it is not threadsafe diff --git a/logstash-core/src/main/java/org/logstash/plugins/CounterMetricImpl.java b/logstash-core/src/main/java/org/logstash/plugins/CounterMetricImpl.java index be652d4df..1d3cd839d 100644 --- a/logstash-core/src/main/java/org/logstash/plugins/CounterMetricImpl.java +++ b/logstash-core/src/main/java/org/logstash/plugins/CounterMetricImpl.java @@ -34,7 +34,7 @@ public class CounterMetricImpl implements CounterMetric { public CounterMetricImpl(final ThreadContext threadContext, final AbstractNamespacedMetricExt metrics, final String metric) { - this.longCounter = LongCounter.fromRubyBase(metrics, threadContext.getRuntime().newSymbol(metric)); + this.longCounter = LongCounter.fromRubyBase(metrics, threadContext.getRuntime().newString(metric).intern()); } @Override diff --git a/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java b/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java index 88af5bac9..bb61ac12a 100644 --- a/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java +++ b/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java @@ -64,7 +64,7 @@ public class NamespacedMetricImpl implements NamespacedMetric { @Override public co.elastic.logstash.api.TimerMetric timer(final String metric) { - return TimerMetric.fromRubyBase(metrics, threadContext.getRuntime().newSymbol(metric)); + return TimerMetric.fromRubyBase(metrics, threadContext.getRuntime().newString(metric).intern()); } @Override @@ -121,7 +121,7 @@ public class NamespacedMetricImpl implements NamespacedMetric { } private RubySymbol getSymbol(final String s) { - return this.threadContext.getRuntime().newSymbol(s); + return this.threadContext.getRuntime().newString(s).intern(); } private IRubyObject convert(final Object o) { diff --git a/logstash-core/src/main/java/org/logstash/plugins/RootMetricImpl.java b/logstash-core/src/main/java/org/logstash/plugins/RootMetricImpl.java index c4ea07f6f..c28d8f1c0 100644 --- a/logstash-core/src/main/java/org/logstash/plugins/RootMetricImpl.java +++ b/logstash-core/src/main/java/org/logstash/plugins/RootMetricImpl.java @@ -23,6 +23,7 @@ package org.logstash.plugins; import co.elastic.logstash.api.Metric; import co.elastic.logstash.api.NamespacedMetric; import org.jruby.RubyArray; +import org.jruby.RubyString; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.instrument.metrics.AbstractMetricExt; @@ -45,7 +46,8 @@ public class RootMetricImpl implements Metric { @Override public NamespacedMetric namespace(final String... key) { final IRubyObject[] rubyfiedKeys = Stream.of(key) - .map(this.threadContext.getRuntime()::newSymbol) + .map(this.threadContext.getRuntime()::newString) + .map(RubyString::intern) .toArray(IRubyObject[]::new); return new NamespacedMetricImpl( diff --git a/logstash-core/src/main/java/org/logstash/plugins/factory/PluginFactoryExt.java b/logstash-core/src/main/java/org/logstash/plugins/factory/PluginFactoryExt.java index c23c3d87e..13d231a21 100644 --- a/logstash-core/src/main/java/org/logstash/plugins/factory/PluginFactoryExt.java +++ b/logstash-core/src/main/java/org/logstash/plugins/factory/PluginFactoryExt.java @@ -253,7 +253,7 @@ public final class PluginFactoryExt extends RubyBasicObject } else { final IRubyObject pluginInstance = ContextualizerExt.initializePlugin(context, executionCntx, klass, rubyArgs); - final AbstractNamespacedMetricExt scopedMetric = typeScopedMetric.namespace(context, RubyUtil.RUBY.newSymbol(id)); + final AbstractNamespacedMetricExt scopedMetric = typeScopedMetric.namespace(context, RubyUtil.RUBY.newString(id).intern()); scopedMetric.gauge(context, MetricKeys.NAME_KEY, pluginInstance.callMethod(context, "config_name")); pluginInstance.callMethod(context, "metric=", scopedMetric); return pluginInstance; diff --git a/logstash-core/src/main/java/org/logstash/plugins/factory/PluginMetricsFactoryExt.java b/logstash-core/src/main/java/org/logstash/plugins/factory/PluginMetricsFactoryExt.java index bb6fd11a0..d8456f93c 100644 --- a/logstash-core/src/main/java/org/logstash/plugins/factory/PluginMetricsFactoryExt.java +++ b/logstash-core/src/main/java/org/logstash/plugins/factory/PluginMetricsFactoryExt.java @@ -53,7 +53,7 @@ public final class PluginMetricsFactoryExt extends RubyBasicObject { @JRubyMethod public AbstractNamespacedMetricExt create(final ThreadContext context, final IRubyObject pluginType) { return getRoot(context).namespace( - context, RubyUtil.RUBY.newSymbol(String.format("%ss", pluginType.asJavaString())) + context, RubyUtil.RUBY.newString(String.format("%ss", pluginType.asJavaString())).intern() ); } } diff --git a/logstash-core/src/test/java/org/logstash/config/ir/PipelineConfigTest.java b/logstash-core/src/test/java/org/logstash/config/ir/PipelineConfigTest.java index 8bd2fb98e..ee44967bb 100644 --- a/logstash-core/src/test/java/org/logstash/config/ir/PipelineConfigTest.java +++ b/logstash-core/src/test/java/org/logstash/config/ir/PipelineConfigTest.java @@ -103,7 +103,7 @@ public class PipelineConfigTest extends RubyEnvTestCase { public void setUp() throws IncompleteSourceWithMetadataException { source = RubyUtil.RUBY.getClass("LogStash::Config::Source::Local"); - pipelineIdSym = RubySymbol.newSymbol(RubyUtil.RUBY, PIPELINE_ID); + pipelineIdSym = RubyUtil.RUBY.newString(PIPELINE_ID).intern(); final SourceCollector sourceCollector = new SourceCollector(); sourceCollector.appendSource("file", "/tmp/1", 0, 0, "input { generator1 }\n", "{\"version\": \"1\"}"); diff --git a/logstash-core/src/test/java/org/logstash/config/ir/compiler/OutputDelegatorTest.java b/logstash-core/src/test/java/org/logstash/config/ir/compiler/OutputDelegatorTest.java index d855251bf..b91256fa9 100644 --- a/logstash-core/src/test/java/org/logstash/config/ir/compiler/OutputDelegatorTest.java +++ b/logstash-core/src/test/java/org/logstash/config/ir/compiler/OutputDelegatorTest.java @@ -216,7 +216,7 @@ public class OutputDelegatorTest extends PluginDelegatorTestCase { Class klazz; StrategyPair(String symbolName, Class c) { - this.symbol = RUBY.newSymbol(symbolName); + this.symbol = RUBY.newString(symbolName).intern(); this.klazz = c; } } diff --git a/logstash-core/src/test/java/org/logstash/config/ir/compiler/PluginDelegatorTestCase.java b/logstash-core/src/test/java/org/logstash/config/ir/compiler/PluginDelegatorTestCase.java index 0d2270f46..187d323af 100644 --- a/logstash-core/src/test/java/org/logstash/config/ir/compiler/PluginDelegatorTestCase.java +++ b/logstash-core/src/test/java/org/logstash/config/ir/compiler/PluginDelegatorTestCase.java @@ -47,7 +47,7 @@ public abstract class PluginDelegatorTestCase extends RubyEnvTestCase { final ThreadContext context = RUBY.getCurrentContext(); @SuppressWarnings("rawtypes") final RubyArray namespaces = RubyArray.newArray(RUBY, 1); - namespaces.add(0, RubySymbol.newSymbol(RUBY, getBaseMetricsPath().split("/")[0])); + namespaces.add(0, RUBY.newString(getBaseMetricsPath().split("/")[0]).intern()); IRubyObject metricWithCollector = runRubyScript("require \"logstash/instrument/collector\"\n" + "metricWithCollector = LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new)"); @@ -70,7 +70,7 @@ public abstract class PluginDelegatorTestCase extends RubyEnvTestCase { RubyHash rh = metricStore; for (String p : path) { - rh = (RubyHash) rh.op_aref(RUBY.getCurrentContext(), RUBY.newSymbol(p)); + rh = (RubyHash) rh.op_aref(RUBY.getCurrentContext(), RUBY.newString(p).intern()); } return rh; } @@ -78,13 +78,13 @@ public abstract class PluginDelegatorTestCase extends RubyEnvTestCase { protected abstract String getBaseMetricsPath(); protected String getMetricStringValue(RubyHash metricStore, String symbolName) { - ConcreteJavaProxy counter = (ConcreteJavaProxy) metricStore.op_aref(RUBY.getCurrentContext(), RUBY.newSymbol(symbolName)); + ConcreteJavaProxy counter = (ConcreteJavaProxy) metricStore.op_aref(RUBY.getCurrentContext(), RUBY.newString(symbolName).intern()); RubyString value = (RubyString) counter.callMethod("value"); return value.asJavaString(); } protected long getMetricLongValue(RubyHash metricStore, String symbolName) { - ConcreteJavaProxy counter = (ConcreteJavaProxy) metricStore.op_aref(RUBY.getCurrentContext(), RUBY.newSymbol(symbolName)); + ConcreteJavaProxy counter = (ConcreteJavaProxy) metricStore.op_aref(RUBY.getCurrentContext(), RUBY.newString(symbolName).intern()); RubyFixnum count = (RubyFixnum) counter.callMethod("value"); return count.getLongValue(); } diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricExtFactory.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricExtFactory.java index 55e4afe56..0026b19f1 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricExtFactory.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricExtFactory.java @@ -58,7 +58,7 @@ public class MetricExtFactory { private RubyModule metricFactoryInterceptor(final String type, final Function javaMetricFactory) { final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); - final IRubyObject interceptType = context.runtime.newSymbol(type); + final IRubyObject interceptType = context.runtime.newString(type).intern(); final IRubyObject metricFactory = JavaUtil.convertJavaToUsableRubyObject(context.runtime, MetricFactory.of(javaMetricFactory)); final IRubyObject interceptorModule = INTERCEPTOR_MODULE_CLASS.newInstance(context, interceptType, metricFactory, Block.NULL_BLOCK); diff --git a/logstash-core/src/test/java/org/logstash/plugins/MetricTestCase.java b/logstash-core/src/test/java/org/logstash/plugins/MetricTestCase.java index fbf92213a..7b766251a 100644 --- a/logstash-core/src/test/java/org/logstash/plugins/MetricTestCase.java +++ b/logstash-core/src/test/java/org/logstash/plugins/MetricTestCase.java @@ -64,19 +64,19 @@ public abstract class MetricTestCase extends RubyEnvTestCase { RubyHash rh = metricStore; for (String p : path) { - rh = (RubyHash) rh.op_aref(RUBY.getCurrentContext(), RUBY.newSymbol(p)); + rh = (RubyHash) rh.op_aref(RUBY.getCurrentContext(), RUBY.newString(p).intern()); } return rh; } protected String getMetricStringValue(RubyHash metricStore, String symbolName) { - ConcreteJavaProxy counter = (ConcreteJavaProxy) metricStore.op_aref(RUBY.getCurrentContext(), RUBY.newSymbol(symbolName)); + ConcreteJavaProxy counter = (ConcreteJavaProxy) metricStore.op_aref(RUBY.getCurrentContext(), RUBY.newString(symbolName).intern()); RubyString value = (RubyString) counter.callMethod("value"); return value.asJavaString(); } protected long getMetricLongValue(RubyHash metricStore, String symbolName) { - ConcreteJavaProxy counter = (ConcreteJavaProxy) metricStore.op_aref(RUBY.getCurrentContext(), RUBY.newSymbol(symbolName)); + ConcreteJavaProxy counter = (ConcreteJavaProxy) metricStore.op_aref(RUBY.getCurrentContext(), RUBY.newString(symbolName).intern()); RubyFixnum count = (RubyFixnum) counter.callMethod("value"); return count.getLongValue(); } diff --git a/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java b/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java index 8c506b6c4..f705089f2 100644 --- a/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java +++ b/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java @@ -27,22 +27,37 @@ import org.jruby.RubyHash; import org.junit.Ignore; import org.junit.Test; +import java.util.Arrays; + import static org.assertj.core.api.Assertions.assertThat; public class NamespacedMetricImplTest extends MetricTestCase { @Test public void testGauge() { - final NamespacedMetric metrics = this.getInstance().namespace("test"); + doTestGauge("test"); + } + @Test + public void testGaugeNested() { + doTestGauge("test", "deep"); + } + + @Test + public void testGaugeUnicode() { + doTestGauge("test", "Ümlãut-åccênt", "okay"); + } + + public void doTestGauge(final String... metricNamespace) { + final NamespacedMetric metrics = this.getInstance().namespace(metricNamespace); metrics.gauge("abc", "def"); { - final RubyHash metricStore = getMetricStore(new String[]{"test"}); + final RubyHash metricStore = getMetricStore(Arrays.copyOf(metricNamespace, metricNamespace.length)); assertThat(this.getMetricStringValue(metricStore, "abc")).isEqualTo("def"); } metrics.gauge("abc", "123"); { - final RubyHash metricStore = getMetricStore(new String[]{"test"}); + final RubyHash metricStore = getMetricStore(Arrays.copyOf(metricNamespace, metricNamespace.length)); assertThat(this.getMetricStringValue(metricStore, "abc")).isEqualTo("123"); } } @@ -125,6 +140,17 @@ public class NamespacedMetricImplTest extends MetricTestCase { assertThat(namespaced2.namespaceName()).containsExactly("test", "abcdef", "12345", "qwerty"); } + @Test + public void testNamespaceUnicodeFragment() { + final NamespacedMetric metrics = this.getInstance().namespace("test", "Ünîcødé", "nÉs†iñG"); + + final NamespacedMetric namespaced = metrics.namespace("abcdef"); + assertThat(namespaced.namespaceName()).containsExactly("test", "Ünîcødé", "nÉs†iñG", "abcdef"); + + final NamespacedMetric namespaced2 = namespaced.namespace("12345", "qwerty"); + assertThat(namespaced2.namespaceName()).containsExactly("test", "Ünîcødé", "nÉs†iñG", "abcdef", "12345", "qwerty"); + } + @Test public void testRoot() { final NamespacedMetric metrics = this.getInstance().namespace("test"); diff --git a/qa/integration/services/logstash_service.rb b/qa/integration/services/logstash_service.rb index 3953a6fda..d9eacc24b 100644 --- a/qa/integration/services/logstash_service.rb +++ b/qa/integration/services/logstash_service.rb @@ -113,21 +113,8 @@ class LogstashService < Service # Can start LS in stdin and can send messages to stdin # Useful to test metrics and such def start_with_stdin(pipeline_config = STDIN_CONFIG) - puts "Starting Logstash #{@logstash_bin} -e #{pipeline_config}" - Bundler.with_unbundled_env do - out = Tempfile.new("duplex") - out.sync = true - @process = build_child_process("-e", pipeline_config) - # pipe STDOUT and STDERR to a file - @process.io.stdout = @process.io.stderr = out - @process.duplex = true - @env_variables.map { |k, v| @process.environment[k] = v} unless @env_variables.nil? - java_home = java.lang.System.getProperty('java.home') - @process.environment['LS_JAVA_HOME'] = java_home - @process.start - wait_for_logstash - puts "Logstash started with PID #{@process.pid}, LS_JAVA_HOME: #{java_home}" if alive? - end + spawn_logstash("-e", pipeline_config) + wait_for_logstash end def write_to_stdin(input) @@ -138,6 +125,7 @@ class LogstashService < Service # Spawn LS as a child process def spawn_logstash(*args) + $stderr.puts "Starting Logstash #{Shellwords.escape(@logstash_bin)} #{Shellwords.join(args)}" Bundler.with_unbundled_env do out = Tempfile.new("duplex") out.sync = true diff --git a/qa/integration/specs/monitoring_api_spec.rb b/qa/integration/specs/monitoring_api_spec.rb index f0fe88767..830ddc0bc 100644 --- a/qa/integration/specs/monitoring_api_spec.rb +++ b/qa/integration/specs/monitoring_api_spec.rb @@ -23,24 +23,62 @@ require "logstash/devutils/rspec/spec_helper" require "stud/try" describe "Test Monitoring API" do + before(:each) do |example| + $stderr.puts("STARTING: #{example.full_description} (#{example.location})") + end + before(:all) { @fixture = Fixture.new(__FILE__) + ruby_encoding_info = %w(external internal locale filesystem).map do |type| + Encoding.find(type)&.name&.then { |name| "#{type}:#{name}" } + end.compact.join(", ") + + $stderr.puts <<~ENCODINGINFO.tr("\n", ' ') + INFO(spec runner process) + Ruby.Encoding=(#{ruby_encoding_info}) + Java.Locale=`#{java.util.Locale::getDefault().toLanguageTag()}` + Java.Charset=`#{java.nio.charset.Charset::defaultCharset().displayName()}` + ENCODINGINFO } + let(:settings_overrides) do + {} + end + + let(:logstash_service) { @fixture.get_service("logstash") } + + before(:each) do + # some settings values cannot be reliably passed on the command line + # because we are not guaranteed that the shell's encoding supports UTF-8. + # Merge our settings into the active settings file, to accommodate feature flags + unless settings_overrides.empty? + settings_file = logstash_service.application_settings_file + FileUtils.cp(settings_file, "#{settings_file}.original") + + base_settings = YAML.load(File.read(settings_file)) || {} + effective_settings = base_settings.merge(settings_overrides) do |key, old_val, new_val| + warn "Overriding setting `#{key}` with `#{new_val}` (was `#{old_val}`)" + new_val + end + + IO.write(settings_file, effective_settings.to_yaml) + end + end + after(:all) { @fixture.teardown } - after(:each) { - @fixture.get_service("logstash").teardown - } + after(:each) do + settings_file = logstash_service.application_settings_file + logstash_service.teardown + FileUtils.mv("#{settings_file}.original", settings_file) if File.exist?("#{settings_file}.original") + end let(:number_of_events) { 5 } let(:max_retry) { 120 } - let(:plugins_config) { "input { stdin {} } filter { mutate { add_tag => 'integration test adding tag' } } output { stdout {} }" } it "can retrieve event stats" do - logstash_service = @fixture.get_service("logstash") logstash_service.start_with_stdin logstash_service.wait_for_logstash number_of_events.times { logstash_service.write_to_stdin("Hello world") } @@ -167,38 +205,179 @@ describe "Test Monitoring API" do end end - it "can retrieve queue stats" do - logstash_service = @fixture.get_service("logstash") - logstash_service.start_with_stdin - logstash_service.wait_for_logstash + shared_examples "pipeline metrics" do + # let(:pipeline_id) { defined?(super()) or fail NotImplementedError } + let(:settings_overrides) do + super().merge({'pipeline.id' => pipeline_id}) + end - Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do - # node_stats can fail if the stats subsystem isn't ready - result = logstash_service.monitoring_api.node_stats rescue nil - expect(result).not_to be_nil - # we use fetch here since we want failed fetches to raise an exception - # and trigger the retry block - queue_stats = result.fetch("pipelines").fetch("main").fetch("queue") - expect(queue_stats).not_to be_nil - if logstash_service.settings.feature_flag == "persistent_queues" - expect(queue_stats["type"]).to eq "persisted" - queue_data_stats = queue_stats.fetch("data") - expect(queue_data_stats["free_space_in_bytes"]).not_to be_nil - expect(queue_data_stats["storage_type"]).not_to be_nil - expect(queue_data_stats["path"]).not_to be_nil - expect(queue_stats["events"]).not_to be_nil - queue_capacity_stats = queue_stats.fetch("capacity") - expect(queue_capacity_stats["page_capacity_in_bytes"]).not_to be_nil - expect(queue_capacity_stats["max_queue_size_in_bytes"]).not_to be_nil - expect(queue_capacity_stats["max_unread_events"]).not_to be_nil - else - expect(queue_stats["type"]).to eq("memory") + it "can retrieve queue stats" do + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_stats can fail if the stats subsystem isn't ready + result = logstash_service.monitoring_api.node_stats rescue nil + expect(result).not_to be_nil + # we use fetch here since we want failed fetches to raise an exception + # and trigger the retry block + queue_stats = result.fetch("pipelines").fetch(pipeline_id).fetch("queue") + expect(queue_stats).not_to be_nil + if logstash_service.settings.feature_flag == "persistent_queues" + expect(queue_stats["type"]).to eq "persisted" + queue_data_stats = queue_stats.fetch("data") + expect(queue_data_stats["free_space_in_bytes"]).not_to be_nil + expect(queue_data_stats["storage_type"]).not_to be_nil + expect(queue_data_stats["path"]).not_to be_nil + expect(queue_stats["events"]).not_to be_nil + queue_capacity_stats = queue_stats.fetch("capacity") + expect(queue_capacity_stats["page_capacity_in_bytes"]).not_to be_nil + expect(queue_capacity_stats["max_queue_size_in_bytes"]).not_to be_nil + expect(queue_capacity_stats["max_unread_events"]).not_to be_nil + else + expect(queue_stats["type"]).to eq("memory") + end end end + + it "retrieves the pipeline flow statuses" do + logstash_service = @fixture.get_service("logstash") + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + number_of_events.times { + logstash_service.write_to_stdin("Testing flow metrics") + sleep(1) + } + + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_stats can fail if the stats subsystem isn't ready + result = logstash_service.monitoring_api.node_stats rescue nil + expect(result).not_to be_nil + # we use fetch here since we want failed fetches to raise an exception + # and trigger the retry block + expect(result).to include('pipelines' => hash_including(pipeline_id => hash_including('flow'))) + flow_status = result.dig("pipelines", pipeline_id, "flow") + expect(flow_status).to_not be_nil + expect(flow_status).to include( + # due to three-decimal-place rounding, it is easy for our worker_concurrency and queue_backpressure + # to be zero, so we are just looking for these to be _populated_ + 'worker_concurrency' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), + 'worker_utilization' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), + 'queue_backpressure' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), + # depending on flow capture interval, our current rate can easily be zero, but our lifetime rates + # should be non-zero so long as pipeline uptime is less than ~10 minutes. + 'input_throughput' => hash_including('current' => a_value >= 0, 'lifetime' => a_value > 0), + 'filter_throughput' => hash_including('current' => a_value >= 0, 'lifetime' => a_value > 0), + 'output_throughput' => hash_including('current' => a_value >= 0, 'lifetime' => a_value > 0) + ) + if logstash_service.settings.feature_flag == "persistent_queues" + expect(flow_status).to include( + 'queue_persisted_growth_bytes' => hash_including('current' => a_kind_of(Numeric), 'lifetime' => a_kind_of(Numeric)), + 'queue_persisted_growth_events' => hash_including('current' => a_kind_of(Numeric), 'lifetime' => a_kind_of(Numeric)) + ) + else + expect(flow_status).to_not include('queue_persisted_growth_bytes') + expect(flow_status).to_not include('queue_persisted_growth_events') + end + end + end + + shared_examples "plugin-level flow metrics" do + let(:settings_overrides) do + super().merge({'config.string' => config_string}) + end + + let(:config_string) do + <<~EOPIPELINE + input { stdin { id => '#{plugin_id_input}' } } + filter { mutate { id => '#{plugin_id_filter}' add_tag => 'integration test adding tag' } } + output { stdout { id => '#{plugin_id_output}' } } + EOPIPELINE + end + + it "retrieves plugin level flow metrics" do + logstash_service.spawn_logstash + logstash_service.wait_for_logstash + number_of_events.times { + logstash_service.write_to_stdin("Testing plugin-level flow metrics") + sleep(1) + } + + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_stats can fail if the stats subsystem isn't ready + result = logstash_service.monitoring_api.node_stats rescue nil + # if the result is nil, we probably aren't ready yet + # our assertion failure will cause Stud to retry + expect(result).not_to be_nil + + expect(result).to include('pipelines' => hash_including(pipeline_id => hash_including('plugins' => hash_including('inputs', 'filters', 'outputs')))) + + input_plugins = result.dig("pipelines", pipeline_id, "plugins", "inputs") + filter_plugins = result.dig("pipelines", pipeline_id, "plugins", "filters") + output_plugins = result.dig("pipelines", pipeline_id, "plugins", "outputs") + expect(input_plugins[0]).to_not be_nil # not ready... + + expect(input_plugins).to include(a_hash_including( + 'id' => plugin_id_input, + 'flow' => a_hash_including( + 'throughput' => hash_including('current' => a_value >= 0, 'lifetime' => a_value > 0) + ) + )) + + expect(filter_plugins).to include(a_hash_including( + 'id' => plugin_id_filter, + 'flow' => a_hash_including( + 'worker_utilization' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), + 'worker_millis_per_event' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), + ) + )) + + expect(output_plugins).to include(a_hash_including( + 'id' => plugin_id_output, + 'flow' => a_hash_including( + 'worker_utilization' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), + 'worker_millis_per_event' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), + ) + )) + end + end + end + + context "with lower-ASCII plugin id's" do + let(:plugin_id_input) { "standard-input" } + let(:plugin_id_filter) { "Mutations" } + let(:plugin_id_output) { "StandardOutput" } + include_examples "plugin-level flow metrics" + end + + context "with unicode plugin id's" do + let(:plugin_id_input) { "입력" } + let(:plugin_id_filter) { "変じる" } + let(:plugin_id_output) { "le-résultat" } + include_examples "plugin-level flow metrics" + end + + end + + context "with lower-ASCII pipeline id" do + let(:pipeline_id) { "main" } + include_examples "pipeline metrics" + end + + context "with unicode pipeline id" do + before(:each) do + if @fixture.settings.feature_flag == "persistent_queues" + skip('behaviour for unicode pipeline ids is unspecified when PQ is enabled') + # NOTE: pipeline ids are used verbatim as a part of the queue path, so the subset + # of unicode characters that are supported depend on the OS and Filesystem. + # The pipeline will fail to start, rendering these monitoring specs useless. + end + end + let(:pipeline_id) { "변환-verändern-変ずる" } + include_examples "pipeline metrics" end it "can configure logging" do - logstash_service = @fixture.get_service("logstash") logstash_service.start_with_stdin logstash_service.wait_for_logstash @@ -246,86 +425,6 @@ describe "Test Monitoring API" do logging_get_assert logstash_service, "INFO", "TRACE" end - it "should retrieve the pipeline flow statuses" do - logstash_service = @fixture.get_service("logstash") - logstash_service.start_with_stdin - logstash_service.wait_for_logstash - number_of_events.times { - logstash_service.write_to_stdin("Testing flow metrics") - sleep(1) - } - - Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do - # node_stats can fail if the stats subsystem isn't ready - result = logstash_service.monitoring_api.node_stats rescue nil - expect(result).not_to be_nil - # we use fetch here since we want failed fetches to raise an exception - # and trigger the retry block - expect(result).to include('pipelines' => hash_including('main' => hash_including('flow'))) - flow_status = result.dig("pipelines", "main", "flow") - expect(flow_status).to_not be_nil - expect(flow_status).to include( - # due to three-decimal-place rounding, it is easy for our worker_concurrency and queue_backpressure - # to be zero, so we are just looking for these to be _populated_ - 'worker_concurrency' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), - 'worker_utilization' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), - 'queue_backpressure' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), - # depending on flow capture interval, our current rate can easily be zero, but our lifetime rates - # should be non-zero so long as pipeline uptime is less than ~10 minutes. - 'input_throughput' => hash_including('current' => a_value >= 0, 'lifetime' => a_value > 0), - 'filter_throughput' => hash_including('current' => a_value >= 0, 'lifetime' => a_value > 0), - 'output_throughput' => hash_including('current' => a_value >= 0, 'lifetime' => a_value > 0) - ) - if logstash_service.settings.feature_flag == "persistent_queues" - expect(flow_status).to include( - 'queue_persisted_growth_bytes' => hash_including('current' => a_kind_of(Numeric), 'lifetime' => a_kind_of(Numeric)), - 'queue_persisted_growth_events' => hash_including('current' => a_kind_of(Numeric), 'lifetime' => a_kind_of(Numeric)) - ) - else - expect(flow_status).to_not include('queue_persisted_growth_bytes') - expect(flow_status).to_not include('queue_persisted_growth_events') - end - end - end - - it "should retrieve plugin level flow metrics" do - logstash_service = @fixture.get_service("logstash") - logstash_service.start_with_stdin(plugins_config) - logstash_service.wait_for_logstash - number_of_events.times { - logstash_service.write_to_stdin("Testing plugin-level flow metrics") - sleep(1) - } - - Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do - # node_stats can fail if the stats subsystem isn't ready - result = logstash_service.monitoring_api.node_stats rescue nil - # if the result is nil, we probably aren't ready yet - # our assertion failure will cause Stud to retry - expect(result).not_to be_nil - - expect(result).to include('pipelines' => hash_including('main' => hash_including('plugins' => hash_including('inputs', 'filters', 'outputs')))) - - input_plugins = result.dig("pipelines", "main", "plugins", "inputs") - filter_plugins = result.dig("pipelines", "main", "plugins", "filters") - output_plugins = result.dig("pipelines", "main", "plugins", "outputs") - expect(input_plugins[0]).to_not be_nil - - input_plugin_flow_status = input_plugins[0].dig("flow") - filter_plugin_flow_status = filter_plugins[0].dig("flow") - output_plugin_flow_status = output_plugins[0].dig("flow") - - expect(input_plugin_flow_status).to include('throughput' => hash_including('current' => a_value >= 0, 'lifetime' => a_value > 0)) - expect(filter_plugin_flow_status).to include( - 'worker_utilization' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), - 'worker_millis_per_event' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), - ) - expect(output_plugin_flow_status).to include( - 'worker_utilization' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), - 'worker_millis_per_event' => hash_including('current' => a_value >= 0, 'lifetime' => a_value >= 0), - ) - end - end private From 04d193d613002b0f7d20e90ba5da8abffb9cc209 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 14:13:53 -0700 Subject: [PATCH 26/32] Update patch plugin versions in gemfile lock (#16258) Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> --- Gemfile.jruby-3.1.lock.release | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index fd4fc606b..6fd5267ec 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -34,8 +34,8 @@ PATH GEM remote: https://rubygems.org/ specs: - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) amazing_print (1.6.0) arr-pm (0.0.12) ast (2.4.2) @@ -199,7 +199,7 @@ GEM json (2.7.2-java) json-schema (2.8.1) addressable (>= 2.4) - jwt (2.8.1) + jwt (2.8.2) base64 kramdown (2.4.0) rexml @@ -237,7 +237,7 @@ GEM logstash-mixin-ecs_compatibility_support (~> 1.3) logstash-mixin-event_support (~> 1.0) logstash-mixin-validator_support (~> 1.0) - logstash-codec-fluent (3.4.2-java) + logstash-codec-fluent (3.4.3-java) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-mixin-event_support (~> 1.0) logstash-mixin-validator_support (~> 1.0) @@ -320,7 +320,7 @@ GEM lru_redux (~> 1.1.0) logstash-filter-drop (3.0.5) logstash-core-plugin-api (>= 1.60, <= 2.99) - logstash-filter-elastic_integration (0.1.9-java) + logstash-filter-elastic_integration (0.1.10-java) logstash-core (>= 8.7.0) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-filter-elasticsearch (3.16.1) @@ -434,7 +434,7 @@ GEM logstash-mixin-ecs_compatibility_support (~> 1.2) logstash-mixin-normalize_config_support (~> 1.0) logstash-mixin-plugin_factory_support - logstash-input-elasticsearch (4.20.2) + logstash-input-elasticsearch (4.20.3) elasticsearch (>= 7.17.9) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-mixin-ca_trusted_fingerprint_support (~> 1.0) @@ -651,7 +651,7 @@ GEM logstash-output-csv (3.0.10) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-output-file - logstash-output-elasticsearch (11.22.6-java) + logstash-output-elasticsearch (11.22.7-java) logstash-core-plugin-api (>= 1.60, <= 2.99) logstash-mixin-ca_trusted_fingerprint_support (~> 1.0) logstash-mixin-deprecation_logger_support (~> 1.0) @@ -733,7 +733,7 @@ GEM mustermann (2.0.2) ruby2_keywords (~> 0.0.1) naught (1.1.0) - net-imap (0.4.12) + net-imap (0.4.14) date net-protocol net-pop (0.1.2) @@ -743,7 +743,7 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.3-java) - nokogiri (1.16.5-java) + nokogiri (1.16.6-java) racc (~> 1.4) octokit (4.25.1) faraday (>= 1, < 3) @@ -751,7 +751,7 @@ GEM openssl_pkcs8_pure (0.0.0.2) paquet (0.2.1) parallel (1.24.0) - parser (3.3.2.0) + parser (3.3.3.0) ast (~> 2.4.1) racc pleaserun (0.0.32) @@ -789,7 +789,7 @@ GEM rspec-mocks (~> 3.13.0) rspec-core (3.13.0) rspec-support (~> 3.13.0) - rspec-expectations (3.13.0) + rspec-expectations (3.13.1) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-mocks (3.13.1) From bd7e40885c60b56b61164094fd35979efdb6f1ac Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Tue, 25 Jun 2024 15:11:22 -0700 Subject: [PATCH 27/32] Backport 16250 8.14 (#16259) * sync notices * backport 16250 Co-authored-by: Mashhur <99575341+mashhurs@users.noreply.github.com> --------- Co-authored-by: Mashhur <99575341+mashhurs@users.noreply.github.com> --- Gemfile.template | 4 + NOTICE.TXT | 190 ++++++++++++++---- .../src/main/resources/licenseMapping.csv | 1 + .../resources/notices/fileutils-NOTICE.txt | 22 ++ 4 files changed, 173 insertions(+), 44 deletions(-) create mode 100644 tools/dependencies-report/src/main/resources/notices/fileutils-NOTICE.txt diff --git a/Gemfile.template b/Gemfile.template index f7141767f..ae2dadb63 100644 --- a/Gemfile.template +++ b/Gemfile.template @@ -20,6 +20,10 @@ gem "gems", "~> 1", :group => :build gem "octokit", "~> 4.25", :group => :build gem "rubyzip", "~> 1", :group => :build gem "stud", "~> 0.0.22", :group => :build +# remove fileutils declaration when start using Ruby 3.2+, by default includes `fileutils-v1.7.0` +# (https://git.ruby-lang.org/ruby.git/commit/?h=ruby_3_2&id=05caafb4731c796890027cafedaac59dc108a23a) +# note that the reason to use 1.7.0 is due to https://github.com/logstash-plugins/logstash-integration-aws/issues/28 +gem "fileutils", "~> 1.7" gem "rubocop", :group => :development gem "belzebuth", :group => :development diff --git a/NOTICE.TXT b/NOTICE.TXT index 9ada823fb..8e4349421 100644 --- a/NOTICE.TXT +++ b/NOTICE.TXT @@ -90,7 +90,7 @@ Copyright © 2019 Red Hat, Inc. All rights reserved. “Red Hat,” is a registe All other trademarks are the property of their respective owners. ========== -Notice for: addressable-2.8.6 +Notice for: addressable-2.8.7 ---------- Copyright © Bob Aman @@ -298,7 +298,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: amazing_print-1.5.0 +Notice for: amazing_print-1.6.0 ---------- MIT License @@ -722,7 +722,7 @@ Notice for: aws-eventstream-1.3.0 limitations under the License. ========== -Notice for: aws-partitions-1.883.0 +Notice for: aws-partitions-1.925.0 ---------- @@ -929,7 +929,7 @@ Notice for: aws-partitions-1.883.0 limitations under the License. ========== -Notice for: aws-sdk-cloudfront-1.87.0 +Notice for: aws-sdk-cloudfront-1.89.0 ---------- @@ -1136,7 +1136,7 @@ Notice for: aws-sdk-cloudfront-1.87.0 limitations under the License. ========== -Notice for: aws-sdk-cloudwatch-1.85.0 +Notice for: aws-sdk-cloudwatch-1.90.0 ---------- @@ -1343,7 +1343,7 @@ Notice for: aws-sdk-cloudwatch-1.85.0 limitations under the License. ========== -Notice for: aws-sdk-core-3.191.0 +Notice for: aws-sdk-core-3.194.2 ---------- @@ -1550,7 +1550,7 @@ Notice for: aws-sdk-core-3.191.0 limitations under the License. ========== -Notice for: aws-sdk-kms-1.77.0 +Notice for: aws-sdk-kms-1.80.0 ---------- @@ -1757,7 +1757,7 @@ Notice for: aws-sdk-kms-1.77.0 limitations under the License. ========== -Notice for: aws-sdk-resourcegroups-1.58.0 +Notice for: aws-sdk-resourcegroups-1.60.0 ---------- @@ -1964,7 +1964,7 @@ Notice for: aws-sdk-resourcegroups-1.58.0 limitations under the License. ========== -Notice for: aws-sdk-s3-1.143.0 +Notice for: aws-sdk-s3-1.149.1 ---------- @@ -2171,7 +2171,7 @@ Notice for: aws-sdk-s3-1.143.0 limitations under the License. ========== -Notice for: aws-sdk-sns-1.71.0 +Notice for: aws-sdk-sns-1.74.0 ---------- @@ -2378,7 +2378,7 @@ Notice for: aws-sdk-sns-1.71.0 limitations under the License. ========== -Notice for: aws-sdk-sqs-1.70.0 +Notice for: aws-sdk-sqs-1.73.0 ---------- @@ -2837,7 +2837,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: bigdecimal-3.1.6 +Notice for: bigdecimal-3.1.8 ---------- # source: https://github.com/ruby/bigdecimal/blob/v3.1.4/LICENSE @@ -2899,7 +2899,7 @@ You can redistribute it and/or modify it under either the terms of the WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. ========== -Notice for: bindata-2.4.15 +Notice for: bindata-2.5.0 ---------- Copyright (C) 2007-2012 Dion Mendel. All rights reserved. @@ -3086,7 +3086,7 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: com.fasterxml.jackson.core:jackson-annotations-2.15.2 +Notice for: com.fasterxml.jackson.core:jackson-annotations-2.15.4 ---------- This copy of Jackson JSON processor annotations is licensed under the @@ -3303,7 +3303,7 @@ http://www.apache.org/licenses/LICENSE-2.0 limitations under the License. ========== -Notice for: com.fasterxml.jackson.core:jackson-core-2.15.2 +Notice for: com.fasterxml.jackson.core:jackson-core-2.15.4 ---------- # Jackson JSON processor @@ -3530,7 +3530,7 @@ from the source code management (SCM) system project uses. See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: com.fasterxml.jackson.core:jackson-databind-2.15.2 +Notice for: com.fasterxml.jackson.core:jackson-databind-2.15.4 ---------- # Jackson JSON processor @@ -3981,7 +3981,7 @@ http://www.apache.org/licenses/LICENSE-2.0 See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: com.fasterxml.jackson.module:jackson-module-afterburner-2.15.2 +Notice for: com.fasterxml.jackson.module:jackson-module-afterburner-2.15.4 ---------- This copy of Jackson JSON processor databind module is licensed under the @@ -5140,7 +5140,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: dalli-3.2.7 +Notice for: dalli-3.2.8 ---------- Copyright (c) Peter M. Goldstein, Mike Perham @@ -5589,7 +5589,7 @@ source: https://github.com/elastic/enterprise-search-ruby/blob/v7.16.0/LICENSE See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: elastic-transport-8.3.1 +Notice for: elastic-transport-8.3.2 ---------- source: https://github.com/elastic/elastic-transport-ruby/blob/v8.3.0/LICENSE @@ -5797,7 +5797,7 @@ source: https://github.com/elastic/elastic-transport-ruby/blob/v8.3.0/LICENSE See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: elasticsearch-7.17.10 +Notice for: elasticsearch-7.17.11 ---------- source: https://github.com/elastic/elasticsearch-ruby/blob/v5.0.4/elasticsearch-api/LICENSE.txt @@ -5817,7 +5817,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: elasticsearch-api-7.17.10 +Notice for: elasticsearch-api-7.17.11 ---------- source: https://github.com/elastic/elasticsearch-ruby/blob/v5.0.4/elasticsearch-transport/LICENSE.txt @@ -5837,7 +5837,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: elasticsearch-transport-7.17.10 +Notice for: elasticsearch-transport-7.17.11 ---------- source: https://github.com/elastic/elasticsearch-ruby/blob/v5.0.4/elasticsearch/LICENSE.txt @@ -5885,7 +5885,7 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: et-orbi-1.2.7 +Notice for: et-orbi-1.2.11 ---------- source: https://github.com/floraison/et-orbi/blob/v1.2.7/LICENSE.txt @@ -6157,7 +6157,33 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: fugit-1.9.0 +Notice for: fileutils-1.7.2 +---------- + +Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. +========== +Notice for: fugit-1.11.0 ---------- source: https://github.com/floraison/fugit/blob/v1.5.2/LICENSE.txt @@ -6292,7 +6318,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: http-cookie-1.0.5 +Notice for: http-cookie-1.0.6 ---------- source: https://github.com/sparklemotion/http-cookie/blob/v1.0.3/LICENSE.txt @@ -6380,7 +6406,7 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: i18n-1.14.1 +Notice for: i18n-1.14.5 ---------- source: https://github.com/svenfuchs/i18n/blob/v0.6.9/MIT-LICENSE @@ -6865,7 +6891,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION of your accepting any such warranty or additional liability. ========== -Notice for: jrjackson-0.4.18 +Notice for: jrjackson-0.4.20 ---------- https://github.com/guyboertje/jrjackson/blob/v0.4.6/README.md @@ -9357,7 +9383,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: jruby-openssl-0.14.2 +Notice for: jruby-openssl-0.14.6 ---------- source: https://github.com/jruby/jruby-openssl/blob/v0.9.21/LICENSE.txt @@ -9417,7 +9443,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: json-2.7.1 +Notice for: json-2.7.2 ---------- source: https://github.com/tmattia/json-generator/blob/v0.1.0/LICENSE.txt @@ -9445,7 +9471,7 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: jwt-2.7.1 +Notice for: jwt-2.8.2 ---------- source: https://github.com/jwt/ruby-jwt/blob/v2.2.2/LICENSE @@ -9594,7 +9620,7 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: method_source-1.0.0 +Notice for: method_source-1.1.0 ---------- source: https://github.com/banister/method_source/blob/v0.9.0/README.markdown#license @@ -9730,7 +9756,7 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: multipart-post-2.3.0 +Notice for: multipart-post-2.4.1 ---------- source: https://github.com/nicksieger/multipart-post/blob/v2.0.0/README.md#license @@ -9870,7 +9896,7 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: net-imap-0.4.9.1 +Notice for: net-imap-0.4.14 ---------- # source: https://github.com/ruby/net-imap/blob/v0.3.7/LICENSE.txt @@ -10014,7 +10040,7 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: net-smtp-0.4.0.1 +Notice for: net-smtp-0.5.0 ---------- # source: https://github.com/ruby/net-smtp/blob/v0.4.0/LICENSE.txt @@ -10042,7 +10068,7 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: nio4r-2.7.0 +Notice for: nio4r-2.7.3 ---------- Released under the MIT license. @@ -10057,7 +10083,7 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: nokogiri-1.16.0 +Notice for: nokogiri-1.16.6 ---------- source: https://github.com/sparklemotion/nokogiri/blob/v1.8.2/LICENSE.md @@ -10303,7 +10329,7 @@ Copyright (C) 1999-2019 by Shigeru Chiba, All rights reserved. This software is distributed under the Mozilla Public License Version 1.1, the GNU Lesser General Public License Version 2.1 or later, or the Apache License Version 2.0. ========== -Notice for: org.jruby:jruby-core-9.4.5.0 +Notice for: org.jruby:jruby-core-9.4.7.0 ---------- JRuby is Copyright (c) 2007-2018 The JRuby project @@ -10586,7 +10612,7 @@ Eclipse Public License - v 2.0 You may add additional accurate notices of copyright ownership. ========== -Notice for: org.logstash:jvm-options-parser-8.13.0 +Notice for: org.logstash:jvm-options-parser-8.14.2 ---------- Copyright (c) 2022 Elasticsearch B.V. @@ -10774,7 +10800,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: public_suffix-5.0.4 +Notice for: public_suffix-5.0.5 ---------- Copyright (c) 2009-2018 Simone Carletti @@ -10889,7 +10915,7 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ========== -Notice for: rack-2.2.8 +Notice for: rack-2.2.9 ---------- The MIT License (MIT) @@ -10940,7 +10966,7 @@ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: rake-13.1.0 +Notice for: rake-13.2.1 ---------- Copyright (c) Jim Weirich @@ -11017,6 +11043,82 @@ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +========== +Notice for: ruby-maven-libs-3.9.6.1 +---------- + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: +(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and +(b) You must cause any modified files to carry prominent notices stating that You changed the files; and +(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Standard License Header +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + ========== Notice for: ruby-progressbar-1.13.0 ---------- @@ -11185,7 +11287,7 @@ See the License for the specific language governing permissions and limitations under the License. ========== -Notice for: sequel-5.76.0 +Notice for: sequel-5.80.0 ---------- Copyright (c) 2007-2008 Sharon Rosner @@ -11752,7 +11854,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: tzinfo-data-1.2023.4 +Notice for: tzinfo-data-1.2024.1 ---------- Copyright (c) 2005-2018 Philip Ross @@ -11776,7 +11878,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========== -Notice for: webhdfs-0.10.2 +Notice for: webhdfs-0.11.0 ---------- Copyright (C) 2012 Fluentd Project diff --git a/tools/dependencies-report/src/main/resources/licenseMapping.csv b/tools/dependencies-report/src/main/resources/licenseMapping.csv index aba8b3bfe..133fb6c5a 100644 --- a/tools/dependencies-report/src/main/resources/licenseMapping.csv +++ b/tools/dependencies-report/src/main/resources/licenseMapping.csv @@ -74,6 +74,7 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL "ffi:",https://github.com/ffi/ffi,BSD-3-CLAUSE "ffi-binary-libfixposix:",https://github.com/byteit101/subspawn,Ruby "filesize:",https://github.com/dominikh,MIT +"fileutils:",https://github.com/ruby/fileutils,BSD-2-Clause "fugit:",https://github.com/floraison/fugit,MIT "gelfd2:",https://github.com/ptqa/gelfd2,Apache-2.0 "gems:",https://github.com/rubygems/gems,MIT diff --git a/tools/dependencies-report/src/main/resources/notices/fileutils-NOTICE.txt b/tools/dependencies-report/src/main/resources/notices/fileutils-NOTICE.txt new file mode 100644 index 000000000..44d33b0b5 --- /dev/null +++ b/tools/dependencies-report/src/main/resources/notices/fileutils-NOTICE.txt @@ -0,0 +1,22 @@ +Copyright (C) 1993-2013 Yukihiro Matsumoto. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. \ No newline at end of file From 3467751f906c6af8c6c204a739345f9f02912e2a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 19:24:41 -0400 Subject: [PATCH 28/32] Doc: Remove include statements for screenshots not rendering properly (#15981) (#16269) (cherry picked from commit cb45cd28cc005a580c81c05ce6032206c5731f3b) Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- .../static/tab-widgets/install-agent.asciidoc | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/docs/static/tab-widgets/install-agent.asciidoc b/docs/static/tab-widgets/install-agent.asciidoc index 046e15389..b19cb0ced 100644 --- a/docs/static/tab-widgets/install-agent.asciidoc +++ b/docs/static/tab-widgets/install-agent.asciidoc @@ -1,10 +1,5 @@ // tag::fleet-managed[] -. When the **Add Agent flyout** appears, stay on the **Enroll in fleet** tab -+ --- -[role="screenshot"] -image::../monitoring/images/integration-agent-add.png[Add agent flyout in {kib}] --- +. When the **Add Agent flyout** appears, stay on the **Enroll in fleet** tab. . Skip the **Select enrollment token** step. The enrollment token you need is already selected. + @@ -20,20 +15,11 @@ It takes about a minute for {agent} to enroll in {fleet}, download the configuration specified in the policy you just created, and start collecting data. --- -[role="screenshot"] -image::../monitoring/images/integration-agent-confirm.png[Agent confirm data] --- // end::fleet-managed[] // tag::standalone[] -. When the **Add Agent flyout** appears, navigate to the **Run standalone** tab -+ --- -[role="screenshot"] -image::../monitoring/images/integration-agent-add-standalone.png[Add agent flyout in {kib}] --- -. Configure the agent. Follow all the instructions in **Install Elastic Agent on your host** +. When the **Add Agent flyout** appears, navigate to the **Run standalone** tab. +. Configure the agent. Follow the instructions in **Install Elastic Agent on your host**. . After unpacking the binary, replace the `elastic-agent.yml` file with that supplied in the Add Agent flyout on the "Run standalone" tab, replacing the values of `ES_USERNAME` and `ES_PASSWORD` appropriately. . Run `sudo ./elastic-agent install` // end::standalone[] \ No newline at end of file From 6b8968476d68aecab028ea5c041d8a16759ace02 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 19:35:12 -0400 Subject: [PATCH 29/32] Doc: Add ecs and datastream requirement for intg filter (#16268) (#16270) (cherry picked from commit 095733c409eaa75e71d46019d7df0df8b6d5caed) Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/static/ea-integrations.asciidoc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/static/ea-integrations.asciidoc b/docs/static/ea-integrations.asciidoc index 0cec8ce3f..3d861fb7d 100644 --- a/docs/static/ea-integrations.asciidoc +++ b/docs/static/ea-integrations.asciidoc @@ -80,3 +80,16 @@ output { <3> <1> Use `filter-elastic_integration` as the first filter in your pipeline <2> You can use additional filters as long as they follow `filter-elastic_integration` <3> Sample config to output data to multiple destinations + +[discrete] +[[es-tips]] +==== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. From 9f71e033124e1d7859bc2a50919acbc0c0899c00 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:31:11 +0300 Subject: [PATCH 30/32] Add retries to aarch64 CI pipeline (#16271) (#16272) Add retries in the aarch64 CI pipeline to reduce noise from transient network failures. Closes https://github.com/elastic/ingest-dev/issues/3510 (cherry picked from commit 7080ec54272d3a436f552dc4b6ad1173bbe25102) Co-authored-by: Dimitrios Liappis --- .buildkite/aarch64_pipeline.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.buildkite/aarch64_pipeline.yml b/.buildkite/aarch64_pipeline.yml index 6e25b565d..fbbe1260f 100644 --- a/.buildkite/aarch64_pipeline.yml +++ b/.buildkite/aarch64_pipeline.yml @@ -17,6 +17,9 @@ steps: source .buildkite/scripts/common/vm-agent.sh ci/unit_tests.sh ruby + retry: + automatic: + - limit: 3 - label: ":java: Java unit tests" key: "java-unit-tests" @@ -28,6 +31,9 @@ steps: source .buildkite/scripts/common/vm-agent.sh ci/unit_tests.sh java + retry: + automatic: + - limit: 3 - label: ":lab_coat: Integration Tests / part 1" key: "integration-tests-part-1" @@ -36,6 +42,9 @@ steps: source .buildkite/scripts/common/vm-agent.sh ci/integration_tests.sh split 0 + retry: + automatic: + - limit: 3 - label: ":lab_coat: Integration Tests / part 2" key: "integration-tests-part-2" @@ -44,6 +53,9 @@ steps: source .buildkite/scripts/common/vm-agent.sh ci/integration_tests.sh split 1 + retry: + automatic: + - limit: 3 - label: ":lab_coat: IT Persistent Queues / part 1" key: "integration-tests-qa-part-1" @@ -53,6 +65,9 @@ steps: source .buildkite/scripts/common/vm-agent.sh export FEATURE_FLAG=persistent_queues ci/integration_tests.sh split 0 + retry: + automatic: + - limit: 3 - label: ":lab_coat: IT Persistent Queues / part 2" key: "integration-tests-qa-part-2" @@ -62,6 +77,9 @@ steps: source .buildkite/scripts/common/vm-agent.sh export FEATURE_FLAG=persistent_queues ci/integration_tests.sh split 1 + retry: + automatic: + - limit: 3 - label: ":lab_coat: x-pack unit tests" key: "x-pack-unit-tests" @@ -70,6 +88,9 @@ steps: source .buildkite/scripts/common/vm-agent.sh x-pack/ci/unit_tests.sh + retry: + automatic: + - limit: 3 - label: ":lab_coat: x-pack integration" key: "integration-tests-x-pack" @@ -78,6 +99,9 @@ steps: source .buildkite/scripts/common/vm-agent.sh x-pack/ci/integration_tests.sh + retry: + automatic: + - limit: 3 - group: "Acceptance Phase" depends_on: "testing-phase" @@ -88,6 +112,9 @@ steps: set -euo pipefail source .buildkite/scripts/common/vm-agent.sh && ci/docker_acceptance_tests.sh {{matrix}} + retry: + automatic: + - limit: 3 matrix: - "full" - "oss" From 7d75395e7766f52998744b3966df1e60764e1e08 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2024 09:35:32 -0700 Subject: [PATCH 31/32] Release notes for 8.14.2 (#16266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update release notes for 8.14.2 * human touch for 8.14.2 release notes * Update docs/static/releasenotes.asciidoc Co-authored-by: João Duarte * Update docs/static/releasenotes.asciidoc Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: Ry Biesemeyer Co-authored-by: Ry Biesemeyer Co-authored-by: João Duarte Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/static/releasenotes.asciidoc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index c877c5bce..7fd591c9e 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,7 @@ This section summarizes the changes in the following releases: +* <> * <> * <> * <> @@ -62,6 +63,36 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-14-2]] +=== Logstash 8.14.2 Release Notes + +[[notable-8.14.2]] +==== Notable issues fixed + +* Fixes a regression from Logstash 8.7.0 that prevented pipelines from starting when they included plugins with unicode ids https://github.com/elastic/logstash/pull/15971[#15971] +* Fixes a regression from Logstash 8.12.0 that prevented pipelines from starting when they included a geoip filter that used the managed databases feature after the databases had been updated https://github.com/elastic/logstash/pull/16222[#16222] +* Fixes an issue with the dead-letter queue that could cause it to emit superfluous warning messages when age-based retention was enabled while determining whether a fully-consumed segment that had already been removed was also expired https://github.com/elastic/logstash/pull/16204[#16204] + +==== Plugins + +*Fluent Codec - 3.4.3* + +* Fix: reduce overhead of unpacking packforward-payloads by reusing a single instance https://github.com/logstash-plugins/logstash-codec-fluent/pull/32[#32] + +*Elastic_integration Filter - 0.1.10* + +* Fixes handling of array-type event fields by treating them as lists https://github.com/elastic/logstash-filter-elastic_integration/pull/146[#146] +* Syncs with Elasticsearch 8.14, including support for new user-provided GeoIP database types `ConnectionType`, `Domain` and `Isp` https://github.com/elastic/logstash-filter-elastic_integration/pull/147[#147] + +*Elasticsearch Input - 4.20.3* + +* [DOC] Update link to bypass redirect, resolving directly to correct content https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/206[#206] + +*Elasticsearch Output - 11.22.7* + +* [DOC] `ssl_key` requires PKCS#8 format https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1181[#1181] + + [[logstash-8-14-1]] === Logstash 8.14.1 Release Notes From 6861ea613a4ef9516a5b147f661476fd58cc50bd Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Thu, 4 Jul 2024 01:07:22 -0700 Subject: [PATCH 32/32] bump to 8.4.3 (#16288) --- Gemfile.jruby-3.1.lock.release | 4 ++-- versions.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.jruby-3.1.lock.release b/Gemfile.jruby-3.1.lock.release index 6fd5267ec..c031270e9 100644 --- a/Gemfile.jruby-3.1.lock.release +++ b/Gemfile.jruby-3.1.lock.release @@ -2,12 +2,12 @@ PATH remote: logstash-core-plugin-api specs: logstash-core-plugin-api (2.1.16-java) - logstash-core (= 8.14.2) + logstash-core (= 8.14.3) PATH remote: logstash-core specs: - logstash-core (8.14.2-java) + logstash-core (8.14.3-java) clamp (~> 1) concurrent-ruby (~> 1, < 1.1.10) down (~> 5.2.0) diff --git a/versions.yml b/versions.yml index a69b0f71d..b63269b2e 100644 --- a/versions.yml +++ b/versions.yml @@ -1,7 +1,7 @@ --- # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var -logstash: 8.14.2 -logstash-core: 8.14.2 +logstash: 8.14.3 +logstash-core: 8.14.3 logstash-core-plugin-api: 2.1.16 bundled_jdk: