diff --git a/docker/Makefile b/docker/Makefile index 01028bb0e..82b265808 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -64,11 +64,14 @@ build-from-local-ubi8-artifacts: venv dockerfile env2yaml (docker kill $(HTTPD); false); -docker kill $(HTTPD) -COPY_FILES = $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml $(ARTIFACTS_DIR)/docker/config/log4j2.properties $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml +COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml +COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties +COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml $(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml: data/logstash/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml: data/logstash/config/logstash-full.yml +$(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties $(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint @@ -84,10 +87,14 @@ docker_paths: mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml mkdir -p $(ARTIFACTS_DIR)/docker/pipeline -COPY_IRONBANK_FILES = $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md +COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml +COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.file.properties $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties +COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/env2yaml.go +COPY_IRONBANK_FILES += $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.mod $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/go.sum $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor/modules.txt $(ARTIFACTS_DIR)/ironbank/LICENSE $(ARTIFACTS_DIR)/ironbank/README.md $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml: data/logstash/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml: data/logstash/config/logstash-full.yml +$(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.file.properties: data/logstash/config/log4j2.file.properties $(ARTIFACTS_DIR)/ironbank/scripts/config/log4j2.properties: data/logstash/config/log4j2.properties $(ARTIFACTS_DIR)/ironbank/scripts/pipeline/default.conf: data/logstash/pipeline/default.conf $(ARTIFACTS_DIR)/ironbank/scripts/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint diff --git a/docker/data/logstash/bin/docker-entrypoint b/docker/data/logstash/bin/docker-entrypoint index 19165f149..e2fd33c29 100755 --- a/docker/data/logstash/bin/docker-entrypoint +++ b/docker/data/logstash/bin/docker-entrypoint @@ -6,6 +6,22 @@ # host system. env2yaml /usr/share/logstash/config/logstash.yml +if [[ -n "$LOG_STYLE" ]]; then + case "$LOG_STYLE" in + console) + # This is the default. Nothing to do. + ;; + file) + # Overwrite the default config with the stack config. Do this as a + # copy, not a move, in case the container is restarted. + cp -f /usr/share/logstash/config/log4j2.file.properties /usr/share/logstash/config/log4j2.properties + ;; + *) + echo "ERROR: LOG_STYLE set to [$LOG_STYLE]. Expected [console] or [file]" >&2 + exit 1 ;; + esac +fi + export LS_JAVA_OPTS="-Dls.cgroup.cpuacct.path.override=/ -Dls.cgroup.cpu.path.override=/ $LS_JAVA_OPTS" if [[ -z $1 ]] || [[ ${1:0:1} == '-' ]] ; then diff --git a/docker/data/logstash/config/log4j2.file.properties b/docker/data/logstash/config/log4j2.file.properties new file mode 100644 index 000000000..234b23db4 --- /dev/null +++ b/docker/data/logstash/config/log4j2.file.properties @@ -0,0 +1,147 @@ +status = error +name = LogstashPropertiesConfig + +appender.console.type = Console +appender.console.name = plain_console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n + +appender.json_console.type = Console +appender.json_console.name = json_console +appender.json_console.layout.type = JSONLayout +appender.json_console.layout.compact = true +appender.json_console.layout.eventEol = true + +appender.rolling.type = RollingFile +appender.rolling.name = plain_rolling +appender.rolling.fileName = ${sys:ls.logs}/logstash-plain.log +appender.rolling.filePattern = ${sys:ls.logs}/logstash-plain-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 100MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = 30 +appender.rolling.avoid_pipelined_filter.type = PipelineRoutingFilter + +appender.json_rolling.type = RollingFile +appender.json_rolling.name = json_rolling +appender.json_rolling.fileName = ${sys:ls.logs}/logstash-json.log +appender.json_rolling.filePattern = ${sys:ls.logs}/logstash-json-%d{yyyy-MM-dd}-%i.log.gz +appender.json_rolling.policies.type = Policies +appender.json_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.json_rolling.policies.time.interval = 1 +appender.json_rolling.policies.time.modulate = true +appender.json_rolling.layout.type = JSONLayout +appender.json_rolling.layout.compact = true +appender.json_rolling.layout.eventEol = true +appender.json_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.json_rolling.policies.size.size = 100MB +appender.json_rolling.strategy.type = DefaultRolloverStrategy +appender.json_rolling.strategy.max = 30 +appender.json_rolling.avoid_pipelined_filter.type = PipelineRoutingFilter + +appender.routing.type = PipelineRouting +appender.routing.name = pipeline_routing_appender +appender.routing.pipeline.type = RollingFile +appender.routing.pipeline.name = appender-${ctx:pipeline.id} +appender.routing.pipeline.fileName = ${sys:ls.logs}/pipeline_${ctx:pipeline.id}.log +appender.routing.pipeline.filePattern = ${sys:ls.logs}/pipeline_${ctx:pipeline.id}.%i.log.gz +appender.routing.pipeline.layout.type = PatternLayout +appender.routing.pipeline.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n +appender.routing.pipeline.policy.type = SizeBasedTriggeringPolicy +appender.routing.pipeline.policy.size = 100MB +appender.routing.pipeline.strategy.type = DefaultRolloverStrategy +appender.routing.pipeline.strategy.max = 30 + +rootLogger.level = ${sys:ls.log.level} +rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console +rootLogger.appenderRef.rolling.ref = ${sys:ls.log.format}_rolling +rootLogger.appenderRef.routing.ref = pipeline_routing_appender + +# Slowlog + +appender.console_slowlog.type = Console +appender.console_slowlog.name = plain_console_slowlog +appender.console_slowlog.layout.type = PatternLayout +appender.console_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n + +appender.json_console_slowlog.type = Console +appender.json_console_slowlog.name = json_console_slowlog +appender.json_console_slowlog.layout.type = JSONLayout +appender.json_console_slowlog.layout.compact = true +appender.json_console_slowlog.layout.eventEol = true + +appender.rolling_slowlog.type = RollingFile +appender.rolling_slowlog.name = plain_rolling_slowlog +appender.rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-plain.log +appender.rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-plain-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling_slowlog.policies.type = Policies +appender.rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling_slowlog.policies.time.interval = 1 +appender.rolling_slowlog.policies.time.modulate = true +appender.rolling_slowlog.layout.type = PatternLayout +appender.rolling_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n +appender.rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling_slowlog.policies.size.size = 100MB +appender.rolling_slowlog.strategy.type = DefaultRolloverStrategy +appender.rolling_slowlog.strategy.max = 30 + +appender.json_rolling_slowlog.type = RollingFile +appender.json_rolling_slowlog.name = json_rolling_slowlog +appender.json_rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-json.log +appender.json_rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-json-%d{yyyy-MM-dd}-%i.log.gz +appender.json_rolling_slowlog.policies.type = Policies +appender.json_rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy +appender.json_rolling_slowlog.policies.time.interval = 1 +appender.json_rolling_slowlog.policies.time.modulate = true +appender.json_rolling_slowlog.layout.type = JSONLayout +appender.json_rolling_slowlog.layout.compact = true +appender.json_rolling_slowlog.layout.eventEol = true +appender.json_rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy +appender.json_rolling_slowlog.policies.size.size = 100MB +appender.json_rolling_slowlog.strategy.type = DefaultRolloverStrategy +appender.json_rolling_slowlog.strategy.max = 30 + +logger.slowlog.name = slowlog +logger.slowlog.level = trace +logger.slowlog.appenderRef.console_slowlog.ref = ${sys:ls.log.format}_console_slowlog +logger.slowlog.appenderRef.rolling_slowlog.ref = ${sys:ls.log.format}_rolling_slowlog +logger.slowlog.additivity = false + +logger.licensereader.name = logstash.licensechecker.licensereader +logger.licensereader.level = error + +# Silence http-client by default +logger.apache_http_client.name = org.apache.http +logger.apache_http_client.level = fatal + +# Deprecation log +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_plain_rolling +appender.deprecation_rolling.fileName = ${sys:ls.logs}/logstash-deprecation.log +appender.deprecation_rolling.filePattern = ${sys:ls.logs}/logstash-deprecation-%d{yyyy-MM-dd}-%i.log.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.deprecation_rolling.policies.time.interval = 1 +appender.deprecation_rolling.policies.time.modulate = true +appender.deprecation_rolling.layout.type = PatternLayout +appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 100MB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 30 + +logger.deprecation.name = org.logstash.deprecation, deprecation +logger.deprecation.level = WARN +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_plain_rolling +logger.deprecation.additivity = false + +logger.deprecation_root.name = deprecation +logger.deprecation_root.level = WARN +logger.deprecation_root.appenderRef.deprecation_rolling.ref = deprecation_plain_rolling +logger.deprecation_root.additivity = false diff --git a/docker/templates/Dockerfile.j2 b/docker/templates/Dockerfile.j2 index 01a752618..311e8c3e9 100644 --- a/docker/templates/Dockerfile.j2 +++ b/docker/templates/Dockerfile.j2 @@ -144,6 +144,7 @@ COPY config/logstash-oss.yml config/logstash.yml COPY config/logstash-full.yml config/logstash.yml {% endif -%} COPY config/log4j2.properties config/ +COPY config/log4j2.file.properties config/ COPY pipeline/default.conf pipeline/logstash.conf RUN chown --recursive logstash:root config/ pipeline/ # Ensure Logstash gets the correct locale by default. @@ -155,6 +156,7 @@ COPY bin/docker-entrypoint /usr/local/bin/ COPY scripts/config/pipelines.yml config/pipelines.yml COPY scripts/config/logstash.yml config/logstash.yml COPY scripts/config/log4j2.properties config/ +COPY scripts/config/log4j2.file.properties config/ COPY scripts/pipeline/default.conf pipeline/logstash.conf RUN chown --recursive logstash:root config/ pipeline/ # Place the startup wrapper script. diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index b8d0e65c9..a14a2c85d 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -603,6 +603,9 @@ namespace "artifact" do File.join(basedir, "config", "log4j2.properties").tap do |path| dir.input("#{path}=/etc/logstash") end + File.join(basedir, "config", "log4j2.file.properties").tap do |path| + dir.input("#{path}=/etc/logstash") + end arch_suffix = bundle_jdk ? map_architecture_for_package_type(platform, jdk_arch) : "no-jdk"