From de3b14f2b4bd775cd99febeaa5db9234394b49a2 Mon Sep 17 00:00:00 2001 From: Phillip Webb Date: Wed, 24 Jul 2024 17:59:45 +0100 Subject: [PATCH] Refine structured logging Refine structured logging to support `Environment`, `ApplicationPid` and `ElasticCommonSchemaService` injection. With these updates we are able to remove the `ApplicationMetadata` class and simplify the parameters passed to the layout/encoder classes. Closes gh-41491 --- .../reference/pages/features/logging.adoc | 18 ++++- ...ticCommonSchemaStructuredLogFormatter.java | 31 ++++---- .../LogstashStructuredLogFormatter.java | 27 ++++--- .../logging/log4j2/StructuredLogLayout.java | 63 +++++----------- .../logback/DefaultLogbackConfiguration.java | 9 --- ...ticCommonSchemaStructuredLogFormatter.java | 30 +++----- .../logging/logback/LogbackLoggingSystem.java | 31 +++++--- .../LogstashStructuredLogFormatter.java | 28 ++++---- .../logging/logback/StructuredLogEncoder.java | 47 +++--------- .../structured/ApplicationMetadata.java | 32 --------- .../ElasticCommonSchemaService.java | 71 +++++++++++++++++++ .../JsonWriterStructuredLogFormatter.java | 65 +++++++++++++++++ .../structured/StructuredLogFormatter.java | 24 ++++++- .../StructuredLogFormatterFactory.java | 11 ++- ...itional-spring-configuration-metadata.json | 58 +++++++++++++++ .../boot/logging/log4j2/log4j2-file.xml | 4 +- .../boot/logging/log4j2/log4j2.xml | 2 +- .../logback/structured-console-appender.xml | 2 - .../logback/structured-file-appender.xml | 2 - ...monSchemaStructuredLogFormatterTests.java} | 15 ++-- ... LogstashStructuredLogFormatterTests.java} | 6 +- .../log4j2/StructuredLoggingLayoutTests.java | 65 +++++++++++------ ...monSchemaStructuredLogFormatterTests.java} | 15 ++-- .../logback/LogbackLoggingSystemTests.java | 16 +++++ ... LogstashStructuredLogFormatterTests.java} | 6 +- .../StructuredLoggingEncoderTests.java | 27 +++++-- .../ElasticCommonSchemaServiceTests.java | 68 ++++++++++++++++++ .../StructuredLogFormatterFactoryTests.java | 30 ++++---- .../log4j2/CustomStructuredLogFormatter.java | 12 ++-- .../CustomStructuredLogFormatter.java | 14 ++-- 30 files changed, 539 insertions(+), 290 deletions(-) delete mode 100644 spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/ApplicationMetadata.java create mode 100644 spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/ElasticCommonSchemaService.java create mode 100644 spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/JsonWriterStructuredLogFormatter.java rename spring-boot-project/spring-boot/src/test/java/org/springframework/boot/logging/log4j2/{Log4j2EcsStructuredLoggingFormatterTests.java => ElasticCommonSchemaStructuredLogFormatterTests.java} (78%) rename spring-boot-project/spring-boot/src/test/java/org/springframework/boot/logging/log4j2/{Log4j2LogstashStructuredLoggingFormatterTests.java => LogstashStructuredLogFormatterTests.java} (89%) rename spring-boot-project/spring-boot/src/test/java/org/springframework/boot/logging/logback/{LogbackEcsStructuredLoggingFormatterTests.java => ElasticCommonSchemaStructuredLogFormatterTests.java} (80%) rename spring-boot-project/spring-boot/src/test/java/org/springframework/boot/logging/logback/{LogbackLogstashStructuredLoggingFormatterTests.java => LogstashStructuredLogFormatterTests.java} (91%) create mode 100644 spring-boot-project/spring-boot/src/test/java/org/springframework/boot/logging/structured/ElasticCommonSchemaServiceTests.java diff --git a/spring-boot-project/spring-boot-docs/src/docs/antora/modules/reference/pages/features/logging.adoc b/spring-boot-project/spring-boot-docs/src/docs/antora/modules/reference/pages/features/logging.adoc index a419cb5b2863..1023adb451fe 100644 --- a/spring-boot-project/spring-boot-docs/src/docs/antora/modules/reference/pages/features/logging.adoc +++ b/spring-boot-project/spring-boot-docs/src/docs/antora/modules/reference/pages/features/logging.adoc @@ -440,7 +440,7 @@ Handling authenticated request == Structured Logging Structured logging is a technique where the log output is written in a well-defined, often machine-readable format. -Spring Boot supports structured logging and has support for the following formats out of the box: +Spring Boot supports structured logging and has support for the following JSON formats out of the box: * xref:#features.logging.structured.ecs[Elastic Common Schema (ECS)] * xref:#features.logging.structured.logstash[Logstash] @@ -474,6 +474,22 @@ A log line looks like this: This format also adds every key value pair contained in the MDC to the JSON object. You can also use the https://www.slf4j.org/manual.html#fluent[SLF4J fluent logging API] to add key value pairs to the logged JSON object with the https://www.slf4j.org/apidocs/org/slf4j/spi/LoggingEventBuilder.html#addKeyValue(java.lang.String,java.lang.Object)[addKeyValue] method. +The `service` values can be customized using `logging.structured.ecs.service` properties: + +[configprops,yaml] +---- +logging: + structured: + ecs: + service: + name: MyService + version: 1.0 + environment: Production + node-name: Primary +---- + +NOTE: configprop:logging.structured.ecs.service.name[] will default to configprop:spring.application.name[] if not specified. + [[features.logging.structured.logstash]] diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/ElasticCommonSchemaStructuredLogFormatter.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/ElasticCommonSchemaStructuredLogFormatter.java index 574581a9cc21..24d953f0674a 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/ElasticCommonSchemaStructuredLogFormatter.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/ElasticCommonSchemaStructuredLogFormatter.java @@ -24,9 +24,11 @@ import org.apache.logging.log4j.util.ReadOnlyStringMap; import org.springframework.boot.json.JsonWriter; -import org.springframework.boot.logging.structured.ApplicationMetadata; import org.springframework.boot.logging.structured.CommonStructuredLogFormat; +import org.springframework.boot.logging.structured.ElasticCommonSchemaService; +import org.springframework.boot.logging.structured.JsonWriterStructuredLogFormatter; import org.springframework.boot.logging.structured.StructuredLogFormatter; +import org.springframework.boot.system.ApplicationPid; import org.springframework.util.ObjectUtils; /** @@ -36,23 +38,19 @@ * @author Moritz Halbritter * @author Phillip Webb */ -class ElasticCommonSchemaStructuredLogFormatter implements StructuredLogFormatter { +class ElasticCommonSchemaStructuredLogFormatter extends JsonWriterStructuredLogFormatter { - private final JsonWriter writer; - - ElasticCommonSchemaStructuredLogFormatter(ApplicationMetadata metadata) { - this.writer = JsonWriter.of((members) -> logEventJson(metadata, members)).withNewLineAtEnd(); + ElasticCommonSchemaStructuredLogFormatter(ApplicationPid pid, ElasticCommonSchemaService service) { + super((members) -> jsonMembers(pid, service, members)); } - private void logEventJson(ApplicationMetadata metadata, JsonWriter.Members members) { - members.add("@timestamp", LogEvent::getInstant).as(this::asTimestamp); + private static void jsonMembers(ApplicationPid pid, ElasticCommonSchemaService service, + JsonWriter.Members members) { + members.add("@timestamp", LogEvent::getInstant).as(ElasticCommonSchemaStructuredLogFormatter::asTimestamp); members.add("log.level", LogEvent::getLevel).as(Level::name); - members.add("process.pid", metadata::pid).whenNotNull(); + members.add("process.pid", pid).when(ApplicationPid::isAvailable).as(ApplicationPid::toLong); members.add("process.thread.name", LogEvent::getThreadName); - members.add("service.name", metadata::name).whenHasLength(); - members.add("service.version", metadata::version).whenHasLength(); - members.add("service.environment", metadata::environment).whenHasLength(); - members.add("service.node.name", metadata::nodeName).whenHasLength(); + service.jsonMembers(members); members.add("log.logger", LogEvent::getLoggerName); members.add("message", LogEvent::getMessage).as(Message::getFormattedMessage); members.add(LogEvent::getContextData) @@ -68,13 +66,8 @@ private void logEventJson(ApplicationMetadata metadata, JsonWriter.Members { - - private JsonWriter writer; +class LogstashStructuredLogFormatter extends JsonWriterStructuredLogFormatter { LogstashStructuredLogFormatter() { - this.writer = JsonWriter.of(this::logEventJson).withNewLineAtEnd(); + super(LogstashStructuredLogFormatter::jsonMembers); } - private void logEventJson(JsonWriter.Members members) { - members.add("@timestamp", LogEvent::getInstant).as(this::asTimestamp); + private static void jsonMembers(JsonWriter.Members members) { + members.add("@timestamp", LogEvent::getInstant).as(LogstashStructuredLogFormatter::asTimestamp); members.add("@version", "1"); members.add("message", LogEvent::getMessage).as(Message::getFormattedMessage); members.add("logger_name", LogEvent::getLoggerName); @@ -60,26 +59,29 @@ private void logEventJson(JsonWriter.Members members) { members.add(LogEvent::getContextData) .whenNot(ReadOnlyStringMap::isEmpty) .usingPairs((contextData, pairs) -> contextData.forEach(pairs::accept)); - members.add("tags", LogEvent::getMarker).whenNotNull().as(this::getMarkers).whenNot(CollectionUtils::isEmpty); + members.add("tags", LogEvent::getMarker) + .whenNotNull() + .as(LogstashStructuredLogFormatter::getMarkers) + .whenNot(CollectionUtils::isEmpty); members.add("stack_trace", LogEvent::getThrownProxy) .whenNotNull() .as(ThrowableProxy::getExtendedStackTraceAsString); } - private String asTimestamp(Instant instant) { + private static String asTimestamp(Instant instant) { java.time.Instant javaInstant = java.time.Instant.ofEpochMilli(instant.getEpochMillisecond()) .plusNanos(instant.getNanoOfMillisecond()); OffsetDateTime offsetDateTime = OffsetDateTime.ofInstant(javaInstant, ZoneId.systemDefault()); return DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(offsetDateTime); } - private Set getMarkers(Marker marker) { + private static Set getMarkers(Marker marker) { Set result = new TreeSet<>(); addMarkers(result, marker); return result; } - private void addMarkers(Set result, Marker marker) { + private static void addMarkers(Set result, Marker marker) { result.add(marker.getName()); if (marker.hasParents()) { for (Marker parent : marker.getParents()) { @@ -88,9 +90,4 @@ private void addMarkers(Set result, Marker marker) { } } - @Override - public String format(LogEvent event) { - return this.writer.writeToString(event); - } - } diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/StructuredLogLayout.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/StructuredLogLayout.java index a604278aa6bc..c471905bac40 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/StructuredLogLayout.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/StructuredLogLayout.java @@ -21,17 +21,21 @@ import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.Node; import org.apache.logging.log4j.core.config.plugins.Plugin; import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute; import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; +import org.apache.logging.log4j.core.config.plugins.PluginLoggerContext; import org.apache.logging.log4j.core.layout.AbstractStringLayout; -import org.springframework.boot.logging.structured.ApplicationMetadata; import org.springframework.boot.logging.structured.CommonStructuredLogFormat; +import org.springframework.boot.logging.structured.ElasticCommonSchemaService; import org.springframework.boot.logging.structured.StructuredLogFormatter; import org.springframework.boot.logging.structured.StructuredLogFormatterFactory; import org.springframework.boot.logging.structured.StructuredLogFormatterFactory.CommonFormatters; +import org.springframework.boot.system.ApplicationPid; +import org.springframework.core.env.Environment; import org.springframework.util.Assert; /** @@ -57,6 +61,11 @@ public String toSerializable(LogEvent event) { return this.formatter.format(event); } + @Override + public byte[] toByteArray(LogEvent event) { + return this.formatter.formatAsBytes(event, (getCharset() != null) ? getCharset() : StandardCharsets.UTF_8); + } + @PluginBuilderFactory static StructuredLogLayout.Builder newBuilder() { return new StructuredLogLayout.Builder(); @@ -64,27 +73,15 @@ static StructuredLogLayout.Builder newBuilder() { static final class Builder implements org.apache.logging.log4j.core.util.Builder { + @PluginLoggerContext + private LoggerContext loggerContext; + @PluginBuilderAttribute private String format; @PluginBuilderAttribute private String charset = StandardCharsets.UTF_8.name(); - @PluginBuilderAttribute - private Long pid; - - @PluginBuilderAttribute - private String serviceName; - - @PluginBuilderAttribute - private String serviceVersion; - - @PluginBuilderAttribute - private String serviceNodeName; - - @PluginBuilderAttribute - private String serviceEnvironment; - Builder setFormat(String format) { this.format = format; return this; @@ -95,38 +92,13 @@ Builder setCharset(String charset) { return this; } - Builder setPid(Long pid) { - this.pid = pid; - return this; - } - - Builder setServiceName(String serviceName) { - this.serviceName = serviceName; - return this; - } - - Builder setServiceVersion(String serviceVersion) { - this.serviceVersion = serviceVersion; - return this; - } - - Builder setServiceNodeName(String serviceNodeName) { - this.serviceNodeName = serviceNodeName; - return this; - } - - Builder setServiceEnvironment(String serviceEnvironment) { - this.serviceEnvironment = serviceEnvironment; - return this; - } - @Override public StructuredLogLayout build() { - ApplicationMetadata applicationMetadata = new ApplicationMetadata(this.pid, this.serviceName, - this.serviceVersion, this.serviceEnvironment, this.serviceNodeName); Charset charset = Charset.forName(this.charset); + Environment environment = Log4J2LoggingSystem.getEnvironment(this.loggerContext); + Assert.state(environment != null, "Unable to find Spring Environment in logger context"); StructuredLogFormatter formatter = new StructuredLogFormatterFactory<>(LogEvent.class, - applicationMetadata, null, this::addCommonFormatters) + environment, null, this::addCommonFormatters) .get(this.format); return new StructuredLogLayout(charset, formatter); } @@ -134,7 +106,8 @@ public StructuredLogLayout build() { private void addCommonFormatters(CommonFormatters commonFormatters) { commonFormatters.add(CommonStructuredLogFormat.ELASTIC_COMMON_SCHEMA, (instantiator) -> new ElasticCommonSchemaStructuredLogFormatter( - instantiator.getArg(ApplicationMetadata.class))); + instantiator.getArg(ApplicationPid.class), + instantiator.getArg(ElasticCommonSchemaService.class))); commonFormatters.add(CommonStructuredLogFormat.LOGSTASH, (instantiator) -> new LogstashStructuredLogFormatter()); } diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/DefaultLogbackConfiguration.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/DefaultLogbackConfiguration.java index 1a96efa6586d..9112fa750b4d 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/DefaultLogbackConfiguration.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/DefaultLogbackConfiguration.java @@ -174,11 +174,6 @@ private Encoder createEncoder(LogbackConfigurator config, String private StructuredLogEncoder createStructuredLoggingEncoder(LogbackConfigurator config, String format) { StructuredLogEncoder encoder = new StructuredLogEncoder(); encoder.setFormat(format); - encoder.setPid(resolveLong(config, "${PID:--1}")); - String applicationName = resolve(config, "${APPLICATION_NAME:-}"); - if (StringUtils.hasLength(applicationName)) { - encoder.setServiceName(applicationName); - } return encoder; } @@ -205,10 +200,6 @@ private int resolveInt(LogbackConfigurator config, String val) { return Integer.parseInt(resolve(config, val)); } - private long resolveLong(LogbackConfigurator config, String val) { - return Long.parseLong(resolve(config, val)); - } - private FileSize resolveFileSize(LogbackConfigurator config, String val) { return FileSize.valueOf(resolve(config, val)); } diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/ElasticCommonSchemaStructuredLogFormatter.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/ElasticCommonSchemaStructuredLogFormatter.java index 3b769a2765d4..2b56f0dc68a0 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/ElasticCommonSchemaStructuredLogFormatter.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/ElasticCommonSchemaStructuredLogFormatter.java @@ -23,9 +23,11 @@ import org.springframework.boot.json.JsonWriter; import org.springframework.boot.json.JsonWriter.PairExtractor; -import org.springframework.boot.logging.structured.ApplicationMetadata; import org.springframework.boot.logging.structured.CommonStructuredLogFormat; +import org.springframework.boot.logging.structured.ElasticCommonSchemaService; +import org.springframework.boot.logging.structured.JsonWriterStructuredLogFormatter; import org.springframework.boot.logging.structured.StructuredLogFormatter; +import org.springframework.boot.system.ApplicationPid; /** * Logback {@link StructuredLogFormatter} for @@ -34,30 +36,23 @@ * @author Moritz Halbritter * @author Phillip Webb */ -class ElasticCommonSchemaStructuredLogFormatter implements StructuredLogFormatter { +class ElasticCommonSchemaStructuredLogFormatter extends JsonWriterStructuredLogFormatter { private static final PairExtractor keyValuePairExtractor = PairExtractor.of((pair) -> pair.key, (pair) -> pair.value); - private JsonWriter writer; - - ElasticCommonSchemaStructuredLogFormatter(ApplicationMetadata metadata, + ElasticCommonSchemaStructuredLogFormatter(ApplicationPid pid, ElasticCommonSchemaService service, ThrowableProxyConverter throwableProxyConverter) { - this.writer = JsonWriter - .of((members) -> loggingEventJson(metadata, throwableProxyConverter, members)) - .withNewLineAtEnd(); + super((members) -> jsonMembers(pid, service, throwableProxyConverter, members)); } - private void loggingEventJson(ApplicationMetadata metadata, ThrowableProxyConverter throwableProxyConverter, - JsonWriter.Members members) { + private static void jsonMembers(ApplicationPid pid, ElasticCommonSchemaService service, + ThrowableProxyConverter throwableProxyConverter, JsonWriter.Members members) { members.add("@timestamp", ILoggingEvent::getInstant); members.add("log.level", ILoggingEvent::getLevel); - members.add("process.pid", metadata::pid).whenNotNull(); + members.add("process.pid", pid).when(ApplicationPid::isAvailable).as(ApplicationPid::toLong); members.add("process.thread.name", ILoggingEvent::getThreadName); - members.add("service.name", metadata::name).whenHasLength(); - members.add("service.version", metadata::version).whenHasLength(); - members.add("service.environment", metadata::environment).whenHasLength(); - members.add("service.node.name", metadata::nodeName).whenHasLength(); + service.jsonMembers(members); members.add("log.logger", ILoggingEvent::getLoggerName); members.add("message", ILoggingEvent::getFormattedMessage); members.addMapEntries(ILoggingEvent::getMDCPropertyMap); @@ -72,9 +67,4 @@ private void loggingEventJson(ApplicationMetadata metadata, ThrowableProxyConver members.add("ecs.version", "8.11"); } - @Override - public String format(ILoggingEvent event) { - return this.writer.writeToString(event); - } - } diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/LogbackLoggingSystem.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/LogbackLoggingSystem.java index 1b3f9a97fde8..61976b36e08c 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/LogbackLoggingSystem.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/LogbackLoggingSystem.java @@ -186,13 +186,13 @@ private void removeDefaultRootHandler() { @Override public void initialize(LoggingInitializationContext initializationContext, String configLocation, LogFile logFile) { LoggerContext loggerContext = getLoggerContext(); + putInitializationContextObjects(loggerContext, initializationContext); if (isAlreadyInitialized(loggerContext)) { return; } if (!initializeFromAotGeneratedArtifactsIfPossible(initializationContext, logFile)) { super.initialize(initializationContext, configLocation, logFile); } - loggerContext.putObject(Environment.class.getName(), initializationContext.getEnvironment()); loggerContext.getTurboFilterList().remove(SUPPRESS_ALL_FILTER); markAsInitialized(loggerContext); if (StringUtils.hasText(System.getProperty(CONFIGURATION_FILE_PROPERTY))) { @@ -211,6 +211,7 @@ private boolean initializeFromAotGeneratedArtifactsIfPossible(LoggingInitializat } LoggerContext loggerContext = getLoggerContext(); stopAndReset(loggerContext); + withLoggingSuppressed(() -> putInitializationContextObjects(loggerContext, initializationContext)); SpringBootJoranConfigurator configurator = new SpringBootJoranConfigurator(initializationContext); configurator.setContext(loggerContext); boolean configuredUsingAotGeneratedArtifacts = configurator.configureUsingAotGeneratedArtifacts(); @@ -222,21 +223,23 @@ private boolean initializeFromAotGeneratedArtifactsIfPossible(LoggingInitializat @Override protected void loadDefaults(LoggingInitializationContext initializationContext, LogFile logFile) { - LoggerContext context = getLoggerContext(); - stopAndReset(context); + LoggerContext loggerContext = getLoggerContext(); + stopAndReset(loggerContext); withLoggingSuppressed(() -> { + putInitializationContextObjects(loggerContext, initializationContext); boolean debug = Boolean.getBoolean("logback.debug"); if (debug) { - StatusListenerConfigHelper.addOnConsoleListenerInstance(context, new OnConsoleStatusListener()); + StatusListenerConfigHelper.addOnConsoleListenerInstance(loggerContext, new OnConsoleStatusListener()); } Environment environment = initializationContext.getEnvironment(); // Apply system properties directly in case the same JVM runs multiple apps - new LogbackLoggingSystemProperties(environment, getDefaultValueResolver(environment), context::putProperty) + new LogbackLoggingSystemProperties(environment, getDefaultValueResolver(environment), + loggerContext::putProperty) .apply(logFile); - LogbackConfigurator configurator = debug ? new DebugLogbackConfigurator(context) - : new LogbackConfigurator(context); + LogbackConfigurator configurator = debug ? new DebugLogbackConfigurator(loggerContext) + : new LogbackConfigurator(loggerContext); new DefaultLogbackConfiguration(logFile).apply(configurator); - context.setPackagingDataEnabled(true); + loggerContext.setPackagingDataEnabled(true); }); } @@ -246,6 +249,7 @@ protected void loadConfiguration(LoggingInitializationContext initializationCont LoggerContext loggerContext = getLoggerContext(); stopAndReset(loggerContext); withLoggingSuppressed(() -> { + putInitializationContextObjects(loggerContext, initializationContext); if (initializationContext != null) { applySystemProperties(initializationContext.getEnvironment(), logFile); } @@ -334,11 +338,18 @@ public void cleanUp() { @Override protected void reinitialize(LoggingInitializationContext initializationContext) { - getLoggerContext().reset(); - getLoggerContext().getStatusManager().clear(); + LoggerContext loggerContext = getLoggerContext(); + loggerContext.reset(); + loggerContext.getStatusManager().clear(); loadConfiguration(initializationContext, getSelfInitializationConfig(), null); } + private void putInitializationContextObjects(LoggerContext loggerContext, + LoggingInitializationContext initializationContext) { + withLoggingSuppressed( + () -> loggerContext.putObject(Environment.class.getName(), initializationContext.getEnvironment())); + } + @Override public List getLoggerConfigurations() { List result = new ArrayList<>(); diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/LogstashStructuredLogFormatter.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/LogstashStructuredLogFormatter.java index 4a50f2f291fc..626a89e8f0ac 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/LogstashStructuredLogFormatter.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/LogstashStructuredLogFormatter.java @@ -34,6 +34,7 @@ import org.springframework.boot.json.JsonWriter; import org.springframework.boot.json.JsonWriter.PairExtractor; import org.springframework.boot.logging.structured.CommonStructuredLogFormat; +import org.springframework.boot.logging.structured.JsonWriterStructuredLogFormatter; import org.springframework.boot.logging.structured.StructuredLogFormatter; /** @@ -42,21 +43,18 @@ * @author Moritz Halbritter * @author Phillip Webb */ -class LogstashStructuredLogFormatter implements StructuredLogFormatter { +class LogstashStructuredLogFormatter extends JsonWriterStructuredLogFormatter { private static final PairExtractor keyValuePairExtractor = PairExtractor.of((pair) -> pair.key, (pair) -> pair.value); - private JsonWriter writer; - LogstashStructuredLogFormatter(ThrowableProxyConverter throwableProxyConverter) { - this.writer = JsonWriter.of((members) -> loggingEventJson(throwableProxyConverter, members)) - .withNewLineAtEnd(); + super((members) -> jsonMembers(throwableProxyConverter, members)); } - private void loggingEventJson(ThrowableProxyConverter throwableProxyConverter, + private static void jsonMembers(ThrowableProxyConverter throwableProxyConverter, JsonWriter.Members members) { - members.add("@timestamp", ILoggingEvent::getInstant).as(this::asTimestamp); + members.add("@timestamp", ILoggingEvent::getInstant).as(LogstashStructuredLogFormatter::asTimestamp); members.add("@version", "1"); members.add("message", ILoggingEvent::getFormattedMessage); members.add("logger_name", ILoggingEvent::getLoggerName); @@ -67,24 +65,27 @@ private void loggingEventJson(ThrowableProxyConverter throwableProxyConverter, members.add(ILoggingEvent::getKeyValuePairs) .whenNotEmpty() .usingExtractedPairs(Iterable::forEach, keyValuePairExtractor); - members.add("tags", ILoggingEvent::getMarkerList).whenNotNull().as(this::getMarkers).whenNotEmpty(); + members.add("tags", ILoggingEvent::getMarkerList) + .whenNotNull() + .as(LogstashStructuredLogFormatter::getMarkers) + .whenNotEmpty(); members.add("stack_trace", (event) -> event) .whenNotNull(ILoggingEvent::getThrowableProxy) .as(throwableProxyConverter::convert); } - private String asTimestamp(Instant instant) { + private static String asTimestamp(Instant instant) { OffsetDateTime offsetDateTime = OffsetDateTime.ofInstant(instant, ZoneId.systemDefault()); return DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(offsetDateTime); } - private Set getMarkers(List markers) { + private static Set getMarkers(List markers) { Set result = new LinkedHashSet<>(); addMarkers(result, markers.iterator()); return result; } - private void addMarkers(Set result, Iterator iterator) { + private static void addMarkers(Set result, Iterator iterator) { while (iterator.hasNext()) { Marker marker = iterator.next(); result.add(marker.getName()); @@ -94,9 +95,4 @@ private void addMarkers(Set result, Iterator iterator) { } } - @Override - public String format(ILoggingEvent event) { - return this.writer.writeToString(event); - } - } diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/StructuredLogEncoder.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/StructuredLogEncoder.java index 04c3e1716993..c4539e9cb053 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/StructuredLogEncoder.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/StructuredLogEncoder.java @@ -24,12 +24,14 @@ import ch.qos.logback.core.encoder.Encoder; import ch.qos.logback.core.encoder.EncoderBase; -import org.springframework.boot.logging.structured.ApplicationMetadata; import org.springframework.boot.logging.structured.CommonStructuredLogFormat; +import org.springframework.boot.logging.structured.ElasticCommonSchemaService; import org.springframework.boot.logging.structured.StructuredLogFormatter; import org.springframework.boot.logging.structured.StructuredLogFormatterFactory; import org.springframework.boot.logging.structured.StructuredLogFormatterFactory.CommonFormatters; +import org.springframework.boot.system.ApplicationPid; import org.springframework.boot.util.Instantiator.AvailableParameters; +import org.springframework.core.env.Environment; import org.springframework.util.Assert; /** @@ -48,42 +50,12 @@ public class StructuredLogEncoder extends EncoderBase { private StructuredLogFormatter formatter; - private Long pid; - - private String serviceName; - - private String serviceVersion; - - private String serviceNodeName; - - private String serviceEnvironment; - private Charset charset = StandardCharsets.UTF_8; public void setFormat(String format) { this.format = format; } - public void setPid(Long pid) { - this.pid = pid; - } - - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - - public void setServiceVersion(String serviceVersion) { - this.serviceVersion = serviceVersion; - } - - public void setServiceNodeName(String serviceNodeName) { - this.serviceNodeName = serviceNodeName; - } - - public void setServiceEnvironment(String serviceEnvironment) { - this.serviceEnvironment = serviceEnvironment; - } - public void setCharset(Charset charset) { this.charset = charset; } @@ -97,10 +69,10 @@ public void start() { } private StructuredLogFormatter createFormatter(String format) { - ApplicationMetadata applicationMetadata = new ApplicationMetadata(this.pid, this.serviceName, - this.serviceVersion, this.serviceEnvironment, this.serviceNodeName); - return new StructuredLogFormatterFactory<>(ILoggingEvent.class, applicationMetadata, - this::addAvailableParameters, this::addCommonFormatters) + Environment environment = (Environment) getContext().getObject(Environment.class.getName()); + Assert.state(environment != null, "Unable to find Spring Environment in logger context"); + return new StructuredLogFormatterFactory<>(ILoggingEvent.class, environment, this::addAvailableParameters, + this::addCommonFormatters) .get(format); } @@ -111,7 +83,8 @@ private void addAvailableParameters(AvailableParameters availableParameters) { private void addCommonFormatters(CommonFormatters commonFormatters) { commonFormatters.add(CommonStructuredLogFormat.ELASTIC_COMMON_SCHEMA, (instantiator) -> new ElasticCommonSchemaStructuredLogFormatter( - instantiator.getArg(ApplicationMetadata.class), + instantiator.getArg(ApplicationPid.class), + instantiator.getArg(ElasticCommonSchemaService.class), instantiator.getArg(ThrowableProxyConverter.class))); commonFormatters.add(CommonStructuredLogFormat.LOGSTASH, (instantiator) -> new LogstashStructuredLogFormatter( instantiator.getArg(ThrowableProxyConverter.class))); @@ -130,7 +103,7 @@ public byte[] headerBytes() { @Override public byte[] encode(ILoggingEvent event) { - return this.formatter.format(event).getBytes(this.charset); + return this.formatter.formatAsBytes(event, (this.charset != null) ? this.charset : StandardCharsets.UTF_8); } @Override diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/ApplicationMetadata.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/ApplicationMetadata.java deleted file mode 100644 index 61bdd9918b5b..000000000000 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/ApplicationMetadata.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2012-2024 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.boot.logging.structured; - -/** - * Metadata about the application. - * - * @param pid the process ID of the application - * @param name the application name - * @param version the version of the application - * @param environment the name of the environment the application is running in - * @param nodeName the name of the node the application is running on - * @author Moritz Halbritter - * @since 3.4.0 - */ -public record ApplicationMetadata(Long pid, String name, String version, String environment, String nodeName) { - -} diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/ElasticCommonSchemaService.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/ElasticCommonSchemaService.java new file mode 100644 index 000000000000..9d448ef759ef --- /dev/null +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/ElasticCommonSchemaService.java @@ -0,0 +1,71 @@ +/* + * Copyright 2012-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.logging.structured; + +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.boot.json.JsonWriter; +import org.springframework.core.env.Environment; +import org.springframework.util.StringUtils; + +/** + * Service details for Elastic Common Schema structured logging. + * + * @param name the application name + * @param version the version of the application + * @param environment the name of the environment the application is running in + * @param nodeName the name of the node the application is running on + * @author Moritz Halbritter + * @author Phillip Webb + * @since 3.4.0 + */ +public record ElasticCommonSchemaService(String name, String version, String environment, String nodeName) { + + static final ElasticCommonSchemaService NONE = new ElasticCommonSchemaService(null, null, null, null); + + private ElasticCommonSchemaService withDefaults(Environment environment) { + String name = withFallbackProperty(environment, this.name, "spring.application.name"); + return new ElasticCommonSchemaService(name, this.version, this.environment, this.nodeName); + } + + private String withFallbackProperty(Environment environment, String value, String property) { + return (!StringUtils.hasLength(value)) ? environment.getProperty(property) : value; + } + + /** + * Add {@link JsonWriter} members for the service. + * @param members the members to add to + */ + public void jsonMembers(JsonWriter.Members members) { + members.add("service.name", this::name).whenHasLength(); + members.add("service.version", this::version).whenHasLength(); + members.add("service.environment", this::environment).whenHasLength(); + members.add("service.node.name", this::nodeName).whenHasLength(); + } + + /** + * Return a new {@link ElasticCommonSchemaService} from bound from properties in the + * given {@link Environment}. + * @param environment the source environment + * @return a new {@link ElasticCommonSchemaService} instance + */ + public static ElasticCommonSchemaService get(Environment environment) { + return Binder.get(environment) + .bind("logging.structured.ecs.service", ElasticCommonSchemaService.class) + .orElse(NONE) + .withDefaults(environment); + } +} diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/JsonWriterStructuredLogFormatter.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/JsonWriterStructuredLogFormatter.java new file mode 100644 index 000000000000..8807ea4ec768 --- /dev/null +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/JsonWriterStructuredLogFormatter.java @@ -0,0 +1,65 @@ +/* + * Copyright 2012-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.logging.structured; + +import java.nio.charset.Charset; +import java.util.function.Consumer; + +import org.springframework.boot.json.JsonWriter; +import org.springframework.boot.json.JsonWriter.Members; + +/** + * Base class for {@link StructuredLogFormatter} implementations that generates JSON using + * a {@link JsonWriter}. + * + * @param the log event type + * @author Phillip Webb + * @since 3.4.0 + */ +public abstract class JsonWriterStructuredLogFormatter implements StructuredLogFormatter { + + private final JsonWriter jsonWriter; + + /** + * Create a new {@link JsonWriterStructuredLogFormatter} instance with the given + * members. + * @param members a consumer which should configure the members + */ + protected JsonWriterStructuredLogFormatter(Consumer> members) { + this(JsonWriter.of(members).withNewLineAtEnd()); + } + + /** + * Create a new {@link JsonWriterStructuredLogFormatter} instance with the given + * {@link JsonWriter}. + * @param jsonWriter the {@link JsonWriter} + */ + protected JsonWriterStructuredLogFormatter(JsonWriter jsonWriter) { + this.jsonWriter = jsonWriter; + } + + @Override + public String format(E event) { + return this.jsonWriter.writeToString(event); + } + + @Override + public byte[] formatAsBytes(E event, Charset charset) { + return this.jsonWriter.write(event).toByteArray(); + } + +} diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatter.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatter.java index 69a249827bdc..eb5193a770cb 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatter.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatter.java @@ -16,14 +16,21 @@ package org.springframework.boot.logging.structured; +import java.nio.charset.Charset; + import ch.qos.logback.classic.pattern.ThrowableProxyConverter; +import org.springframework.boot.system.ApplicationPid; +import org.springframework.core.env.Environment; + /** * Formats a log event to a structured log message. *

* Implementing classes can declare the following parameter types in the constructor: *

    - *
  • {@link ApplicationMetadata}
    • + *
  • {@link Environment}
    • + *
  • {@link ApplicationPid}
    • + *
  • {@link ElasticCommonSchemaService}
    • *
* When using Logback, implementing classes can also use the following parameter types in * the constructor: @@ -35,13 +42,24 @@ * @author Moritz Halbritter * @since 3.4.0 */ +@FunctionalInterface public interface StructuredLogFormatter { /** - * Formats the given log event. + * Formats the given log event to a String. * @param event the log event to write - * @return the formatted log event + * @return the formatted log event String */ String format(E event); + /** + * Formats the given log event to a byte array. + * @param event the log event to write + * @param charset the charset + * @return the formatted log event bytes + */ + default byte[] formatAsBytes(E event, Charset charset) { + return format(event).getBytes(charset); + } + } diff --git a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatterFactory.java b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatterFactory.java index c2a51ec103c1..5cb42e400e91 100644 --- a/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatterFactory.java +++ b/spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/structured/StructuredLogFormatterFactory.java @@ -21,10 +21,12 @@ import java.util.TreeMap; import java.util.function.Consumer; +import org.springframework.boot.system.ApplicationPid; import org.springframework.boot.util.Instantiator; import org.springframework.boot.util.Instantiator.AvailableParameters; import org.springframework.boot.util.Instantiator.FailureHandler; import org.springframework.core.GenericTypeResolver; +import org.springframework.core.env.Environment; import org.springframework.util.Assert; /** @@ -56,16 +58,19 @@ public class StructuredLogFormatterFactory { /** * Create a new {@link StructuredLogFormatterFactory} instance. * @param logEventType the log event type - * @param applicationMetadata an {@link ApplicationMetadata} instance for injection + * @param environment the Spring {@link Environment} * @param availableParameters callback used to configure available parameters for the * specific logging system * @param commonFormatters callback used to define supported common formatters */ - public StructuredLogFormatterFactory(Class logEventType, ApplicationMetadata applicationMetadata, + public StructuredLogFormatterFactory(Class logEventType, Environment environment, Consumer availableParameters, Consumer> commonFormatters) { this.logEventType = logEventType; this.instantiator = new Instantiator<>(StructuredLogFormatter.class, (allAvailableParameters) -> { - allAvailableParameters.add(ApplicationMetadata.class, applicationMetadata); + allAvailableParameters.add(Environment.class, environment); + allAvailableParameters.add(ApplicationPid.class, (type) -> new ApplicationPid()); + allAvailableParameters.add(ElasticCommonSchemaService.class, + (type) -> ElasticCommonSchemaService.get(environment)); if (availableParameters != null) { availableParameters.accept(allAvailableParameters); } diff --git a/spring-boot-project/spring-boot/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/spring-boot-project/spring-boot/src/main/resources/META-INF/additional-spring-configuration-metadata.json index 27eebf2c81d1..abe86991ad75 100644 --- a/spring-boot-project/spring-boot/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/spring-boot-project/spring-boot/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -223,6 +223,26 @@ "sourceType": "org.springframework.boot.context.logging.LoggingApplicationListener", "defaultValue": true }, + { + "name": "logging.structured.ecs.service.environment", + "type": "java.lang.String", + "description": "Structured ECS service environment." + }, + { + "name": "logging.structured.ecs.service.name", + "type": "java.lang.String", + "description": "Structured ECS service name (defaults to 'spring.application.name')." + }, + { + "name": "logging.structured.ecs.service.node-name", + "type": "java.lang.String", + "description": "Structured ECS service node name." + }, + { + "name": "logging.structured.ecs.service.version", + "type": "java.lang.String", + "description": "Structured ECS service version." + }, { "name": "logging.structured.format.console", "type": "java.lang.String", @@ -597,6 +617,44 @@ } ] }, + { + "name": "logging.structured.format.console", + "values": [ + { + "value": "ecs" + }, + { + "value": "logstash" + } + ], + "providers": [ + { + "name": "handle-as", + "parameters": { + "target": "java.lang.Class" + } + } + ] + }, + { + "name": "logging.structured.format.file", + "values": [ + { + "value": "ecs" + }, + { + "value": "logstash" + } + ], + "providers": [ + { + "name": "handle-as", + "parameters": { + "target": "java.lang.Class" + } + } + ] + }, { "name": "spring.config.import", "values": [ diff --git a/spring-boot-project/spring-boot/src/main/resources/org/springframework/boot/logging/log4j2/log4j2-file.xml b/spring-boot-project/spring-boot/src/main/resources/org/springframework/boot/logging/log4j2/log4j2-file.xml index f6a0e81b33a6..a1387fc0042c 100644 --- a/spring-boot-project/spring-boot/src/main/resources/org/springframework/boot/logging/log4j2/log4j2-file.xml +++ b/spring-boot-project/spring-boot/src/main/resources/org/springframework/boot/logging/log4j2/log4j2-file.xml @@ -11,7 +11,7 @@ - + diff --git a/spring-boot-project/spring-boot/src/main/resources/org/springframework/boot/logging/log4j2/log4j2.xml b/spring-boot-project/spring-boot/src/main/resources/org/springframework/boot/logging/log4j2/log4j2.xml index a994ab43897c..cb94b2ff67cd 100644 --- a/spring-boot-project/spring-boot/src/main/resources/org/springframework/boot/logging/log4j2/log4j2.xml +++ b/spring-boot-project/spring-boot/src/main/resources/org/springframework/boot/logging/log4j2/log4j2.xml @@ -11,7 +11,7 @@