add socket log appender, enable json format for it

core/deployment/src/main/java/io/quarkus/deployment/builditem/LogSocketFormatBuildItem.java

@@ -0,0 +1,36 @@
+package io.quarkus.deployment.builditem;
+
+import java.util.Optional;
+import java.util.logging.Formatter;
+
+import org.wildfly.common.Assert;
+
+import io.quarkus.builder.item.MultiBuildItem;
+import io.quarkus.runtime.RuntimeValue;
+
+/**
+ * The socket format build item. Producing this item will cause the logging subsystem to disregard its
+ * socket logging formatting configuration and use the formatter provided instead. If multiple formatters
+ * are enabled at runtime, a warning message is printed and only one is used.
+ */
+public final class LogSocketFormatBuildItem extends MultiBuildItem {
+    private final RuntimeValue<Optional<Formatter>> formatterValue;
+
+    /**
+     * Construct a new instance.
+     *
+     * @param formatterValue the optional formatter runtime value to use (must not be {@code null})
+     */
+    public LogSocketFormatBuildItem(final RuntimeValue<Optional<Formatter>> formatterValue) {
+        this.formatterValue = Assert.checkNotNullParam("formatterValue", formatterValue);
+    }
+
+    /**
+     * Get the formatter value.
+     *
+     * @return the formatter value
+     */
+    public RuntimeValue<Optional<Formatter>> getFormatterValue() {
+        return formatterValue;
+    }
+}

core/deployment/src/main/java/io/quarkus/deployment/logging/LoggingResourceProcessor.java

@@ -72,6 +72,7 @@
 import io.quarkus.deployment.builditem.LogConsoleFormatBuildItem;
 import io.quarkus.deployment.builditem.LogFileFormatBuildItem;
 import io.quarkus.deployment.builditem.LogHandlerBuildItem;
+import io.quarkus.deployment.builditem.LogSocketFormatBuildItem;
 import io.quarkus.deployment.builditem.LogSyslogFormatBuildItem;
 import io.quarkus.deployment.builditem.NamedLogHandlersBuildItem;
 import io.quarkus.deployment.builditem.RunTimeConfigurationDefaultBuildItem;
@@ -244,6 +245,7 @@ LoggingSetupBuildItem setupLoggingRuntimeInit(RecorderContext context, LoggingSe
             List<LogConsoleFormatBuildItem> consoleFormatItems,
             List<LogFileFormatBuildItem> fileFormatItems,
             List<LogSyslogFormatBuildItem> syslogFormatItems,
+            List<LogSocketFormatBuildItem> socketFormatItems,
             Optional<ConsoleFormatterBannerBuildItem> possibleBannerBuildItem,
             List<LogStreamBuildItem> logStreamBuildItems,
             BuildProducer<ShutdownListenerBuildItem> shutdownListenerBuildItemBuildProducer,
@@ -285,6 +287,8 @@ LoggingSetupBuildItem setupLoggingRuntimeInit(RecorderContext context, LoggingSe
                     .map(LogFileFormatBuildItem::getFormatterValue).collect(Collectors.toList());
             List<RuntimeValue<Optional<Formatter>>> possibleSyslogFormatters = syslogFormatItems.stream()
                     .map(LogSyslogFormatBuildItem::getFormatterValue).collect(Collectors.toList());
+            List<RuntimeValue<Optional<Formatter>>> possibleSocketFormatters = socketFormatItems.stream()
+                    .map(LogSocketFormatBuildItem::getFormatterValue).collect(Collectors.toList());
 
             context.registerSubstitution(InheritableLevel.ActualLevel.class, String.class, InheritableLevel.Substitution.class);
             context.registerSubstitution(InheritableLevel.Inherited.class, String.class, InheritableLevel.Substitution.class);
@@ -303,6 +307,7 @@ LoggingSetupBuildItem setupLoggingRuntimeInit(RecorderContext context, LoggingSe
                             categoryMinLevelDefaults.content, alwaysEnableLogStream,
                             streamingDevUiLogHandler, handlers, namedHandlers,
                             possibleConsoleFormatters, possibleFileFormatters, possibleSyslogFormatters,
+                            possibleSocketFormatters,
                             possibleSupplier, launchModeBuildItem.getLaunchMode(), true)));
             LogConfig logConfig = new LogConfig();
             ConfigInstantiator.handleObject(logConfig);

core/runtime/src/main/java/io/quarkus/runtime/logging/LogConfig.java

@@ -59,6 +59,14 @@ public final class LogConfig {
     @ConfigDocSection
     public SyslogConfig syslog;
 
+    /**
+     * Socket logging.
+     * <p>
+     * Logging to a socket is also supported but not enabled by default.
+     */
+    @ConfigDocSection
+    public SocketConfig socket;
+
     /**
      * Logging categories.
      * <p>
@@ -97,6 +105,15 @@ public final class LogConfig {
     @ConfigDocSection
     public Map<String, SyslogConfig> syslogHandlers;
 
+    /**
+     * Socket handlers.
+     * <p>
+     * The named socket handlers configured here can be linked to one or more categories.
+     */
+    @ConfigItem(name = "handler.socket")
+    @ConfigDocSection
+    public Map<String, SocketConfig> socketHandlers;
+
     /**
      * Log cleanup filters - internal use.
      */

core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java

@@ -41,6 +41,7 @@
 import org.jboss.logmanager.handlers.FileHandler;
 import org.jboss.logmanager.handlers.PeriodicSizeRotatingFileHandler;
 import org.jboss.logmanager.handlers.SizeRotatingFileHandler;
+import org.jboss.logmanager.handlers.SocketHandler;
 import org.jboss.logmanager.handlers.SyslogHandler;
 
 import io.quarkus.bootstrap.logging.InitialConfigurator;
@@ -86,6 +87,7 @@ public static void handleFailedStart(RuntimeValue<Optional<Supplier<String>>> ba
                 Collections.emptyList(),
                 Collections.emptyList(),
                 Collections.emptyList(),
+                Collections.emptyList(),
                 Collections.emptyList(), banner, LaunchMode.DEVELOPMENT, false);
     }
 
@@ -99,6 +101,7 @@ public ShutdownListener initializeLogging(LogConfig config, LogBuildTimeConfig b
             final List<RuntimeValue<Optional<Formatter>>> possibleConsoleFormatters,
             final List<RuntimeValue<Optional<Formatter>>> possibleFileFormatters,
             final List<RuntimeValue<Optional<Formatter>>> possibleSyslogFormatters,
+            final List<RuntimeValue<Optional<Formatter>>> possibleSocketFormatters,
             final RuntimeValue<Optional<Supplier<String>>> possibleBannerSupplier,
             LaunchMode launchMode,
             boolean includeFilters) {
@@ -183,6 +186,14 @@ public void close() throws SecurityException {
             }
         }
 
+        if (config.socket.enable) {
+            final Handler socketHandler = configureSocketHandler(config.socket, errorManager, cleanupFiler,
+                    namedFilters, possibleSocketFormatters, includeFilters);
+            if (socketHandler != null) {
+                handlers.add(socketHandler);
+            }
+        }
+
         if ((launchMode.isDevOrTest() || enableWebStream)
                 && streamingDevUiConsoleHandler != null
                 && streamingDevUiConsoleHandler.getValue().isPresent()) {
@@ -201,7 +212,7 @@ public void close() throws SecurityException {
 
         Map<String, Handler> namedHandlers = shouldCreateNamedHandlers(config, additionalNamedHandlers)
                 ? createNamedHandlers(config, consoleRuntimeConfig.getValue(), additionalNamedHandlers,
-                        possibleConsoleFormatters, possibleFileFormatters, possibleSyslogFormatters,
+                        possibleConsoleFormatters, possibleFileFormatters, possibleSyslogFormatters, possibleSocketFormatters,
                         errorManager, cleanupFiler, namedFilters, launchMode,
                         shutdownNotifier, includeFilters)
                 : Collections.emptyMap();
@@ -312,7 +323,8 @@ public static void initializeBuildTimeLogging(LogConfig config, LogBuildTimeConf
         }
 
         Map<String, Handler> namedHandlers = createNamedHandlers(config, consoleConfig, Collections.emptyList(),
-                Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), errorManager, logCleanupFilter,
+                Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(),
+                errorManager, logCleanupFilter,
                 Collections.emptyMap(), launchMode, dummy, false);
 
         for (Map.Entry<String, CategoryConfig> entry : categories.entrySet()) {
@@ -399,6 +411,7 @@ private static Map<String, Handler> createNamedHandlers(LogConfig config, Consol
             List<RuntimeValue<Optional<Formatter>>> possibleConsoleFormatters,
             List<RuntimeValue<Optional<Formatter>>> possibleFileFormatters,
             List<RuntimeValue<Optional<Formatter>>> possibleSyslogFormatters,
+            List<RuntimeValue<Optional<Formatter>>> possibleSocketFormatters,
             ErrorManager errorManager, LogCleanupFilter cleanupFilter,
             Map<String, Filter> namedFilters, LaunchMode launchMode,
             ShutdownNotifier shutdownHandler, boolean includeFilters) {
@@ -433,6 +446,17 @@ private static Map<String, Handler> createNamedHandlers(LogConfig config, Consol
                 addToNamedHandlers(namedHandlers, syslogHandler, sysLogConfigEntry.getKey());
             }
         }
+        for (Entry<String, SocketConfig> socketConfigEntry : config.socketHandlers.entrySet()) {
+            SocketConfig namedSocketConfig = socketConfigEntry.getValue();
+            if (!namedSocketConfig.enable) {
+                continue;
+            }
+            final Handler socketHandler = configureSocketHandler(namedSocketConfig, errorManager, cleanupFilter,
+                    namedFilters, possibleSocketFormatters, includeFilters);
+            if (socketHandler != null) {
+                addToNamedHandlers(namedHandlers, socketHandler, socketConfigEntry.getKey());
+            }
+        }
 
         Map<String, Handler> additionalNamedHandlersMap;
         if (additionalNamedHandlers.isEmpty()) {
@@ -742,6 +766,53 @@ private static Handler configureSyslogHandler(final SyslogConfig config, final E
         }
     }
 
+    private static Handler configureSocketHandler(final SocketConfig config,
+            final ErrorManager errorManager,
+            final LogCleanupFilter logCleanupFilter,
+            final Map<String, Filter> namedFilters,
+            final List<RuntimeValue<Optional<Formatter>>> possibleSocketFormatters,
+            final boolean includeFilters) {
+        try {
+            final SocketHandler handler = new SocketHandler(config.endpoint.getHostString(), config.endpoint.getPort());
+            handler.setProtocol(config.protocol);
+            handler.setBlockOnReconnect(config.blockOnReconnect);
+            handler.setLevel(config.level);
+
+            Formatter formatter = null;
+            boolean formatterWarning = false;
+            for (RuntimeValue<Optional<Formatter>> value : possibleSocketFormatters) {
+                if (formatter != null) {
+                    formatterWarning = true;
+                }
+                final Optional<Formatter> val = value.getValue();
+                if (val.isPresent()) {
+                    formatter = val.get();
+                }
+            }
+            if (formatter == null) {
+                formatter = new PatternFormatter(config.format);
+            }
+            handler.setFormatter(formatter);
+
+            handler.setErrorManager(errorManager);
+            handler.setFilter(logCleanupFilter);
+            applyFilter(includeFilters, errorManager, logCleanupFilter, config.filter, namedFilters, handler);
+
+            if (formatterWarning) {
+                handler.getErrorManager().error("Multiple socket formatters were activated", null,
+                        ErrorManager.GENERIC_FAILURE);
+            }
+
+            if (config.async.enable) {
+                return createAsyncHandler(config.async, config.level, handler);
+            }
+            return handler;
+        } catch (IOException e) {
+            errorManager.error("Failed to create socket handler", e, ErrorManager.OPEN_FAILURE);
+            return null;
+        }
+    }
+
     private static AsyncHandler createAsyncHandler(AsyncConfig asyncConfig, Level level, Handler handler) {
         final AsyncHandler asyncHandler = new AsyncHandler(asyncConfig.queueLength);
         asyncHandler.setOverflowAction(asyncConfig.overflow);

core/runtime/src/main/java/io/quarkus/runtime/logging/SocketConfig.java

@@ -0,0 +1,64 @@
+package io.quarkus.runtime.logging;
+
+import java.net.InetSocketAddress;
+import java.util.Optional;
+import java.util.logging.Level;
+
+import org.jboss.logmanager.handlers.SocketHandler.Protocol;
+
+import io.quarkus.runtime.annotations.ConfigGroup;
+import io.quarkus.runtime.annotations.ConfigItem;
+
+@ConfigGroup
+public class SocketConfig {
+
+    /**
+     * If socket logging should be enabled
+     */
+    @ConfigItem
+    boolean enable;
+
+    /**
+     *
+     * The IP address and port of the server receiving the logs
+     */
+    @ConfigItem(defaultValue = "localhost:4560")
+    InetSocketAddress endpoint;
+
+    /**
+     * Sets the protocol used to connect to the syslog server
+     */
+    @ConfigItem(defaultValue = "tcp")
+    Protocol protocol;
+
+    /**
+     * Enables or disables blocking when attempting to reconnect a
+     * {@link Protocol#TCP
+     * TCP} or {@link Protocol#SSL_TCP SSL TCP} protocol
+     */
+    @ConfigItem
+    boolean blockOnReconnect;
+
+    /**
+     * The log message format
+     */
+    @ConfigItem(defaultValue = "%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] (%t) %s%e%n")
+    String format;
+
+    /**
+     * The log level specifying, which message levels will be logged by socket logger
+     */
+    @ConfigItem(defaultValue = "ALL")
+    Level level;
+
+    /**
+     * The name of the filter to link to the file handler.
+     */
+    @ConfigItem
+    Optional<String> filter;
+
+    /**
+     * Socket async logging config
+     */
+    AsyncConfig async;
+}

docs/src/main/asciidoc/centralized-log-management.adoc

@@ -236,6 +236,83 @@ networks:
 
 Launch your application, you should see your logs arriving inside the Elastic Stack; you can use Kibana available at http://localhost:5601/ to access them.
 
+
+[[logstash_ecs]]
+== GELF alternative: Send logs to Logstash in the ECS (Elastic Common Schema) format
+
+You can also send your logs to Logstash using a TCP input in the https://www.elastic.co/guide/en/ecs-logging/overview/current/intro.html[ECS] format.
+To achieve this we will use the `quarkus-logging-json` extension to format the logs in JSON format and the socket handler to send them to Logstash.
+
+For this you can use the same `docker-compose.yml` file as above but with a different Logstash pipeline configuration.
+
+[source]
+----
+input {
+  tcp {
+    port => 4560
+    coded => json
+  }
+}
+
+filter {
+  if ![span][id] and [mdc][spanId] {
+    mutate { rename => { "[mdc][spanId]" => "[span][id]" } }
+  }
+  if ![trace][id] and [mdc][traceId] {
+    mutate { rename => {"[mdc][traceId]" => "[trace][id]"} }
+  }
+}
+
+output {
+  stdout {}
+  elasticsearch {
+    hosts => ["http://elasticsearch:9200"]
+  }
+}
+----
+
+Then configure your application to log in JSON format instead of GELF
+----
+<dependency>
+    <groupId>io.quarkus</groupId>
+    <artifactId>quarkus-logging-json</artifactId>
+</dependency>
+----
+
+and specify the host and port of your Logstash endpoint. To be ECS compliant, some keys need to be renamed.
+
+[source, properties]
+----
+# to keep the logs in the usual format in the console
+quarkus.log.console.json=false
+
+quarkus.log.socket.enable=true
+quarkus.log.socket.json=true
+quarkus.log.socket.endpoint=localhost:4560
+
+# to have the exception serialized into a single text element
+quarkus.log.socket.json.exception-output-type=formatted
+
+# override some keys to be ECS compliant
+quarkus.log.socket.json.key-overrides=timestamp=@timestamp,\
+	logger_name=log.logger,\
+	level=log.level,\
+	process_id=process.pid,\
+	process_name=process.name,\
+	thread_name=process.thread.name,\
+	thread_id=process.thread.id,\
+	sequence=event.sequence,\
+	host_name=host.hostname,\
+	stack_trace=error.stack_trace
+quarkus.log.socket.json.excluded-keys=loggerClassName
+quarkus.log.socket.json.additional-field."service.environment".value=${quarkus.profile}
+quarkus.log.socket.json.additional-field."service.name".value=${quarkus.application.name}
+quarkus.log.socket.json.additional-field."service.version".value=${quarkus.application.version}
+quarkus.log.socket.json.additional-field."data_stream.type".value=logs
+quarkus.log.socket.json.additional-field."ecs.version".value=1.12.2
+----
+
+
 == Send logs to Fluentd (EFK)
 
 First, you need to create a Fluentd image with the needed plugins: elasticsearch and input-gelf.
@@ -422,6 +499,7 @@ quarkus.log.syslog.hostname=quarkus-test
 
 Launch your application, you should see your logs arriving inside EFK: you can use Kibana available at http://localhost:5601/ to access them.
 
+
 == Elasticsearch indexing consideration
 
 Be careful that, by default, Elasticsearch will automatically map unknown fields (if not disabled in the index settings) by detecting their type.