Skip to content

Commit

Permalink
[Enhancement] support json log4j logging
Browse files Browse the repository at this point in the history
Fixes #48464

Signed-off-by: Kevin Xiaohua Cai <[email protected]>
  • Loading branch information
kevincai committed Jul 19, 2024
1 parent 2a33dc9 commit 3706d45
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 26 deletions.
2 changes: 2 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/common/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ public class Config extends ConfigBase {
@ConfField
public static boolean sys_log_to_console = false;

@ConfField(comment = "Log4j layout format. Valid choices: plaintext, json")
public static String sys_log_format = "plaintext";
/**
* audit_log_dir:
* This specifies FE audit log dir.
Expand Down
71 changes: 45 additions & 26 deletions fe/fe-core/src/main/java/com/starrocks/common/Log4jConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

package com.starrocks.common;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import groovy.lang.Tuple3;
Expand Down Expand Up @@ -61,12 +62,10 @@ public class Log4jConfig extends XmlConfiguration {
"<Configuration status=\"info\" packages=\"com.starrocks.common\">\n" +
" <Appenders>\n" +
" <Console name=\"ConsoleErr\" target=\"SYSTEM_ERR\" follow=\"true\">\n" +
" <PatternLayout pattern=\"%d{yyyy-MM-dd HH:mm:ss.SSSXXX} %p (%t|%tid) [%C{1}.%M():%L] %m%n\"/>\n" +
" ${syslog_default_layout}\n" +
" </Console>\n" +
" <RollingFile name=\"Sys\" fileName=\"${sys_log_dir}/fe.log\" filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
" <Pattern>%d{yyyy-MM-dd HH:mm:ss.SSSXXX} %p (%t|%tid) [%C{1}.%M():%L] %m%n</Pattern>\n" +
" </PatternLayout>\n" +
" ${syslog_default_layout}\n" +
" <Policies>\n" +
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${sys_roll_maxsize}MB\"/>\n" +
Expand All @@ -79,9 +78,7 @@ public class Log4jConfig extends XmlConfiguration {
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" <RollingFile name=\"SysWF\" fileName=\"${sys_log_dir}/fe.warn.log\" filePattern=\"${sys_log_dir}/fe.warn.log.${sys_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
" <Pattern>%d{yyyy-MM-dd HH:mm:ss.SSSXXX} %p (%t|%tid) [%C{1}.%M():%L] %m%n %ex</Pattern>\n" +
" </PatternLayout>\n" +
" ${syslog_warning_layout}\n" +
" <Policies>\n" +
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${sys_roll_maxsize}MB\"/>\n" +
Expand All @@ -94,9 +91,7 @@ public class Log4jConfig extends XmlConfiguration {
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" <RollingFile name=\"Auditfile\" fileName=\"${audit_log_dir}/fe.audit.log\" filePattern=\"${audit_log_dir}/fe.audit.log.${audit_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
" <Pattern>%d{yyyy-MM-dd HH:mm:ss.SSSXXX} [%c{1}] %m%n</Pattern>\n" +
" </PatternLayout>\n" +
" ${syslog_audit_layout}\n" +
" <Policies>\n" +
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${audit_roll_maxsize}MB\"/>\n" +
Expand All @@ -109,9 +104,7 @@ public class Log4jConfig extends XmlConfiguration {
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" <RollingFile name=\"dumpFile\" fileName=\"${dump_log_dir}/fe.dump.log\" filePattern=\"${dump_log_dir}/fe.dump.log.${dump_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
" <Pattern>%d{yyyy-MM-dd HH:mm:ss.SSSXXX} [%c{1}] %m%n</Pattern>\n" +
" </PatternLayout>\n" +
" ${syslog_dump_layout}\n" +
" <Policies>\n" +
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${dump_roll_maxsize}MB\"/>\n" +
Expand All @@ -124,9 +117,7 @@ public class Log4jConfig extends XmlConfiguration {
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" <RollingFile name=\"BigQueryFile\" fileName=\"${big_query_log_dir}/fe.big_query.log\" filePattern=\"${big_query_log_dir}/fe.big_query.log.${big_query_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
" <Pattern>%d{yyyy-MM-dd HH:mm:ss.SSSXXX} [%c{1}] %m%n</Pattern>\n" +
" </PatternLayout>\n" +
" ${syslog_bigquery_layout}\n" +
" <Policies>\n" +
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${big_query_roll_maxsize}MB\"/>\n" +
Expand All @@ -139,9 +130,7 @@ public class Log4jConfig extends XmlConfiguration {
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" <RollingFile name=\"ProfileFile\" fileName=\"${profile_log_dir}/fe.profile.log\" filePattern=\"${profile_log_dir}/fe.profile.log.${profile_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
" <Pattern>%d{yyyy-MM-dd HH:mm:ss.SSSXXX} [%c{1}] %m%n</Pattern>\n" +
" </PatternLayout>\n" +
" ${syslog_profile_layout}\n" +
" <Policies>\n" +
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${profile_log_roll_size_mb}MB\"/>\n" +
Expand All @@ -154,9 +143,7 @@ public class Log4jConfig extends XmlConfiguration {
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" <RollingFile name=\"InternalFile\" fileName=\"${internal_log_dir}/fe.internal.log\" filePattern=\"${internal_log_dir}/fe.internal.log.${internal_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
" <Pattern>%d{yyyy-MM-dd HH:mm:ss.SSSXXX} %p (%t|%tid) [%C{1}.%M():%L] %m%n</Pattern>\n" +
" </PatternLayout>\n" +
" ${syslog_default_layout}\n" +
" <Policies>\n" +
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${internal_roll_maxsize}MB\"/>\n" +
Expand Down Expand Up @@ -211,7 +198,8 @@ public class Log4jConfig extends XmlConfiguration {
private static String[] bigQueryModules;
private static String[] internalModules;

private static void reconfig() throws IOException {
@VisibleForTesting
static String generateActiveLog4jXmlConfig() throws IOException {
Map<String, String> properties = Maps.newHashMap();

// sys log config
Expand Down Expand Up @@ -244,7 +232,7 @@ private static void reconfig() throws IOException {
// big query log config
properties.put("big_query_log_dir", Config.big_query_log_dir);
properties.put("big_query_roll_maxsize", String.valueOf(Config.log_roll_size_mb));
properties.put("big_query_roll_num", String.valueOf(Config.big_query_log_roll_num));
properties.put("big_query_log_roll_num", String.valueOf(Config.big_query_log_roll_num));
properties.put("big_query_log_delete_age", String.valueOf(Config.big_query_log_delete_age));
properties.put("big_query_file_pattern",
getIntervalPattern("big_query_log_roll_interval", Config.big_query_log_roll_interval));
Expand All @@ -260,15 +248,46 @@ private static void reconfig() throws IOException {
// internal log config
properties.put("internal_log_dir", Config.internal_log_dir);
properties.put("internal_roll_maxsize", String.valueOf(Config.log_roll_size_mb));
properties.put("internal_roll_num", String.valueOf(Config.internal_log_roll_num));
properties.put("internal_log_roll_num", String.valueOf(Config.internal_log_roll_num));
properties.put("internal_log_delete_age", String.valueOf(Config.internal_log_delete_age));
properties.put("internal_file_pattern",
getIntervalPattern("big_query_log_roll_interval", Config.internal_log_roll_interval));

// appender layout
final String jsonLoggingConfValue = "json";
if (jsonLoggingConfValue.equalsIgnoreCase(Config.sys_log_format)) {
// json logging
String jsonLayout =
"<JSONLayout compact=\"true\" eventEol=\"true\" properties=\"true\" stacktraceAsString=\"true\" />";
properties.put("syslog_default_layout", jsonLayout);
properties.put("syslog_warning_layout", jsonLayout);
properties.put("syslog_audit_layout", jsonLayout);
properties.put("syslog_dump_layout", jsonLayout);
properties.put("syslog_bigquery_layout", jsonLayout);
properties.put("syslog_profile_layout", jsonLayout);
} else {
// fallback to plaintext logging
properties.put("syslog_default_layout",
"<PatternLayout charset=\"UTF-8\" pattern=\"%d{yyyy-MM-dd HH:mm:ss.SSSXXX} %p (%t|%tid) [%C{1}.%M():%L] %m%n\"/>");
properties.put("syslog_warning_layout",
"<PatternLayout charset=\"UTF-8\" pattern=\"%d{yyyy-MM-dd HH:mm:ss.SSSXXX} %p (%t|%tid) [%C{1}.%M():%L] %m%n %ex\"/>");
properties.put("syslog_audit_layout",
"<PatternLayout charset=\"UTF-8\" pattern=\"%d{yyyy-MM-dd HH:mm:ss.SSSXXX} [%c{1}] %m%n\"/>");
properties.put("syslog_dump_layout",
"<PatternLayout charset=\"UTF-8\" pattern=\"%d{yyyy-MM-dd HH:mm:ss.SSSXXX} [%c{1}] %m%n\"/>");
properties.put("syslog_bigquery_layout",
"<PatternLayout charset=\"UTF-8\" pattern=\"%d{yyyy-MM-dd HH:mm:ss.SSSXXX} [%c{1}] %m%n\"/>");
properties.put("syslog_profile_layout",
"<PatternLayout charset=\"UTF-8\" pattern=\"%d{yyyy-MM-dd HH:mm:ss.SSSXXX} [%c{1}] %m%n\"/>");
}

String xmlConfTemplate = generateXmlConfTemplate();
strSub = new StrSubstitutor(new Interpolator(properties));
xmlConfTemplate = strSub.replace(xmlConfTemplate);
return strSub.replace(xmlConfTemplate);
}

private static void reconfig() throws IOException {
String xmlConfTemplate = generateActiveLog4jXmlConfig();
if (!FeConstants.runningUnitTest && !FeConstants.isReplayFromQueryDump) {
System.out.println("=====");
System.out.println(xmlConfTemplate);
Expand Down
72 changes: 72 additions & 0 deletions fe/fe-core/src/test/java/com/starrocks/common/Log4jConfigTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Copyright 2021-present StarRocks, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.starrocks.common;

import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Log4jConfigTest {
private String logFormat;

@Before
public void setUp() throws IOException {
logFormat = Config.sys_log_format;
Log4jConfig.initLogging();
}

@After
public void tearDown() {
Config.sys_log_format = logFormat;
}

@Test
public void testJsonLoggingFormatConfig() throws IOException {
// ${([^}]*)}
String regStr = "\\$\\{([^\\}]*)\\}";
String name = "";
// default plaintext configuration
Config.sys_log_format = "plaintext";
{
String xmlConfig = Log4jConfig.generateActiveLog4jXmlConfig();
Assert.assertFalse(xmlConfig.contains("<JSONLayout"));
Assert.assertTrue(xmlConfig.contains("<PatternLayout"));
// no unresolved variable
Matcher matcher = Pattern.compile(regStr, Pattern.MULTILINE).matcher(xmlConfig);
if (matcher.find()) {
name = matcher.group(1);
Assert.fail(String.format("Unexpected of unresolved variables:'%s' in the final xmlConfig", name));
}
}
// check the json configuration
Config.sys_log_format = "json";
{
String xmlConfig = Log4jConfig.generateActiveLog4jXmlConfig();
Assert.assertTrue(xmlConfig.contains("<JSONLayout"));
Assert.assertFalse(xmlConfig.contains("<PatternLayout"));
// no unresolved variable
Matcher matcher = Pattern.compile(regStr, Pattern.MULTILINE).matcher(xmlConfig);
if (matcher.find()) {
name = matcher.group(1);
Assert.fail(String.format("Unexpected of unresolved variables:'%s' in the final xmlConfig", name));
}
}
}
}

0 comments on commit 3706d45

Please sign in to comment.