support compress

config.go

@@ -35,6 +35,9 @@ type FileLogConfig struct {
 	MaxDays int `toml:"max-days" json:"max-days"`
 	// Maximum number of old log files to retain.
 	MaxBackups int `toml:"max-backups" json:"max-backups"`
+	// Compress function for rotated files.
+	// Currently only `gzip` and empty are supported, empty means compression disabled.
+	Compress string `toml:"compress" json:"compress"`
 }
 
 // Config serializes log related config in toml/json.
@@ -103,7 +106,7 @@ func (cfg *Config) buildOptions(errSink zapcore.WriteSyncer) []zap.Option {
 
 	if cfg.Sampling != nil {
 		opts = append(opts, zap.WrapCore(func(core zapcore.Core) zapcore.Core {
-			return zapcore.NewSampler(core, time.Second, int(cfg.Sampling.Initial), int(cfg.Sampling.Thereafter))
+			return zapcore.NewSamplerWithOptions(core, time.Second, int(cfg.Sampling.Initial), int(cfg.Sampling.Thereafter))
 		}))
 	}
 	return opts

go.mod

@@ -1,13 +1,12 @@
 module github.com/pingcap/log
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/pingcap/errors v0.11.0
 	github.com/stretchr/testify v1.7.0
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.19.0
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 go 1.16

go.sum

@@ -1,5 +1,3 @@
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -46,8 +44,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

log.go

@@ -182,13 +182,24 @@ func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) {
 		cfg.MaxSize = defaultLogMaxSize
 	}
 
+	compress := false
+	switch cfg.Compress {
+	case "":
+		compress = false
+	case "gzip":
+		compress = true
+	default:
+		return nil, fmt.Errorf("can't set compress to `%s`", cfg.Compress)
+	}
+
 	// use lumberjack to logrotate
 	return &lumberjack.Logger{
 		Filename:   cfg.Filename,
 		MaxSize:    cfg.MaxSize,
 		MaxBackups: cfg.MaxBackups,
 		MaxAge:     cfg.MaxDays,
 		LocalTime:  true,
+		Compress:   compress,
 	}, nil
 }
 

zap_log_test.go

@@ -235,6 +235,58 @@ func TestLogJSON(t *testing.T) {
 		"{\"level\":\"INFO\",\"caller\":\"zap_log_test.go:233\",\"message\":\"new connection\",\"connID\":\"1\",\"traceID\":\"dse1121\"}")
 }
 
+func TestRotateLogWithCompress(t *testing.T) {
+	tempDir := t.TempDir()
+	conf := &Config{
+		Level: "info",
+		File: FileLogConfig{
+			Filename: tempDir + "/test.log",
+			MaxSize:  1,
+			Compress: "gzip",
+		},
+	}
+	logger, _, err := InitLogger(conf)
+	require.NoError(t, err)
+
+	var data []byte
+	for i := 1; i <= 1*1024*1024; i++ {
+		if i%1000 != 0 {
+			data = append(data, 'd')
+			continue
+		}
+		logger.Info(string(data))
+		data = data[:0]
+	}
+	// Waiting rotation finished, it's async
+	for {
+		files, _ := os.ReadDir(tempDir)
+		if len(files) == 2 {
+			for _, f := range files {
+				info, err := f.Info()
+				require.NoError(t, err)
+				// Many duplicate logs, the file size after compress should less than 512KB
+				require.Less(t, info.Size(), int64(512*1024))
+			}
+			break
+		}
+		time.Sleep(time.Millisecond)
+	}
+}
+
+func TestCompressError(t *testing.T) {
+	tempDir := t.TempDir()
+	conf := &Config{
+		Level: "info",
+		File: FileLogConfig{
+			Filename: tempDir + "/test.log",
+			MaxSize:  1,
+			Compress: "xxx",
+		},
+	}
+	_, _, err := InitLogger(conf)
+	require.Error(t, err)
+}
+
 // testLogSpy is a testing.TB that captures logged messages.
 type testLogSpy struct {
 	testing.TB