From f5e8c20ad70c63797b9c9eebc3edeae25f7f72ee Mon Sep 17 00:00:00 2001 From: zyy17 Date: Thu, 7 Nov 2024 00:13:51 +0800 Subject: [PATCH] refactor: add Filters in LoggingSpec (#203) --- apis/v1alpha1/common.go | 7 +++++ apis/v1alpha1/zz_generated.deepcopy.go | 5 ++++ .../greptime.io_greptimedbclusters.yaml | 24 ++++++++++++++++ .../greptime.io_greptimedbstandalones.yaml | 4 +++ docs/api-references/docs.md | 1 + .../cluster/configure-logging/cluster.yaml | 5 +++- manifests/bundle.yaml | 28 +++++++++++++++++++ manifests/crds.yaml | 28 +++++++++++++++++++ pkg/dbconfig/common.go | 12 +++++++- 9 files changed, 112 insertions(+), 2 deletions(-) diff --git a/apis/v1alpha1/common.go b/apis/v1alpha1/common.go index 7bb05b28..dcfd64f1 100644 --- a/apis/v1alpha1/common.go +++ b/apis/v1alpha1/common.go @@ -458,6 +458,13 @@ type LoggingSpec struct { // +kubebuilder:validation:Enum:={"info", "error", "warn", "debug"} Level LoggingLevel `json:"level,omitempty"` + // Filters is the filters of the logging. + // User can use [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html) to filter the logging. + // We can use `target[span{field=value}]=level` to filter the logging by target and span field. + // For example, "mito2=debug" will filter the logging of target `mito2` to `debug` level. + // +optional + Filters []string `json:"filters,omitempty"` + // LogsDir is the directory path of the logs. // +optional LogsDir string `json:"logsDir,omitempty"` diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index dd49a942..426eba9a 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -613,6 +613,11 @@ func (in *LogPipeline) DeepCopy() *LogPipeline { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoggingSpec) DeepCopyInto(out *LoggingSpec) { *out = *in + if in.Filters != nil { + in, out := &in.Filters, &out.Filters + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.PersistentWithData != nil { in, out := &in.PersistentWithData, &out.PersistentWithData *out = new(bool) diff --git a/config/crd/resources/greptime.io_greptimedbclusters.yaml b/config/crd/resources/greptime.io_greptimedbclusters.yaml index bdd7ee0e..a7b3adf5 100644 --- a/config/crd/resources/greptime.io_greptimedbclusters.yaml +++ b/config/crd/resources/greptime.io_greptimedbclusters.yaml @@ -2901,6 +2901,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -5811,6 +5815,10 @@ spec: type: string logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -8704,6 +8712,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -11630,6 +11642,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -11681,6 +11697,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -17457,6 +17477,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json diff --git a/config/crd/resources/greptime.io_greptimedbstandalones.yaml b/config/crd/resources/greptime.io_greptimedbstandalones.yaml index f2d30c4c..6b42a2a8 100644 --- a/config/crd/resources/greptime.io_greptimedbstandalones.yaml +++ b/config/crd/resources/greptime.io_greptimedbstandalones.yaml @@ -2914,6 +2914,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index b4fe9ee6..43d77cda 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -519,6 +519,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `level` _[LoggingLevel](#logginglevel)_ | Level is the level of the logging. | | Enum: [info error warn debug]
| +| `filters` _string array_ | Filters is the filters of the logging.
User can use [EnvFilter](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html) to filter the logging.
We can use `target[span\{field=value\}]=level` to filter the logging by target and span field.
For example, "mito2=debug" will filter the logging of target `mito2` to `debug` level. | | | | `logsDir` _string_ | LogsDir is the directory path of the logs. | | | | `persistentWithData` _boolean_ | PersistentWithData indicates whether to persist the log with the datanode data storage. It **ONLY** works for the datanode component.
If false, the log will be stored in ephemeral storage. | | | | `onlyLogToStdout` _boolean_ | OnlyLogToStdout indicates whether to only log to stdout. If true, the log will not be stored in the storage even if the storage is configured. | | | diff --git a/examples/cluster/configure-logging/cluster.yaml b/examples/cluster/configure-logging/cluster.yaml index a5814aba..cab0c33f 100644 --- a/examples/cluster/configure-logging/cluster.yaml +++ b/examples/cluster/configure-logging/cluster.yaml @@ -5,7 +5,7 @@ metadata: spec: logging: format: json - level: debug + level: info logsDir: /data/greptimedb/logs onlyLogToStdout: false base: @@ -19,3 +19,6 @@ spec: - "etcd.etcd-cluster.svc.cluster.local:2379" datanode: replicas: 1 + logging: + filters: + - mito2=debug diff --git a/manifests/bundle.yaml b/manifests/bundle.yaml index e4074d17..cb025379 100644 --- a/manifests/bundle.yaml +++ b/manifests/bundle.yaml @@ -2907,6 +2907,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -5817,6 +5821,10 @@ spec: type: string logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -8710,6 +8718,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -11636,6 +11648,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -11687,6 +11703,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -17463,6 +17483,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -20842,6 +20866,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json diff --git a/manifests/crds.yaml b/manifests/crds.yaml index 17f6d60a..a829a812 100644 --- a/manifests/crds.yaml +++ b/manifests/crds.yaml @@ -2900,6 +2900,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -5810,6 +5814,10 @@ spec: type: string logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -8703,6 +8711,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -11629,6 +11641,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -11680,6 +11696,10 @@ spec: type: integer logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -17456,6 +17476,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json @@ -20835,6 +20859,10 @@ spec: type: object logging: properties: + filters: + items: + type: string + type: array format: enum: - json diff --git a/pkg/dbconfig/common.go b/pkg/dbconfig/common.go index 87e69566..d2045ccb 100644 --- a/pkg/dbconfig/common.go +++ b/pkg/dbconfig/common.go @@ -16,7 +16,9 @@ package dbconfig import ( "encoding/base64" + "fmt" "strconv" + "strings" "k8s.io/klog/v2" "k8s.io/utils/pointer" @@ -176,7 +178,7 @@ func (c *LoggingConfig) ConfigureLogging(spec *v1alpha1.LoggingSpec) { c.Dir = pointer.String(spec.LogsDir) } - c.Level = pointer.String(string(spec.Level)) + c.Level = pointer.String(c.levelWithFilters(string(spec.Level), spec.Filters)) c.LogFormat = pointer.String(string(spec.Format)) if spec.SlowQuery != nil { @@ -193,3 +195,11 @@ func (c *LoggingConfig) ConfigureLogging(spec *v1alpha1.LoggingSpec) { c.SlowQuerySampleRatio = pointer.Float64(ration) } } + +// levelWithFilters returns the level with filters. For example, it will output "info,mito2=debug" if the level is "info" and the filters are ["mito2=debug"]. +func (c *LoggingConfig) levelWithFilters(level string, filters []string) string { + if len(filters) > 0 { + return fmt.Sprintf("%s,%s", level, strings.Join(filters, ",")) + } + return level +}