From 0c1d6bf830d2cab2a60598aa4580f21d0be9b8d0 Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 16 Oct 2024 15:39:21 +0800 Subject: [PATCH 1/2] fix: load config with default value --- config/config.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index 6247c3f..d61d87e 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "fmt" + "io/ioutil" "os" ) @@ -59,11 +60,13 @@ func LoadConfig() (*Config, error) { return nil, err } defer f.Close() - decoder := json.NewDecoder(f) - err = decoder.Decode(&conf) + confByte, err := ioutil.ReadAll(f) if err != nil { - fmt.Println("Error decoding JSON:", err) - return &conf, err + return nil, fmt.Errorf("read config file failed: %v", err) + } + err = json.Unmarshal(confByte, &conf) + if err != nil { + return nil, fmt.Errorf("unmarshal config failed: %v", err) } return &conf, nil From a13cf8642c1848b9c813226cc6534d976accd00e Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 16 Oct 2024 15:56:45 +0800 Subject: [PATCH 2/2] update --- README.md | 2 +- config/conf.json | 7 +++---- config/config.go | 3 +++ go.mod | 3 ++- go.sum | 7 +++++++ 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 346b92a..29d7237 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Config the config file `config/conf.json` "KafkaConsumerGroup": "test", "mockData": "", "isJsonTransform": true, - "databendDSN": "https://cloudapp:password@tn3ftqihs--medium-p8at.gw.aws-us-east-2.default.databend.com:443", + "databendDSN": "https://host:port@host:port", "databendTable": "default.kfk_test", "batchSize": 1, "batchMaxInterval": 5, diff --git a/config/conf.json b/config/conf.json index a061b92..70b7d53 100644 --- a/config/conf.json +++ b/config/conf.json @@ -6,8 +6,8 @@ "saslUser": "ccun_fbf463a8ca06693300193b59e1c7b0ce65c77f593bc1dab9baf080eb1bdb8f1d", "saslPassword": "ccp_c731c6250eff4cb7b8ef0a702fc33b050baff3213ae845e0f10b2fb71db7f18a", "mockData": "", - "isJsonTransform": false, - "databendDSN": "https://cloudapp:password@tn3ftqihs--medium-p8at.gw.aws-us-east-2.default.databend.com:443", + "isJsonTransform": true, + "databendDSN": "https://user:password@host:port", "databendTable": "default.kfk_test", "batchSize": 10, "batchMaxInterval": 5, @@ -19,6 +19,5 @@ "minBytes": 1024, "maxBytes": 1048576, "maxWait": 10, - "useReplaceMode": false, - "userStage": "~" + "useReplaceMode": false } \ No newline at end of file diff --git a/config/config.go b/config/config.go index d61d87e..87a28ff 100644 --- a/config/config.go +++ b/config/config.go @@ -5,6 +5,8 @@ import ( "fmt" "io/ioutil" "os" + + "github.com/mcuadros/go-defaults" ) type Config struct { @@ -68,6 +70,7 @@ func LoadConfig() (*Config, error) { if err != nil { return nil, fmt.Errorf("unmarshal config failed: %v", err) } + defaults.SetDefaults(&conf) return &conf, nil } diff --git a/go.mod b/go.mod index e722c09..8842ba9 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module github.com/databendcloud/bend-ingest-kafka go 1.21 require ( + github.com/avast/retry-go v3.0.0+incompatible github.com/codesuki/go-time-series v0.0.0-20210430055340-c4c8d8fa61d4 github.com/datafuselabs/databend-go v0.6.5 github.com/google/uuid v1.3.0 + github.com/mcuadros/go-defaults v1.2.0 github.com/pkg/errors v0.9.1 github.com/segmentio/kafka-go v0.4.39 github.com/sirupsen/logrus v1.9.0 @@ -18,7 +20,6 @@ require ( github.com/BurntSushi/toml v1.2.1 // indirect github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect github.com/apache/thrift v0.14.2 // indirect - github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect diff --git a/go.sum b/go.sum index 0458ae3..124ec10 100644 --- a/go.sum +++ b/go.sum @@ -112,7 +112,12 @@ github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQan github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc= +github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= @@ -335,6 +340,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=