fix: load config with default value

README.md

@@ -61,7 +61,7 @@ Config the config file `config/conf.json`
   "KafkaConsumerGroup": "test",
   "mockData": "",
   "isJsonTransform": true,
-  "databendDSN": "https://cloudapp:password@tn3ftqihs--medium-p8at.gw.aws-us-east-2.default.databend.com:443",
+  "databendDSN": "https://host:port@host:port",
   "databendTable": "default.kfk_test",
   "batchSize": 1,
   "batchMaxInterval": 5,

config/conf.json

@@ -6,8 +6,8 @@
   "saslUser": "ccun_fbf463a8ca06693300193b59e1c7b0ce65c77f593bc1dab9baf080eb1bdb8f1d",
   "saslPassword": "ccp_c731c6250eff4cb7b8ef0a702fc33b050baff3213ae845e0f10b2fb71db7f18a",
   "mockData": "",
-  "isJsonTransform": false,
-  "databendDSN": "https://cloudapp:password@tn3ftqihs--medium-p8at.gw.aws-us-east-2.default.databend.com:443",
+  "isJsonTransform": true,
+  "databendDSN": "https://user:password@host:port",
   "databendTable": "default.kfk_test",
   "batchSize": 10,
   "batchMaxInterval": 5,
@@ -19,6 +19,5 @@
   "minBytes": 1024,
   "maxBytes": 1048576,
   "maxWait": 10,
-  "useReplaceMode": false,
-  "userStage": "~"
+  "useReplaceMode": false
 }

config/config.go

@@ -3,7 +3,10 @@ package config
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
+
+	"github.com/mcuadros/go-defaults"
 )
 
 type Config struct {
@@ -59,12 +62,15 @@ func LoadConfig() (*Config, error) {
 		return nil, err
 	}
 	defer f.Close()
-	decoder := json.NewDecoder(f)
-	err = decoder.Decode(&conf)
+	confByte, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, fmt.Errorf("read config file failed: %v", err)
+	}
+	err = json.Unmarshal(confByte, &conf)
 	if err != nil {
-		fmt.Println("Error decoding JSON:", err)
-		return &conf, err
+		return nil, fmt.Errorf("unmarshal config failed: %v", err)
 	}
+	defaults.SetDefaults(&conf)
 
 	return &conf, nil
 }

go.mod

@@ -3,9 +3,11 @@ module github.com/databendcloud/bend-ingest-kafka
 go 1.21
 
 require (
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/codesuki/go-time-series v0.0.0-20210430055340-c4c8d8fa61d4
 	github.com/datafuselabs/databend-go v0.6.5
 	github.com/google/uuid v1.3.0
+	github.com/mcuadros/go-defaults v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/segmentio/kafka-go v0.4.39
 	github.com/sirupsen/logrus v1.9.0
@@ -18,7 +20,6 @@ require (
 	github.com/BurntSushi/toml v1.2.1 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect
 	github.com/apache/thrift v0.14.2 // indirect
-	github.com/avast/retry-go v3.0.0+incompatible // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect

go.sum

@@ -112,7 +112,12 @@ github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQan
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc=
+github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
@@ -335,6 +340,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
 gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=