Merge branch 'feature/feat-minio-storage' into develop

.env

@@ -0,0 +1,18 @@
+# Increase or decrease based on the available host memory (in bytes)
+
+MEM_LIMIT=8073741824
+
+
+MYSQL_PASSWORD=qwe123
+MYSQL_PORT=3306
+
+# Port to expose minio to the host
+MINIO_CONSOLE_PORT=9001
+MINIO_PORT=9000
+
+MINIO_USER=begonia
+MINIO_PASSWORD=begoniabegonia
+
+
+TIMEZONE='Asia/Shanghai'
+

.github/workflows/go.yml

@@ -3,10 +3,14 @@ name: Go
 on:
   push:
     paths:
-      - '**.go'
+      - "**.go"
+      - "**.yml"
+      - "**.mod"
   pull_request:
     paths:
-      - '**.go'
+      - "**.go"
+      - "**.yml"
+      - "**.mod"
 
 jobs:
   test:
@@ -17,16 +21,44 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.20.7
+          go-version: 1.22.4
 
       - name: Start Docker Composer
         run: docker-compose up -d
 
       - name: Init MySQL
         run: |
-          go run ./cmd/begonia init -e dev
-          cat ~/.begonia/admin-app.json
-
+          go mod tidy
+          go run ./cmd/begonia init -c ./config/settings.yml -e test
+          cat ~/.begonia/admin-app.test.json
+      - name: Init Minio
+        run: |
+          go install github.com/minio/mc@latest
+          cat << EOF > /tmp/policy.json
+          {
+            "Version": "2012-10-17",
+            "Statement": [
+              {
+                "Effect": "Allow",
+                "Action": [
+                  "s3:*"
+                ],
+                "Resource": [
+                  "arn:aws:s3:::*"
+                ]
+              }
+            ]
+          }
+          EOF
+      - name: Configure Minio
+        run: |
+          mc alias set myminio http://127.0.0.1:9000 begonia begoniabegonia
+          mc admin user add myminio begonia-2 begoniabegonia
+          mc admin user svcacct add \
+            --access-key "begonia" \
+            --secret-key "begoniabegonia" \
+            --policy "/tmp/policy.json" \
+            myminio begonia-2
       - name: Type a temporary tag
         run: git tag test-$(date '+%Y%m%d%H%M%S')
 
@@ -45,7 +77,7 @@ jobs:
           cd grpc-gateway
           go install ./protoc-gen-grpc-gateway
       - name: Test
-        run: go test -v -gcflags=-l -coverprofile=coverage.txt -covermode=atomic -ldflags -X=github.com/begonia-org/begonia.Version=$(git describe --tags --abbrev=0)\ -X=github.com/begonia-org/begonia.BuildTime=$(date '+%Y%m%d%H%M%S')\ -X=github.com/begonia-org/begonia.Commit=$(git rev-parse --short HEAD)\ -X=github.com/begonia-org/begonia.Env=test ./...
+        run: go test -p 1 -v -gcflags=-l -coverprofile=coverage.txt -covermode=atomic -ldflags -X=github.com/begonia-org/begonia.Version=$(git describe --tags --abbrev=0)\ -X=github.com/begonia-org/begonia.BuildTime=$(date '+%Y%m%d%H%M%S')\ -X=github.com/begonia-org/begonia.Commit=$(git rev-parse --short HEAD)\ -X=github.com/begonia-org/begonia.Env=test ./...
 
       - name: Upload coverage to Codecov
         uses: codecov/[email protected]

README.md

@@ -28,7 +28,8 @@ Begonia is an HTTP to gRPC reverse proxy server, which registers services define
 - Allows requests with `application/x-www-form-urlencoded` and `multipart/form-data` parameter formats.
 - Rich built-in middleware, such as APIKEY verification, AKSK verification, and `go-playground/validator` for parameter validation middleware.parameter verification middleware
 - Dynamic registration, updating, and deletion of gRPC service routes based on the protoc descriptor_set_out.
-
+# Architecture
+![Architecture](docs/begonia.png)
 # Getting Started
 
 ### Installation
@@ -62,19 +63,19 @@ docker compose up -d
 #### 2. Initialize the database
 
 ```bash
-begonia init -e dev
+begonia init -e dev -c config/settings.yml
 ```
 
 #### 3. Start the service
 
 ```bash
-begonia start -e dev
+begonia start -e dev -c config/settings.yml
 ```
 
 #### 4. Register the service
 
 ```bash
-go run . endpoint add -n "example" -d /data/work/begonia-org/begonia/example/example.pb -p 127.0.0.1:1949 -p 127.0.0.1:2024
+begonia endpoint add -n "example" -d /data/work/begonia-org/begonia/example/example.pb -p 127.0.0.1:1949 -p 127.0.0.1:2024
 ```
 
 #### 5. Test request service

README_ZH.md

@@ -29,6 +29,9 @@ Begonia 是一个 HTTP 到 gRPC 的反向代理服务器，它基于 protoc 生
 - 丰富的内置中间件，例如 APIKEY 校验、AKSK 校验，`go-playground/validator`参数校验中间件
 - 基于 protoc `descriptor_set_out` 实现gRPC服务路由的动态注册、更新和删除
 
+# 架构
+![架构](docs/begonia.png)
+
 # 开始
 
 ### 安装
@@ -62,19 +65,19 @@ docker compose up -d
 #### 2、初始化数据库
 
 ```bash
-begonia init -e dev
+begonia init -e dev -c config/settings.yml
 ```
 
 #### 3、启动服务
 
 ```bash
-begonia start -e dev
+begonia start -e dev -c config/settings.yml
 ```
 
 #### 4、注册服务
 
 ```bash
-go run . endpoint add  -n "example" -d /data/work/begonia-org/begonia/example/example.pb -p 127.0.0.1:1949  -p 127.0.0.1:2024
+begonia endpoint add  -n "example" -d /data/work/begonia-org/begonia/example/example.pb -p 127.0.0.1:1949  -p 127.0.0.1:2024
 ```
 
 #### 5、测试请求服务

cmd/begonia/endpoint.go

@@ -18,14 +18,15 @@ var accessKey string
 var secret string
 var addr string
 
-func readInitAPP() {
+func readInitAPP(env string) {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		log.Fatalf(err.Error())
 		return
 	}
+
 	path := filepath.Join(homeDir, ".begonia")
-	path = filepath.Join(path, "admin-app.json")
+	path = filepath.Join(path, fmt.Sprintf("admin-app.%s.json", env))
 	file, err := os.Open(path)
 	if err != nil {
 		log.Fatalf(err.Error())
@@ -66,8 +67,8 @@ func readInitAPP() {
 	addr = gw.Addr
 
 }
-func RegisterEndpoint(name string, endpoints []string, pbFile string, opts ...client.EndpointOption) {
-	readInitAPP()
+func RegisterEndpoint(env,name string, endpoints []string, pbFile string, opts ...client.EndpointOption) {
+	readInitAPP(env)
 	pb, err := os.ReadFile(pbFile)
 	if err != nil {
 		panic(err)
@@ -102,8 +103,25 @@ func RegisterEndpoint(name string, endpoints []string, pbFile string, opts ...cl
 	log.Printf("#####################Add Endpoint Success#####################")
 	log.Printf("#####################ID:%s####################################", rsp.Id)
 }
-func DeleteEndpoint(id string) {
-	readInitAPP()
+func UpdateEndpoint(env,id string, mask []string, opts ...client.EndpointOption) {
+	readInitAPP(env)
+	apiClient := client.NewEndpointAPI(addr, accessKey, secret)
+	log.Printf("#####################Update Endpoint###########################")
+	patch := &endpoint.EndpointSrvUpdateRequest{}
+	patch.UniqueKey = id
+	for _, opt := range opts {
+		opt(patch)
+	}
+	_, err := apiClient.PatchEndpointConfig(context.Background(), patch)
+	if err != nil {
+		log.Fatalf(err.Error())
+		panic(err.Error())
+	}
+	log.Printf("#####################Update Endpoint %s Success#####################", id)
+
+}
+func DeleteEndpoint(env,id string) {
+	readInitAPP(env)
 	apiClient := client.NewEndpointAPI(addr, accessKey, secret)
 	log.Printf("#####################Delete Endpoint:%s#####################", id)
 	_, err := apiClient.DeleteEndpointConfig(context.Background(), id)

cmd/begonia/main.go

@@ -10,13 +10,15 @@ import (
 	"github.com/begonia-org/begonia/config"
 	"github.com/begonia-org/begonia/gateway"
 	"github.com/begonia-org/begonia/internal"
+	api "github.com/begonia-org/go-sdk/api/endpoint/v1"
 	"github.com/begonia-org/go-sdk/client"
 	"github.com/spf13/cobra"
 )
 
 // var ProviderSet = wire.NewSet(NewMasterCmd)
 func addCommonCommand(cmd *cobra.Command) *cobra.Command {
 	cmd.Flags().StringP("env", "e", "dev", "Runtime Environment")
+	cmd.Flags().StringP("config", "c", "./config/settings.yml", "Config file path")
 	return cmd
 }
 func NewInitCmd() *cobra.Command {
@@ -25,10 +27,14 @@ func NewInitCmd() *cobra.Command {
 		Short: "Init Database",
 		Run: func(cmd *cobra.Command, args []string) {
 			env, _ := cmd.Flags().GetString("env")
-			config := config.ReadConfig(env)
+			cnf, err := cmd.Flags().GetString("config")
+			if err != nil {
+				log.Fatalf("failed to get config: %v", err)
+			}
+			config := config.ReadConfigWithDir(env, cnf)
 			operator := internal.InitOperatorApp(config)
 			log.Printf("init database")
-			err := operator.Init()
+			err = operator.Init()
 			if err != nil {
 				log.Fatalf("failed to init database: %v", err)
 			}
@@ -45,7 +51,11 @@ func NewGatewayCmd() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			endpoint, _ := cmd.Flags().GetString("endpoint")
 			env, _ := cmd.Flags().GetString("env")
-			config := config.ReadConfig(env)
+			cnf, err := cmd.Flags().GetString("config")
+			if err != nil {
+				log.Fatalf("failed to get config: %v", err)
+			}
+			config := config.ReadConfigWithDir(env, cnf)
 			worker := internal.New(config, gateway.Log, endpoint)
 			hd, _ := os.UserHomeDir()
 			_ = os.WriteFile(hd+"/.begonia/gateway.json", []byte(fmt.Sprintf(`{"addr":"http://%s"}`, endpoint)), 0666)
@@ -65,8 +75,9 @@ func NewEndpointDelCmd() *cobra.Command {
 
 		Run: func(cmd *cobra.Command, args []string) {
 			id, _ := cmd.Flags().GetString("id")
+			env, _ := cmd.Flags().GetString("env")
 
-			DeleteEndpoint(id)
+			DeleteEndpoint(env,id)
 		},
 	}
 	cmd.Flags().StringP("id", "i", "", "ID Of Your Service")
@@ -85,18 +96,75 @@ func NewEndpointAddCmd() *cobra.Command {
 			tags, _ := cmd.Flags().GetStringArray("tags")
 			balance, _ := cmd.Flags().GetString("balance")
 			endpoints, _ := cmd.Flags().GetStringArray("endpoint")
+			env,_:=cmd.Flags().GetString("env")
 
-			RegisterEndpoint(name, endpoints, desc, client.WithBalance(strings.ToUpper(balance)), client.WithTags(tags))
+			RegisterEndpoint(env,name, endpoints, desc, client.WithBalance(strings.ToUpper(balance)), client.WithTags(tags))
 		},
 	}
+	cmd = newWriteEndpointCmd(cmd)
+	_ = cmd.MarkFlagRequired("name")
+	_ = cmd.MarkFlagRequired("endpoint")
+	_ = cmd.MarkFlagRequired("desc")
+	return cmd
+}
+func newWriteEndpointCmd(cmd *cobra.Command) *cobra.Command {
 	cmd.Flags().StringArrayP("endpoint", "p", []string{}, "Endpoint Of Your Service (example:127.0.0.1:1949)")
 	cmd.Flags().StringP("name", "n", "", "Service Name")
 	cmd.Flags().StringP("desc", "d", "", "Descriptions Set Of Your Service (example:./example/example.pb)")
 	cmd.Flags().StringArrayP("tags", "t", []string{}, "Tags Of Your Service")
 	cmd.Flags().StringP("balance", "b", "RR", "Balance Type Of Your Service (options: RR WRR LC WLC CH SED NQ)")
-	_ = cmd.MarkFlagRequired("name")
-	_ = cmd.MarkFlagRequired("endpoint")
-	_ = cmd.MarkFlagRequired("desc")
+
+	return cmd
+}
+func NewEndpointUpdateCmd() *cobra.Command {
+	var cmd = &cobra.Command{
+		Use:   "update",
+		Short: "Update Service To Gateway",
+
+		Run: func(cmd *cobra.Command, args []string) {
+			id, _ := cmd.Flags().GetString("id")
+			name, _ := cmd.Flags().GetString("name")
+			desc, _ := cmd.Flags().GetString("desc")
+			tags, _ := cmd.Flags().GetStringArray("tags")
+			balance, _ := cmd.Flags().GetString("balance")
+			endpoints, _ := cmd.Flags().GetStringArray("endpoint")
+			mask := make([]string, 0)
+			options := make([]client.EndpointOption, 0)
+			if cmd.Flags().Changed("name") {
+				options = append(options, client.WithName(name))
+				mask = append(mask, "name")
+			}
+
+			if cmd.Flags().Changed("desc") {
+				options = append(options, client.WithDescription(desc))
+				mask = append(mask, "description")
+			}
+			if cmd.Flags().Changed("tags") {
+				options = append(options, client.WithTags(tags))
+				mask = append(mask, "tags")
+			}
+			if cmd.Flags().Changed("balance") {
+				options = append(options, client.WithBalance(strings.ToUpper(balance)))
+				mask = append(mask, "balance")
+			}
+			if cmd.Flags().Changed("endpoints") {
+				meta := make([]*api.EndpointMeta, 0)
+				for i, v := range endpoints {
+					meta = append(meta, &api.EndpointMeta{
+						Addr:   v,
+						Weight: int32(i),
+					})
+				}
+				options = append(options, client.WithEndpoints(meta))
+				mask = append(mask, "endpoints")
+			}
+			env,_:=cmd.Flags().GetString("env")
+			UpdateEndpoint(env,id, mask, options...)
+		},
+	}
+	cmd = newWriteEndpointCmd(cmd)
+	cmd.Flags().StringP("id", "i", "", "ID Of Your Service")
+	_ = cmd.MarkFlagRequired("id")
 	return cmd
 }
 func NewEndpointCmd() *cobra.Command {
@@ -109,6 +177,7 @@ func NewEndpointCmd() *cobra.Command {
 
 	cmd.AddCommand(NewEndpointAddCmd())
 	cmd.AddCommand(NewEndpointDelCmd())
+	cmd.AddCommand(NewEndpointUpdateCmd())
 	return cmd
 }
 func NewBegoniaInfoCmd() *cobra.Command {
@@ -132,8 +201,8 @@ func main() {
 	rootCmd.AddCommand(cmd)
 	rootCmd.AddCommand(NewBegoniaInfoCmd())
 	rootCmd.AddCommand(addCommonCommand(NewInitCmd()))
-	rootCmd.AddCommand(NewEndpointCmd())
+	rootCmd.AddCommand(addCommonCommand(NewEndpointCmd()))
 	if err := cmd.Execute(); err != nil {
-		log.Fatalf("failed to start master: %v", err)
+		log.Fatalf("failed to start begonia: %v", err)
 	}
 }

codecov.yml

@@ -0,0 +1,5 @@
+ignore:
+  - "cmd"
+  - "version.go"
+  - "example"
+  - "config"

config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"log"
 	"path/filepath"
 	"runtime"
 
@@ -10,5 +11,11 @@ import (
 func ReadConfig(env string) *tiga.Configuration {
 	_, filename, _, _ := runtime.Caller(0)
 	dir := filepath.Dir(filename)
+	log.Printf("config dir: %s", dir)
+	return tiga.InitSettings(env, dir)
+}
+func ReadConfigWithDir(env, filename string) *tiga.Configuration {
+	dir := filepath.Dir(filename)
+	log.Printf("config dir: %s", dir)
 	return tiga.InitSettings(env, dir)
 }