Merge pull request #2 from shijilin0116/main

Add support for synchronizing to xlsx files

README.md

@@ -13,7 +13,7 @@
 - 数据同步（接口必须支持翻页）
   - 全量同步
     - [x] 同步为 csv 格式
-    - [ ] 同步为 xls/xlsx 格式
+    - [x] 同步为 xls/xlsx 格式
     - [x] 同步到数据库
     - [x] 同步到模型
   - 增量同步（接口必须支持ts增量参数）
@@ -103,6 +103,7 @@ Todo
 
 - [Init & CallAPI](example/example.go)
 - [SyncToCSV](example/example_csv.go)
+- [SyncToXlSX](example/example_xlsx.go)
 - [SyncToModel](example/example_model.go)
 - [SyncToDB](example/example_db.go)
 

example/example.go

@@ -33,6 +33,7 @@ func main() {
 	fmt.Println(string(res))
 
 	exampleSyncToCSV()
+	exampleSyncToXLSX()
 	exampleSyncToModel()
 	exampleSyncToDB()
 	//exampleSyncPerformance()

example/example_csv.go

@@ -27,8 +27,8 @@ func exampleSyncToCSV() {
 	// 同步到 csv
 	// csv 模式下，所有字段都会转为 string
 	csvFile := "test.csv"
-
-	rows, err := sdk.SyncToCSV(csvFile, api)
+	mode := "csv"
+	rows, err := sdk.SyncToFile(mode, csvFile, api)
 	if err != nil {
 		fmt.Println(err)
 		return

example/example_xlsx.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/ecnu/ecnu-openapi-sdk-go/sdk"
+)
+
+func exampleSyncToXLSX() {
+	/*
+		   type APIConfig struct {
+		   	APIPath        string `json:"api_path"`   // 接口的地址，例如 /api/v1/organization/list, 也可以追加参数，例如 /api/v1/organization/list?departmentId=0445
+		   	PageSize       int    `json:"page_size"`  // 翻页参数会自动添加，默认 pageSize 是 2000，最大值是 10000。
+			BatchSize      int    `json:"data_batch"` // 批量写入数据时的批次大小，默认是100。给的太大可能会数据库报错，请根据实际情况调整。
+			UpdatedAtField string                     // 增量同步时，数据库内的时间戳字段名，默认是 updated_at
+		   }
+
+	*/
+
+	// 配置待同步的接口
+	api := sdk.APIConfig{
+		APIPath:  "/api/v1/sync/fakewithts",
+		PageSize: 2000,
+	}
+
+	api.SetParam("ts", "0")
+	// 同步到 xlsx
+	// xlsx 模式下，所有字段都会转为 string
+	xlsxFile := "test.xlsx"
+	mode := "xlsx"
+	rows, err := sdk.SyncToFile(mode, xlsxFile, api)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Printf("XLSX：组织机构同步 %d 条数据\n", rows)
+}

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/liamylian/jsontime/v2 v2.0.0
+	github.com/tealeg/xlsx/v3 v3.3.4
 	golang.org/x/oauth2 v0.12.0
 	gorm.io/driver/mysql v1.5.2
 	gorm.io/driver/postgres v1.5.3
@@ -13,20 +14,29 @@ require (
 )
 
 require (
+	github.com/frankban/quicktest v1.14.6 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/btree v1.0.0 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.4.3 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/microsoft/go-mssqldb v1.6.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/peterbourgon/diskv/v3 v3.0.1 // indirect
+	github.com/rogpeppe/fastuuid v1.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa // indirect
 	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect

go.sum

@@ -16,11 +16,14 @@ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0/go.mod h
 github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0 h1:HCc0+LpPfpCKs6LGGLAhwBARt9632unrVcI6i8s/8os=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
 github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
@@ -35,8 +38,11 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -63,6 +69,10 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/liamylian/jsontime/v2 v2.0.0 h1:3if2kDW/boymUdO+4Qj/m4uaXMBSF6np9KEgg90cwH0=
@@ -80,10 +90,20 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
 github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/peterbourgon/diskv/v3 v3.0.1 h1:x06SQA46+PKIUftmEujdwSEpIx8kR+M9eLYsUxeYveU=
+github.com/peterbourgon/diskv/v3 v3.0.1/go.mod h1:kJ5Ny7vLdARGU3WUuy6uzO6T0nb/2gWcT1JiBvRmb5o=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa h1:2cO3RojjYl3hVTbEvJVqrMaFmORhL6O06qdW42toftk=
+github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa/go.mod h1:Yjr3bdWaVWyME1kha7X0jsz3k2DgXNa1Pj3XGyUAbx8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -96,6 +116,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/tealeg/xlsx/v3 v3.3.4 h1:+ekdnOtVHfCGadxXXuQv1JK9uXSweMpqICsHJ9macR4=
+github.com/tealeg/xlsx/v3 v3.3.4/go.mod h1:KV4FTFtvGy0TBlOivJLZu/YNZk6e0Qtk7eOSglWksuA=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -144,6 +166,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
@@ -165,6 +188,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

sdk/parse.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	xlsx "github.com/tealeg/xlsx/v3"
 	"os"
 	"reflect"
 	"sort"
@@ -21,8 +22,83 @@ func init() {
 	jsontime.AddLocaleAlias("shanghai", timeZoneShanghai)
 }
 
+// ParseRowsToXSLX
+func parseRowsToXLSX(rows []interface{}, filename string) error {
+	// 首先，检查 rows 是否为空
+	if len(rows) == 0 {
+		return errors.New("rows is empty")
+	}
+
+	// 然后，检查 filename 是否为空
+	if filename == "" {
+		return errors.New("filename is empty")
+	}
+
+	// 创建一个新的xlsx文件
+	xlsxFile := xlsx.NewFile()
+
+	// 创建一个新的sheet
+	sheet, err := xlsxFile.AddSheet("Sheet1")
+	if err != nil {
+		return err
+	}
+
+	// 创建一个新的行
+	xlsxRow := sheet.AddRow()
+
+	// 遍历 rows 中的每个元素，将其转换为 map[string]interface{}
+	for i, row := range rows {
+		// 将 row 转换为 json 字节
+		jsonBytes, err := json.Marshal(row)
+		if err != nil {
+			return err
+		}
+
+		// 创建一个新的 map[string]interface{}
+		m := make(map[string]interface{})
+
+		// 解码 json 字节到新 map 中
+		err = json.Unmarshal(jsonBytes, &m)
+		if err != nil {
+			return err
+		}
+
+		// 对 map 中的 key 进行排序
+		var keys []string
+		for k := range m {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+
+		// 对于第一行，写入 header
+		if i == 0 {
+			for _, key := range keys {
+				cell := xlsxRow.AddCell()
+				cell.SetString(key)
+			}
+		}
+
+		// 创建一个新的行
+		xlsxRow = sheet.AddRow()
+
+		// 将 values 写入xlsx
+		for _, key := range keys {
+			cell := xlsxRow.AddCell()
+			cell.SetString(fmt.Sprintf("%v", m[key]))
+		}
+	}
+
+	// 保存xlsx文件
+	err = xlsxFile.Save(filename)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // ParseRowsToCSV
-func ParseRowsToCSV(rows []interface{}, filename string) error {
+func parseRowsToCSV(rows []interface{}, filename string) error {
 	// 首先，检查 rows 是否为空
 	if len(rows) == 0 {
 		return errors.New("rows is empty")

sdk/sdk_test.go

@@ -34,7 +34,7 @@ func Test_ParseRowsToCSV(t *testing.T) {
 
 	filename := "test.csv"
 
-	if err := ParseRowsToCSV(rows, filename); err != nil {
+	if err := parseRowsToCSV(rows, filename); err != nil {
 		t.Error(err)
 	}
 

sdk/sync.go

@@ -2,6 +2,7 @@ package sdk
 
 import (
 	"database/sql"
+	"errors"
 	"net/url"
 	"reflect"
 	"strings"
@@ -57,7 +58,12 @@ func (api *APIConfig) ParamEncode() string {
 	return api.params.Encode()
 }
 
-func SyncToCSV(csvFileName string, api APIConfig) (int64, error) {
+func SyncToCSV(fileName string, api APIConfig) (int64, error) {
+	mode := "csv"
+	return SyncToFile(mode, fileName, api)
+}
+
+func SyncToFile(mode string, fileName string, api APIConfig) (int64, error) {
 	c := GetOpenAPIClient()
 	api.SetDefault()
 	apiPath := api.APIPath
@@ -72,7 +78,15 @@ func SyncToCSV(csvFileName string, api APIConfig) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	if err = ParseRowsToCSV(rows, csvFileName); err != nil {
+	switch mode {
+	case "csv":
+		err = parseRowsToCSV(rows, fileName)
+	case "xlsx":
+		err = parseRowsToXLSX(rows, fileName)
+	default:
+		return 0, errors.New("not support mode: csv or xlsx")
+	}
+	if err != nil {
 		return 0, err
 	}
 	return int64(len(rows)), nil