api定义: 审核规则支持csv格式导入导出

Makefile

@@ -62,6 +62,8 @@ RPM_BUILD_IMAGE ?= rpmbuild/centos7
 ## Static Parameter, should not be overwrite
 GOBIN = ${shell pwd}/bin
 PARSER_PATH   = ${shell pwd}/vendor/github.com/pingcap/parser
+LOCALE_PATH   = ${shell pwd}/sqle/locale
+PLUGIN_LOCALE_PATH   = ${shell pwd}/sqle/driver/mysql/plocale
 
 ## Arm Build
 ARM_CGO_BUILD_FLAG =
@@ -70,6 +72,23 @@ ifeq ($(EDITION)_$(GOARCH),ee_arm64)
 endif
 
 default: install
+
+######################################## i18n ##########################################################
+install_i18n_tool:
+	GOBIN=$(GOBIN) go install -v github.com/nicksnyder/go-i18n/v2/goi18n@latest
+
+extract_i18n:
+	cd ${LOCALE_PATH} && $(GOBIN)/goi18n extract -sourceLanguage zh
+	cd ${PLUGIN_LOCALE_PATH} && $(GOBIN)/goi18n extract -sourceLanguage zh
+
+start_trans_i18n:
+	cd ${LOCALE_PATH} && touch translate.en.toml && $(GOBIN)/goi18n merge -sourceLanguage=zh active.*.toml
+	cd ${PLUGIN_LOCALE_PATH} && touch translate.en.toml && $(GOBIN)/goi18n merge -sourceLanguage=zh active.*.toml
+
+end_trans_i18n:
+	cd ${LOCALE_PATH} && $(GOBIN)/goi18n merge active.en.toml translate.en.toml && rm -rf translate.en.toml
+	cd ${PLUGIN_LOCALE_PATH} && $(GOBIN)/goi18n merge active.en.toml translate.en.toml && rm -rf translate.en.toml
+
 ######################################## Code Check ####################################################
 ## Static Code Analysis
 vet: swagger

go.mod

@@ -65,10 +65,12 @@ require (
 )
 
 require (
+	github.com/BurntSushi/toml v1.3.2
 	github.com/aliyun/credentials-go v1.1.2
 	github.com/hashicorp/go-version v1.7.0
 	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.69
-	golang.org/x/text v0.13.0
+	github.com/nicksnyder/go-i18n/v2 v2.4.0
+	golang.org/x/text v0.14.0
 	gorm.io/driver/mysql v1.4.7
 	gorm.io/gorm v1.24.3
 )

go.sum

@@ -20,7 +20,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I=
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
@@ -630,6 +631,8 @@ github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8
 github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k=
 github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8=
 github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
+github.com/nicksnyder/go-i18n/v2 v2.4.0 h1:3IcvPOAvnCKwNm0TB0dLDTuawWEj+ax/RERNC+diLMM=
+github.com/nicksnyder/go-i18n/v2 v2.4.0/go.mod h1:nxYSZE9M0bf3Y70gPQjN9ha7XNHX7gMc814+6wVyEI4=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
@@ -1141,8 +1144,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

sqle/api/app.go

@@ -8,6 +8,7 @@ import (
 	"github.com/actiontech/dms/pkg/dms-common/api/accesstoken"
 	dmsV1 "github.com/actiontech/dms/pkg/dms-common/api/dms/v1"
 	jwtPkg "github.com/actiontech/dms/pkg/dms-common/api/jwt"
+	"github.com/actiontech/sqle/sqle/locale"
 
 	// "github.com/actiontech/sqle/sqle/api/cloudbeaver_wrapper"
 	"github.com/actiontech/sqle/sqle/api/controller"
@@ -107,9 +108,9 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
 	})
 
 	v1Router := e.Group(apiV1)
-	v1Router.Use(sqleMiddleware.JWTTokenAdapter(), sqleMiddleware.JWTWithConfig(dmsV1.JwtSigningKey), sqleMiddleware.VerifyUserIsDisabled(), sqleMiddleware.OperationLogRecord(), accesstoken.CheckLatestAccessToken(controller.GetDMSServerAddress(), jwtPkg.GetTokenDetailFromContextWithOldJwt))
+	v1Router.Use(sqleMiddleware.JWTTokenAdapter(), sqleMiddleware.JWTWithConfig(dmsV1.JwtSigningKey), sqleMiddleware.VerifyUserIsDisabled(), sqleMiddleware.OperationLogRecord(), accesstoken.CheckLatestAccessToken(controller.GetDMSServerAddress(), jwtPkg.GetTokenDetailFromContextWithOldJwt), locale.EchoMiddlewareI18nByAcceptLanguage())
 	v2Router := e.Group(apiV2)
-	v2Router.Use(sqleMiddleware.JWTTokenAdapter(), sqleMiddleware.JWTWithConfig(dmsV1.JwtSigningKey), sqleMiddleware.VerifyUserIsDisabled(), sqleMiddleware.OperationLogRecord(), accesstoken.CheckLatestAccessToken(controller.GetDMSServerAddress(), jwtPkg.GetTokenDetailFromContextWithOldJwt))
+	v2Router.Use(sqleMiddleware.JWTTokenAdapter(), sqleMiddleware.JWTWithConfig(dmsV1.JwtSigningKey), sqleMiddleware.VerifyUserIsDisabled(), sqleMiddleware.OperationLogRecord(), accesstoken.CheckLatestAccessToken(controller.GetDMSServerAddress(), jwtPkg.GetTokenDetailFromContextWithOldJwt), locale.EchoMiddlewareI18nByAcceptLanguage())
 
 	// v1 admin api, just admin user can access.
 	{

sqle/api/controller/v1/audit_plan.go

@@ -19,6 +19,7 @@ import (
 	"github.com/actiontech/sqle/sqle/driver"
 	driverV2 "github.com/actiontech/sqle/sqle/driver/v2"
 	"github.com/actiontech/sqle/sqle/errors"
+	"github.com/actiontech/sqle/sqle/locale"
 	"github.com/actiontech/sqle/sqle/log"
 	"github.com/actiontech/sqle/sqle/model"
 	"github.com/actiontech/sqle/sqle/notification"
@@ -1463,7 +1464,7 @@ func GetAuditPlanReportSQLsV1(c echo.Context) error {
 	for i, auditPlanReportSQL := range auditPlanReportSQLs {
 		auditPlanReportSQLsResV1[i] = AuditPlanReportSQLResV1{
 			SQL:         auditPlanReportSQL.SQL,
-			AuditResult: auditPlanReportSQL.AuditResults.String(),
+			AuditResult: auditPlanReportSQL.AuditResults.String(c.Request().Context()),
 			Number:      auditPlanReportSQL.Number,
 		}
 	}
@@ -1474,10 +1475,13 @@ func GetAuditPlanReportSQLsV1(c echo.Context) error {
 	})
 }
 
-func spliceAuditResults(auditResults []model.AuditResult) string {
+func spliceAuditResults(ctx context.Context, auditResults []model.AuditResult) string {
+	lang := locale.GetLangTagFromCtx(ctx)
 	results := []string{}
 	for _, auditResult := range auditResults {
-		results = append(results, fmt.Sprintf("[%v]%v", auditResult.Level, auditResult.Message))
+		results = append(results,
+			fmt.Sprintf("[%v]%v", auditResult.Level, auditResult.GetAuditMsgByLangTag(lang.String())),
+		)
 	}
 	return strings.Join(results, "\n")
 }
@@ -1517,16 +1521,17 @@ func ExportAuditPlanReportV1(c echo.Context) error {
 		return controller.JSONBaseErrorReq(c, fmt.Errorf("the audit plan corresponding to the report was not found"))
 	}
 
+	ctx := c.Request().Context()
 	baseInfo := [][]string{
-		{"扫描任务名称", auditPlanName},
-		{"报告生成时间", reportInfo.CreatedAt.Format("2006/01/02 15:04")},
-		{"审核结果评分", strconv.FormatInt(int64(reportInfo.Score), 10)},
-		{"审核通过率", fmt.Sprintf("%v%%", reportInfo.PassRate*100)},
-		{"所属项目", projectName},
-		{"扫描任务创建人", dms.GetUserNameWithDelTag(reportInfo.AuditPlan.CreateUserID)},
-		{"扫描任务类型", reportInfo.AuditPlan.Type},
-		{"数据库类型", reportInfo.AuditPlan.DBType},
-		{"审核的数据库", reportInfo.AuditPlan.InstanceDatabase},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportTaskName), auditPlanName},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportGenerationTime), reportInfo.CreatedAt.Format("2006/01/02 15:04")},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportResultRating), strconv.FormatInt(int64(reportInfo.Score), 10)},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportApprovalRate), fmt.Sprintf("%v%%", reportInfo.PassRate*100)},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportBelongingProject), projectName},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportCreator), dms.GetUserNameWithDelTag(reportInfo.AuditPlan.CreateUserID)},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportType), reportInfo.AuditPlan.Type},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportDbType), reportInfo.AuditPlan.DBType},
+		{locale.ShouldLocalizeMsg(ctx, locale.APExportDatabase), reportInfo.AuditPlan.InstanceDatabase},
 	}
 	err = csvWriter.WriteAll(baseInfo)
 	if err != nil {
@@ -1539,14 +1544,18 @@ func ExportAuditPlanReportV1(c echo.Context) error {
 		return controller.JSONBaseErrorReq(c, err)
 	}
 
-	err = csvWriter.Write([]string{"编号", "SQL", "审核结果"})
+	err = csvWriter.Write([]string{
+		locale.ShouldLocalizeMsg(ctx, locale.APExportNumber), // 编号
+		"SQL",
+		locale.ShouldLocalizeMsg(ctx, locale.APExportAuditResult), // 审核结果
+	})
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)
 	}
 
 	sqlInfo := [][]string{}
 	for idx, sql := range reportInfo.AuditPlanReportSQLs {
-		sqlInfo = append(sqlInfo, []string{strconv.Itoa(idx + 1), sql.SQL, spliceAuditResults(sql.AuditResults)})
+		sqlInfo = append(sqlInfo, []string{strconv.Itoa(idx + 1), sql.SQL, spliceAuditResults(ctx, sql.AuditResults)})
 	}
 
 	err = csvWriter.WriteAll(sqlInfo)
@@ -1556,7 +1565,7 @@ func ExportAuditPlanReportV1(c echo.Context) error {
 
 	csvWriter.Flush()
 
-	fileName := fmt.Sprintf("扫描任务报告_%s_%s.csv", auditPlanName, time.Now().Format("20060102150405"))
+	fileName := fmt.Sprintf("audit_plan_report_%s_%s.csv", auditPlanName, time.Now().Format("20060102150405"))
 	c.Response().Header().Set(echo.HeaderContentDisposition, mime.FormatMediaType("attachment", map[string]string{
 		"filename": fileName,
 	}))

sqle/api/controller/v1/instance.go

@@ -446,6 +446,7 @@ func GetInstanceTips(c echo.Context) error {
 // @router /v1/projects/{project_name}/instances/{instance_name}/rules [get]
 func GetInstanceRules(c echo.Context) error {
 	s := model.GetStorage()
+	ctx := c.Request().Context()
 
 	instanceName := c.Param("instance_name")
 	projectUid, err := dms.GetPorjectUIDByName(context.TODO(), c.Param("project_name"))
@@ -475,7 +476,7 @@ func GetInstanceRules(c echo.Context) error {
 	}
 	return c.JSON(http.StatusOK, &GetRulesResV1{
 		BaseRes: controller.NewBaseReq(nil),
-		Data:    convertRulesToRes(rules),
+		Data:    convertRulesToRes(ctx, rules),
 	})
 }
 

sqle/api/controller/v1/instance_audit_plan.go

@@ -17,6 +17,7 @@ import (
 	v1 "github.com/actiontech/dms/pkg/dms-common/api/dms/v1"
 	dmsCommonJwt "github.com/actiontech/dms/pkg/dms-common/api/jwt"
 	"github.com/actiontech/sqle/sqle/api/controller"
+	scannerCmd "github.com/actiontech/sqle/sqle/cmd/scannerd/command"
 	"github.com/actiontech/sqle/sqle/config"
 	dms "github.com/actiontech/sqle/sqle/dms"
 	"github.com/actiontech/sqle/sqle/driver"
@@ -191,7 +192,7 @@ func CreateInstanceAuditPlan(c echo.Context) error {
 		AuditPlans:   auditPlans,
 		ActiveStatus: model.ActiveStatusNormal,
 	}
-	err = s.Save(ap)
+	err = s.SaveInstanceAuditPlan(ap)
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)
 	}
@@ -781,16 +782,18 @@ func GetInstanceAuditPlanOverview(c echo.Context) error {
 
 		typeBase := ConvertAuditPlanTypeToRes(v.ID, v.Type)
 		resAuditPlan := InstanceAuditPlanInfo{
-			ID:                 v.ID,
-			Type:               typeBase,
-			DBType:             detail.DBType,
-			InstanceName:       inst.Name,
-			ExecCmd:            execCmd,
-			RuleTemplate:       ruleTemplate,
-			TotalSQLNums:       totalSQLNums,
-			UnsolvedSQLNums:    unsolvedSQLNums,
-			LastCollectionTime: v.LastCollectionTime,
-			ActiveStatus:       v.ActiveStatus,
+			ID:              v.ID,
+			Type:            typeBase,
+			DBType:          detail.DBType,
+			InstanceName:    inst.Name,
+			ExecCmd:         execCmd,
+			RuleTemplate:    ruleTemplate,
+			TotalSQLNums:    totalSQLNums,
+			UnsolvedSQLNums: unsolvedSQLNums,
+			ActiveStatus:    v.ActiveStatus,
+		}
+		if v.AuditPlanTaskInfo != nil {
+			resAuditPlan.LastCollectionTime = v.AuditPlanTaskInfo.LastCollectionTime
 		}
 		resAuditPlans = append(resAuditPlans, resAuditPlan)
 	}
@@ -820,8 +823,23 @@ func GetAuditPlanExecCmd(projectName string, iap *model.InstanceAuditPlan, ap *m
 		return ""
 	}
 
-	cmd := `./scannerd %s --project=%s --host=%s --port=%s  --audit_plan_id=%d  --token=%s`
-	return fmt.Sprintf(cmd, ap.Type, projectName, ip, port, ap.ID, iap.Token)
+	scannerd, err := scannerCmd.GetScannerdCmd(ap.Type)
+	if err != nil {
+		logger.Infof("get scannerd %s failed %s", ap.Type, err)
+		return ""
+	}
+	cmd, err := scannerd.GenCommand("./scannerd", map[string]string{
+		scannerCmd.FlagHost:        ip,
+		scannerCmd.FlagPort:        port,
+		scannerCmd.FlagProject:     projectName,
+		scannerCmd.FlagToken:       iap.Token,
+		scannerCmd.FlagAuditPlanID: fmt.Sprint(ap.ID),
+	})
+	if err != nil {
+		logger.Infof("generate scannerd %s command failed %s", ap.Type, err)
+		return ""
+	}
+	return cmd
 }
 
 type UpdateInstanceAuditPlanStatusReqV1 struct {

sqle/api/controller/v1/pipeline.go

@@ -52,6 +52,10 @@ func (p *pipelineNodeDetail) fillWith(node *pipeline.PipelineNode) {
 	if node == nil {
 		return
 	}
+	integrationInfo, err := node.IntegrationInfo()
+	if err != nil {
+		integrationInfo = err.Error()
+	}
 	p.ID = node.ID
 	p.Name = node.Name
 	p.Type = node.NodeType
@@ -61,7 +65,7 @@ func (p *pipelineNodeDetail) fillWith(node *pipeline.PipelineNode) {
 	p.ObjectType = node.ObjectType
 	p.AuditMethod = node.AuditMethod
 	p.RuleTemplateName = node.RuleTemplateName
-	p.IntegrationInfo = node.IntegrationInfo()
+	p.IntegrationInfo = integrationInfo
 }
 
 // pipelineNodeBase 流水线节点基础信息
@@ -332,7 +336,7 @@ func UpdatePipeline(c echo.Context) error {
 
 	var pipelineSvc pipeline.PipelineSvc
 	svcPipeline := req.convertToSvcPipeline(projectUid, uint(pipelineID))
-	
+
 	err = pipelineSvc.CheckRuleTemplate(svcPipeline)
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)