Skip to content

Commit

Permalink
Merge pull request #2690 from actiontech/issue-2675-4
Browse files Browse the repository at this point in the history 
feat: get global workflow can filter by statuses, project priority, instance id and project id
  • Loading branch information
@LordofAvernus
LordofAvernus authored Oct 18, 2024
2 parents f43ddb5 + 459cb55 commit 82f5c6d
Show file tree
Hide file tree
Showing 30 changed files with 329 additions and 5,261 deletions.
3 changes: 1 addition & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.19
require (
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/Masterminds/semver/v3 v3.1.1
github.com/actiontech/dms v0.0.0-20241010085644-dd7513f0051e
github.com/actiontech/dms v0.0.0-20241017075851-7c8f25660c39
github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62
github.com/actiontech/mybatis-mapper-2-sql v0.5.1-0.20240806065717-29cde7000ef5
github.com/agiledragon/gomonkey v2.0.2+incompatible
Expand Down Expand Up @@ -66,7 +66,6 @@ require (

require (
github.com/aliyun/credentials-go v1.1.2
github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
github.com/hashicorp/go-version v1.7.0
github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.69
github.com/nicksnyder/go-i18n/v2 v2.4.0
Expand Down
8 changes: 2 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,8 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/actiontech/dms v0.0.0-20241010022353-9b4591e71b26 h1:DVUD1u/1vC6yHqj171JCYgyVhFNWlQGp2y/wzKEGhJo=
github.com/actiontech/dms v0.0.0-20241010022353-9b4591e71b26/go.mod h1:Jm+tDoBYmX8R8zqgPJRq1NwZXdxO/F+HxVRJVF65WXA=
github.com/actiontech/dms v0.0.0-20241010085644-dd7513f0051e h1:eEnYDdYR3f8lvlUtCI0bfd/vp0QUiJTf8TzGkwvshUw=
github.com/actiontech/dms v0.0.0-20241010085644-dd7513f0051e/go.mod h1:Jm+tDoBYmX8R8zqgPJRq1NwZXdxO/F+HxVRJVF65WXA=
github.com/actiontech/dms v0.0.0-20241017075851-7c8f25660c39 h1:9EEP+D1p1i58sAlXQnTUPV/czYAwK5/kHKPVugBXm20=
github.com/actiontech/dms v0.0.0-20241017075851-7c8f25660c39/go.mod h1:Jm+tDoBYmX8R8zqgPJRq1NwZXdxO/F+HxVRJVF65WXA=
github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62 h1:JM7WnLzlvXOGE90KKd+aigi+qUDS+U5dLwQMNpTKZxE=
github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62/go.mod h1:adDZHhAf2LRMx2h0JzofPXn12x2XlyQjVE116KXquwo=
github.com/actiontech/mybatis-mapper-2-sql v0.5.1-0.20240806065717-29cde7000ef5 h1:vyQVrkYPzUV9d7gSvOWoezwWMTiC4jc3f3Hpianefq0=
Expand Down Expand Up @@ -305,8 +303,6 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ=
github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
github.com/goccy/go-graphviz v0.0.5/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk=
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
Expand Down
35 changes: 18 additions & 17 deletions sqle/api/controller/v1/sql_manage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package v1
import (
"context"

dmsV1 "github.com/actiontech/dms/pkg/dms-common/api/dms/v1"
"github.com/actiontech/sqle/sqle/api/controller"
"github.com/actiontech/sqle/sqle/locale"
"github.com/labstack/echo/v4"
Expand Down Expand Up @@ -349,11 +350,11 @@ func convertSQLAnalysisResultToRes(ctx context.Context, res *AnalysisResult, raw
}

type GetGlobalSqlManageListReq struct {
FilterProjectUid *string `query:"filter_project_uid" json:"filter_project_uid,omitempty"`
FilterInstanceId *string `query:"filter_instance_id" json:"filter_instance_id,omitempty"`
FilterProjectPriority *string `query:"filter_project_priority" json:"filter_project_priority,omitempty" enums:"high,medium,low"`
PageIndex uint32 `query:"page_index" valid:"required" json:"page_index"`
PageSize uint32 `query:"page_size" valid:"required" json:"page_size"`
FilterProjectUid *string `query:"filter_project_uid" json:"filter_project_uid,omitempty"`
FilterInstanceId *string `query:"filter_instance_id" json:"filter_instance_id,omitempty"`
FilterProjectPriority *dmsV1.ProjectPriority `query:"filter_project_priority" json:"filter_project_priority,omitempty" enums:"high,medium,low"`
PageIndex uint32 `query:"page_index" valid:"required" json:"page_index"`
PageSize uint32 `query:"page_size" valid:"required" json:"page_size"`
}

type GetGlobalSqlManageListResp struct {
Expand All @@ -363,18 +364,18 @@ type GetGlobalSqlManageListResp struct {
}

type GlobalSqlManage struct {
Id uint64 `json:"id"`
Sql string `json:"sql"`
Source *Source `json:"source"`
AuditResult []*AuditResult `json:"audit_result"`
ProjectName string `json:"project_name"`
ProjectUid string `json:"project_uid"`
InstanceName string `json:"instance_name"`
InstanceId string `json:"instance_id"`
Status string `json:"status" enums:"unhandled,solved,ignored,manual_audited"`
ProjectPriority string `json:"project_priority" enums:"high,medium,low"`
FirstAppearTimeStamp string `json:"first_appear_timestamp"`
ProblemDescriptions []string `json:"problem_descriptions"` // 根据来源信息拼接
Id uint64 `json:"id"`
Sql string `json:"sql"`
Source *Source `json:"source"`
AuditResult []*AuditResult `json:"audit_result"`
ProjectName string `json:"project_name"`
ProjectUid string `json:"project_uid"`
InstanceName string `json:"instance_name"`
InstanceId string `json:"instance_id"`
Status string `json:"status" enums:"unhandled,solved,ignored,manual_audited"`
ProjectPriority dmsV1.ProjectPriority `json:"project_priority" enums:"high,medium,low"`
FirstAppearTimeStamp string `json:"first_appear_timestamp"`
ProblemDescriptions []string `json:"problem_descriptions"` // 根据来源信息拼接
}

// GetGlobalSqlManageList
Expand Down
193 changes: 159 additions & 34 deletions sqle/api/controller/v1/workflow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/actiontech/sqle/sqle/server"
"github.com/actiontech/sqle/sqle/utils"

"github.com/actiontech/dms/pkg/dms-common/dmsobject"
"github.com/labstack/echo/v4"
)

Expand Down Expand Up @@ -494,24 +495,24 @@ func CheckWorkflowCanCommit(template *model.WorkflowTemplate, tasks []*model.Tas
}

type GetWorkflowsReqV1 struct {
FilterSubject string `json:"filter_subject" query:"filter_subject"`
FilterWorkflowID string `json:"filter_workflow_id" query:"filter_workflow_id"`
FilterCreateTimeFrom string `json:"filter_create_time_from" query:"filter_create_time_from"`
FilterCreateTimeTo string `json:"filter_create_time_to" query:"filter_create_time_to"`
FilterCreateUserId string `json:"filter_create_user_id" query:"filter_create_user_id"`
FilterStatus string `json:"filter_status" query:"filter_status" valid:"omitempty,oneof=wait_for_audit wait_for_execution rejected canceled executing exec_failed finished"`
FilterStatusList []string `json:"filter_status_list" query:"filter_status_list" validate:"dive,oneof=wait_for_audit wait_for_execution rejected canceled executing exec_failed finished"`
FilterCurrentStepAssigneeUserId string `json:"filter_current_step_assignee_user_id" query:"filter_current_step_assignee_user_id"`
FilterTaskInstanceId string `json:"filter_task_instance_id" query:"filter_task_instance_id"`
FilterTaskExecuteStartTimeFrom string `json:"filter_task_execute_start_time_from" query:"filter_task_execute_start_time_from"`
FilterTaskExecuteStartTimeTo string `json:"filter_task_execute_start_time_to" query:"filter_task_execute_start_time_to"`
FilterSqlVersionID *uint `json:"filter_sql_version_id" query:"filter_sql_version_id"`
FilterProjectUid string `json:"filter_project_uid" query:"filter_project_uid"`
FilterInstanceId string `json:"filter_instance_id" query:"filter_instance_id"`
FilterProjectPriority string `json:"filter_project_priority" query:"filter_project_priority" valid:"omitempty,oneof=high medium low"`
PageIndex uint32 `json:"page_index" query:"page_index" valid:"required"`
PageSize uint32 `json:"page_size" query:"page_size" valid:"required"`
FuzzyKeyword string `json:"fuzzy_keyword" query:"fuzzy_keyword"`
FilterSubject string `json:"filter_subject" query:"filter_subject"`
FilterWorkflowID string `json:"filter_workflow_id" query:"filter_workflow_id"`
FilterCreateTimeFrom string `json:"filter_create_time_from" query:"filter_create_time_from"`
FilterCreateTimeTo string `json:"filter_create_time_to" query:"filter_create_time_to"`
FilterCreateUserId string `json:"filter_create_user_id" query:"filter_create_user_id"`
FilterStatus string `json:"filter_status" query:"filter_status" valid:"omitempty,oneof=wait_for_audit wait_for_execution rejected canceled executing exec_failed finished"`
FilterStatusList []string `json:"filter_status_list" query:"filter_status_list" validate:"dive,oneof=wait_for_audit wait_for_execution rejected canceled executing exec_failed finished"`
FilterCurrentStepAssigneeUserId string `json:"filter_current_step_assignee_user_id" query:"filter_current_step_assignee_user_id"`
FilterTaskInstanceId string `json:"filter_task_instance_id" query:"filter_task_instance_id"`
FilterTaskExecuteStartTimeFrom string `json:"filter_task_execute_start_time_from" query:"filter_task_execute_start_time_from"`
FilterTaskExecuteStartTimeTo string `json:"filter_task_execute_start_time_to" query:"filter_task_execute_start_time_to"`
FilterSqlVersionID *uint `json:"filter_sql_version_id" query:"filter_sql_version_id"`
FilterProjectUid string `json:"filter_project_uid" query:"filter_project_uid"`
FilterInstanceId string `json:"filter_instance_id" query:"filter_instance_id"`
FilterProjectPriority dmsV1.ProjectPriority `json:"filter_project_priority" query:"filter_project_priority" valid:"omitempty,oneof=high medium low"`
PageIndex uint32 `json:"page_index" query:"page_index" valid:"required"`
PageSize uint32 `json:"page_size" query:"page_size" valid:"required"`
FuzzyKeyword string `json:"fuzzy_keyword" query:"fuzzy_keyword"`
}

type GetWorkflowsResV1 struct {
Expand All @@ -521,23 +522,23 @@ type GetWorkflowsResV1 struct {
}

type WorkflowDetailResV1 struct {
ProjectName string `json:"project_name"`
ProjectUid string `json:"project_uid,omitempty"`
ProjectPriority string `json:"project_priority"`
Name string `json:"workflow_name"`
WorkflowId string `json:"workflow_id" `
Desc string `json:"desc"`
SqlVersionName []string `json:"sql_version_name,omitempty"`
CreateUser string `json:"create_user_name"`
CreateTime *time.Time `json:"create_time"`
CurrentStepType string `json:"current_step_type,omitempty" enums:"sql_review,sql_execute"`
CurrentStepAssigneeUser []string `json:"current_step_assignee_user_name_list,omitempty"`
Status string `json:"status" enums:"wait_for_audit,wait_for_execution,rejected,canceled,exec_failed,executing,finished"`
InstanceInfo []InstanceInfo `json:"instance_info,omitempty"`
ProjectName string `json:"project_name"`
ProjectUid string `json:"project_uid,omitempty"`
ProjectPriority dmsV1.ProjectPriority `json:"project_priority"`
Name string `json:"workflow_name"`
WorkflowId string `json:"workflow_id" `
Desc string `json:"desc"`
SqlVersionName []string `json:"sql_version_name,omitempty"`
CreateUser string `json:"create_user_name"`
CreateTime *time.Time `json:"create_time"`
CurrentStepType string `json:"current_step_type,omitempty" enums:"sql_review,sql_execute"`
CurrentStepAssigneeUser []string `json:"current_step_assignee_user_name_list,omitempty"`
Status string `json:"status" enums:"wait_for_audit,wait_for_execution,rejected,canceled,exec_failed,executing,finished"`
InstanceInfo []InstanceInfo `json:"instance_info,omitempty"`
}

type InstanceInfo struct {
InstanceId int64 `json:"instance_id,omitempty"`
InstanceId string `json:"instance_id,omitempty"`
InstanceName string `json:"instance_name,omitempty"`
}

Expand Down Expand Up @@ -591,6 +592,25 @@ func GetGlobalWorkflowsV1(c echo.Context) error {
"check_user_can_access": user.Name != model.DefaultAdminUser, // dms-todo: 判断是否是超级管理员
"limit": limit,
"offset": offset,
"filter_status_list": req.FilterStatusList, // 根据SQL工单的状态筛选多个状态的工单
"filter_project_id": req.FilterProjectUid, // 根据项目id筛选某些一个项目下的多个工单
"filter_instance_id": req.FilterInstanceId, // 根据工单记录的数据源id，筛选包含该数据源的工单，多数据源情况下，一旦包含该数据源，则被选中
}

projectMap := make(map[string] /* project uid */ *dmsV1.ListProject)
// 若根据项目优先级筛选，则先请求dms，获取优先级对应的项目信息
if req.FilterProjectPriority != "" {
data["filter_project_id_list"], projectMap, err = loadProjectsByPriority(c.Request().Context(), req.FilterProjectPriority)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}
if len(projectMap) == 0 {
return c.JSON(http.StatusOK, GetWorkflowsResV1{
BaseRes: controller.NewBaseReq(nil),
Data: []*WorkflowDetailResV1{},
TotalNums: 0,
})
}
}

s := model.GetStorage()
Expand All @@ -604,10 +624,33 @@ func GetGlobalWorkflowsV1(c echo.Context) error {
2. 用户相关代码需要从DMS获取
*/

// 从dms获取工单对应的项目信息，若该信息已经在根据项目优先级筛选时被加载，则不需要重复加载
if req.FilterProjectPriority == "" {
projectMap, err = loadProjectsByWorkflows(c.Request().Context(), workflows)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}
}
// 从dms获取工单对应的数据源信息
instanceMap, err := loadInstanceByWorkflows(c.Request().Context(), workflows)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}

workflowsResV1 := make([]*WorkflowDetailResV1, 0, len(workflows))
for _, workflow := range workflows {
instanceInfos := make([]InstanceInfo, 0, len(workflow.InstanceIds))
for _, id := range workflow.InstanceIds {
instanceInfos = append(instanceInfos, InstanceInfo{
InstanceId: id,
InstanceName: instanceMap[id].Name,
})
}
workflowRes := &WorkflowDetailResV1{
ProjectName: workflow.ProjectId, // dms-todo: 临时使用id代替name
ProjectName: projectMap[workflow.ProjectId].Name,
ProjectUid: workflow.ProjectId,
ProjectPriority: projectMap[workflow.ProjectId].ProjectPriority,
InstanceInfo: instanceInfos,
Name: workflow.Subject,
WorkflowId: workflow.WorkflowId,
Desc: workflow.Desc,
Expand All @@ -627,6 +670,88 @@ func GetGlobalWorkflowsV1(c echo.Context) error {
})
}

// 根据项目优先级从 dms 系统中获取相应的项目列表，并返回项目ID列表和项目映射
func loadProjectsByPriority(ctx context.Context, priority dmsV1.ProjectPriority) (projectIds []string, projectMap map[string] /* project uid */ *dmsV1.ListProject, err error) {
projectMap = make(map[string]*dmsV1.ListProject)
// 如果根据项目优先级筛选SQL工单，则先获取项目优先级，根据优先级对应的项目ID进行筛选
projects, _, err := dmsobject.ListProjects(ctx, controller.GetDMSServerAddress(), dmsV1.ListProjectReq{
PageSize: 999,
PageIndex: 1,
FilterByProjectPriority: priority,
})
if err != nil {
return nil, nil, err
}
for _, project := range projects {
if _, exist := projectMap[project.ProjectUid]; !exist {
projectMap[project.ProjectUid] = project
}
projectIds = append(projectIds, project.ProjectUid)
}
return projectIds, projectMap, nil
}

// 根据工单列表中的项目ID从 dms 系统中获取对应的项目信息，并返回项目映射
func loadProjectsByWorkflows(ctx context.Context, workflows []*model.WorkflowListDetail) (projectMap map[string] /* project uid */ *dmsV1.ListProject, err error) {
projectMap = make(map[string]*dmsV1.ListProject)
if len(workflows) == 0 {
return projectMap, nil
}

var projectIds []string
for _, workflow := range workflows {
if _, exist := projectMap[workflow.ProjectId]; !exist {
projectIds = append(projectIds, workflow.ProjectId)
projectMap[workflow.ProjectId] = nil
}
}
// get project priority from dms
projects, _, err := dmsobject.ListProjects(ctx, controller.GetDMSServerAddress(), dmsV1.ListProjectReq{
PageSize: uint32(len(projectIds)),
PageIndex: 1,
FilterByProjectUids: projectIds,
})
if err != nil {
return nil, err
}
projectMap = make(map[string] /* project uid */ *dmsV1.ListProject)
for _, project := range projects {
projectMap[project.ProjectUid] = project
}
return projectMap, nil
}

// 根据工单列表中的实例ID从 dms 系统中获取对应的数据源实例信息，并返回实例映射
func loadInstanceByWorkflows(ctx context.Context, workflows []*model.WorkflowListDetail) (instanceMap map[string] /* instance id */ *dmsV1.ListDBService, err error) {
instanceMap = make(map[string]*dmsV1.ListDBService)
if len(workflows) == 0 {
return instanceMap, nil
}

var instanceIdList []string
for _, workflow := range workflows {
for _, id := range workflow.InstanceIds {
if _, exist := instanceMap[id]; !exist {
instanceIdList = append(instanceIdList, id)
instanceMap[id] = nil
}
}
}
// get instances from dms
instances, _, err := dmsobject.ListDbServices(ctx, controller.GetDMSServerAddress(), dmsV1.ListDBServiceReq{
PageSize: uint32(len(instanceIdList)),
PageIndex: 1,
FilterByDBServiceIds: instanceIdList,
})
if err != nil {
return nil, err
}
for _, instance := range instances {
instanceMap[instance.DBServiceUid] = instance
}
return instanceMap, nil
}

// GetWorkflowsV1
// @Summary 获取工单列表
// @Description get workflow list
Expand Down Expand Up @@ -723,7 +848,7 @@ func GetWorkflowsV1(c echo.Context) error {
CurrentStepType: workflow.CurrentStepType.String,
CurrentStepAssigneeUser: CurrentStepAssigneeUserNames,
Status: workflow.Status,
SqlVersionName: workflow.SqlVersionName,
SqlVersionName: workflow.SqlVersionNames,
}
workflowsResV1 = append(workflowsResV1, workflowRes)
}
Expand Down
Loading

0 comments on commit 82f5c6d

Please sign in to comment.