From 92019bfb5d502cd86adb965c025cf4e2ac57d252 Mon Sep 17 00:00:00 2001 From: sheyanjie Date: Tue, 7 Nov 2023 19:55:05 +0800 Subject: [PATCH 1/3] add col db and resource --- api/audit.go | 23 ++++++++++++++--------- api/audit_test.go | 8 ++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/api/audit.go b/api/audit.go index f0b246f..e7497fe 100644 --- a/api/audit.go +++ b/api/audit.go @@ -6,13 +6,14 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "strings" + "time" + "github.com/gin-gonic/gin" "github.com/taosdata/taoskeeper/db" "github.com/taosdata/taoskeeper/infrastructure/config" "github.com/taosdata/taoskeeper/infrastructure/log" - "net/http" - "strings" - "time" ) var auditLogger = log.GetLogger("audit") @@ -32,6 +33,8 @@ type AuditInfo struct { ClusterID string `json:"cluster_id"` User string `json:"user"` Operation string `json:"operation"` + Db string `json:"db"` + Resource string `json:"resource"` ClientAdd string `json:"client_add"` // client address Details string `json:"details"` } @@ -108,12 +111,12 @@ func parseSql(audit AuditInfo) string { } ts := time.UnixMilli(audit.Timestamp).Format(time.RFC3339) return fmt.Sprintf( - "insert into %s using operations tags ('%s') values ('%s', '%s', '%s', '%s', '%s')", - getTableName(audit), audit.ClusterID, ts, audit.User, audit.Operation, audit.ClientAdd, details) + "insert into %s using operations_v2 tags ('%s') values ('%s', '%s', '%s', '%s', '%s', '%s', '%s')", + getTableName(audit), audit.ClusterID, ts, audit.User, audit.Operation, audit.Db, audit.Resource, audit.ClientAdd, details) } func getTableName(audit AuditInfo) string { - return fmt.Sprintf("t_operations_%s", audit.ClusterID) + return fmt.Sprintf("t_operations_v2_%s", audit.ClusterID) } func (a *Audit) initConnect() error { @@ -163,8 +166,8 @@ func (a *Audit) createDBSql() string { return buf.String() } -var createTableSql = "create stable if not exists operations " + - "(ts timestamp, user_name varchar(25), operation varchar(20), client_address varchar(25), details varchar(65444)) " + +var createTableSql = "create stable if not exists operations_v2 " + + "(ts timestamp, user_name varchar(25), operation varchar(20), db varchar(65), resource varchar(193), client_address varchar(25), details varchar(60000)) " + "tags (cluster_id varchar(64))" func (a *Audit) createSTables() error { @@ -174,6 +177,8 @@ func (a *Audit) createSTables() error { _, err := a.conn.Exec(context.Background(), createTableSql) if err != nil { auditLogger.Error("## create stable error ", "error", err) + return err } - return err + + return nil } diff --git a/api/audit_test.go b/api/audit_test.go index fb4b017..9c3fcfa 100644 --- a/api/audit_test.go +++ b/api/audit_test.go @@ -43,13 +43,13 @@ func TestAudit(t *testing.T) { { name: "1", ts: 1692840000000, - data: `{"timestamp": 1692840000000, "cluster_id": "cluster_id", "user": "user", "operation": "operation", "client_add": "localhost:30000", "details": "detail"}`, + data: `{"timestamp": 1692840000000, "cluster_id": "cluster_id", "user": "user", "operation": "operation", "db":"dbnamea", "resource":"resourcenamea", "client_add": "localhost:30000", "details": "detail"}`, expect: "detail", }, { name: "2", ts: 1692850000000, - data: "{\"timestamp\": 1692850000000, \"cluster_id\": \"cluster_id\", \"user\": \"user\", \"operation\": \"operation\", \"client_add\": \"localhost:30000\", \"details\": \"create database `meter` buffer 32 cachemodel 'none' duration 50d keep 3650d single_stable 0 wal_retention_period 3600 precision 'ms'\"}", + data: "{\"timestamp\": 1692850000000, \"cluster_id\": \"cluster_id\", \"user\": \"user\", \"operation\": \"operation\", \"db\":\"dbnameb\", \"resource\":\"resourcenameb\", \"client_add\": \"localhost:30000\", \"details\": \"create database `meter` buffer 32 cachemodel 'none' duration 50d keep 3650d single_stable 0 wal_retention_period 3600 precision 'ms'\"}", expect: "create database `meter` buffer 32 cachemodel 'none' duration 50d keep 3650d single_stable 0 wal_retention_period 3600 precision 'ms'", }, } @@ -57,7 +57,7 @@ func TestAudit(t *testing.T) { conn, err := db.NewConnectorWithDb(c.TDengine.Username, c.TDengine.Password, c.TDengine.Host, c.TDengine.Port, c.Audit.Database.Name) assert.NoError(t, err) defer func() { - _, _ = conn.Query(context.Background(), "drop database if exists audit.operations") + _, _ = conn.Query(context.Background(), "drop database if exists audit.operations_v2") }() for _, c := range cases { @@ -68,7 +68,7 @@ func TestAudit(t *testing.T) { router.ServeHTTP(w, req) assert.Equal(t, 200, w.Code) - data, err := conn.Query(context.Background(), fmt.Sprintf("select ts, details from audit.operations where ts=%d", c.ts)) + data, err := conn.Query(context.Background(), fmt.Sprintf("select ts, details from audit.operations_v2 where ts=%d", c.ts)) assert.NoError(t, err) assert.Equal(t, 1, len(data.Data)) assert.Equal(t, c.expect, data.Data[0][1]) From 658e3cb97516bcf4ef1ad8c074434ec4ac216af6 Mon Sep 17 00:00:00 2001 From: sheyanjie Date: Wed, 8 Nov 2023 09:54:45 +0800 Subject: [PATCH 2/3] update --- api/audit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/audit.go b/api/audit.go index e7497fe..f191e89 100644 --- a/api/audit.go +++ b/api/audit.go @@ -167,7 +167,7 @@ func (a *Audit) createDBSql() string { } var createTableSql = "create stable if not exists operations_v2 " + - "(ts timestamp, user_name varchar(25), operation varchar(20), db varchar(65), resource varchar(193), client_address varchar(25), details varchar(60000)) " + + "(ts timestamp, user_name varchar(25), operation varchar(20), db varchar(65), resource varchar(193), client_address varchar(25), details varchar(50000)) " + "tags (cluster_id varchar(64))" func (a *Audit) createSTables() error { From 6aa089dac135ec83eda200ebbd53da836e21f2ef Mon Sep 17 00:00:00 2001 From: sheyanjie Date: Wed, 8 Nov 2023 20:27:18 +0800 Subject: [PATCH 3/3] update --- api/audit.go | 6 ++++++ api/audit_test.go | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/api/audit.go b/api/audit.go index f191e89..d6bb673 100644 --- a/api/audit.go +++ b/api/audit.go @@ -18,6 +18,8 @@ import ( var auditLogger = log.GetLogger("audit") +const MAX_DETAIL_LEN = 50000 + type Audit struct { username string password string @@ -109,6 +111,10 @@ func parseSql(audit AuditInfo) string { if strings.Contains(details, "\"") { details = strings.ReplaceAll(details, "\"", "\\\"") } + if len(details) > MAX_DETAIL_LEN { + details = details[:MAX_DETAIL_LEN] + } + ts := time.UnixMilli(audit.Timestamp).Format(time.RFC3339) return fmt.Sprintf( "insert into %s using operations_v2 tags ('%s') values ('%s', '%s', '%s', '%s', '%s', '%s', '%s')", diff --git a/api/audit_test.go b/api/audit_test.go index 9c3fcfa..f31fa4c 100644 --- a/api/audit_test.go +++ b/api/audit_test.go @@ -33,6 +33,8 @@ func TestAudit(t *testing.T) { err = a.Init(router) assert.NoError(t, err) + longDetails := strings.Repeat("0123456789", 5000) + cases := []struct { name string ts int64 @@ -49,7 +51,13 @@ func TestAudit(t *testing.T) { { name: "2", ts: 1692850000000, - data: "{\"timestamp\": 1692850000000, \"cluster_id\": \"cluster_id\", \"user\": \"user\", \"operation\": \"operation\", \"db\":\"dbnameb\", \"resource\":\"resourcenameb\", \"client_add\": \"localhost:30000\", \"details\": \"create database `meter` buffer 32 cachemodel 'none' duration 50d keep 3650d single_stable 0 wal_retention_period 3600 precision 'ms'\"}", + data: `{"timestamp": 1692850000000, "cluster_id": "cluster_id", "user": "user", "operation": "operation", "db":"dbnamea", "resource":"resourcenamea", "client_add": "localhost:30000", "details": "` + longDetails + `"}`, + expect: longDetails[:50000], + }, + { + name: "3", + ts: 1692860000000, + data: "{\"timestamp\": 1692860000000, \"cluster_id\": \"cluster_id\", \"user\": \"user\", \"operation\": \"operation\", \"db\":\"dbnameb\", \"resource\":\"resourcenameb\", \"client_add\": \"localhost:30000\", \"details\": \"create database `meter` buffer 32 cachemodel 'none' duration 50d keep 3650d single_stable 0 wal_retention_period 3600 precision 'ms'\"}", expect: "create database `meter` buffer 32 cachemodel 'none' duration 50d keep 3650d single_stable 0 wal_retention_period 3600 precision 'ms'", }, } @@ -57,7 +65,7 @@ func TestAudit(t *testing.T) { conn, err := db.NewConnectorWithDb(c.TDengine.Username, c.TDengine.Password, c.TDengine.Host, c.TDengine.Port, c.Audit.Database.Name) assert.NoError(t, err) defer func() { - _, _ = conn.Query(context.Background(), "drop database if exists audit.operations_v2") + _, _ = conn.Query(context.Background(), "drop stable if exists audit.operations_v2") }() for _, c := range cases {