support vector and upgrade library (#91)

Signed-off-by: wjhuang2016 <[email protected]>

ddl/datatype.go

@@ -57,6 +57,7 @@ const (
 
 	KindEnum
 	KindSet
+	KindVector
 )
 
 var ALLFieldType = map[int]string{
@@ -93,9 +94,10 @@ var ALLFieldType = map[int]string{
 	KindTIMESTAMP: "TIMESTAMP",
 	KindYEAR:      "YEAR",
 
-	KindJSON: "JSON",
-	KindEnum: "ENUM",
-	KindSet:  "SET",
+	KindJSON:   "JSON",
+	KindEnum:   "ENUM",
+	KindSet:    "SET",
+	KindVector: "VECTOR",
 }
 
 // TestFieldType is use to control what kind of data types to test.
@@ -131,6 +133,13 @@ var TestFieldType = []int{
 	//KindJSON,    // have `admin check table when index is virtual generated column` bug unfixed.
 	KindEnum,
 	KindSet,
+
+	// More likelihood to vector type.
+	KindVector,
+	KindVector,
+	KindVector,
+	KindVector,
+	KindVector,
 }
 
 var AllowPartitionType = []int{
@@ -216,6 +225,8 @@ func GetMaxLenByKind(kind int) int {
 		return EnumMaxLen
 	case KindSet:
 		return SetMaxLen
+	case KindVector:
+		return 1600
 	}
 	return 0
 }

ddl/ddl.go

@@ -11,7 +11,7 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/PingCAP-QE/clustered-index-rand-test/sqlgen"
+	"github.com/PingCAP-QE/schrddl/sqlgenerator"
 	"github.com/juju/errors"
 	"github.com/ngaut/log"
 	"github.com/pingcap/tidb/parser/model"
@@ -283,7 +283,7 @@ func NewDDLCase(cfg *DDLCaseConfig) *DDLCase {
 			dmlOps:    make([]dmlTestOpExecutor, 0),
 			caseIndex: i,
 			stop:      0,
-			tableMap:  make(map[string]*sqlgen.Table),
+			tableMap:  make(map[string]*sqlgenerator.Table),
 		}
 	}
 	b := &DDLCase{
@@ -626,7 +626,7 @@ func (c *testCase) readDataFromTiDB() error {
 				return errors.Trace(err)
 			}
 
-			log.Infof("[ddl] [instance %d] rows.Columns():%v, len(cols):%v", c.caseIndex, cols, len(cols))
+			//log.Infof("[ddl] [instance %d] rows.Columns():%v, len(cols):%v", c.caseIndex, cols, len(cols))
 
 			// See https://stackoverflow.com/questions/14477941/read-select-columns-into-string-in-go
 			rawResult := make([][]byte, len(cols))

ddl/ddl_ops.go

@@ -88,7 +88,7 @@ func (c *testCase) generateDDLOps() error {
 	if err := c.generateMultiSchemaChange(2); err != nil {
 		return errors.Trace(err)
 	}
-	if err := c.generateSetTilfahReplica(0); err != nil {
+	if err := c.generateSetTilfahReplica(2 * defaultTime); err != nil {
 		return errors.Trace(err)
 	}
 	return nil
@@ -567,7 +567,7 @@ func (c *testCase) execParaDDLSQL(taskCh chan *ddlJobTask, num int) error {
 			}
 
 			if !ddlIgnoreError(ddlErr) {
-				log.Infof("[ddl] [instance %d] TiDB execute %s , err %v, elapsed time:%v", c.caseIndex, task.sql, err, time.Since(opStart).Seconds())
+				log.Infof("[ddl] [instance %d] TiDB execute %s , err %v, elapsed time:%v", c.caseIndex, task.sql, ddlErr, time.Since(opStart).Seconds())
 				task.err = ddlErr
 				unExpectedErr = ddlErr
 				// No need to update schema.
@@ -742,7 +742,7 @@ func (c *testCase) generateAddTable(repeat int) error {
 }
 
 func (c *testCase) prepareAddTable(cfg interface{}, taskCh chan *ddlJobTask) error {
-	columnCount := rand.Intn(c.cfg.TablesToCreate) + 2
+	columnCount := rand.Intn(5) + 2
 	tableColumns := arraylist.New()
 	var partitionColumn *ddlTestColumn
 	for i := 0; i < columnCount; i++ {
@@ -779,7 +779,7 @@ func (c *testCase) prepareAddTable(cfg interface{}, taskCh chan *ddlJobTask) err
 	charset, collate := c.pickupRandomCharsetAndCollate()
 
 	tableInfo := ddlTestTable{
-		name:            uuid.NewV4().String()[:8],
+		name:            "t" + uuid.NewV4().String()[:8],
 		columns:         tableColumns,
 		partitionColumn: partitionColumn,
 		indexes:         make([]*ddlTestIndex, 0),
@@ -818,7 +818,7 @@ func (c *testCase) prepareAddTable(cfg interface{}, taskCh chan *ddlJobTask) err
 	sql += fmt.Sprintf("COMMENT '%s' CHARACTER SET '%s' COLLATE '%s'",
 		tableInfo.comment, charset, collate)
 
-	if rand.Intn(3) == 0 && partitionColumn != nil {
+	if rand.Intn(30) == 0 && partitionColumn != nil {
 		sql += fmt.Sprintf(" partition by hash(`%s`) partitions %d ", partitionColumn.name, rand.Intn(10)+1)
 	}
 
@@ -1268,21 +1268,28 @@ func (c *testCase) prepareAddIndex(ctx interface{}, taskCh chan *ddlJobTask) err
 		perm := rand.Perm(table.columns.Size())[:numberOfColumns]
 		for _, idx := range perm {
 			column := getColumnFromArrayList(table.columns, idx)
+			if column.k == KindVector {
+				continue
+			}
 			if column.canBeIndex() {
 				index.columns = append(index.columns, column)
 			}
 		}
 	}
 
 	needGlobal := index.uniques && table.partitionColumn != nil
+	isVector := true
 
-	for _, column := range index.columns {
+	for i, column := range index.columns {
 		if !checkAddDropColumn(ctx, column) {
 			return nil
 		}
 		if table.partitionColumn != nil && column.name == table.partitionColumn.name {
 			needGlobal = false
 		}
+		if column.k != KindVector || i > 0 {
+			isVector = false
+		}
 	}
 
 	if len(index.columns) == 0 {
@@ -1315,6 +1322,10 @@ func (c *testCase) prepareAddIndex(ctx interface{}, taskCh chan *ddlJobTask) err
 		sql += " GLOBAL"
 	}
 
+	if isVector {
+		sql = fmt.Sprintf("ALTER TABLE `%s` ADD VECTOR INDEX `%s` ((VEC_COSINE_DISTANCE(`%s`))) USING HNSW", table.name, index.name, index.columns[0].name)
+	}
+
 	arg := &ddlIndexJobArg{index: &index}
 	task := &ddlJobTask{
 		k:       ddlAddIndex,
@@ -2048,8 +2059,9 @@ func (c *testCase) prepareSetTiflashReplica(_ interface{}, taskCh chan *ddlJobTa
 		return nil
 	}
 
-	cnt := rand.Intn(6)
-	sql := fmt.Sprintf("ALTER TABLE `%s` SET TIFLASH REPLICA %d", table.name, cnt)
+	cnt := 1
+	//sql := fmt.Sprintf("ALTER TABLE `%s` SET TIFLASH REPLICA %d", table.name, cnt)
+	sql := fmt.Sprintf("ALTER TABLE `%s` SET TIFLASH REPLICA 1", table.name)
 	task := &ddlJobTask{
 		k:       ddlSetTiflashReplica,
 		sql:     sql,

ddl/dml_ops.go

@@ -9,7 +9,7 @@ import (
 	"sort"
 	"strconv"
 
-	"github.com/PingCAP-QE/clustered-index-rand-test/sqlgen"
+	"github.com/PingCAP-QE/schrddl/sqlgenerator"
 	"github.com/juju/errors"
 	"github.com/ngaut/log"
 )
@@ -244,9 +244,9 @@ func (c *testCase) prepareInsert(cfg interface{}, taskCh chan *dmlJobTask) error
 		return nil
 	}
 
-	state := sqlgen.NewState()
+	state := sqlgenerator.NewState()
 	state.Tables = append(state.Tables, table.mapTableToRandTestTable())
-	sql, err := sqlgen.CommonInsertOrReplace.Eval(state)
+	sql, err := sqlgenerator.CommonInsertOrReplace.Eval(state)
 	if err != nil {
 		return err
 	}
@@ -270,7 +270,8 @@ func (c *testCase) prepareUpdate(cfg interface{}, taskCh chan *dmlJobTask) error
 	c.tablesLock.Lock()
 	defer c.tablesLock.Unlock()
 
-	state := sqlgen.NewState()
+	state := sqlgenerator.NewState()
+	state.SetWeight(sqlgenerator.Limit, 1000)
 	for _, table := range c.tableMap {
 		state.Tables = append(state.Tables, table)
 	}
@@ -279,7 +280,7 @@ func (c *testCase) prepareUpdate(cfg interface{}, taskCh chan *dmlJobTask) error
 		return nil
 	}
 
-	sql, err := sqlgen.CommonUpdate.Eval(state)
+	sql, err := sqlgenerator.CommonUpdate.Eval(state)
 	if err != nil {
 		return err
 	}
@@ -303,7 +304,8 @@ func (c *testCase) prepareDelete(cfg interface{}, taskCh chan *dmlJobTask) error
 	c.tablesLock.Lock()
 	defer c.tablesLock.Unlock()
 
-	state := sqlgen.NewState()
+	state := sqlgenerator.NewState()
+	state.SetWeight(sqlgenerator.Limit, 1000)
 	for _, table := range c.tableMap {
 		state.Tables = append(state.Tables, table)
 	}
@@ -312,7 +314,7 @@ func (c *testCase) prepareDelete(cfg interface{}, taskCh chan *dmlJobTask) error
 		return nil
 	}
 
-	sql, err := sqlgen.CommonDelete.Eval(state)
+	sql, err := sqlgenerator.CommonDelete.Eval(state)
 	if err != nil {
 		return err
 	}
@@ -340,13 +342,13 @@ func (c *testCase) prepareSelect(cfg interface{}, taskCh chan *dmlJobTask) error
 		return nil
 	}
 
-	state := sqlgen.NewState()
-	state.SetWeight(sqlgen.WindowFunctionOverW, 0)
+	state := sqlgenerator.NewState()
+	state.SetWeight(sqlgenerator.WindowFunctionOverW, 0)
 	for _, table := range c.tableMap {
 		state.Tables = append(state.Tables, table)
 	}
 
-	query, err := sqlgen.Query.Eval(state)
+	query, err := sqlgenerator.Query.Eval(state)
 	if err != nil {
 		return err
 	}