Range Query Optimization (For sequential Vindex types)

examples/vtexplain/sample_schema.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS `t1` (
+  `c1` bigint unsigned NOT NULL,
+  `c2` bigint unsigned NOT NULL,
+  PRIMARY KEY (`c1`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

examples/vtexplain/sample_vschema.json

@@ -0,0 +1,20 @@
+{
+    "ks1": {
+        "sharded": true,
+        "vindexes": {
+            "binary_vdx": {
+                "type": "binary"
+            }
+        },
+        "tables": {
+            "t1": {
+                "columnVindexes": [
+                    {
+                        "column": "c1",
+                        "name": "binary_vdx"
+                    }
+                ]
+            }
+        }
+    }
+}

go/vt/key/key.go

@@ -90,6 +90,11 @@ func Empty(id []byte) bool {
 // KeyRange helper methods
 //
 
+// Make a Key Range
+func NewKeyRange(start []byte, end []byte) *topodatapb.KeyRange {
+	return &topodatapb.KeyRange{Start: start, End: end}
+}
+
 // KeyRangeAdd adds two adjacent KeyRange values (in any order) into a single value. If the values are not adjacent,
 // it returns false.
 func KeyRangeAdd(a, b *topodatapb.KeyRange) (*topodatapb.KeyRange, bool) {

go/vt/vtgate/engine/routing.go

@@ -51,6 +51,9 @@ const (
 	// IN is for routing a statement to a multi shard.
 	// Requires: A Vindex, and a multi Values.
 	IN
+	// Between is for routing a statement to a multi shard
+	// Requires: A Vindex, and a multi Values.
+	Between
 	// MultiEqual is used for routing queries with IN with tuple clause
 	// Requires: A Vindex, and a multi Tuple Values.
 	MultiEqual
@@ -78,6 +81,7 @@ var opName = map[Opcode]string{
 	EqualUnique:   "EqualUnique",
 	Equal:         "Equal",
 	IN:            "IN",
+	Between:       "Between",
 	MultiEqual:    "MultiEqual",
 	Scatter:       "Scatter",
 	DBA:           "DBA",
@@ -157,6 +161,14 @@ func (rp *RoutingParameters) findRoute(ctx context.Context, vcursor VCursor, bin
 		default:
 			return rp.in(ctx, vcursor, bindVars)
 		}
+	case Between:
+		switch rp.Vindex.(type) {
+		case vindexes.SingleColumn:
+			return rp.between(ctx, vcursor, bindVars)
+		default:
+			// Only SingleColumn vindex supported.
+			return nil, nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "opcode: %v not supported", rp.Opcode)
+		}
 	case MultiEqual:
 		switch rp.Vindex.(type) {
 		case vindexes.MultiColumn:
@@ -396,6 +408,19 @@ func (rp *RoutingParameters) inMultiCol(ctx context.Context, vcursor VCursor, bi
 	return rss, shardVarsMultiCol(bindVars, mapVals, isSingleVal), nil
 }
 
+func (rp *RoutingParameters) between(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) ([]*srvtopo.ResolvedShard, []map[string]*querypb.BindVariable, error) {
+	env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor)
+	value, err := env.Evaluate(rp.Values[0])
+	if err != nil {
+		return nil, nil, err
+	}
+	rss, values, err := resolveShardsBetween(ctx, vcursor, rp.Vindex.(vindexes.Between), rp.Keyspace, value.TupleValues())
+	if err != nil {
+		return nil, nil, err
+	}
+	return rss, shardVars(bindVars, values), nil
+}
+
 func (rp *RoutingParameters) multiEqual(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) ([]*srvtopo.ResolvedShard, []map[string]*querypb.BindVariable, error) {
 	env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor)
 	value, err := env.Evaluate(rp.Values[0])
@@ -520,6 +545,24 @@ func buildMultiColumnVindexValues(shardsValues [][][]sqltypes.Value) [][][]*quer
 	return shardsIds
 }
 
+func resolveShardsBetween(ctx context.Context, vcursor VCursor, vindex vindexes.Between, keyspace *vindexes.Keyspace, vindexKeys []sqltypes.Value) ([]*srvtopo.ResolvedShard, [][]*querypb.Value, error) {
+	// Convert vindexKeys to []*querypb.Value
+	ids := make([]*querypb.Value, len(vindexKeys))
+	for i, vik := range vindexKeys {
+		ids[i] = sqltypes.ValueToProto(vik)
+	}
+
+	// RangeMap using the Vindex
+	destinations, err := vindex.RangeMap(ctx, vcursor, vindexKeys[0], vindexKeys[1])
+	if err != nil {
+		return nil, nil, err
+
+	}
+
+	// And use the Resolver to map to ResolvedShards.
+	return vcursor.ResolveDestinations(ctx, keyspace.Name, ids, destinations)
+}
+
 func shardVars(bv map[string]*querypb.BindVariable, mapVals [][]*querypb.Value) []map[string]*querypb.BindVariable {
 	shardVars := make([]map[string]*querypb.BindVariable, len(mapVals))
 	for i, vals := range mapVals {

go/vt/vtgate/planbuilder/operators/sharded_routing.go

@@ -223,6 +223,9 @@ func (tr *ShardedRouting) resetRoutingLogic(ctx *plancontext.PlanningContext) Ro
 func (tr *ShardedRouting) searchForNewVindexes(ctx *plancontext.PlanningContext, predicate sqlparser.Expr) (Routing, bool) {
 	newVindexFound := false
 	switch node := predicate.(type) {
+	case *sqlparser.BetweenExpr:
+		return tr.planBetweenOp(ctx, node)
+
 	case *sqlparser.ComparisonExpr:
 		return tr.planComparison(ctx, node)
 
@@ -234,6 +237,24 @@ func (tr *ShardedRouting) searchForNewVindexes(ctx *plancontext.PlanningContext,
 	return nil, newVindexFound
 }
 
+func (tr *ShardedRouting) planBetweenOp(ctx *plancontext.PlanningContext, node *sqlparser.BetweenExpr) (routing Routing, foundNew bool) {
+	// x BETWEEN a AND b => x >= a AND x <= b
+	column, ok := node.Left.(*sqlparser.ColName)
+	if !ok {
+		return nil, false
+	}
+	vals := []sqlparser.Expr{}
+	vals = append(vals, node.From, node.To)
+	var vdValue sqlparser.ValTuple = vals
+	opcode := func(*vindexes.ColumnVindex) engine.Opcode { return engine.Between }
+
+	val := makeEvalEngineExpr(ctx, vdValue)
+	if val == nil {
+		return nil, false
+	}
+	return nil, tr.haveMatchingVindex(ctx, node, vdValue, column, val, opcode, justTheVindex)
+}
+
 func (tr *ShardedRouting) planComparison(ctx *plancontext.PlanningContext, cmp *sqlparser.ComparisonExpr) (routing Routing, foundNew bool) {
 	switch cmp.Operator {
 	case sqlparser.EqualOp:

go/vt/vtgate/vindexes/binary.go

@@ -30,6 +30,7 @@ var (
 	_ Reversible      = (*Binary)(nil)
 	_ Hashing         = (*Binary)(nil)
 	_ ParamValidating = (*Binary)(nil)
+	_ Between         = (*Binary)(nil)
 )
 
 // Binary is a vindex that converts binary bits to a keyspace id.
@@ -108,6 +109,20 @@ func (*Binary) ReverseMap(_ VCursor, ksids [][]byte) ([]sqltypes.Value, error) {
 	return reverseIds, nil
 }
 
+// RangeMap can map ids to key.Destination objects.
+func (vind *Binary) RangeMap(ctx context.Context, vcursor VCursor, startId sqltypes.Value, endId sqltypes.Value) ([]key.Destination, error) {
+	startKsId, err := vind.Hash(startId)
+	if err != nil {
+		return nil, err
+	}
+	endKsId, err := vind.Hash(endId)
+	if err != nil {
+		return nil, err
+	}
+	out := []key.Destination{&key.DestinationKeyRange{KeyRange: key.NewKeyRange(startKsId, endKsId)}}
+	return out, nil
+}
+
 // UnknownParams implements the ParamValidating interface.
 func (vind *Binary) UnknownParams() []string {
 	return vind.unknownParams

go/vt/vtgate/vindexes/binary_test.go

@@ -145,3 +145,18 @@ func TestBinaryReverseMap(t *testing.T) {
 		t.Errorf("ReverseMap(): %v, want %s", err, wantErr)
 	}
 }
+
+func TestBinaryRangeMap(t *testing.T) {
+
+	startInterval := "0x01"
+	endInterval := "0x10"
+
+	got, err := binOnlyVindex.(Between).RangeMap(context.Background(), nil, sqltypes.NewHexNum([]byte(startInterval)),
+		sqltypes.NewHexNum([]byte(endInterval)))
+	require.NoError(t, err)
+	want := "DestinationKeyRange(01-10)"
+	if !reflect.DeepEqual(got[0].String(), want) {
+		t.Errorf("RangeMap(): %+v, want %+v", got, want)
+	}
+
+}

go/vt/vtgate/vindexes/numeric.go

@@ -31,6 +31,7 @@ var (
 	_ Reversible      = (*Numeric)(nil)
 	_ Hashing         = (*Numeric)(nil)
 	_ ParamValidating = (*Numeric)(nil)
+	_ Between         = (*Numeric)(nil)
 )
 
 // Numeric defines a bit-pattern mapping of a uint64 to the KeyspaceId.
@@ -108,6 +109,20 @@ func (*Numeric) ReverseMap(_ VCursor, ksids [][]byte) ([]sqltypes.Value, error)
 	return reverseIds, nil
 }
 
+// RangeMap implements Between.
+func (vind *Numeric) RangeMap(ctx context.Context, vcursor VCursor, startId sqltypes.Value, endId sqltypes.Value) ([]key.Destination, error) {
+	startKsId, err := vind.Hash(startId)
+	if err != nil {
+		return nil, err
+	}
+	endKsId, err := vind.Hash(endId)
+	if err != nil {
+		return nil, err
+	}
+	out := []key.Destination{&key.DestinationKeyRange{KeyRange: key.NewKeyRange(startKsId, endKsId)}}
+	return out, nil
+}
+
 // UnknownParams implements the ParamValidating interface.
 func (vind *Numeric) UnknownParams() []string {
 	return vind.unknownParams

go/vt/vtgate/vindexes/vindex.go

@@ -130,6 +130,13 @@ type (
 		ReverseMap(vcursor VCursor, ks [][]byte) ([]sqltypes.Value, error)
 	}
 
+	// A Between vindex is an optional interface one that maps to a keyspace range
+	// instead of a single keyspace id. It's being used to reduce the fan out for
+	// 'BETWEEN' expressions.
+	Between interface {
+		RangeMap(ctx context.Context, vcursor VCursor, startId sqltypes.Value, endId sqltypes.Value) ([]key.Destination, error)
+	}
+
 	// A Prefixable vindex is one that maps the prefix of a id to a keyspace range
 	// instead of a single keyspace id. It's being used to reduced the fan out for
 	// 'LIKE' expressions.