diff --git a/go/test/endtoend/vreplication/global_routing_test.go b/go/test/endtoend/vreplication/global_routing_test.go index 25339d46f1e..1674c426bb4 100644 --- a/go/test/endtoend/vreplication/global_routing_test.go +++ b/go/test/endtoend/vreplication/global_routing_test.go @@ -17,9 +17,11 @@ limitations under the License. package vreplication import ( + "bytes" "fmt" "strings" "testing" + "text/template" "time" "github.com/stretchr/testify/require" @@ -28,15 +30,34 @@ import ( vttablet "vitess.io/vitess/go/vt/vttablet/common" ) -/* -* Create unsharded keyspace with two tables, t1,t2,t3, empty vschema. Confirm global routing works. Also try @primary, @replica -* Add another unsharded keyspace with t2,t4,t5. Check what happens -* Add MoveTables into sharded keyspace moving t2, t4 . Check what happens on Create/SwitchRead/SwitchWrites/Complete -* Check global routing for each with an expectation. -* First BEFORE and then AFTEr the logic change - */ +type tgrTestConfig struct { + ksU1, ksU2, ksS1 string + ksU1Tables, ksU2Tables, ksS1Tables []string +} + +var grTestConfig tgrTestConfig = tgrTestConfig{ + ksU1: "unsharded1", + ksU2: "unsharded2", + ksS1: "sharded1", + ksU1Tables: []string{"t1", "t2", "t3"}, + ksU2Tables: []string{"t2", "t4", "t5"}, + ksS1Tables: []string{"t2", "t4", "t6"}, +} + +type grTestExpectations struct { + postKsU1, postKsU2, postKsS1 func(t *testing.T) +} + +type grTestCase struct { + markAsGlobal bool + unshardedHasVSchema bool +} + +type grHelpers struct { + t *testing.T +} -func getSchema(tables []string) string { +func (h *grHelpers) getSchema(tables []string) string { var createSQL string for _, table := range tables { createSQL += "CREATE TABLE " + table + " (id int primary key, val varchar(32)) ENGINE=InnoDB;\n" @@ -44,15 +65,8 @@ func getSchema(tables []string) string { return createSQL } -func insertData(t *testing.T, keyspace string, table string, id int, val string) { - vtgateConn, cancel := getVTGateConn() - defer cancel() - _, err := vtgateConn.ExecuteFetch(fmt.Sprintf("insert into %s.%s(id, val) values(%d, '%s')", keyspace, table, id, val), 1, false) - require.NoError(t, err) -} - -var ksS1VSchema = ` -{ +func (h *grHelpers) getShardedVSchema(tables []string) string { + const vSchemaTmpl = `{ "sharded": true, "vindexes": { "reverse_bits": { @@ -60,15 +74,9 @@ var ksS1VSchema = ` } }, "tables": { - "t2": { - "column_vindexes": [ - { - "column": "id", - "name": "reverse_bits" - } - ] - }, - "t4": { + {{- range $i, $table := .Tables}} + {{- if gt $i 0}},{{end}} + "{{ $table }}": { "column_vindexes": [ { "column": "id", @@ -76,11 +84,30 @@ var ksS1VSchema = ` } ] } + {{- end}} } } ` + type VSchemaData struct { + Tables []string + } + tmpl, err := template.New("vschema").Parse(vSchemaTmpl) + require.NoError(h.t, err) + var buf bytes.Buffer + err = tmpl.Execute(&buf, VSchemaData{tables}) + require.NoError(h.t, err) + return buf.String() +} + +func (h *grHelpers) insertData(t *testing.T, keyspace string, table string, id int, val string) { + vtgateConn, cancel := getVTGateConn() + defer cancel() + _, err := vtgateConn.ExecuteFetch(fmt.Sprintf("insert into %s.%s(id, val) values(%d, '%s')", + keyspace, table, id, val), 1, false) + require.NoError(t, err) +} -func isGlobal(t *testing.T, tables []string, expectedVal string) bool { +func (h *grHelpers) isGlobal(t *testing.T, tables []string, expectedVal string) bool { vtgateConn, cancel := getVTGateConn() defer cancel() var err error @@ -100,7 +127,7 @@ func isGlobal(t *testing.T, tables []string, expectedVal string) bool { return asExpected } -func isNotGlobal(t *testing.T, tables []string) bool { +func (h *grHelpers) isNotGlobal(t *testing.T, tables []string) bool { vtgateConn, cancel := getVTGateConn() defer cancel() var err error @@ -119,7 +146,7 @@ func isNotGlobal(t *testing.T, tables []string) bool { return asExpected } -func isAmbiguous(t *testing.T, tables []string) bool { +func (h *grHelpers) isAmbiguous(t *testing.T, tables []string) bool { vtgateConn, cancel := getVTGateConn() defer cancel() var err error @@ -137,98 +164,62 @@ func isAmbiguous(t *testing.T, tables []string) bool { return asExpected } -type tGlobalRoutingTestConfig struct { - ksU1, ksU2, ksS1 string - ksU1Tables, ksU2Tables, ksS1Tables []string -} - -var globalRoutingTestConfig tGlobalRoutingTestConfig = tGlobalRoutingTestConfig{ - ksU1: "unsharded1", - ksU2: "unsharded2", - ksS1: "sharded1", - ksU1Tables: []string{"t1", "t2", "t3"}, - ksU2Tables: []string{"t2", "t4", "t5"}, - ksS1Tables: []string{"t2", "t4"}, -} - -type tGlobalRoutingTestExpectationFuncs struct { - postKsU1, postKsU2, postKsS1 func(t *testing.T) -} - -type globalRoutingTestCase struct { - markAsGlobal bool - unshardedHaveVSchema bool -} - -func setExpectations(t *testing.T) *map[globalRoutingTestCase]*tGlobalRoutingTestExpectationFuncs { - var exp = make(map[globalRoutingTestCase]*tGlobalRoutingTestExpectationFuncs) - exp[globalRoutingTestCase{unshardedHaveVSchema: false, markAsGlobal: false}] = &tGlobalRoutingTestExpectationFuncs{ +func (h *grHelpers) getExpectations() *map[grTestCase]*grTestExpectations { + var exp = make(map[grTestCase]*grTestExpectations) + exp[grTestCase{unshardedHasVSchema: false, markAsGlobal: false}] = &grTestExpectations{ postKsU1: func(t *testing.T) { - require.True(t, isGlobal(t, []string{"t1", "t2", "t3"}, globalRoutingTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t1", "t2", "t3"}, grTestConfig.ksU1)) }, postKsU2: func(t *testing.T) { - require.True(t, isNotGlobal(t, []string{"t1", "t2", "t3"})) - require.True(t, isNotGlobal(t, []string{"t4", "t5"})) + require.True(t, h.isNotGlobal(t, []string{"t1", "t2", "t3"})) + require.True(t, h.isNotGlobal(t, []string{"t4", "t5"})) }, postKsS1: func(t *testing.T) { - require.True(t, isGlobal(t, []string{"t2", "t4"}, globalRoutingTestConfig.ksS1)) - require.True(t, isNotGlobal(t, []string{"t1", "t3"})) - require.True(t, isNotGlobal(t, []string{"t5"})) + require.True(t, h.isGlobal(t, []string{"t2", "t4"}, grTestConfig.ksS1)) + require.True(t, h.isNotGlobal(t, []string{"t1", "t3"})) + require.True(t, h.isNotGlobal(t, []string{"t5"})) + require.True(t, h.isGlobal(t, []string{"t6"}, grTestConfig.ksS1)) }, } - exp[globalRoutingTestCase{unshardedHaveVSchema: false, markAsGlobal: true}] = &tGlobalRoutingTestExpectationFuncs{ + exp[grTestCase{unshardedHasVSchema: false, markAsGlobal: true}] = &grTestExpectations{ postKsU1: func(t *testing.T) { - require.True(t, isGlobal(t, []string{"t1", "t2", "t3"}, globalRoutingTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t1", "t2", "t3"}, grTestConfig.ksU1)) }, postKsU2: func(t *testing.T) { - require.True(t, isGlobal(t, []string{"t1", "t3"}, globalRoutingTestConfig.ksU1)) - require.True(t, isGlobal(t, []string{"t4", "t5"}, globalRoutingTestConfig.ksU2)) - require.True(t, isAmbiguous(t, []string{"t2"})) + require.True(t, h.isGlobal(t, []string{"t1", "t3"}, grTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t4", "t5"}, grTestConfig.ksU2)) + require.True(t, h.isAmbiguous(t, []string{"t2"})) }, postKsS1: func(t *testing.T) { - require.True(t, isGlobal(t, []string{"t2", "t4"}, globalRoutingTestConfig.ksS1)) - require.True(t, isGlobal(t, []string{"t1", "t3"}, globalRoutingTestConfig.ksU1)) - require.True(t, isGlobal(t, []string{"t5"}, globalRoutingTestConfig.ksU2)) + require.True(t, h.isGlobal(t, []string{"t2", "t4"}, grTestConfig.ksS1)) + require.True(t, h.isGlobal(t, []string{"t1", "t3"}, grTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t5"}, grTestConfig.ksU2)) + require.True(t, h.isGlobal(t, []string{"t6"}, grTestConfig.ksS1)) }, } - exp[globalRoutingTestCase{unshardedHaveVSchema: true, markAsGlobal: false}] = &tGlobalRoutingTestExpectationFuncs{ + exp[grTestCase{unshardedHasVSchema: true, markAsGlobal: false}] = &grTestExpectations{ postKsU1: func(t *testing.T) { - require.True(t, isGlobal(t, []string{"t1", "t2", "t3"}, globalRoutingTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t1", "t2", "t3"}, grTestConfig.ksU1)) }, postKsU2: func(t *testing.T) { - require.True(t, isGlobal(t, []string{"t1", "t3"}, globalRoutingTestConfig.ksU1)) - require.True(t, isGlobal(t, []string{"t4", "t5"}, globalRoutingTestConfig.ksU2)) - require.True(t, isAmbiguous(t, []string{"t2"})) + require.True(t, h.isGlobal(t, []string{"t1", "t3"}, grTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t4", "t5"}, grTestConfig.ksU2)) + require.True(t, h.isAmbiguous(t, []string{"t2"})) }, postKsS1: func(t *testing.T) { - require.True(t, isAmbiguous(t, []string{"t2", "t4"})) - require.True(t, isGlobal(t, []string{"t1", "t3"}, globalRoutingTestConfig.ksU1)) - require.True(t, isGlobal(t, []string{"t5"}, globalRoutingTestConfig.ksU2)) + require.True(t, h.isAmbiguous(t, []string{"t2", "t4"})) + require.True(t, h.isGlobal(t, []string{"t1", "t3"}, grTestConfig.ksU1)) + require.True(t, h.isGlobal(t, []string{"t5"}, grTestConfig.ksU2)) }, } - exp[globalRoutingTestCase{unshardedHaveVSchema: true, markAsGlobal: true}] = - exp[globalRoutingTestCase{unshardedHaveVSchema: true, markAsGlobal: false}] + exp[grTestCase{unshardedHasVSchema: true, markAsGlobal: true}] = + exp[grTestCase{unshardedHasVSchema: true, markAsGlobal: false}] return &exp } -func TestGlobalRouting(t *testing.T) { - exp := *setExpectations(t) - testCases := []globalRoutingTestCase{ - {unshardedHaveVSchema: false, markAsGlobal: true}, - {unshardedHaveVSchema: false, markAsGlobal: false}, - {unshardedHaveVSchema: true, markAsGlobal: true}, - {unshardedHaveVSchema: true, markAsGlobal: false}, - } - for _, tc := range testCases { - funcs := exp[tc] - require.NotNil(t, funcs) - testGlobalRouting(t, tc.markAsGlobal, tc.unshardedHaveVSchema, funcs) - } -} - -func getUnshardedVschema(unshardedHaveVSchema bool, tables []string) string { - if !unshardedHaveVSchema { +func (h *grHelpers) getUnshardedVschema(unshardedHasVSchema bool, tables []string) string { + if !unshardedHasVSchema { return "" } vschema := `{"tables": {` @@ -242,7 +233,31 @@ func getUnshardedVschema(unshardedHaveVSchema bool, tables []string) string { return vschema } -func testGlobalRouting(t *testing.T, markAsGlobal, unshardedHaveVSchema bool, funcs *tGlobalRoutingTestExpectationFuncs) { +func (h *grHelpers) rebuildGraphs(t *testing.T) { + err := vc.VtctldClient.ExecuteCommand("RebuildVSchemaGraph") + require.NoError(t, err) + err = vc.VtctldClient.ExecuteCommand("RebuildKeyspaceGraph", grTestConfig.ksU1, grTestConfig.ksU2) + require.NoError(t, err) +} + +func TestGlobalRouting(t *testing.T) { + h := grHelpers{t} + exp := *h.getExpectations() + testCases := []grTestCase{ + {unshardedHasVSchema: false, markAsGlobal: true}, + {unshardedHasVSchema: false, markAsGlobal: false}, + {unshardedHasVSchema: true, markAsGlobal: true}, + {unshardedHasVSchema: true, markAsGlobal: false}, + } + for _, tc := range testCases { + funcs := exp[tc] + require.NotNil(t, funcs) + testGlobalRouting(t, tc.markAsGlobal, tc.unshardedHasVSchema, funcs) + } +} + +func testGlobalRouting(t *testing.T, markAsGlobal, unshardedHasVSchema bool, funcs *grTestExpectations) { + h := grHelpers{t: t} setSidecarDBName("_vt") vttablet.InitVReplicationConfigDefaults() extraVTGateArgs = append(extraVTGateArgs, fmt.Sprintf("--mark_unique_unsharded_tables_as_global=%t", markAsGlobal)) @@ -250,39 +265,43 @@ func testGlobalRouting(t *testing.T, markAsGlobal, unshardedHaveVSchema bool, fu vc = NewVitessCluster(t, nil) defer vc.TearDown() zone1 := vc.Cells["zone1"] - config := globalRoutingTestConfig - vc.AddKeyspace(t, []*Cell{zone1}, config.ksU1, "0", getUnshardedVschema(unshardedHaveVSchema, config.ksU1Tables), - getSchema(config.ksU1Tables), 1, 0, 100, nil) + config := grTestConfig + vc.AddKeyspace(t, []*Cell{zone1}, config.ksU1, "0", h.getUnshardedVschema(unshardedHasVSchema, config.ksU1Tables), + h.getSchema(config.ksU1Tables), 1, 0, 100, nil) verifyClusterHealth(t, vc) for _, table := range config.ksU1Tables { - insertData(t, config.ksU1, table, 1, config.ksU1) + h.insertData(t, config.ksU1, table, 1, config.ksU1) + vtgateConn, cancel := getVTGateConn() + waitForRowCount(t, vtgateConn, config.ksU1+"@replica", table, 1) + cancel() } time.Sleep(5 * time.Second) + funcs.postKsU1(t) - vc.AddKeyspace(t, []*Cell{zone1}, config.ksU2, "0", getUnshardedVschema(unshardedHaveVSchema, config.ksU2Tables), - getSchema(config.ksU2Tables), 1, 0, 200, nil) + vc.AddKeyspace(t, []*Cell{zone1}, config.ksU2, "0", h.getUnshardedVschema(unshardedHasVSchema, config.ksU2Tables), + h.getSchema(config.ksU2Tables), 1, 0, 200, nil) verifyClusterHealth(t, vc) for _, table := range config.ksU2Tables { - insertData(t, config.ksU2, table, 1, config.ksU2) + h.insertData(t, config.ksU2, table, 1, config.ksU2) + vtgateConn, cancel := getVTGateConn() + waitForRowCount(t, vtgateConn, config.ksU2+"@replica", table, 1) + cancel() } - time.Sleep(5 * time.Second) // FIXME: wait for the mysql replication to catch up on the replica - rebuild(t) + time.Sleep(5 * time.Second) + h.rebuildGraphs(t) funcs.postKsU2(t) - vc.AddKeyspace(t, []*Cell{zone1}, config.ksS1, "-80,80-", ksS1VSchema, getSchema(config.ksS1Tables), 1, 0, 300, nil) + vc.AddKeyspace(t, []*Cell{zone1}, config.ksS1, "-80,80-", h.getShardedVSchema(config.ksS1Tables), h.getSchema(config.ksS1Tables), + 1, 0, 300, nil) verifyClusterHealth(t, vc) for _, table := range config.ksS1Tables { - insertData(t, config.ksS1, table, 1, config.ksS1) + h.insertData(t, config.ksS1, table, 1, config.ksS1) + vtgateConn, cancel := getVTGateConn() + waitForRowCount(t, vtgateConn, config.ksS1+"@replica", table, 1) + cancel() } time.Sleep(5 * time.Second) - rebuild(t) + h.rebuildGraphs(t) funcs.postKsS1(t) } - -func rebuild(t *testing.T) { - err := vc.VtctldClient.ExecuteCommand("RebuildVSchemaGraph") - require.NoError(t, err) - err = vc.VtctldClient.ExecuteCommand("RebuildKeyspaceGraph", globalRoutingTestConfig.ksU1, globalRoutingTestConfig.ksU2) - require.NoError(t, err) -} diff --git a/go/vt/vtgate/vindexes/global_routing.md b/go/vt/vtgate/vindexes/global_routing.md deleted file mode 100644 index d6be38c5f16..00000000000 --- a/go/vt/vtgate/vindexes/global_routing.md +++ /dev/null @@ -1,102 +0,0 @@ -# RFC: Enhancements to Global Routing for Unsharded Tables - -## Overview - -This RFC proposes enhancements to the global routing mechanism in Vitess. The goal is to ensure -that unique tables from keyspaces without defined vschemas are globally routable. This document discusses the -current global -routing features, the proposed changes, and provides examples to illustrate the impact of these changes. - -## Motivation - -Vitess has two ways of addressing tables: using qualified names where the keyspace is specified or using unqualified -table names. Example: `keyspace1.table1` vs. `table1`. Tables are currently only globally routable if - -* there is only one keyspace in the cluster, which is unsharded, or -* if there are multiple keyspaces and the unique tables are defined in the `vschema` for all keyspaces from which you - want tables to be globally routable. - -This has a catastrophic consequences of this logic. One example: - -* User has a single unsharded keyspace `unsharded1` and are using unqualified table names, because their app had been - written - using unqualified names while targeting a regular MySQL db. `vtgate` correctly routes this because there is only - one unsharded keyspace: there is no vschema to consult because it was not necessary at this point to create one - for the unsharded keyspace. -* User wants to reshard some large tables. Say, A `MoveTables` workflow is started into a sharded - keyspace, -say, - `sharded1`, which, obviously, has a vschema with all tables defined. Say `table2` is moved in this workflow but - `table1` continues to live in `unsharded1` - So tables with the same name `table2` exist in both the user's db as well as in `sharded1`. Routing rules have - already been setup so that the unqualified tables are routed to `unsharded1`. `sharded1` is still not-serving, so - the tables in `sharded1` are "invisible" to `vtgate`. -* When the `MoveTables`has caught up and the user does a `SwitchWrites` (i.e.`SwitchTraffic` for primary) `sharded1` - is now serving, *but* routing rules are updated to point to `sharded`, so the global routing _just_ works -* The problem comes when user does a `Complete` on the workflow to clean things up. - -A similar problem also holds if the user started with just one unsharded keyspace in Vitess and uses MoveTables to move -some of the tables into a sharded keyspace. - -## Current Global Routing Features - -### Global Routable Tables - -In Vitess, a table is considered globally routable if it meets the following criteria: - -- The table exists in a single unsharded keyspace. -- The table exists in multiple unsharded keyspaces but is identical in schema and vindex configuration. - -### Ambiguous Tables - -A table is considered ambiguous if: - -- The table exists in multiple unsharded keyspaces with different schemas or vindex configurations. -- The table exists in both sharded and unsharded keyspaces. - -### Example - -Consider the following keyspaces and tables: - -- `keyspace1` (unsharded): `table1`, `table2` -- `keyspace2` (unsharded): `table2`, `table3` -- `keyspace3` (sharded): `table4` - -In this scenario: - -- `table1` is globally routable. -- `table2` is ambiguous because it exists in multiple unsharded keyspaces with potentially different configurations. -- `table3` is globally routable. -- `table4` is globally routable because it exists in a sharded keyspace for which vschema is defined. - -## Proposed Changes - -The proposed changes aim to make tables from keyspaces without defined vschemas globally routable. Specifically, the -changes include: - -1. **Automatic Inclusion of Tables from Keyspaces without Vschemas**: Tables from keyspaces that do not have vschemas - defined will be automatically included in the global routing table. -2. **Conflict Resolution**: In case of conflicts (i.e., tables with the same name in multiple keyspaces), the table will - be marked as ambiguous. - -### Example - -Consider the following keyspaces and tables after the proposed changes: - -- `keyspace1` (unsharded, no vschema): `table1`, `table2` -- `keyspace2` (unsharded, no vschema): `table2`, `table3` -- `keyspace3` (sharded): `table4` - -In this scenario: - -- `table1` is globally routable. -- `table2` is ambiguous because it exists in multiple unsharded keyspaces. -- `table3` is globally routable. -- `table4` is not globally routable because it exists in a sharded keyspace. - -## Benefits - -- **Improved Global Routing**: Ensures that tables from keyspaces without vschemas are included in the global routing - table, preventing hard-down situations as detailed above. -- **Simplified Configuration**: Reduces the need for explicit vschema definitions for unsharded keyspaces, simplifying - the configuration process.