diff --git a/go/cmd/mysqlctl/command/init.go b/go/cmd/mysqlctl/command/init.go index 71a9661aa80..14d8e5f6d29 100644 --- a/go/cmd/mysqlctl/command/init.go +++ b/go/cmd/mysqlctl/command/init.go @@ -49,7 +49,7 @@ var initArgs = struct { func commandInit(cmd *cobra.Command, args []string) error { // Generate my.cnf from scratch and use it to find mysqld. - mysqld, cnf, err := mysqlctl.CreateMysqldAndMycnf(tabletUID, mysqlSocket, mysqlPort) + mysqld, cnf, err := mysqlctl.CreateMysqldAndMycnf(tabletUID, mysqlSocket, mysqlPort, collationEnv) if err != nil { return fmt.Errorf("failed to initialize mysql config: %v", err) } diff --git a/go/cmd/mysqlctl/command/init_config.go b/go/cmd/mysqlctl/command/init_config.go index 70e751e02cb..36687482e08 100644 --- a/go/cmd/mysqlctl/command/init_config.go +++ b/go/cmd/mysqlctl/command/init_config.go @@ -40,7 +40,7 @@ var InitConfig = &cobra.Command{ func commandInitConfig(cmd *cobra.Command, args []string) error { // Generate my.cnf from scratch and use it to find mysqld. - mysqld, cnf, err := mysqlctl.CreateMysqldAndMycnf(tabletUID, mysqlSocket, mysqlPort) + mysqld, cnf, err := mysqlctl.CreateMysqldAndMycnf(tabletUID, mysqlSocket, mysqlPort, collationEnv) if err != nil { return fmt.Errorf("failed to initialize mysql config: %v", err) } diff --git a/go/cmd/mysqlctl/command/reinit_config.go b/go/cmd/mysqlctl/command/reinit_config.go index b06642c8203..fd7523c0411 100644 --- a/go/cmd/mysqlctl/command/reinit_config.go +++ b/go/cmd/mysqlctl/command/reinit_config.go @@ -41,7 +41,7 @@ var ReinitConfig = &cobra.Command{ func commandReinitConfig(cmd *cobra.Command, args []string) error { // There ought to be an existing my.cnf, so use it to find mysqld. - mysqld, cnf, err := mysqlctl.OpenMysqldAndMycnf(tabletUID) + mysqld, cnf, err := mysqlctl.OpenMysqldAndMycnf(tabletUID, collationEnv) if err != nil { return fmt.Errorf("failed to find mysql config: %v", err) } diff --git a/go/cmd/mysqlctl/command/root.go b/go/cmd/mysqlctl/command/root.go index 4f5626ef7e6..78b3a623666 100644 --- a/go/cmd/mysqlctl/command/root.go +++ b/go/cmd/mysqlctl/command/root.go @@ -23,21 +23,22 @@ import ( "vitess.io/vitess/go/acl" vtcmd "vitess.io/vitess/go/cmd" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/servenv" ) var ( - mysqlPort = 3306 - tabletUID = uint32(41983) - mysqlSocket string + mysqlPort = 3306 + tabletUID = uint32(41983) + mysqlSocket string + collationEnv *collations.Environment Root = &cobra.Command{ Use: "mysqlctl", Short: "mysqlctl initializes and controls mysqld with Vitess-specific configuration.", Long: "`mysqlctl` is a command-line client used for managing `mysqld` instances.\n\n" + - "It is responsible for bootstrapping tasks such as generating a configuration file for `mysqld` and initializing the instance and its data directory.\n" + "The `mysqld_safe` watchdog is utilized when present.\n" + "This helps ensure that `mysqld` is automatically restarted after failures.", @@ -74,4 +75,6 @@ func init() { Root.PersistentFlags().StringVar(&mysqlSocket, "mysql_socket", mysqlSocket, "Path to the mysqld socket file.") acl.RegisterFlags(Root.PersistentFlags()) + + collationEnv = collations.NewEnvironment(servenv.MySQLServerVersion()) } diff --git a/go/cmd/mysqlctl/command/shutdown.go b/go/cmd/mysqlctl/command/shutdown.go index 6e2e3a74a61..321d4a9b35f 100644 --- a/go/cmd/mysqlctl/command/shutdown.go +++ b/go/cmd/mysqlctl/command/shutdown.go @@ -44,7 +44,7 @@ var shutdownArgs = struct { func commandShutdown(cmd *cobra.Command, args []string) error { // There ought to be an existing my.cnf, so use it to find mysqld. - mysqld, cnf, err := mysqlctl.OpenMysqldAndMycnf(tabletUID) + mysqld, cnf, err := mysqlctl.OpenMysqldAndMycnf(tabletUID, collationEnv) if err != nil { return fmt.Errorf("failed to find mysql config: %v", err) } diff --git a/go/cmd/mysqlctl/command/start.go b/go/cmd/mysqlctl/command/start.go index 397909e0966..aef404d0a8e 100644 --- a/go/cmd/mysqlctl/command/start.go +++ b/go/cmd/mysqlctl/command/start.go @@ -45,7 +45,7 @@ var startArgs = struct { func commandStart(cmd *cobra.Command, args []string) error { // There ought to be an existing my.cnf, so use it to find mysqld. - mysqld, cnf, err := mysqlctl.OpenMysqldAndMycnf(tabletUID) + mysqld, cnf, err := mysqlctl.OpenMysqldAndMycnf(tabletUID, collationEnv) if err != nil { return fmt.Errorf("failed to find mysql config: %v", err) } diff --git a/go/cmd/mysqlctl/command/teardown.go b/go/cmd/mysqlctl/command/teardown.go index 4ad0539bdd1..89d7b3b5f6d 100644 --- a/go/cmd/mysqlctl/command/teardown.go +++ b/go/cmd/mysqlctl/command/teardown.go @@ -47,7 +47,7 @@ var teardownArgs = struct { func commandTeardown(cmd *cobra.Command, args []string) error { // There ought to be an existing my.cnf, so use it to find mysqld. - mysqld, cnf, err := mysqlctl.OpenMysqldAndMycnf(tabletUID) + mysqld, cnf, err := mysqlctl.OpenMysqldAndMycnf(tabletUID, collationEnv) if err != nil { return fmt.Errorf("failed to find mysql config: %v", err) } diff --git a/go/cmd/mysqlctld/cli/mysqlctld.go b/go/cmd/mysqlctld/cli/mysqlctld.go index 51a0c47f56e..7a5ff6a5ce6 100644 --- a/go/cmd/mysqlctld/cli/mysqlctld.go +++ b/go/cmd/mysqlctld/cli/mysqlctld.go @@ -28,6 +28,7 @@ import ( "github.com/spf13/cobra" "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -40,9 +41,10 @@ var ( mysqld *mysqlctl.Mysqld cnf *mysqlctl.Mycnf - mysqlPort = 3306 - tabletUID = uint32(41983) - mysqlSocket string + mysqlPort = 3306 + tabletUID = uint32(41983) + mysqlSocket string + collationEnv *collations.Environment // mysqlctl init flags waitTime = 5 * time.Minute @@ -90,6 +92,8 @@ func init() { Main.Flags().DurationVar(&shutdownWaitTime, "shutdown-wait-time", shutdownWaitTime, "How long to wait for mysqld shutdown") acl.RegisterFlags(Main.Flags()) + + collationEnv = collations.NewEnvironment(servenv.MySQLServerVersion()) } func run(cmd *cobra.Command, args []string) error { @@ -110,7 +114,7 @@ func run(cmd *cobra.Command, args []string) error { log.Infof("mycnf file (%s) doesn't exist, initializing", mycnfFile) var err error - mysqld, cnf, err = mysqlctl.CreateMysqldAndMycnf(tabletUID, mysqlSocket, mysqlPort) + mysqld, cnf, err = mysqlctl.CreateMysqldAndMycnf(tabletUID, mysqlSocket, mysqlPort, collationEnv) if err != nil { cancel() return fmt.Errorf("failed to initialize mysql config: %w", err) @@ -126,7 +130,7 @@ func run(cmd *cobra.Command, args []string) error { log.Infof("mycnf file (%s) already exists, starting without init", mycnfFile) var err error - mysqld, cnf, err = mysqlctl.OpenMysqldAndMycnf(tabletUID) + mysqld, cnf, err = mysqlctl.OpenMysqldAndMycnf(tabletUID, collationEnv) if err != nil { cancel() return fmt.Errorf("failed to find mysql config: %w", err) diff --git a/go/cmd/vtadmin/main.go b/go/cmd/vtadmin/main.go index 210e2edb918..ff76fe2f3b6 100644 --- a/go/cmd/vtadmin/main.go +++ b/go/cmd/vtadmin/main.go @@ -24,6 +24,8 @@ import ( "github.com/spf13/cobra" + _flag "vitess.io/vitess/go/internal/flag" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -35,8 +37,6 @@ import ( vtadminhttp "vitess.io/vitess/go/vt/vtadmin/http" "vitess.io/vitess/go/vt/vtadmin/http/debug" "vitess.io/vitess/go/vt/vtadmin/rbac" - - _flag "vitess.io/vitess/go/internal/flag" ) var ( @@ -138,13 +138,14 @@ func run(cmd *cobra.Command, args []string) { log.Warningf("no cache-refresh-key set; forcing cache refreshes will not be possible") } cache.SetCacheRefreshKey(cacheRefreshKey) + collationEnv := collations.NewEnvironment(servenv.MySQLServerVersion()) s := vtadmin.NewAPI(clusters, vtadmin.Options{ GRPCOpts: opts, HTTPOpts: httpOpts, RBAC: rbacConfig, EnableDynamicClusters: enableDynamicClusters, - }) + }, collationEnv) bootSpan.Finish() if err := s.ListenAndServe(); err != nil { @@ -208,6 +209,8 @@ func main() { rootCmd.Flags().AddGoFlag(flag.Lookup("stderrthreshold")) rootCmd.Flags().AddGoFlag(flag.Lookup("log_dir")) + servenv.RegisterMySQLServerFlags(rootCmd.Flags()) + if err := rootCmd.Execute(); err != nil { log.Fatal(err) } diff --git a/go/cmd/vtbackup/cli/vtbackup.go b/go/cmd/vtbackup/cli/vtbackup.go index d55cf643de4..4700d93eea1 100644 --- a/go/cmd/vtbackup/cli/vtbackup.go +++ b/go/cmd/vtbackup/cli/vtbackup.go @@ -29,11 +29,11 @@ import ( "github.com/spf13/cobra" - "vitess.io/vitess/go/mysql/replication" - "vitess.io/vitess/go/acl" "vitess.io/vitess/go/cmd" "vitess.io/vitess/go/exit" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" @@ -93,6 +93,8 @@ var ( keepAliveTimeout time.Duration disableRedoLog bool + collationEnv *collations.Environment + // Deprecated, use "Phase" instead. deprecatedDurationByPhase = stats.NewGaugesWithSingleLabel( "DurationByPhaseSeconds", @@ -215,6 +217,8 @@ func init() { Main.Flags().BoolVar(&disableRedoLog, "disable-redo-log", disableRedoLog, "Disable InnoDB redo log during replication-from-primary phase of backup.") acl.RegisterFlags(Main.Flags()) + + collationEnv = collations.NewEnvironment(servenv.MySQLServerVersion()) } func run(_ *cobra.Command, args []string) error { @@ -327,7 +331,7 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back }() // Start up mysqld as if we are mysqlctld provisioning a fresh tablet. - mysqld, mycnf, err := mysqlctl.CreateMysqldAndMycnf(tabletAlias.Uid, mysqlSocket, mysqlPort) + mysqld, mycnf, err := mysqlctl.CreateMysqldAndMycnf(tabletAlias.Uid, mysqlSocket, mysqlPort, collationEnv) if err != nil { return fmt.Errorf("failed to initialize mysql config: %v", err) } diff --git a/go/cmd/vtcombo/cli/main.go b/go/cmd/vtcombo/cli/main.go index 6912a886b18..c9fde3d2252 100644 --- a/go/cmd/vtcombo/cli/main.go +++ b/go/cmd/vtcombo/cli/main.go @@ -30,6 +30,8 @@ import ( "github.com/spf13/cobra" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/acl" "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/vt/dbconfigs" @@ -78,6 +80,7 @@ In particular, it contains: tpb vttestpb.VTTestTopology ts *topo.Server + collationEnv *collations.Environment resilientServer *srvtopo.ResilientServer ) @@ -114,6 +117,8 @@ func init() { // We're going to force the value later, so don't even bother letting the // user know about this flag. Main.Flags().MarkHidden("tablet_protocol") + + collationEnv = collations.NewEnvironment(servenv.MySQLServerVersion()) } func startMysqld(uid uint32) (mysqld *mysqlctl.Mysqld, cnf *mysqlctl.Mycnf, err error) { @@ -123,7 +128,7 @@ func startMysqld(uid uint32) (mysqld *mysqlctl.Mysqld, cnf *mysqlctl.Mycnf, err mycnfFile := mysqlctl.MycnfFile(uid) if _, statErr := os.Stat(mycnfFile); os.IsNotExist(statErr) { - mysqld, cnf, err = mysqlctl.CreateMysqldAndMycnf(uid, "", mysqlPort) + mysqld, cnf, err = mysqlctl.CreateMysqldAndMycnf(uid, "", mysqlPort, collationEnv) if err != nil { return nil, nil, fmt.Errorf("failed to initialize mysql config :%w", err) } @@ -131,7 +136,7 @@ func startMysqld(uid uint32) (mysqld *mysqlctl.Mysqld, cnf *mysqlctl.Mycnf, err return nil, nil, fmt.Errorf("failed to initialize mysql :%w", err) } } else { - mysqld, cnf, err = mysqlctl.OpenMysqldAndMycnf(uid) + mysqld, cnf, err = mysqlctl.OpenMysqldAndMycnf(uid, collationEnv) if err != nil { return nil, nil, fmt.Errorf("failed to find mysql config: %w", err) } @@ -205,7 +210,7 @@ func run(cmd *cobra.Command, args []string) (err error) { mysqld.SetReadOnly(false) } else { - dbconfigs.GlobalDBConfigs.InitWithSocket("") + dbconfigs.GlobalDBConfigs.InitWithSocket("", collationEnv) mysqld.Mysqld = mysqlctl.NewMysqld(&dbconfigs.GlobalDBConfigs) servenv.OnClose(mysqld.Close) } @@ -217,7 +222,7 @@ func run(cmd *cobra.Command, args []string) (err error) { // to be the "internal" protocol that InitTabletMap registers. cmd.Flags().Set("tablet_manager_protocol", "internal") cmd.Flags().Set("tablet_protocol", "internal") - uid, err := vtcombo.InitTabletMap(ts, &tpb, mysqld, &dbconfigs.GlobalDBConfigs, schemaDir, startMysql) + uid, err := vtcombo.InitTabletMap(ts, &tpb, mysqld, &dbconfigs.GlobalDBConfigs, schemaDir, startMysql, collationEnv) if err != nil { // ensure we start mysql in the event we fail here if startMysql { @@ -242,8 +247,8 @@ func run(cmd *cobra.Command, args []string) (err error) { } } - wr := wrangler.New(logutil.NewConsoleLogger(), ts, nil) - newUID, err := vtcombo.CreateKs(ctx, ts, &tpb, mysqld, &dbconfigs.GlobalDBConfigs, schemaDir, ks, true, uid, wr) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, nil, collationEnv) + newUID, err := vtcombo.CreateKs(ctx, ts, &tpb, mysqld, &dbconfigs.GlobalDBConfigs, schemaDir, ks, true, uid, wr, collationEnv) if err != nil { return err } @@ -291,11 +296,12 @@ func run(cmd *cobra.Command, args []string) (err error) { vtgate.QueryLogHandler = "/debug/vtgate/querylog" vtgate.QueryLogzHandler = "/debug/vtgate/querylogz" vtgate.QueryzHandler = "/debug/vtgate/queryz" + // pass nil for healthcheck, it will get created - vtg := vtgate.Init(context.Background(), nil, resilientServer, tpb.Cells[0], tabletTypesToWait, plannerVersion) + vtg := vtgate.Init(context.Background(), nil, resilientServer, tpb.Cells[0], tabletTypesToWait, plannerVersion, collationEnv) // vtctld configuration and init - err = vtctld.InitVtctld(ts) + err = vtctld.InitVtctld(ts, collationEnv) if err != nil { return err } diff --git a/go/cmd/vtcombo/cli/plugin_grpcvtctlserver.go b/go/cmd/vtcombo/cli/plugin_grpcvtctlserver.go index 8b7f918bc58..d815805f60a 100644 --- a/go/cmd/vtcombo/cli/plugin_grpcvtctlserver.go +++ b/go/cmd/vtcombo/cli/plugin_grpcvtctlserver.go @@ -24,7 +24,7 @@ import ( func init() { servenv.OnRun(func() { if servenv.GRPCCheckServiceMap("vtctl") { - grpcvtctlserver.StartServer(servenv.GRPCServer, ts) + grpcvtctlserver.StartServer(servenv.GRPCServer, ts, collationEnv) } }) } diff --git a/go/cmd/vtctl/vtctl.go b/go/cmd/vtctl/vtctl.go index e95f484cf4f..b44938eb776 100644 --- a/go/cmd/vtctl/vtctl.go +++ b/go/cmd/vtctl/vtctl.go @@ -31,6 +31,7 @@ import ( "vitess.io/vitess/go/cmd" "vitess.io/vitess/go/cmd/vtctldclient/command" "vitess.io/vitess/go/exit" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -126,7 +127,6 @@ func main() { ts := topo.Open() defer ts.Close() - ctx, cancel := context.WithTimeout(context.Background(), waitTime) installSignalHandlers(cancel) @@ -170,8 +170,8 @@ func main() { fallthrough default: log.Warningf("WARNING: vtctl should only be used for VDiff v1 workflows. Please use VDiff v2 and consider using vtctldclient for all other commands.") - - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + collationEnv := collations.NewEnvironment(servenv.MySQLServerVersion()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collationEnv) if args[0] == "--" { vtctl.PrintDoubleDashDeprecationNotice(wr) diff --git a/go/cmd/vtctld/cli/cli.go b/go/cmd/vtctld/cli/cli.go index f7fef555896..86d6b0adf0d 100644 --- a/go/cmd/vtctld/cli/cli.go +++ b/go/cmd/vtctld/cli/cli.go @@ -20,14 +20,16 @@ import ( "github.com/spf13/cobra" "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vtctld" ) var ( - ts *topo.Server - Main = &cobra.Command{ + ts *topo.Server + collationEnv *collations.Environment + Main = &cobra.Command{ Use: "vtctld", Short: "The Vitess cluster management daemon.", Long: `vtctld provides web and gRPC interfaces to manage a single Vitess cluster. @@ -59,8 +61,9 @@ func run(cmd *cobra.Command, args []string) error { ts = topo.Open() defer ts.Close() + collationEnv = collations.NewEnvironment(servenv.MySQLServerVersion()) // Init the vtctld core - if err := vtctld.InitVtctld(ts); err != nil { + if err := vtctld.InitVtctld(ts, collationEnv); err != nil { return err } diff --git a/go/cmd/vtctld/cli/plugin_grpcvtctlserver.go b/go/cmd/vtctld/cli/plugin_grpcvtctlserver.go index 8b7f918bc58..d815805f60a 100644 --- a/go/cmd/vtctld/cli/plugin_grpcvtctlserver.go +++ b/go/cmd/vtctld/cli/plugin_grpcvtctlserver.go @@ -24,7 +24,7 @@ import ( func init() { servenv.OnRun(func() { if servenv.GRPCCheckServiceMap("vtctl") { - grpcvtctlserver.StartServer(servenv.GRPCServer, ts) + grpcvtctlserver.StartServer(servenv.GRPCServer, ts, collationEnv) } }) } diff --git a/go/cmd/vtctld/cli/schema.go b/go/cmd/vtctld/cli/schema.go index 68dc47b2b6f..31d51d5be9f 100644 --- a/go/cmd/vtctld/cli/schema.go +++ b/go/cmd/vtctld/cli/schema.go @@ -71,7 +71,7 @@ func initSchema() { return } ctx := context.Background() - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collationEnv) _, err = schemamanager.Run( ctx, controller, diff --git a/go/cmd/vtexplain/cli/vtexplain.go b/go/cmd/vtexplain/cli/vtexplain.go index 8b0622cf8a3..fdda3061492 100644 --- a/go/cmd/vtexplain/cli/vtexplain.go +++ b/go/cmd/vtexplain/cli/vtexplain.go @@ -22,6 +22,7 @@ import ( "os" "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vtexplain" @@ -78,9 +79,7 @@ If no keyspace name is present, VTExplain will return the following error: ` + "```\n", Example: "Explain how Vitess will execute the query `SELECT * FROM users` using the VSchema contained in `vschemas.json` and the database schema `schema.sql`:\n\n" + "```\nvtexplain --vschema-file vschema.json --schema-file schema.sql --sql \"SELECT * FROM users\"\n```\n\n" + - "Explain how the example will execute on 128 shards using Row-based replication:\n\n" + - "```\nvtexplain -- -shards 128 --vschema-file vschema.json --schema-file schema.sql --replication-mode \"ROW\" --output-mode text --sql \"INSERT INTO users (user_id, name) VALUES(1, 'john')\"\n```\n", Args: cobra.NoArgs, PreRunE: servenv.CobraPreRunE, @@ -175,7 +174,8 @@ func parseAndRun() error { Target: dbName, } - vte, err := vtexplain.Init(context.Background(), vschema, schema, ksShardMap, opts) + collationEnv := collations.NewEnvironment(servenv.MySQLServerVersion()) + vte, err := vtexplain.Init(context.Background(), vschema, schema, ksShardMap, opts, collationEnv) if err != nil { return err } diff --git a/go/cmd/vtgate/cli/cli.go b/go/cmd/vtgate/cli/cli.go index bcd280890e5..c81570524f5 100644 --- a/go/cmd/vtgate/cli/cli.go +++ b/go/cmd/vtgate/cli/cli.go @@ -23,6 +23,8 @@ import ( "github.com/spf13/cobra" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/acl" "vitess.io/vitess/go/exit" "vitess.io/vitess/go/vt/discovery" @@ -157,9 +159,10 @@ func run(cmd *cobra.Command, args []string) error { } plannerVersion, _ := plancontext.PlannerNameToVersion(plannerName) + collationEnv := collations.NewEnvironment(servenv.MySQLServerVersion()) // pass nil for HealthCheck and it will be created - vtg := vtgate.Init(context.Background(), nil, resilientServer, cell, tabletTypes, plannerVersion) + vtg := vtgate.Init(context.Background(), nil, resilientServer, cell, tabletTypes, plannerVersion, collationEnv) servenv.OnRun(func() { // Flags are parsed now. Parse the template using the actual flag value and overwrite the current template. diff --git a/go/cmd/vttablet/cli/cli.go b/go/cmd/vttablet/cli/cli.go index d68856be9b6..7efa66e58c8 100644 --- a/go/cmd/vttablet/cli/cli.go +++ b/go/cmd/vttablet/cli/cli.go @@ -25,6 +25,8 @@ import ( "github.com/spf13/cobra" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/acl" "vitess.io/vitess/go/vt/binlog" "vitess.io/vitess/go/vt/dbconfigs" @@ -110,14 +112,15 @@ func run(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to parse --tablet-path: %w", err) } + collationEnv := collations.NewEnvironment(servenv.MySQLServerVersion()) // config and mycnf initializations are intertwined. - config, mycnf, err := initConfig(tabletAlias) + config, mycnf, err := initConfig(tabletAlias, collationEnv) if err != nil { return err } ts := topo.Open() - qsc, err := createTabletServer(context.Background(), config, ts, tabletAlias) + qsc, err := createTabletServer(context.Background(), config, ts, tabletAlias, collationEnv) if err != nil { ts.Close() return err @@ -136,7 +139,7 @@ func run(cmd *cobra.Command, args []string) error { if servenv.GRPCPort() != 0 { gRPCPort = int32(servenv.GRPCPort()) } - tablet, err := tabletmanager.BuildTabletFromInput(tabletAlias, int32(servenv.Port()), gRPCPort, config.DB) + tablet, err := tabletmanager.BuildTabletFromInput(tabletAlias, int32(servenv.Port()), gRPCPort, config.DB, collationEnv) if err != nil { return fmt.Errorf("failed to parse --tablet-path: %w", err) } @@ -148,8 +151,9 @@ func run(cmd *cobra.Command, args []string) error { DBConfigs: config.DB.Clone(), QueryServiceControl: qsc, UpdateStream: binlog.NewUpdateStream(ts, tablet.Keyspace, tabletAlias.Cell, qsc.SchemaEngine()), - VREngine: vreplication.NewEngine(config, ts, tabletAlias.Cell, mysqld, qsc.LagThrottler()), - VDiffEngine: vdiff.NewEngine(config, ts, tablet), + VREngine: vreplication.NewEngine(config, ts, tabletAlias.Cell, mysqld, qsc.LagThrottler(), collationEnv), + VDiffEngine: vdiff.NewEngine(config, ts, tablet, collationEnv), + CollationEnv: collationEnv, } if err := tm.Start(tablet, config); err != nil { ts.Close() @@ -169,7 +173,7 @@ func run(cmd *cobra.Command, args []string) error { return nil } -func initConfig(tabletAlias *topodatapb.TabletAlias) (*tabletenv.TabletConfig, *mysqlctl.Mycnf, error) { +func initConfig(tabletAlias *topodatapb.TabletAlias, collationEnv *collations.Environment) (*tabletenv.TabletConfig, *mysqlctl.Mycnf, error) { tabletenv.Init() // Load current config after tabletenv.Init, because it changes it. config := tabletenv.NewCurrentConfig() @@ -211,9 +215,9 @@ func initConfig(tabletAlias *topodatapb.TabletAlias) (*tabletenv.TabletConfig, * // If connection parameters were specified, socketFile will be empty. // Otherwise, the socketFile (read from mycnf) will be used to initialize // dbconfigs. - config.DB.InitWithSocket(socketFile) + config.DB.InitWithSocket(socketFile, collationEnv) for _, cfg := range config.ExternalConnections { - cfg.InitWithSocket("") + cfg.InitWithSocket("", collationEnv) } return config, mycnf, nil } @@ -237,7 +241,7 @@ func extractOnlineDDL() error { return nil } -func createTabletServer(ctx context.Context, config *tabletenv.TabletConfig, ts *topo.Server, tabletAlias *topodatapb.TabletAlias) (*tabletserver.TabletServer, error) { +func createTabletServer(ctx context.Context, config *tabletenv.TabletConfig, ts *topo.Server, tabletAlias *topodatapb.TabletAlias, collationEnv *collations.Environment) (*tabletserver.TabletServer, error) { if tableACLConfig != "" { // To override default simpleacl, other ACL plugins must set themselves to be default ACL factory tableacl.Register("simpleacl", &simpleacl.Factory{}) @@ -246,7 +250,7 @@ func createTabletServer(ctx context.Context, config *tabletenv.TabletConfig, ts } // creates and registers the query service - qsc := tabletserver.NewTabletServer(ctx, "", config, ts, tabletAlias) + qsc := tabletserver.NewTabletServer(ctx, "", config, ts, tabletAlias, collationEnv) servenv.OnRun(func() { qsc.Register() addStatusParts(qsc) diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 82895637c69..a7ad1445c2b 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -94,6 +94,7 @@ Flags: --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) --logtostderr log to standard error instead of files --max-stack-size int configure the maximum stack size in bytes (default 67108864) + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) --opentsdb_uri string URI of opentsdb /api/put method diff --git a/go/mysql/client.go b/go/mysql/client.go index db1fd0cb68f..108ef6f774b 100644 --- a/go/mysql/client.go +++ b/go/mysql/client.go @@ -229,11 +229,6 @@ func (c *Conn) clientHandshake(params *ConnParams) error { c.Capabilities = capabilities & (CapabilityClientDeprecateEOF) } - charset, err := collations.Local().ParseConnectionCharset(params.Charset) - if err != nil { - return err - } - // Handle switch to SSL if necessary. if params.SslEnabled() { // If client asked for SSL, but server doesn't support it, @@ -270,7 +265,7 @@ func (c *Conn) clientHandshake(params *ConnParams) error { } // Send the SSLRequest packet. - if err := c.writeSSLRequest(capabilities, charset, params); err != nil { + if err := c.writeSSLRequest(capabilities, uint8(params.Charset), params); err != nil { return err } @@ -302,7 +297,7 @@ func (c *Conn) clientHandshake(params *ConnParams) error { // Build and send our handshake response 41. // Note this one will never have SSL flag on. - if err := c.writeHandshakeResponse41(capabilities, scrambledPassword, charset, params); err != nil { + if err := c.writeHandshakeResponse41(capabilities, scrambledPassword, uint8(params.Charset), params); err != nil { return err } diff --git a/go/mysql/collations/cached_size.go b/go/mysql/collations/cached_size.go new file mode 100644 index 00000000000..630bf41230a --- /dev/null +++ b/go/mysql/collations/cached_size.go @@ -0,0 +1,111 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by Sizegen. DO NOT EDIT. + +package collations + +import ( + "math" + "reflect" + "unsafe" + + hack "vitess.io/vitess/go/hack" +) + +//go:nocheckptr +func (cached *Environment) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(48) + } + // field byName map[string]vitess.io/vitess/go/mysql/collations.ID + if cached.byName != nil { + size += int64(48) + hmap := reflect.ValueOf(cached.byName) + numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) + numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) + size += hack.RuntimeAllocSize(int64(numOldBuckets * 160)) + if len(cached.byName) > 0 || numBuckets > 1 { + size += hack.RuntimeAllocSize(int64(numBuckets * 160)) + } + for k := range cached.byName { + size += hack.RuntimeAllocSize(int64(len(k))) + } + } + // field byCharset map[string]*vitess.io/vitess/go/mysql/collations.colldefaults + if cached.byCharset != nil { + size += int64(48) + hmap := reflect.ValueOf(cached.byCharset) + numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) + numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) + size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) + if len(cached.byCharset) > 0 || numBuckets > 1 { + size += hack.RuntimeAllocSize(int64(numBuckets * 208)) + } + for k, v := range cached.byCharset { + size += hack.RuntimeAllocSize(int64(len(k))) + if v != nil { + size += hack.RuntimeAllocSize(int64(4)) + } + } + } + // field byCharsetName map[vitess.io/vitess/go/mysql/collations.ID]string + if cached.byCharsetName != nil { + size += int64(48) + hmap := reflect.ValueOf(cached.byCharsetName) + numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) + numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) + size += hack.RuntimeAllocSize(int64(numOldBuckets * 160)) + if len(cached.byCharsetName) > 0 || numBuckets > 1 { + size += hack.RuntimeAllocSize(int64(numBuckets * 160)) + } + for _, v := range cached.byCharsetName { + size += hack.RuntimeAllocSize(int64(len(v))) + } + } + // field unsupported map[string]vitess.io/vitess/go/mysql/collations.ID + if cached.unsupported != nil { + size += int64(48) + hmap := reflect.ValueOf(cached.unsupported) + numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) + numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) + size += hack.RuntimeAllocSize(int64(numOldBuckets * 160)) + if len(cached.unsupported) > 0 || numBuckets > 1 { + size += hack.RuntimeAllocSize(int64(numBuckets * 160)) + } + for k := range cached.unsupported { + size += hack.RuntimeAllocSize(int64(len(k))) + } + } + // field byID map[vitess.io/vitess/go/mysql/collations.ID]string + if cached.byID != nil { + size += int64(48) + hmap := reflect.ValueOf(cached.byID) + numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) + numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) + size += hack.RuntimeAllocSize(int64(numOldBuckets * 160)) + if len(cached.byID) > 0 || numBuckets > 1 { + size += hack.RuntimeAllocSize(int64(numBuckets * 160)) + } + for _, v := range cached.byID { + size += hack.RuntimeAllocSize(int64(len(v))) + } + } + return size +} diff --git a/go/mysql/collations/env.go b/go/mysql/collations/env.go index 91fc2a8bd8c..9fe87230649 100644 --- a/go/mysql/collations/env.go +++ b/go/mysql/collations/env.go @@ -248,10 +248,10 @@ func (env *Environment) CollationAlias(collation string) (string, bool) { // to a Collation ID, with the exception that it can only fit in 1 byte. // For MySQL 8.0+ environments, the default charset is `utf8mb4_0900_ai_ci`. // For older MySQL environments, the default charset is `utf8mb4_general_ci`. -func (env *Environment) DefaultConnectionCharset() uint8 { +func (env *Environment) DefaultConnectionCharset() ID { switch env.version { case collverMySQL8: - return uint8(CollationUtf8mb4ID) + return CollationUtf8mb4ID default: return 45 } @@ -267,7 +267,7 @@ func (env *Environment) DefaultConnectionCharset() uint8 { // handshake. // - empty, in which case the default connection charset for this MySQL version // is returned. -func (env *Environment) ParseConnectionCharset(csname string) (uint8, error) { +func (env *Environment) ParseConnectionCharset(csname string) (ID, error) { if csname == "" { return env.DefaultConnectionCharset(), nil } @@ -282,7 +282,7 @@ func (env *Environment) ParseConnectionCharset(csname string) (uint8, error) { if collid == 0 || collid > 255 { return 0, fmt.Errorf("unsupported connection charset: %q", csname) } - return uint8(collid), nil + return collid, nil } func (env *Environment) AllCollationIDs() []ID { diff --git a/go/mysql/collations/integration/charset_test.go b/go/mysql/collations/integration/charset_test.go index 8a4d12a0e4d..b1b747e768b 100644 --- a/go/mysql/collations/integration/charset_test.go +++ b/go/mysql/collations/integration/charset_test.go @@ -45,7 +45,7 @@ func TestLocalEncodings(t *testing.T) { defer conn.Close() for _, tc := range cases { - local := collations.Local().LookupByName(tc.collation) + local := collations.MySQL8().LookupByName(tc.collation) remote := remote.NewCollation(conn, tc.collation) verifyTranscoding(t, colldata.Lookup(local), remote, tc.input) } diff --git a/go/mysql/collations/integration/coercion_test.go b/go/mysql/collations/integration/coercion_test.go index dad55bcafad..c194b48c071 100644 --- a/go/mysql/collations/integration/coercion_test.go +++ b/go/mysql/collations/integration/coercion_test.go @@ -54,7 +54,7 @@ type testConcat struct { } func (tc *testConcat) Expression() string { - env := collations.Local() + env := collations.MySQL8() return fmt.Sprintf("CONCAT((_%s X'%x' COLLATE %q), (_%s X'%x' COLLATE %q))", colldata.Lookup(tc.left.Collation).Charset().Name(), tc.left.Text, env.LookupName(tc.left.Collation), colldata.Lookup(tc.right.Collation).Charset().Name(), tc.right.Text, env.LookupName(tc.right.Collation), @@ -63,7 +63,7 @@ func (tc *testConcat) Expression() string { func (tc *testConcat) Test(t *testing.T, remote *RemoteCoercionResult, local collations.TypedCollation, coercion1, coercion2 colldata.Coercion) { localCollation := colldata.Lookup(local.Collation) - remoteName := collations.Local().LookupName(remote.Collation) + remoteName := collations.MySQL8().LookupName(remote.Collation) assert.Equal(t, remoteName, localCollation.Name(), "bad collation resolved: local is %s, remote is %s", localCollation.Name(), remoteName) assert.Equal(t, remote.Coercibility, local.Coercibility, "bad coercibility resolved: local is %d, remote is %d", local.Coercibility, remote.Coercibility) @@ -85,8 +85,8 @@ func (tc *testConcat) Test(t *testing.T, remote *RemoteCoercionResult, local col rEBytes, err := remote.Expr.ToBytes() require.NoError(t, err) - assert.True(t, bytes.Equal(concat.Bytes(), rEBytes), "failed to concatenate text;\n\tCONCAT(%v COLLATE %s, %v COLLATE %s) = \n\tCONCAT(%v, %v) COLLATE %s = \n\t\t%v\n\n\texpected: %v", tc.left.Text, collations.Local().LookupName(tc.left.Collation), - tc.right.Text, collations.Local().LookupName(tc.right.Collation), leftText, rightText, localCollation.Name(), + assert.True(t, bytes.Equal(concat.Bytes(), rEBytes), "failed to concatenate text;\n\tCONCAT(%v COLLATE %s, %v COLLATE %s) = \n\tCONCAT(%v, %v) COLLATE %s = \n\t\t%v\n\n\texpected: %v", tc.left.Text, collations.MySQL8().LookupName(tc.left.Collation), + tc.right.Text, collations.MySQL8().LookupName(tc.right.Collation), leftText, rightText, localCollation.Name(), concat.Bytes(), rEBytes) } @@ -96,7 +96,7 @@ type testComparison struct { } func (tc *testComparison) Expression() string { - env := collations.Local() + env := collations.MySQL8() return fmt.Sprintf("(_%s X'%x' COLLATE %q) = (_%s X'%x' COLLATE %q)", env.LookupCharsetName(tc.left.Collation), tc.left.Text, env.LookupName(tc.left.Collation), env.LookupCharsetName(tc.right.Collation), tc.right.Text, env.LookupName(tc.right.Collation), @@ -135,7 +135,7 @@ func TestComparisonSemantics(t *testing.T) { t.Skipf("The behavior of Coercion Semantics is not correct before 8.0.31") } - for _, coll := range colldata.All(collations.Local()) { + for _, coll := range colldata.All(collations.MySQL8()) { text := verifyTranscoding(t, coll, remote.NewCollation(conn, coll.Name()), []byte(BaseString)) testInputs = append(testInputs, &TextWithCollation{Text: text, Collation: coll.ID()}) } @@ -175,7 +175,7 @@ func TestComparisonSemantics(t *testing.T) { Coercibility: 0, Repertoire: collations.RepertoireASCII, } - resultLocal, coercionLocal1, coercionLocal2, errLocal := colldata.Merge(collations.Local(), left, right, + resultLocal, coercionLocal1, coercionLocal2, errLocal := colldata.Merge(collations.MySQL8(), left, right, colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, @@ -194,7 +194,7 @@ func TestComparisonSemantics(t *testing.T) { query := fmt.Sprintf("SELECT CAST((%s) AS BINARY), COLLATION(%s), COERCIBILITY(%s)", expr, expr, expr) resultRemote, errRemote := conn.ExecuteFetch(query, 1, false) - env := collations.Local() + env := collations.MySQL8() if errRemote != nil { require.True(t, strings.Contains(errRemote.Error(), "Illegal mix of collations"), "query %s failed: %v", query, errRemote) @@ -212,7 +212,7 @@ func TestComparisonSemantics(t *testing.T) { continue } - remoteCollation := collations.Local().LookupByName(resultRemote.Rows[0][1].ToString()) + remoteCollation := collations.MySQL8().LookupByName(resultRemote.Rows[0][1].ToString()) remoteCI, _ := resultRemote.Rows[0][2].ToInt64() remoteTest.Test(t, &RemoteCoercionResult{ Expr: resultRemote.Rows[0][0], diff --git a/go/mysql/collations/integration/collations_test.go b/go/mysql/collations/integration/collations_test.go index e5362608e75..8db7140d6c1 100644 --- a/go/mysql/collations/integration/collations_test.go +++ b/go/mysql/collations/integration/collations_test.go @@ -38,7 +38,6 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/remote" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/sqlparser" ) @@ -46,9 +45,7 @@ var collationEnv *collations.Environment func init() { // We require MySQL 8.0 collations for the comparisons in the tests - mySQLVersion := "8.0.0" - servenv.SetMySQLServerVersionForTest(mySQLVersion) - collationEnv = collations.NewEnvironment(mySQLVersion) + collationEnv = collations.NewEnvironment("8.0.30") } func getSQLQueries(t *testing.T, testfile string) []string { diff --git a/go/mysql/collations/integration/helpers_test.go b/go/mysql/collations/integration/helpers_test.go index d436280f04b..a5d2bb0cc36 100644 --- a/go/mysql/collations/integration/helpers_test.go +++ b/go/mysql/collations/integration/helpers_test.go @@ -52,7 +52,7 @@ func testRemoteWeights(t *testing.T, golden io.Writer, cases []testweight) { for _, tc := range cases { t.Run(tc.collation, func(t *testing.T) { - local := collations.Local().LookupByName(tc.collation) + local := collations.MySQL8().LookupByName(tc.collation) remote := remote.NewCollation(conn, tc.collation) localResult := colldata.Lookup(local).WeightString(nil, tc.input, 0) remoteResult := remote.WeightString(nil, tc.input, 0) @@ -85,7 +85,7 @@ func testRemoteComparison(t *testing.T, golden io.Writer, cases []testcmp) { for _, tc := range cases { t.Run(tc.collation, func(t *testing.T) { - local := collations.Local().LookupByName(tc.collation) + local := collations.MySQL8().LookupByName(tc.collation) remote := remote.NewCollation(conn, tc.collation) localResult := normalizecmp(colldata.Lookup(local).Collate(tc.left, tc.right, false)) remoteResult := remote.Collate(tc.left, tc.right, false) diff --git a/go/mysql/collations/integration/weight_string_test.go b/go/mysql/collations/integration/weight_string_test.go index 666856ca38b..ad4ad4270fc 100644 --- a/go/mysql/collations/integration/weight_string_test.go +++ b/go/mysql/collations/integration/weight_string_test.go @@ -60,7 +60,7 @@ func TestWeightStringsComprehensive(t *testing.T) { conn := mysqlconn(t) defer conn.Close() - allCollations := colldata.All(collations.Local()) + allCollations := colldata.All(collations.MySQL8()) sort.Slice(allCollations, func(i, j int) bool { return allCollations[i].ID() < allCollations[j].ID() }) @@ -104,7 +104,7 @@ func TestCJKWeightStrings(t *testing.T) { conn := mysqlconn(t) defer conn.Close() - allCollations := colldata.All(collations.Local()) + allCollations := colldata.All(collations.MySQL8()) testdata, _ := filepath.Glob("../internal/charset/testdata/*.txt") for _, testfile := range testdata { cs := filepath.Base(testfile) diff --git a/go/mysql/collations/integration/wildcard_test.go b/go/mysql/collations/integration/wildcard_test.go index 6475a35dd21..6a0271218dc 100644 --- a/go/mysql/collations/integration/wildcard_test.go +++ b/go/mysql/collations/integration/wildcard_test.go @@ -79,7 +79,7 @@ func TestRemoteWildcardMatches(t *testing.T) { {"Ǎḅeçd", "a%bd"}, } - for _, local := range colldata.All(collations.Local()) { + for _, local := range colldata.All(collations.MySQL8()) { t.Run(local.Name(), func(t *testing.T) { var remote = remote.NewCollation(conn, local.Name()) var err error diff --git a/go/mysql/collations/local.go b/go/mysql/collations/local.go index 3cf81b270c7..090420e07a7 100644 --- a/go/mysql/collations/local.go +++ b/go/mysql/collations/local.go @@ -19,37 +19,14 @@ limitations under the License. package collations import ( - "sync" - - "vitess.io/vitess/go/internal/flag" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/servenv" ) -var defaultEnv *Environment -var defaultEnvInit sync.Once - -// Local is the default collation Environment for Vitess. This depends -// on the value of the `mysql_server_version` flag passed to this Vitess process. -func Local() *Environment { - defaultEnvInit.Do(func() { - if !flag.Parsed() { - panic("collations.Local() called too early") - } - defaultEnv = NewEnvironment(servenv.MySQLServerVersion()) - }) - return defaultEnv -} - -// Default returns the default collation for this Vitess process. -// This is based on the local collation environment, which is based on the user's configured -// MySQL version for this Vitess deployment. -func Default() ID { - return ID(Local().DefaultConnectionCharset()) -} - -func DefaultCollationForType(t sqltypes.Type) ID { - return CollationForType(t, Default()) +// MySQL8 is the collation Environment for MySQL 8. This should +// only be used for testing where we know it's safe to use this +// version, and we don't need a specific other version. +func MySQL8() *Environment { + return fetchCacheEnvironment(collverMySQL8) } func CollationForType(t sqltypes.Type, fallback ID) ID { diff --git a/go/mysql/collations/tools/maketestdata/maketestdata.go b/go/mysql/collations/tools/maketestdata/maketestdata.go index edad1c840a3..7adee5d5dfd 100644 --- a/go/mysql/collations/tools/maketestdata/maketestdata.go +++ b/go/mysql/collations/tools/maketestdata/maketestdata.go @@ -167,7 +167,7 @@ func main() { fs := pflag.NewFlagSet("maketestdata", pflag.ExitOnError) flag.Parse(fs) - var defaults = collations.Local() + var defaults = collations.MySQL8() var collationsForLanguage = make(map[testutil.Lang][]collations.ID) var allcollations = colldata.All(defaults) for lang := range testutil.KnownLanguages { diff --git a/go/mysql/conn.go b/go/mysql/conn.go index 85a8ffd4027..402906b9b75 100644 --- a/go/mysql/conn.go +++ b/go/mysql/conn.go @@ -217,9 +217,6 @@ type Conn struct { closing bool } -// splitStatementFunciton is the function that is used to split the statement in case of a multi-statement query. -var splitStatementFunction = sqlparser.SplitStatementToPieces - // PrepareData is a buffer used for store prepare statement meta data type PrepareData struct { ParamsType []int32 @@ -1235,7 +1232,7 @@ func (c *Conn) handleComPrepare(handler Handler, data []byte) (kontinue bool) { var queries []string if c.Capabilities&CapabilityClientMultiStatements != 0 { var err error - queries, err = splitStatementFunction(query) + queries, err = sqlparser.SplitStatementToPieces(query) if err != nil { log.Errorf("Conn %v: Error splitting query: %v", c, err) return c.writeErrorPacketFromErrorAndLog(err) @@ -1363,7 +1360,7 @@ func (c *Conn) handleComQuery(handler Handler, data []byte) (kontinue bool) { var queries []string var err error if c.Capabilities&CapabilityClientMultiStatements != 0 { - queries, err = splitStatementFunction(query) + queries, err = sqlparser.SplitStatementToPieces(query) if err != nil { log.Errorf("Conn %v: Error splitting query: %v", c, err) return c.writeErrorPacketFromErrorAndLog(err) diff --git a/go/mysql/conn_flaky_test.go b/go/mysql/conn_flaky_test.go index 0057aff5aa6..e829131698b 100644 --- a/go/mysql/conn_flaky_test.go +++ b/go/mysql/conn_flaky_test.go @@ -31,17 +31,13 @@ import ( "testing" "time" - "vitess.io/vitess/go/mysql/replication" - "vitess.io/vitess/go/mysql/sqlerror" - "vitess.io/vitess/go/vt/sqlparser" - "github.com/stretchr/testify/assert" - - "vitess.io/vitess/go/test/utils" - "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/utils" querypb "vitess.io/vitess/go/vt/proto/query" ) @@ -878,14 +874,6 @@ func TestMultiStatement(t *testing.T) { func TestMultiStatementOnSplitError(t *testing.T) { listener, sConn, cConn := createSocketPair(t) - // Set the splitStatementFunction to return an error. - splitStatementFunction = func(blob string) (pieces []string, err error) { - return nil, fmt.Errorf("Error in split statements") - } - defer func() { - // Set the splitStatementFunction to the correct function back - splitStatementFunction = sqlparser.SplitStatementToPieces - }() sConn.Capabilities |= CapabilityClientMultiStatements defer func() { listener.Close() @@ -893,7 +881,7 @@ func TestMultiStatementOnSplitError(t *testing.T) { cConn.Close() }() - err := cConn.WriteComQuery("select 1;select 2") + err := cConn.WriteComQuery("broken>'query 1;parse" } -func (compareGT) compare(left, right eval) (boolean, error) { - cmp, isNull, err := evalCompareAll(left, right, false) +func (compareGT) compare(collationEnv *collations.Environment, left, right eval) (boolean, error) { + cmp, isNull, err := evalCompareAll(left, right, false, collationEnv) return makeboolean2(cmp > 0, isNull), err } func (compareGE) String() string { return ">=" } -func (compareGE) compare(left, right eval) (boolean, error) { - cmp, isNull, err := evalCompareAll(left, right, false) +func (compareGE) compare(collationEnv *collations.Environment, left, right eval) (boolean, error) { + cmp, isNull, err := evalCompareAll(left, right, false, collationEnv) return makeboolean2(cmp >= 0, isNull), err } func (compareNullSafeEQ) String() string { return "<=>" } -func (compareNullSafeEQ) compare(left, right eval) (boolean, error) { - cmp, err := evalCompareNullSafe(left, right) +func (compareNullSafeEQ) compare(collationEnv *collations.Environment, left, right eval) (boolean, error) { + cmp, err := evalCompareNullSafe(left, right, collationEnv) return makeboolean(cmp == 0), err } @@ -164,7 +164,7 @@ func compareAsJSON(l, r sqltypes.Type) bool { return l == sqltypes.TypeJSON || r == sqltypes.TypeJSON } -func evalCompareNullSafe(lVal, rVal eval) (int, error) { +func evalCompareNullSafe(lVal, rVal eval, collationEnv *collations.Environment) (int, error) { if lVal == nil { if rVal == nil { return 0, nil @@ -175,18 +175,18 @@ func evalCompareNullSafe(lVal, rVal eval) (int, error) { return 1, nil } if left, right, ok := compareAsTuples(lVal, rVal); ok { - return evalCompareTuplesNullSafe(left.t, right.t) + return evalCompareTuplesNullSafe(left.t, right.t, collationEnv) } - n, err := evalCompare(lVal, rVal) + n, err := evalCompare(lVal, rVal, collationEnv) return n, err } -func evalCompareMany(left, right []eval, fulleq bool) (int, bool, error) { +func evalCompareMany(left, right []eval, fulleq bool, collationEnv *collations.Environment) (int, bool, error) { // For row comparisons, (a, b) = (x, y) is equivalent to: (a = x) AND (b = y) var seenNull bool for idx, lResult := range left { rResult := right[idx] - n, isNull, err := evalCompareAll(lResult, rResult, fulleq) + n, isNull, err := evalCompareAll(lResult, rResult, fulleq, collationEnv) if err != nil { return 0, false, err } @@ -203,20 +203,20 @@ func evalCompareMany(left, right []eval, fulleq bool) (int, bool, error) { return 0, seenNull, nil } -func evalCompareAll(lVal, rVal eval, fulleq bool) (int, bool, error) { +func evalCompareAll(lVal, rVal eval, fulleq bool, collationEnv *collations.Environment) (int, bool, error) { if lVal == nil || rVal == nil { return 0, true, nil } if left, right, ok := compareAsTuples(lVal, rVal); ok { - return evalCompareMany(left.t, right.t, fulleq) + return evalCompareMany(left.t, right.t, fulleq, collationEnv) } - n, err := evalCompare(lVal, rVal) + n, err := evalCompare(lVal, rVal, collationEnv) return n, false, err } // For more details on comparison expression evaluation and type conversion: // - https://dev.mysql.com/doc/refman/8.0/en/type-conversion.html -func evalCompare(left, right eval) (comp int, err error) { +func evalCompare(left, right eval, collationEnv *collations.Environment) (comp int, err error) { lt := left.SQLType() rt := right.SQLType() @@ -224,7 +224,7 @@ func evalCompare(left, right eval) (comp int, err error) { case compareAsDates(lt, rt): return compareDates(left.(*evalTemporal), right.(*evalTemporal)), nil case compareAsStrings(lt, rt): - return compareStrings(left, right) + return compareStrings(left, right, collationEnv) case compareAsSameNumericType(lt, rt) || compareAsDecimal(lt, rt): return compareNumeric(left, right) case compareAsDateAndString(lt, rt): @@ -269,12 +269,12 @@ func fallbackBinary(t sqltypes.Type) bool { return false } -func evalCompareTuplesNullSafe(left, right []eval) (int, error) { +func evalCompareTuplesNullSafe(left, right []eval, collationEnv *collations.Environment) (int, error) { if len(left) != len(right) { panic("did not typecheck cardinality") } for idx, lResult := range left { - res, err := evalCompareNullSafe(lResult, right[idx]) + res, err := evalCompareNullSafe(lResult, right[idx], collationEnv) if err != nil { return 0, err } @@ -302,7 +302,7 @@ func (c *ComparisonExpr) eval(env *ExpressionEnv) (eval, error) { if _, ok := c.Op.(compareNullSafeEQ); !ok && right == nil { return nil, nil } - cmp, err := c.Op.compare(left, right) + cmp, err := c.Op.compare(env.collationEnv, left, right) if err != nil { return nil, err } @@ -312,25 +312,25 @@ func (c *ComparisonExpr) eval(env *ExpressionEnv) (eval, error) { func (expr *ComparisonExpr) compileAsTuple(c *compiler) (ctype, error) { switch expr.Op.(type) { case compareNullSafeEQ: - c.asm.CmpTupleNullsafe() + c.asm.CmpTupleNullsafe(c.collationEnv) return ctype{Type: sqltypes.Int64, Col: collationNumeric, Flag: flagIsBoolean}, nil case compareEQ: - c.asm.CmpTuple(true) + c.asm.CmpTuple(c.collationEnv, true) c.asm.Cmp_eq_n() case compareNE: - c.asm.CmpTuple(true) + c.asm.CmpTuple(c.collationEnv, true) c.asm.Cmp_ne_n() case compareLT: - c.asm.CmpTuple(false) + c.asm.CmpTuple(c.collationEnv, false) c.asm.Cmp_lt_n() case compareLE: - c.asm.CmpTuple(false) + c.asm.CmpTuple(c.collationEnv, false) c.asm.Cmp_le_n() case compareGT: - c.asm.CmpTuple(false) + c.asm.CmpTuple(c.collationEnv, false) c.asm.Cmp_gt_n() case compareGE: - c.asm.CmpTuple(false) + c.asm.CmpTuple(c.collationEnv, false) c.asm.Cmp_ge_n() default: panic("invalid comparison operator") @@ -455,14 +455,14 @@ func (expr *ComparisonExpr) compile(c *compiler) (ctype, error) { return cmptype, nil } -func evalInExpr(lhs eval, rhs *evalTuple) (boolean, error) { +func evalInExpr(collationEnv *collations.Environment, lhs eval, rhs *evalTuple) (boolean, error) { if lhs == nil { return boolNULL, nil } var foundNull, found bool for _, rtuple := range rhs.t { - numeric, isNull, err := evalCompareAll(lhs, rtuple, true) + numeric, isNull, err := evalCompareAll(lhs, rtuple, true, collationEnv) if err != nil { return boolNULL, err } @@ -496,7 +496,7 @@ func (i *InExpr) eval(env *ExpressionEnv) (eval, error) { if !ok { return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "rhs of an In operation should be a tuple") } - in, err := evalInExpr(left, rtuple) + in, err := evalInExpr(env.collationEnv, left, rtuple) if err != nil { return nil, err } @@ -553,7 +553,7 @@ func (expr *InExpr) compile(c *compiler) (ctype, error) { if err != nil { return ctype{}, err } - c.asm.In_slow(expr.Negate) + c.asm.In_slow(c.collationEnv, expr.Negate) } return ctype{Type: sqltypes.Int64, Col: collationNumeric, Flag: flagIsBoolean | (nullableFlags(lhs.Flag) | (rt.Flag & flagNullable))}, nil @@ -580,7 +580,7 @@ func (l *LikeExpr) eval(env *ExpressionEnv) (eval, error) { } var col collations.TypedCollation - left, right, col, err = mergeAndCoerceCollations(left, right) + left, right, col, err = mergeAndCoerceCollations(left, right, env.collationEnv) if err != nil { return nil, err } @@ -633,10 +633,9 @@ func (expr *LikeExpr) compile(c *compiler) (ctype, error) { var merged collations.TypedCollation var coerceLeft colldata.Coercion var coerceRight colldata.Coercion - var env = collations.Local() if lt.Col.Collation != rt.Col.Collation { - merged, coerceLeft, coerceRight, err = colldata.Merge(env, lt.Col, rt.Col, colldata.CoercionOptions{ + merged, coerceLeft, coerceRight, err = colldata.Merge(c.collationEnv, lt.Col, rt.Col, colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, }) diff --git a/go/vt/vtgate/evalengine/expr_convert.go b/go/vt/vtgate/evalengine/expr_convert.go index 5b2d82b707f..f4dab09dafe 100644 --- a/go/vt/vtgate/evalengine/expr_convert.go +++ b/go/vt/vtgate/evalengine/expr_convert.go @@ -31,11 +31,13 @@ type ( Length, Scale int HasLength, HasScale bool Collation collations.ID + CollationEnv *collations.Environment } ConvertUsingExpr struct { UnaryExpr - Collation collations.ID + Collation collations.ID + CollationEnv *collations.Environment } ) diff --git a/go/vt/vtgate/evalengine/expr_env.go b/go/vt/vtgate/evalengine/expr_env.go index 1c92b0a45ee..b349cd01c04 100644 --- a/go/vt/vtgate/evalengine/expr_env.go +++ b/go/vt/vtgate/evalengine/expr_env.go @@ -21,6 +21,8 @@ import ( "strings" "time" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/config" "vitess.io/vitess/go/mysql/datetime" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/callerid" @@ -31,6 +33,7 @@ type VCursor interface { TimeZone() *time.Location GetKeyspace() string SQLMode() string + CollationEnv() *collations.Environment } type ( @@ -44,10 +47,11 @@ type ( Fields []*querypb.Field // internal state - now time.Time - vc VCursor - user *querypb.VTGateCallerID - sqlmode SQLMode + now time.Time + vc VCursor + user *querypb.VTGateCallerID + sqlmode SQLMode + collationEnv *collations.Environment } ) @@ -88,12 +92,12 @@ func (env *ExpressionEnv) Evaluate(expr Expr) (EvalResult, error) { return env.EvaluateVM(p) } e, err := expr.eval(env) - return EvalResult{e}, err + return EvalResult{v: e, collationEnv: env.collationEnv}, err } func (env *ExpressionEnv) EvaluateAST(expr Expr) (EvalResult, error) { e, err := expr.eval(env) - return EvalResult{e}, err + return EvalResult{v: e, collationEnv: env.collationEnv}, err } func (env *ExpressionEnv) TypeOf(expr Expr) (Type, error) { @@ -113,9 +117,34 @@ func (env *ExpressionEnv) SetTime(now time.Time) { } } +type emptyVCursor struct { + collationEnv *collations.Environment + tz *time.Location +} + +func (e *emptyVCursor) TimeZone() *time.Location { + return e.tz +} + +func (e *emptyVCursor) GetKeyspace() string { + return "" +} + +func (e *emptyVCursor) SQLMode() string { + return config.DefaultSQLMode +} + +func (e *emptyVCursor) CollationEnv() *collations.Environment { + return e.collationEnv +} + +func NewEmptyVCursor(collationEnv *collations.Environment, tz *time.Location) VCursor { + return &emptyVCursor{collationEnv: collationEnv, tz: tz} +} + // EmptyExpressionEnv returns a new ExpressionEnv with no bind vars or row -func EmptyExpressionEnv() *ExpressionEnv { - return NewExpressionEnv(context.Background(), nil, nil) +func EmptyExpressionEnv(collationEnv *collations.Environment) *ExpressionEnv { + return NewExpressionEnv(context.Background(), nil, NewEmptyVCursor(collationEnv, time.Local)) } // NewExpressionEnv returns an expression environment with no current row, but with bindvars @@ -123,9 +152,8 @@ func NewExpressionEnv(ctx context.Context, bindVars map[string]*querypb.BindVari env := &ExpressionEnv{BindVars: bindVars, vc: vc} env.user = callerid.ImmediateCallerIDFromContext(ctx) env.SetTime(time.Now()) - if vc != nil { - env.sqlmode = ParseSQLMode(vc.SQLMode()) - } + env.sqlmode = ParseSQLMode(vc.SQLMode()) + env.collationEnv = vc.CollationEnv() return env } diff --git a/go/vt/vtgate/evalengine/expr_logical.go b/go/vt/vtgate/evalengine/expr_logical.go index 7fe836d7164..c9332e613c3 100644 --- a/go/vt/vtgate/evalengine/expr_logical.go +++ b/go/vt/vtgate/evalengine/expr_logical.go @@ -17,7 +17,6 @@ limitations under the License. package evalengine import ( - "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" ) @@ -586,7 +585,6 @@ func (is *IsExpr) compile(c *compiler) (ctype, error) { func (c *CaseExpr) eval(env *ExpressionEnv) (eval, error) { var ta typeAggregation var ca collationAggregation - var local = collations.Local() var result eval var matched = false @@ -606,7 +604,7 @@ func (c *CaseExpr) eval(env *ExpressionEnv) (eval, error) { return nil, err } ta.addEval(then) - if err := ca.add(local, evalCollation(then)); err != nil { + if err := ca.add(evalCollation(then), env.collationEnv); err != nil { return nil, err } @@ -621,7 +619,7 @@ func (c *CaseExpr) eval(env *ExpressionEnv) (eval, error) { return nil, err } ta.addEval(e) - if err := ca.add(local, evalCollation(e)); err != nil { + if err := ca.add(evalCollation(e), env.collationEnv); err != nil { return nil, err } if !matched { @@ -676,7 +674,6 @@ func (c *CaseExpr) simplify(env *ExpressionEnv) error { func (cs *CaseExpr) compile(c *compiler) (ctype, error) { var ca collationAggregation var ta typeAggregation - var local = collations.Local() for _, wt := range cs.cases { when, err := wt.when.compile(c) @@ -694,7 +691,7 @@ func (cs *CaseExpr) compile(c *compiler) (ctype, error) { } ta.add(then.Type, then.Flag) - if err := ca.add(local, then.Col); err != nil { + if err := ca.add(then.Col, c.collationEnv); err != nil { return ctype{}, err } } @@ -706,7 +703,7 @@ func (cs *CaseExpr) compile(c *compiler) (ctype, error) { } ta.add(els.Type, els.Flag) - if err := ca.add(local, els.Col); err != nil { + if err := ca.add(els.Col, c.collationEnv); err != nil { return ctype{}, err } } diff --git a/go/vt/vtgate/evalengine/fn_compare.go b/go/vt/vtgate/evalengine/fn_compare.go index 835d84c3d39..02c2532fc9a 100644 --- a/go/vt/vtgate/evalengine/fn_compare.go +++ b/go/vt/vtgate/evalengine/fn_compare.go @@ -32,7 +32,7 @@ type ( CallExpr } - multiComparisonFunc func(args []eval, cmp int) (eval, error) + multiComparisonFunc func(collationEnv *collations.Environment, args []eval, cmp int) (eval, error) builtinMultiComparison struct { CallExpr @@ -58,9 +58,8 @@ func (b *builtinCoalesce) eval(env *ExpressionEnv) (eval, error) { func (b *builtinCoalesce) compile(c *compiler) (ctype, error) { var ( - ta typeAggregation - ca collationAggregation - local = collations.Local() + ta typeAggregation + ca collationAggregation ) f := flagNullable @@ -73,7 +72,7 @@ func (b *builtinCoalesce) compile(c *compiler) (ctype, error) { f = 0 } ta.add(tt.Type, tt.Flag) - if err := ca.add(local, tt.Col); err != nil { + if err := ca.add(tt.Col, c.collationEnv); err != nil { return ctype{}, err } } @@ -115,7 +114,7 @@ func getMultiComparisonFunc(args []eval) multiComparisonFunc { for _, arg := range args { if arg == nil { - return func(args []eval, cmp int) (eval, error) { + return func(collationEnv *collations.Environment, args []eval, cmp int) (eval, error) { return nil, nil } } @@ -166,7 +165,7 @@ func getMultiComparisonFunc(args []eval) multiComparisonFunc { panic("unexpected argument type") } -func compareAllInteger_u(args []eval, cmp int) (eval, error) { +func compareAllInteger_u(_ *collations.Environment, args []eval, cmp int) (eval, error) { x := args[0].(*evalUint64) for _, arg := range args[1:] { y := arg.(*evalUint64) @@ -177,7 +176,7 @@ func compareAllInteger_u(args []eval, cmp int) (eval, error) { return x, nil } -func compareAllInteger_i(args []eval, cmp int) (eval, error) { +func compareAllInteger_i(_ *collations.Environment, args []eval, cmp int) (eval, error) { x := args[0].(*evalInt64) for _, arg := range args[1:] { y := arg.(*evalInt64) @@ -188,7 +187,7 @@ func compareAllInteger_i(args []eval, cmp int) (eval, error) { return x, nil } -func compareAllFloat(args []eval, cmp int) (eval, error) { +func compareAllFloat(_ *collations.Environment, args []eval, cmp int) (eval, error) { candidateF, ok := evalToFloat(args[0]) if !ok { return nil, errDecimalOutOfRange @@ -213,7 +212,7 @@ func evalDecimalPrecision(e eval) int32 { return 0 } -func compareAllDecimal(args []eval, cmp int) (eval, error) { +func compareAllDecimal(_ *collations.Environment, args []eval, cmp int) (eval, error) { decExtreme := evalToDecimal(args[0], 0, 0).dec precExtreme := evalDecimalPrecision(args[0]) @@ -230,14 +229,12 @@ func compareAllDecimal(args []eval, cmp int) (eval, error) { return newEvalDecimalWithPrec(decExtreme, precExtreme), nil } -func compareAllText(args []eval, cmp int) (eval, error) { - env := collations.Local() - +func compareAllText(collationEnv *collations.Environment, args []eval, cmp int) (eval, error) { var charsets = make([]charset.Charset, 0, len(args)) var ca collationAggregation for _, arg := range args { col := evalCollation(arg) - if err := ca.add(env, col); err != nil { + if err := ca.add(col, collationEnv); err != nil { return nil, err } charsets = append(charsets, colldata.Lookup(col.Collation).Charset()) @@ -265,7 +262,7 @@ func compareAllText(args []eval, cmp int) (eval, error) { return newEvalText(b1, tc), nil } -func compareAllBinary(args []eval, cmp int) (eval, error) { +func compareAllBinary(_ *collations.Environment, args []eval, cmp int) (eval, error) { candidateB := args[0].ToRawBytes() for _, arg := range args[1:] { @@ -283,17 +280,15 @@ func (call *builtinMultiComparison) eval(env *ExpressionEnv) (eval, error) { if err != nil { return nil, err } - return getMultiComparisonFunc(args)(args, call.cmp) + return getMultiComparisonFunc(args)(env.collationEnv, args, call.cmp) } func (call *builtinMultiComparison) compile_c(c *compiler, args []ctype) (ctype, error) { - env := collations.Local() - var ca collationAggregation var f typeFlag for _, arg := range args { f |= nullableFlags(arg.Flag) - if err := ca.add(env, arg.Col); err != nil { + if err := ca.add(arg.Col, c.collationEnv); err != nil { return ctype{}, err } } diff --git a/go/vt/vtgate/evalengine/fn_regexp.go b/go/vt/vtgate/evalengine/fn_regexp.go index 4897ba63f6a..87923423424 100644 --- a/go/vt/vtgate/evalengine/fn_regexp.go +++ b/go/vt/vtgate/evalengine/fn_regexp.go @@ -91,7 +91,7 @@ func position(val *evalInt64, limit int64, f string) (int64, error) { return pos, nil } -func evalRegexpCollation(input, pat eval, f string) (eval, eval, collations.TypedCollation, icuregex.RegexpFlag, error) { +func evalRegexpCollation(env *collations.Environment, input, pat eval, f string) (eval, eval, collations.TypedCollation, icuregex.RegexpFlag, error) { var typedCol collations.TypedCollation var err error @@ -101,7 +101,6 @@ func evalRegexpCollation(input, pat eval, f string) (eval, eval, collations.Type patCol := patBytes.col.Collation if (inputCol == collations.CollationBinaryID && patCol != collations.CollationBinaryID) || (inputCol != collations.CollationBinaryID && patCol == collations.CollationBinaryID) { - env := collations.Local() inputColName := env.LookupName(inputCol) patColName := env.LookupName(patCol) return nil, nil, typedCol, 0, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.CharacterSetMismatch, "Character set '%s' cannot be used in conjunction with '%s' in call to %s.", inputColName, patColName, f) @@ -109,13 +108,13 @@ func evalRegexpCollation(input, pat eval, f string) (eval, eval, collations.Type } } - input, pat, typedCol, err = mergeAndCoerceCollations(input, pat) + input, pat, typedCol, err = mergeAndCoerceCollations(input, pat, env) if err != nil { return nil, nil, collations.TypedCollation{}, 0, err } var flags icuregex.RegexpFlag - var collation = collations.Local().LookupName(typedCol.Collation) + collation := env.LookupName(typedCol.Collation) if strings.Contains(collation, "_ci") { flags |= icuregex.CaseInsensitive } @@ -123,11 +122,10 @@ func evalRegexpCollation(input, pat eval, f string) (eval, eval, collations.Type return input, pat, typedCol, flags, nil } -func compileRegexpCollation(input, pat ctype, f string) (collations.TypedCollation, icuregex.RegexpFlag, error) { +func compileRegexpCollation(env *collations.Environment, input, pat ctype, f string) (collations.TypedCollation, icuregex.RegexpFlag, error) { var merged collations.TypedCollation var err error - env := collations.Local() if input.isTextual() && pat.isTextual() { inputCol := input.Col.Collation patCol := pat.Col.Collation @@ -140,7 +138,7 @@ func compileRegexpCollation(input, pat ctype, f string) (collations.TypedCollati } if input.Col.Collation != pat.Col.Collation { - merged, _, _, err = mergeCollations(input.Col, pat.Col, input.Type, pat.Type) + merged, _, _, err = mergeCollations(input.Col, pat.Col, input.Type, pat.Type, env) } else { merged = input.Col } @@ -218,7 +216,7 @@ func compileConstantRegex(c *compiler, args TupleExpr, pat, mt int, cs collation return nil, c.unsupported(pattern) } var err error - staticEnv := EmptyExpressionEnv() + staticEnv := EmptyExpressionEnv(c.collationEnv) pattern, err = simplifyExpr(staticEnv, pattern) if err != nil { return nil, err @@ -278,7 +276,7 @@ func (r *builtinRegexpLike) eval(env *ExpressionEnv) (eval, error) { return nil, err } - input, pat, typedCol, flags, err := evalRegexpCollation(input, pat, "regexp_like") + input, pat, typedCol, flags, err := evalRegexpCollation(env.collationEnv, input, pat, "regexp_like") if err != nil { return nil, err } @@ -348,7 +346,7 @@ func (r *builtinRegexpLike) compile(c *compiler) (ctype, error) { skips = append(skips, c.compileNullCheckArg(f, 2)) } - merged, flags, err := compileRegexpCollation(input, pat, "regexp_like") + merged, flags, err := compileRegexpCollation(c.collationEnv, input, pat, "regexp_like") if err != nil { return ctype{}, err } @@ -387,7 +385,7 @@ func (r *builtinRegexpInstr) eval(env *ExpressionEnv) (eval, error) { return nil, err } - input, pat, typedCol, flags, err := evalRegexpCollation(input, pat, "regexp_instr") + input, pat, typedCol, flags, err := evalRegexpCollation(env.collationEnv, input, pat, "regexp_instr") if err != nil { return nil, err } @@ -555,7 +553,7 @@ func (r *builtinRegexpInstr) compile(c *compiler) (ctype, error) { } } - merged, flags, err := compileRegexpCollation(input, pat, "regexp_instr") + merged, flags, err := compileRegexpCollation(c.collationEnv, input, pat, "regexp_instr") if err != nil { return ctype{}, err } @@ -594,7 +592,7 @@ func (r *builtinRegexpSubstr) eval(env *ExpressionEnv) (eval, error) { return nil, err } - input, pat, typedCol, flags, err := evalRegexpCollation(input, pat, "regexp_substr") + input, pat, typedCol, flags, err := evalRegexpCollation(env.collationEnv, input, pat, "regexp_substr") if err != nil { return nil, err } @@ -732,7 +730,7 @@ func (r *builtinRegexpSubstr) compile(c *compiler) (ctype, error) { } } - merged, flags, err := compileRegexpCollation(input, pat, "regexp_substr") + merged, flags, err := compileRegexpCollation(c.collationEnv, input, pat, "regexp_substr") if err != nil { return ctype{}, err } @@ -828,7 +826,7 @@ func (r *builtinRegexpReplace) eval(env *ExpressionEnv) (eval, error) { return nil, err } - input, pat, typedCol, flags, err := evalRegexpCollation(input, pat, "regexp_replace") + input, pat, typedCol, flags, err := evalRegexpCollation(env.collationEnv, input, pat, "regexp_replace") if err != nil { return nil, err } @@ -972,7 +970,7 @@ func (r *builtinRegexpReplace) compile(c *compiler) (ctype, error) { } } - merged, flags, err := compileRegexpCollation(input, pat, "regexp_replace") + merged, flags, err := compileRegexpCollation(c.collationEnv, input, pat, "regexp_replace") if err != nil { return ctype{}, err } diff --git a/go/vt/vtgate/evalengine/fn_string.go b/go/vt/vtgate/evalengine/fn_string.go index 97d461a44d6..fb75642b164 100644 --- a/go/vt/vtgate/evalengine/fn_string.go +++ b/go/vt/vtgate/evalengine/fn_string.go @@ -396,7 +396,7 @@ func (c *builtinCollation) eval(env *ExpressionEnv) (eval, error) { // the collation of a `COLLATION` expr is hardcoded to `utf8mb3_general_ci`, // not to the default collation of our connection. this is probably a bug in MySQL, but we match it - return newEvalText([]byte(collations.Local().LookupName(col.Collation)), collationUtf8mb3), nil + return newEvalText([]byte(env.collationEnv.LookupName(col.Collation)), collationUtf8mb3), nil } func (expr *builtinCollation) compile(c *compiler) (ctype, error) { @@ -407,7 +407,7 @@ func (expr *builtinCollation) compile(c *compiler) (ctype, error) { skip := c.asm.jumpFrom() - c.asm.Fn_COLLATION(collationUtf8mb3) + c.asm.Fn_COLLATION(c.collationEnv, collationUtf8mb3) c.asm.jumpDestination(skip) return ctype{Type: sqltypes.VarChar, Col: collationUtf8mb3}, nil @@ -755,7 +755,7 @@ func (l *builtinStrcmp) eval(env *ExpressionEnv) (eval, error) { col1 := evalCollation(left) col2 := evalCollation(right) - mcol, _, _, err := colldata.Merge(collations.Local(), col1, col2, colldata.CoercionOptions{ + mcol, _, _, err := colldata.Merge(env.collationEnv, col1, col2, colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, }) @@ -795,7 +795,7 @@ func (expr *builtinStrcmp) compile(c *compiler) (ctype, error) { if sqltypes.IsNumber(lt.Type) || sqltypes.IsNumber(rt.Type) { mcol = collationNumeric } else { - mcol, _, _, err = colldata.Merge(collations.Local(), lt.Col, rt.Col, colldata.CoercionOptions{ + mcol, _, _, err = colldata.Merge(c.collationEnv, lt.Col, rt.Col, colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, }) @@ -966,7 +966,6 @@ func concatConvert(buf []byte, str *evalBytes, tc collations.TypedCollation) ([] } func (call *builtinConcat) eval(env *ExpressionEnv) (eval, error) { - local := collations.Local() var ca collationAggregation tt := sqltypes.VarChar @@ -979,7 +978,7 @@ func (call *builtinConcat) eval(env *ExpressionEnv) (eval, error) { args = append(args, a) tt = concatSQLType(a.SQLType(), tt) - err = ca.add(local, evalCollation(a)) + err = ca.add(evalCollation(a), env.collationEnv) if err != nil { return nil, err } @@ -1014,7 +1013,6 @@ func (call *builtinConcat) eval(env *ExpressionEnv) (eval, error) { } func (call *builtinConcat) compile(c *compiler) (ctype, error) { - local := collations.Local() var ca collationAggregation tt := sqltypes.VarChar var f typeFlag @@ -1031,7 +1029,7 @@ func (call *builtinConcat) compile(c *compiler) (ctype, error) { args = append(args, a) tt = concatSQLType(a.Type, tt) - err = ca.add(local, a.Col) + err = ca.add(a.Col, c.collationEnv) if err != nil { return ctype{}, err } @@ -1073,7 +1071,6 @@ type builtinConcatWs struct { } func (call *builtinConcatWs) eval(env *ExpressionEnv) (eval, error) { - local := collations.Local() var ca collationAggregation tt := sqltypes.VarChar @@ -1093,7 +1090,7 @@ func (call *builtinConcatWs) eval(env *ExpressionEnv) (eval, error) { args = append(args, a) tt = concatSQLType(a.SQLType(), tt) - err = ca.add(local, evalCollation(a)) + err = ca.add(evalCollation(a), env.collationEnv) if err != nil { return nil, err } @@ -1143,7 +1140,6 @@ func (call *builtinConcatWs) eval(env *ExpressionEnv) (eval, error) { } func (call *builtinConcatWs) compile(c *compiler) (ctype, error) { - local := collations.Local() var ca collationAggregation tt := sqltypes.VarChar @@ -1156,7 +1152,7 @@ func (call *builtinConcatWs) compile(c *compiler) (ctype, error) { } tt = concatSQLType(a.Type, tt) - err = ca.add(local, a.Col) + err = ca.add(a.Col, c.collationEnv) if err != nil { return ctype{}, err } diff --git a/go/vt/vtgate/evalengine/format.go b/go/vt/vtgate/evalengine/format.go index db473fd418e..ee6b66b192a 100644 --- a/go/vt/vtgate/evalengine/format.go +++ b/go/vt/vtgate/evalengine/format.go @@ -206,12 +206,12 @@ func (tuple TupleExpr) format(buf *sqlparser.TrackedBuffer) { func (c *CollateExpr) format(buf *sqlparser.TrackedBuffer) { formatExpr(buf, c, c.Inner, true) buf.WriteLiteral(" COLLATE ") - buf.WriteString(collations.Local().LookupName(c.TypedCollation.Collation)) + buf.WriteString(c.CollationEnv.LookupName(c.TypedCollation.Collation)) } func (i *IntroducerExpr) format(buf *sqlparser.TrackedBuffer) { buf.WriteString("_") - buf.WriteString(collations.Local().LookupName(i.TypedCollation.Collation)) + buf.WriteString(i.CollationEnv.LookupName(i.TypedCollation.Collation)) formatExpr(buf, i, i.Inner, true) } @@ -294,7 +294,7 @@ func (c *ConvertExpr) format(buf *sqlparser.TrackedBuffer) { } if c.Collation != collations.Unknown { buf.WriteLiteral(" character set ") - buf.WriteString(collations.Local().LookupName(c.Collation)) + buf.WriteString(c.CollationEnv.LookupName(c.Collation)) } buf.WriteByte(')') } @@ -303,7 +303,7 @@ func (c *ConvertUsingExpr) format(buf *sqlparser.TrackedBuffer) { buf.WriteLiteral("convert(") formatExpr(buf, c, c.Inner, true) buf.WriteLiteral(" using ") - buf.WriteString(collations.Local().LookupName(c.Collation)) + buf.WriteString(c.CollationEnv.LookupName(c.Collation)) buf.WriteByte(')') } diff --git a/go/vt/vtgate/evalengine/integration/comparison_test.go b/go/vt/vtgate/evalengine/integration/comparison_test.go index 44c02b2a5a5..bda73a6d5f4 100644 --- a/go/vt/vtgate/evalengine/integration/comparison_test.go +++ b/go/vt/vtgate/evalengine/integration/comparison_test.go @@ -44,8 +44,6 @@ import ( ) var ( - collationEnv *collations.Environment - debugGolden = false debugNormalize = true debugSimplify = time.Now().UnixNano()&1 != 0 @@ -82,7 +80,7 @@ func normalizeValue(v sqltypes.Value, coll collations.ID) sqltypes.Value { return v } -func compareRemoteExprEnv(t *testing.T, env *evalengine.ExpressionEnv, conn *mysql.Conn, expr string, fields []*querypb.Field, cmp *testcases.Comparison) { +func compareRemoteExprEnv(t *testing.T, collationEnv *collations.Environment, env *evalengine.ExpressionEnv, conn *mysql.Conn, expr string, fields []*querypb.Field, cmp *testcases.Comparison) { t.Helper() localQuery := "SELECT " + expr @@ -145,7 +143,7 @@ func compareRemoteExprEnv(t *testing.T, env *evalengine.ExpressionEnv, conn *mys var localVal, remoteVal sqltypes.Value var localCollation, remoteCollation collations.ID if localErr == nil { - v := local.Value(collations.Default()) + v := local.Value(collations.MySQL8().DefaultConnectionCharset()) if debugCheckCollations { if v.IsNull() { localCollation = collations.CollationBinaryID @@ -220,6 +218,10 @@ func (vc *vcursor) SQLMode() string { return config.DefaultSQLMode } +func (vc *vcursor) CollationEnv() *collations.Environment { + return collations.MySQL8() +} + func initTimezoneData(t *testing.T, conn *mysql.Conn) { // We load the timezone information into MySQL. The evalengine assumes // our backend MySQL is configured with the timezone information as well @@ -253,7 +255,7 @@ func TestMySQL(t *testing.T) { // We require MySQL 8.0 collations for the comparisons in the tests servenv.SetMySQLServerVersionForTest(conn.ServerVersion) - collationEnv = collations.NewEnvironment(conn.ServerVersion) + collationEnv := collations.NewEnvironment(conn.ServerVersion) servenv.OnParse(registerFlags) initTimezoneData(t, conn) @@ -265,7 +267,7 @@ func TestMySQL(t *testing.T) { env := evalengine.NewExpressionEnv(ctx, nil, &vcursor{}) tc.Run(func(query string, row []sqltypes.Value) { env.Row = row - compareRemoteExprEnv(t, env, conn, query, tc.Schema, tc.Compare) + compareRemoteExprEnv(t, collationEnv, env, conn, query, tc.Schema, tc.Compare) }) }) } diff --git a/go/vt/vtgate/evalengine/integration/fuzz_test.go b/go/vt/vtgate/evalengine/integration/fuzz_test.go index 657fbcb7c68..9eecd89f11a 100644 --- a/go/vt/vtgate/evalengine/integration/fuzz_test.go +++ b/go/vt/vtgate/evalengine/integration/fuzz_test.go @@ -19,7 +19,6 @@ limitations under the License. package integration import ( - "context" "encoding/json" "fmt" "math/rand" @@ -147,6 +146,7 @@ func evaluateLocalEvalengine(env *evalengine.ExpressionEnv, query string, fields cfg := &evalengine.Config{ ResolveColumn: evalengine.FieldResolver(fields).Column, Collation: collations.CollationUtf8mb4ID, + CollationEnv: collations.MySQL8(), NoConstantFolding: !debugSimplify, } expr, err = evalengine.Translate(astExpr, cfg) @@ -200,7 +200,7 @@ func TestGenerateFuzzCases(t *testing.T) { compareWithMySQL := func(expr sqlparser.Expr) *mismatch { query := "SELECT " + sqlparser.String(expr) - env := evalengine.NewExpressionEnv(context.Background(), nil, nil) + env := evalengine.EmptyExpressionEnv(collations.MySQL8()) eval, localErr := evaluateLocalEvalengine(env, query, nil) remote, remoteErr := conn.ExecuteFetch(query, 1, false) @@ -218,7 +218,7 @@ func TestGenerateFuzzCases(t *testing.T) { remoteErr: remoteErr, } if localErr == nil { - res.localVal = eval.Value(collations.Default()) + res.localVal = eval.Value(collations.MySQL8().DefaultConnectionCharset()) } if remoteErr == nil { res.remoteVal = remote.Rows[0][0] @@ -333,7 +333,7 @@ func compareResult(local, remote Result, cmp *testcases.Comparison) error { var localCollationName string var remoteCollationName string - env := collations.Local() + env := collations.MySQL8() if coll := local.Collation; coll != collations.Unknown { localCollationName = env.LookupName(coll) } diff --git a/go/vt/vtgate/evalengine/mysql_test.go b/go/vt/vtgate/evalengine/mysql_test.go index bfa503d82dd..08bdac28f1e 100644 --- a/go/vt/vtgate/evalengine/mysql_test.go +++ b/go/vt/vtgate/evalengine/mysql_test.go @@ -17,7 +17,6 @@ limitations under the License. package evalengine import ( - "context" "encoding/json" "errors" "os" @@ -70,6 +69,7 @@ func convert(t *testing.T, query string, simplify bool) (Expr, error) { cfg := &Config{ Collation: collations.CollationUtf8mb4ID, + CollationEnv: collations.MySQL8(), NoConstantFolding: !simplify, } @@ -89,7 +89,7 @@ func testSingle(t *testing.T, query string) (EvalResult, error) { if err != nil { return EvalResult{}, err } - return NewExpressionEnv(context.Background(), nil, nil).Evaluate(converted) + return EmptyExpressionEnv(collations.MySQL8()).Evaluate(converted) } func TestMySQLGolden(t *testing.T) { @@ -141,11 +141,11 @@ func TestMySQLGolden(t *testing.T) { continue } if tc.Error != "" { - t.Errorf("query %d: %s\nmysql err: %s\nvitess val: %s", testcount, tc.Query, tc.Error, eval.Value(collations.Default())) + t.Errorf("query %d: %s\nmysql err: %s\nvitess val: %s", testcount, tc.Query, tc.Error, eval.Value(collations.MySQL8().DefaultConnectionCharset())) continue } if eval.String() != tc.Value { - t.Errorf("query %d: %s\nmysql val: %s\nvitess val: %s", testcount, tc.Query, tc.Value, eval.Value(collations.Default())) + t.Errorf("query %d: %s\nmysql val: %s\nvitess val: %s", testcount, tc.Query, tc.Value, eval.Value(collations.MySQL8().DefaultConnectionCharset())) continue } ok++ @@ -159,5 +159,5 @@ func TestMySQLGolden(t *testing.T) { func TestDebug1(t *testing.T) { // Debug eval, err := testSingle(t, `SELECT _latin1 0xFF regexp _latin1 '[[:lower:]]' COLLATE latin1_bin`) - t.Logf("eval=%s err=%v coll=%s", eval.String(), err, collations.Local().LookupName(eval.Collation())) + t.Logf("eval=%s err=%v coll=%s", eval.String(), err, collations.MySQL8().LookupName(eval.Collation())) } diff --git a/go/vt/vtgate/evalengine/perf_test.go b/go/vt/vtgate/evalengine/perf_test.go index 10974cd313d..0f14397d5de 100644 --- a/go/vt/vtgate/evalengine/perf_test.go +++ b/go/vt/vtgate/evalengine/perf_test.go @@ -33,6 +33,7 @@ func BenchmarkCompilerExpressions(b *testing.B) { ResolveColumn: fields.Column, ResolveType: fields.Type, Collation: collations.CollationUtf8mb4ID, + CollationEnv: collations.MySQL8(), } translated, err := evalengine.Translate(expr, cfg) diff --git a/go/vt/vtgate/evalengine/translate.go b/go/vt/vtgate/evalengine/translate.go index ea38f116de2..917fa5e5199 100644 --- a/go/vt/vtgate/evalengine/translate.go +++ b/go/vt/vtgate/evalengine/translate.go @@ -334,7 +334,7 @@ func (ast *astCompiler) translateCollateExpr(collate *sqlparser.CollateExpr) (IR if err != nil { return nil, err } - coll := collations.Local().LookupByName(collate.Collation) + coll := ast.cfg.CollationEnv.LookupByName(collate.Collation) if coll == collations.Unknown { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Unknown collation: '%s'", collate.Collation) } @@ -345,6 +345,7 @@ func (ast *astCompiler) translateCollateExpr(collate *sqlparser.CollateExpr) (IR Coercibility: collations.CoerceExplicit, Repertoire: collations.RepertoireUnicode, }, + CollationEnv: ast.cfg.CollationEnv, }, nil } @@ -358,7 +359,7 @@ func (ast *astCompiler) translateIntroducerExpr(introduced *sqlparser.Introducer if strings.ToLower(introduced.CharacterSet) == "_binary" { collation = collations.CollationBinaryID } else { - defaultCollation := collations.Local().DefaultCollationForCharset(introduced.CharacterSet[1:]) + defaultCollation := ast.cfg.CollationEnv.DefaultCollationForCharset(introduced.CharacterSet[1:]) if defaultCollation == collations.Unknown { panic(fmt.Sprintf("unknown character set: %s", introduced.CharacterSet)) } @@ -389,6 +390,7 @@ func (ast *astCompiler) translateIntroducerExpr(introduced *sqlparser.Introducer Coercibility: collations.CoerceExplicit, Repertoire: collations.RepertoireUnicode, }, + CollationEnv: ast.cfg.CollationEnv, }, nil default: panic("character set introducers are only supported for literals and arguments") @@ -420,7 +422,7 @@ func (ast *astCompiler) translateUnaryExpr(unary *sqlparser.UnaryExpr) (IR, erro case sqlparser.TildaOp: return &BitwiseNotExpr{UnaryExpr: UnaryExpr{expr}}, nil case sqlparser.NStringOp: - return &ConvertExpr{UnaryExpr: UnaryExpr{expr}, Type: "NCHAR", Collation: collations.CollationUtf8mb3ID}, nil + return &ConvertExpr{UnaryExpr: UnaryExpr{expr}, Type: "NCHAR", Collation: collations.CollationUtf8mb3ID, CollationEnv: ast.cfg.CollationEnv}, nil default: return nil, translateExprNotSupported(unary) } @@ -570,16 +572,10 @@ type Config struct { NoConstantFolding bool NoCompilation bool SQLMode SQLMode + CollationEnv *collations.Environment } func Translate(e sqlparser.Expr, cfg *Config) (Expr, error) { - if cfg == nil { - cfg = &Config{} - } - if cfg.Collation == collations.Unknown { - cfg.Collation = collations.Default() - } - ast := astCompiler{cfg: cfg} expr, err := ast.translateExpr(e) @@ -592,7 +588,7 @@ func Translate(e sqlparser.Expr, cfg *Config) (Expr, error) { } if !cfg.NoConstantFolding { - staticEnv := EmptyExpressionEnv() + staticEnv := EmptyExpressionEnv(cfg.CollationEnv) expr, err = simplifyExpr(staticEnv, expr) if err != nil { return nil, err @@ -604,14 +600,15 @@ func Translate(e sqlparser.Expr, cfg *Config) (Expr, error) { } if len(ast.untyped) == 0 && !cfg.NoCompilation { - comp := compiler{collation: cfg.Collation, sqlmode: cfg.SQLMode} + comp := compiler{collation: cfg.Collation, collationEnv: cfg.CollationEnv, sqlmode: cfg.SQLMode} return comp.compile(expr) } return &UntypedExpr{ - ir: expr, - collation: cfg.Collation, - needTypes: ast.untyped, + ir: expr, + collation: cfg.Collation, + collationEnv: cfg.CollationEnv, + needTypes: ast.untyped, }, nil } @@ -627,9 +624,14 @@ type typedExpr struct { err error } -func (typed *typedExpr) compile(expr IR, collation collations.ID, sqlmode SQLMode) (*CompiledExpr, error) { +func (typed *typedExpr) compile(expr IR, collation collations.ID, collationEnv *collations.Environment, sqlmode SQLMode) (*CompiledExpr, error) { typed.once.Do(func() { - comp := compiler{collation: collation, dynamicTypes: typed.types, sqlmode: sqlmode} + comp := compiler{ + collation: collation, + collationEnv: collationEnv, + dynamicTypes: typed.types, + sqlmode: sqlmode, + } typed.compiled, typed.err = comp.compile(expr) }) return typed.compiled, typed.err @@ -646,7 +648,8 @@ type UntypedExpr struct { // ir is the translated IR for the expression ir IR // collation is the default collation for the translated expression - collation collations.ID + collation collations.ID + collationEnv *collations.Environment // needTypes are the IR nodes in ir that could not be typed ahead of time: these must // necessarily be either Column or BindVariable nodes, as all other nodes can always // be statically typed. The dynamicTypeOffset field on each node is the offset of @@ -696,7 +699,7 @@ func (u *UntypedExpr) Compile(env *ExpressionEnv) (*CompiledExpr, error) { if err != nil { return nil, err } - return typed.compile(u.ir, u.collation, env.sqlmode) + return typed.compile(u.ir, u.collation, u.collationEnv, env.sqlmode) } func (u *UntypedExpr) typeof(env *ExpressionEnv) (ctype, error) { diff --git a/go/vt/vtgate/evalengine/translate_convert.go b/go/vt/vtgate/evalengine/translate_convert.go index 29216716b2b..133315e69af 100644 --- a/go/vt/vtgate/evalengine/translate_convert.go +++ b/go/vt/vtgate/evalengine/translate_convert.go @@ -32,7 +32,7 @@ func (ast *astCompiler) binaryCollationForCollation(collation collations.ID) col if binary == nil { return collations.Unknown } - return collations.Local().BinaryCollationForCharset(binary.Charset().Name()) + return ast.cfg.CollationEnv.BinaryCollationForCharset(binary.Charset().Name()) } func (ast *astCompiler) translateConvertCharset(charset string, binary bool) (collations.ID, error) { @@ -47,7 +47,7 @@ func (ast *astCompiler) translateConvertCharset(charset string, binary bool) (co return collation, nil } charset = strings.ToLower(charset) - collationID := collations.Local().DefaultCollationForCharset(charset) + collationID := ast.cfg.CollationEnv.DefaultCollationForCharset(charset) if collationID == collations.Unknown { return collations.Unknown, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Unknown character set: '%s'", charset) } @@ -66,6 +66,7 @@ func (ast *astCompiler) translateConvertExpr(expr sqlparser.Expr, convertType *s err error ) + convert.CollationEnv = ast.cfg.CollationEnv convert.Inner, err = ast.translateExpr(expr) if err != nil { return nil, err @@ -123,6 +124,7 @@ func (ast *astCompiler) translateConvertUsingExpr(expr *sqlparser.ConvertUsingEx err error ) + using.CollationEnv = ast.cfg.CollationEnv using.Inner, err = ast.translateExpr(expr.Expr) if err != nil { return nil, err diff --git a/go/vt/vtgate/evalengine/translate_test.go b/go/vt/vtgate/evalengine/translate_test.go index 377f34db8f2..b721bf3597f 100644 --- a/go/vt/vtgate/evalengine/translate_test.go +++ b/go/vt/vtgate/evalengine/translate_test.go @@ -20,6 +20,7 @@ import ( "context" "strings" "testing" + "time" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" @@ -125,7 +126,8 @@ func TestTranslateSimplification(t *testing.T) { cfg := &Config{ ResolveColumn: fields.Column, - Collation: collations.Default(), + Collation: collations.MySQL8().DefaultConnectionCharset(), + CollationEnv: collations.MySQL8(), NoConstantFolding: true, NoCompilation: true, } @@ -301,7 +303,10 @@ func TestEvaluate(t *testing.T) { stmt, err := sqlparser.Parse("select " + test.expression) require.NoError(t, err) astExpr := stmt.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr - sqltypesExpr, err := Translate(astExpr, &Config{Collation: collations.Default()}) + sqltypesExpr, err := Translate(astExpr, &Config{ + Collation: collations.MySQL8().DefaultConnectionCharset(), + CollationEnv: collations.MySQL8(), + }) require.Nil(t, err) require.NotNil(t, sqltypesExpr) env := NewExpressionEnv(context.Background(), map[string]*querypb.BindVariable{ @@ -311,14 +316,14 @@ func TestEvaluate(t *testing.T) { "uint32_bind_variable": sqltypes.Uint32BindVariable(21), "uint64_bind_variable": sqltypes.Uint64BindVariable(22), "float_bind_variable": sqltypes.Float64BindVariable(2.2), - }, nil) + }, NewEmptyVCursor(collations.MySQL8(), time.Local)) // When r, err := env.Evaluate(sqltypesExpr) // Then require.NoError(t, err) - assert.Equal(t, test.expected, r.Value(collations.Default()), "expected %s", test.expected.String()) + assert.Equal(t, test.expected, r.Value(collations.MySQL8().DefaultConnectionCharset()), "expected %s", test.expected.String()) }) } } @@ -346,12 +351,16 @@ func TestEvaluateTuple(t *testing.T) { stmt, err := sqlparser.Parse("select " + test.expression) require.NoError(t, err) astExpr := stmt.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr - sqltypesExpr, err := Translate(astExpr, &Config{Collation: collations.Default()}) + collationEnv := collations.MySQL8() + sqltypesExpr, err := Translate(astExpr, &Config{ + Collation: collationEnv.DefaultConnectionCharset(), + CollationEnv: collationEnv, + }) require.Nil(t, err) require.NotNil(t, sqltypesExpr) // When - r, err := EmptyExpressionEnv().Evaluate(sqltypesExpr) + r, err := EmptyExpressionEnv(collationEnv).Evaluate(sqltypesExpr) // Then require.NoError(t, err) @@ -383,7 +392,10 @@ func TestTranslationFailures(t *testing.T) { stmt, err := sqlparser.Parse("select " + testcase.expression) require.NoError(t, err) astExpr := stmt.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr - _, err = Translate(astExpr, &Config{Collation: collations.Default()}) + _, err = Translate(astExpr, &Config{ + Collation: collations.MySQL8().DefaultConnectionCharset(), + CollationEnv: collations.MySQL8(), + }) require.EqualError(t, err, testcase.expectedErr) }) } @@ -419,7 +431,11 @@ func TestCardinalityWithBindVariables(t *testing.T) { } astExpr := stmt.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr - _, err = Translate(astExpr, &Config{Collation: collations.Default(), NoCompilation: true}) + _, err = Translate(astExpr, &Config{ + Collation: collations.MySQL8().DefaultConnectionCharset(), + CollationEnv: collations.MySQL8(), + NoCompilation: true, + }) return err }() diff --git a/go/vt/vtgate/evalengine/vm.go b/go/vt/vtgate/evalengine/vm.go index df1d6aa6405..28c3af70e0e 100644 --- a/go/vt/vtgate/evalengine/vm.go +++ b/go/vt/vtgate/evalengine/vm.go @@ -87,12 +87,12 @@ func (env *ExpressionEnv) EvaluateVM(p *CompiledExpr) (EvalResult, error) { goto err } } - return EvalResult{env.vm.stack[env.vm.sp-1]}, nil + return EvalResult{v: env.vm.stack[env.vm.sp-1], collationEnv: env.collationEnv}, nil err: if env.vm.err == errDeoptimize { e, err := p.ir.eval(env) - return EvalResult{e}, err + return EvalResult{v: e, collationEnv: env.collationEnv}, err } - return EvalResult{}, env.vm.err + return EvalResult{collationEnv: env.collationEnv}, env.vm.err } diff --git a/go/vt/vtgate/evalengine/weights_test.go b/go/vt/vtgate/evalengine/weights_test.go index 7e43315f7df..9a34e6e9e81 100644 --- a/go/vt/vtgate/evalengine/weights_test.go +++ b/go/vt/vtgate/evalengine/weights_test.go @@ -77,7 +77,7 @@ func TestTinyWeightStrings(t *testing.T) { return cmp } - cmp, err := NullsafeCompare(a, b, tc.col) + cmp, err := NullsafeCompare(a, b, collations.MySQL8(), tc.col) require.NoError(t, err) fullComparisons++ @@ -88,7 +88,7 @@ func TestTinyWeightStrings(t *testing.T) { a := items[i] b := items[i+1] - cmp, err := NullsafeCompare(a, b, tc.col) + cmp, err := NullsafeCompare(a, b, collations.MySQL8(), tc.col) require.NoError(t, err) if cmp > 0 { @@ -161,7 +161,7 @@ func TestWeightStrings(t *testing.T) { v2, err := valueToEvalCast(b.value, typ, tc.col, 0) require.NoError(t, err) - cmp, err := evalCompareNullSafe(v1, v2) + cmp, err := evalCompareNullSafe(v1, v2, collations.MySQL8()) require.NoError(t, err) if cmp > 0 { diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index 40c50420586..b3a64911d56 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -122,6 +122,8 @@ type Executor struct { warmingReadsPercent int warmingReadsChannel chan bool + + collationEnv *collations.Environment } var executorOnce sync.Once @@ -152,6 +154,7 @@ func NewExecutor( noScatter bool, pv plancontext.PlannerVersion, warmingReadsPercent int, + collationEnv *collations.Environment, ) *Executor { e := &Executor{ serv: serv, @@ -168,6 +171,7 @@ func NewExecutor( plans: plans, warmingReadsPercent: warmingReadsPercent, warmingReadsChannel: make(chan bool, warmingReadsConcurrency), + collationEnv: collationEnv, } vschemaacl.Init() @@ -504,11 +508,15 @@ func (e *Executor) addNeededBindVars(vcursor *vcursorImpl, bindVarNeeds *sqlpars return err } - evalExpr, err := evalengine.Translate(expr, nil) + evalExpr, err := evalengine.Translate(expr, &evalengine.Config{ + Collation: vcursor.collation, + CollationEnv: vcursor.collationEnv, + SQLMode: evalengine.ParseSQLMode(vcursor.SQLMode()), + }) if err != nil { return err } - evaluated, err := evalengine.EmptyExpressionEnv().Evaluate(evalExpr) + evaluated, err := evalengine.NewExpressionEnv(context.Background(), nil, vcursor).Evaluate(evalExpr) if err != nil { return err } @@ -1335,7 +1343,7 @@ func (e *Executor) prepare(ctx context.Context, safeSession *SafeSession, sql st func (e *Executor) handlePrepare(ctx context.Context, safeSession *SafeSession, sql string, bindVars map[string]*querypb.BindVariable, logStats *logstats.LogStats) ([]*querypb.Field, error) { query, comments := sqlparser.SplitMarginComments(sql) - vcursor, _ := newVCursorImpl(safeSession, comments, e, logStats, e.vm, e.VSchema(), e.resolver.resolver, e.serv, e.warnShardedOnly, e.pv) + vcursor, _ := newVCursorImpl(safeSession, comments, e, logStats, e.vm, e.VSchema(), e.resolver.resolver, e.serv, e.warnShardedOnly, e.pv, e.collationEnv) stmt, reservedVars, err := parseAndValidateQuery(query) if err != nil { diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index 8baffdfde09..ab149fdb26d 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -27,6 +27,8 @@ import ( "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sidecardb" "vitess.io/vitess/go/vt/vtgate/logstats" @@ -184,7 +186,7 @@ func createExecutorEnvCallback(t testing.TB, eachShard func(shard, ks string, ta // one-off queries from thrashing the cache. Disable the doorkeeper in the tests to prevent flakiness. plans := theine.NewStore[PlanCacheKey, *engine.Plan](queryPlanCacheMemory, false) - executor = NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0) + executor = NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0, collations.MySQL8()) executor.SetQueryLogger(queryLogger) key.AnyShardPicker = DestinationAnyShardPickerFirstShard{} @@ -231,7 +233,7 @@ func createCustomExecutor(t testing.TB, vschema string) (executor *Executor, sbc queryLogger := streamlog.New[*logstats.LogStats]("VTGate", queryLogBufferSize) plans := DefaultPlanCache() - executor = NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0) + executor = NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0, collations.MySQL8()) executor.SetQueryLogger(queryLogger) t.Cleanup(func() { @@ -268,7 +270,7 @@ func createCustomExecutorSetValues(t testing.TB, vschema string, values []*sqlty sbclookup = hc.AddTestTablet(cell, "0", 1, KsTestUnsharded, "0", topodatapb.TabletType_PRIMARY, true, 1, nil) queryLogger := streamlog.New[*logstats.LogStats]("VTGate", queryLogBufferSize) plans := DefaultPlanCache() - executor = NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0) + executor = NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0, collations.MySQL8()) executor.SetQueryLogger(queryLogger) t.Cleanup(func() { @@ -293,7 +295,7 @@ func createExecutorEnvWithPrimaryReplicaConn(t testing.TB, ctx context.Context, replica = hc.AddTestTablet(cell, "0-replica", 1, KsTestUnsharded, "0", topodatapb.TabletType_REPLICA, true, 1, nil) queryLogger := streamlog.New[*logstats.LogStats]("VTGate", queryLogBufferSize) - executor = NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, DefaultPlanCache(), nil, false, querypb.ExecuteOptions_Gen4, warmingReadsPercent) + executor = NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, DefaultPlanCache(), nil, false, querypb.ExecuteOptions_Gen4, warmingReadsPercent, collations.MySQL8()) executor.SetQueryLogger(queryLogger) t.Cleanup(func() { diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index 09467e85407..f83b3965494 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -170,8 +170,8 @@ func TestSystemVariablesMySQLBelow80(t *testing.T) { sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar(""), @@ -208,8 +208,8 @@ func TestSystemVariablesWithSetVarDisabled(t *testing.T) { sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar(""), @@ -250,8 +250,8 @@ func TestSetSystemVariablesTx(t *testing.T) { sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar(""), @@ -293,8 +293,8 @@ func TestSetSystemVariables(t *testing.T) { lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar(""), @@ -327,7 +327,7 @@ func TestSetSystemVariables(t *testing.T) { lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "sql_safe_updates", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "sql_safe_updates", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("0"), @@ -350,7 +350,7 @@ func TestSetSystemVariables(t *testing.T) { lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "max_tmp_tables", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "max_tmp_tables", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("4"), @@ -373,7 +373,7 @@ func TestSetSystemVariables(t *testing.T) { lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "max_tmp_tables", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "max_tmp_tables", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("1"), @@ -402,8 +402,8 @@ func TestSetSystemVariablesWithReservedConnection(t *testing.T) { sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("only_full_group_by"), @@ -614,7 +614,7 @@ func TestStreamBuffering(t *testing.T) { sbclookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "col", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "col", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), @@ -646,7 +646,7 @@ func TestStreamBuffering(t *testing.T) { wantResults := []*sqltypes.Result{{ Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "col", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "col", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, }, { Rows: [][]sqltypes.Value{{ @@ -690,7 +690,7 @@ func TestStreamLimitOffset(t *testing.T) { conn.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, {Name: "weight_string(id)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, Rows: returnRows[shard], @@ -719,7 +719,7 @@ func TestStreamLimitOffset(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ @@ -792,14 +792,14 @@ func TestSelectSystemVariables(t *testing.T) { {Name: "@@skip_query_plan_cache", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, {Name: "@@enable_system_settings", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, {Name: "@@sql_select_limit", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "@@transaction_mode", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "@@workload", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "@@read_after_write_gtid", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "@@transaction_mode", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "@@workload", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "@@read_after_write_gtid", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, {Name: "@@read_after_write_timeout", Type: sqltypes.Float64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "@@session_track_gtids", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "@@ddl_strategy", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "@@migration_context", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, - {Name: "@@socket", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "@@session_track_gtids", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "@@ddl_strategy", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "@@migration_context", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, + {Name: "@@socket", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, {Name: "@@query_timeout", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, Rows: [][]sqltypes.Value{{ @@ -884,7 +884,7 @@ func TestSelectUserDefinedVariable(t *testing.T) { require.NoError(t, err) wantResult = &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "@foo", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "@foo", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("bar"), @@ -1077,7 +1077,7 @@ func TestSelectDatabase(t *testing.T) { map[string]*querypb.BindVariable{}) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "database()", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "database()", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("TestExecutor@primary"), @@ -1563,7 +1563,7 @@ func TestStreamSelectIN(t *testing.T) { func createExecutor(ctx context.Context, serv *sandboxTopo, cell string, resolver *Resolver) *Executor { queryLogger := streamlog.New[*logstats.LogStats]("VTGate", queryLogBufferSize) plans := DefaultPlanCache() - ex := NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0) + ex := NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0, collations.MySQL8()) ex.SetQueryLogger(queryLogger) return ex } @@ -1920,7 +1920,7 @@ func TestSelectScatterOrderByVarChar(t *testing.T) { sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -1954,7 +1954,7 @@ func TestSelectScatterOrderByVarChar(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, InsertID: 0, } @@ -2052,7 +2052,7 @@ func TestStreamSelectScatterOrderByVarChar(t *testing.T) { sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2080,7 +2080,7 @@ func TestStreamSelectScatterOrderByVarChar(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, } for i := 0; i < 4; i++ { @@ -3189,7 +3189,7 @@ func TestStreamOrderByLimitWithMultipleResults(t *testing.T) { } queryLogger := streamlog.New[*logstats.LogStats]("VTGate", queryLogBufferSize) plans := DefaultPlanCache() - executor := NewExecutor(ctx, serv, cell, resolver, true, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0) + executor := NewExecutor(ctx, serv, cell, resolver, true, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0, collations.MySQL8()) executor.SetQueryLogger(queryLogger) defer executor.Close() // some sleep for all goroutines to start diff --git a/go/vt/vtgate/executor_stream_test.go b/go/vt/vtgate/executor_stream_test.go index 5ef00fd0691..aa55ddcdb31 100644 --- a/go/vt/vtgate/executor_stream_test.go +++ b/go/vt/vtgate/executor_stream_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/streamlog" "vitess.io/vitess/go/test/utils" querypb "vitess.io/vitess/go/vt/proto/query" @@ -68,7 +69,7 @@ func TestStreamSQLSharded(t *testing.T) { queryLogger := streamlog.New[*logstats.LogStats]("VTGate", queryLogBufferSize) plans := DefaultPlanCache() - executor := NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0) + executor := NewExecutor(ctx, serv, cell, resolver, false, false, testBufferSize, plans, nil, false, querypb.ExecuteOptions_Gen4, 0, collations.MySQL8()) executor.SetQueryLogger(queryLogger) defer executor.Close() diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 38746dea779..7b3f159a5e7 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -667,7 +667,7 @@ func TestExecutorShow(t *testing.T) { append(buildVarCharRow( "utf8mb4", "UTF-8 Unicode", - collations.Local().LookupName(collations.Default())), + collations.MySQL8().LookupName(collations.MySQL8().DefaultConnectionCharset())), sqltypes.NewUint32(4)), }, } @@ -712,7 +712,7 @@ func TestExecutorShow(t *testing.T) { append(buildVarCharRow( "utf8mb4", "UTF-8 Unicode", - collations.Local().LookupName(collations.Default())), + collations.MySQL8().LookupName(collations.MySQL8().DefaultConnectionCharset())), sqltypes.NewUint32(4)), }, } @@ -763,7 +763,7 @@ func TestExecutorShow(t *testing.T) { wantqr = &sqltypes.Result{ Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, - {Name: "value", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "value", Type: sqltypes.VarChar, Charset: uint32(collations.MySQL8().DefaultConnectionCharset())}, }, Rows: [][]sqltypes.Value{ {sqltypes.NewInt32(1), sqltypes.NewVarChar("foo")}, @@ -1627,8 +1627,8 @@ var pv = querypb.ExecuteOptions_Gen4 func TestGetPlanUnnormalized(t *testing.T) { r, _, _, _, ctx := createExecutorEnv(t) - emptyvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) - unshardedvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + emptyvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) + unshardedvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) query1 := "select * from music_user_map where id = 1" plan1, logStats1 := getPlanCached(t, ctx, r, emptyvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, false) @@ -1711,7 +1711,7 @@ func TestGetPlanCacheUnnormalized(t *testing.T) { t.Run("Cache", func(t *testing.T) { r, _, _, _, ctx := createExecutorEnv(t) - emptyvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + emptyvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) query1 := "select * from music_user_map where id = 1" _, logStats1 := getPlanCached(t, ctx, r, emptyvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, true) @@ -1735,7 +1735,7 @@ func TestGetPlanCacheUnnormalized(t *testing.T) { // Skip cache using directive r, _, _, _, ctx := createExecutorEnv(t) - unshardedvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + unshardedvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) query1 := "insert /*vt+ SKIP_QUERY_PLAN_CACHE=1 */ into user(id) values (1), (2)" getPlanCached(t, ctx, r, unshardedvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, false) @@ -1746,12 +1746,12 @@ func TestGetPlanCacheUnnormalized(t *testing.T) { assertCacheSize(t, r.plans, 1) // the target string will be resolved and become part of the plan cache key, which adds a new entry - ksIDVc1, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "[deadbeef]"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + ksIDVc1, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "[deadbeef]"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) getPlanCached(t, ctx, r, ksIDVc1, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, false) assertCacheSize(t, r.plans, 2) // the target string will be resolved and become part of the plan cache key, as it's an unsharded ks, it will be the same entry as above - ksIDVc2, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "[beefdead]"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + ksIDVc2, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "[beefdead]"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) getPlanCached(t, ctx, r, ksIDVc2, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, false) assertCacheSize(t, r.plans, 2) }) @@ -1761,7 +1761,7 @@ func TestGetPlanCacheNormalized(t *testing.T) { t.Run("Cache", func(t *testing.T) { r, _, _, _, ctx := createExecutorEnv(t) r.normalize = true - emptyvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + emptyvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) query1 := "select * from music_user_map where id = 1" _, logStats1 := getPlanCached(t, ctx, r, emptyvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, true /* skipQueryPlanCache */) @@ -1778,7 +1778,7 @@ func TestGetPlanCacheNormalized(t *testing.T) { // Skip cache using directive r, _, _, _, ctx := createExecutorEnv(t) r.normalize = true - unshardedvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + unshardedvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) query1 := "insert /*vt+ SKIP_QUERY_PLAN_CACHE=1 */ into user(id) values (1), (2)" getPlanCached(t, ctx, r, unshardedvc, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, false) @@ -1789,12 +1789,12 @@ func TestGetPlanCacheNormalized(t *testing.T) { assertCacheSize(t, r.plans, 1) // the target string will be resolved and become part of the plan cache key, which adds a new entry - ksIDVc1, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "[deadbeef]"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + ksIDVc1, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "[deadbeef]"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) getPlanCached(t, ctx, r, ksIDVc1, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, false) assertCacheSize(t, r.plans, 2) // the target string will be resolved and become part of the plan cache key, as it's an unsharded ks, it will be the same entry as above - ksIDVc2, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "[beefdead]"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + ksIDVc2, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "[beefdead]"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) getPlanCached(t, ctx, r, ksIDVc2, query1, makeComments(" /* comment */"), map[string]*querypb.BindVariable{}, false) assertCacheSize(t, r.plans, 2) }) @@ -1804,8 +1804,8 @@ func TestGetPlanNormalized(t *testing.T) { r, _, _, _, ctx := createExecutorEnv(t) r.normalize = true - emptyvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) - unshardedvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + emptyvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) + unshardedvc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: KsTestUnsharded + "@unknown"}), makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) query1 := "select * from music_user_map where id = 1" query2 := "select * from music_user_map where id = 2" @@ -1862,7 +1862,7 @@ func TestGetPlanPriority(t *testing.T) { r.normalize = true logStats := logstats.NewLogStats(ctx, "Test", "", "", nil) - vCursor, err := newVCursorImpl(session, makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv) + vCursor, err := newVCursorImpl(session, makeComments(""), r, nil, r.vm, r.VSchema(), r.resolver.resolver, nil, false, pv, collations.MySQL8()) assert.NoError(t, err) stmt, err := sqlparser.Parse(testCase.sql) diff --git a/go/vt/vtgate/plan_execute.go b/go/vt/vtgate/plan_execute.go index 5d2414ac275..65a29f62e12 100644 --- a/go/vt/vtgate/plan_execute.go +++ b/go/vt/vtgate/plan_execute.go @@ -97,7 +97,7 @@ func (e *Executor) newExecute( } } - vcursor, err := newVCursorImpl(safeSession, comments, e, logStats, e.vm, vs, e.resolver.resolver, e.serv, e.warnShardedOnly, e.pv) + vcursor, err := newVCursorImpl(safeSession, comments, e, logStats, e.vm, vs, e.resolver.resolver, e.serv, e.warnShardedOnly, e.pv, e.collationEnv) if err != nil { return err } diff --git a/go/vt/vtgate/planbuilder/collations_test.go b/go/vt/vtgate/planbuilder/collations_test.go index 01b86125921..7325d189e43 100644 --- a/go/vt/vtgate/planbuilder/collations_test.go +++ b/go/vt/vtgate/planbuilder/collations_test.go @@ -67,7 +67,7 @@ func (tc *collationTestCase) addCollationsToSchema(vschema *vschemawrapper.VSche func TestOrderedAggregateCollations(t *testing.T) { collid := func(collname string) collations.ID { - return collations.Local().LookupByName(collname) + return collations.MySQL8().LookupByName(collname) } testCases := []collationTestCase{ { diff --git a/go/vt/vtgate/planbuilder/ddl.go b/go/vt/vtgate/planbuilder/ddl.go index 41e5d64346e..e0555fbf6b5 100644 --- a/go/vt/vtgate/planbuilder/ddl.go +++ b/go/vt/vtgate/planbuilder/ddl.go @@ -147,6 +147,7 @@ func buildDDLPlans(ctx context.Context, sql string, ddlStatement sqlparser.DDLSt TargetDestination: destination, DDL: ddlStatement, SQL: query, + CollationEnv: vschema.CollationEnv(), }, nil } diff --git a/go/vt/vtgate/planbuilder/expression_converter.go b/go/vt/vtgate/planbuilder/expression_converter.go index 865c515ecbd..d0f6e017409 100644 --- a/go/vt/vtgate/planbuilder/expression_converter.go +++ b/go/vt/vtgate/planbuilder/expression_converter.go @@ -30,6 +30,8 @@ import ( type expressionConverter struct { tabletExpressions []sqlparser.Expr + collationEnv *collations.Environment + collation collations.ID } func booleanValues(astExpr sqlparser.Expr) evalengine.Expr { @@ -81,7 +83,10 @@ func (ec *expressionConverter) convert(astExpr sqlparser.Expr, boolean, identifi return evalExpr, nil } } - evalExpr, err := evalengine.Translate(astExpr, nil) + evalExpr, err := evalengine.Translate(astExpr, &evalengine.Config{ + Collation: ec.collation, + CollationEnv: ec.collationEnv, + }) if err != nil { if !strings.Contains(err.Error(), evalengine.ErrTranslateExprNotSupported) { return nil, err diff --git a/go/vt/vtgate/planbuilder/expression_converter_test.go b/go/vt/vtgate/planbuilder/expression_converter_test.go index 3c0c25b6003..5cd38685ac2 100644 --- a/go/vt/vtgate/planbuilder/expression_converter_test.go +++ b/go/vt/vtgate/planbuilder/expression_converter_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/evalengine" ) @@ -49,7 +50,10 @@ func TestConversion(t *testing.T) { require.NoError(t, err) slct := statement.(*sqlparser.Select) exprs := extract(slct.SelectExprs) - ec := &expressionConverter{} + ec := &expressionConverter{ + collationEnv: collations.MySQL8(), + collation: collations.MySQL8().DefaultConnectionCharset(), + } var result []evalengine.Expr for _, expr := range exprs { evalExpr, err := ec.convert(expr, false, false) diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 1adcc61972a..abbda050b0c 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/slice" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" @@ -254,13 +255,14 @@ func transformAggregator(ctx *plancontext.PlanningContext, op *operators.Aggrega oa := &orderedAggregate{ resultsBuilder: newResultsBuilder(plan, nil), + collationEnv: ctx.VSchema.CollationEnv(), } for _, aggr := range op.Aggregations { if aggr.OpCode == opcode.AggregateUnassigned { return nil, vterrors.VT12001(fmt.Sprintf("in scatter query: aggregation function '%s'", sqlparser.String(aggr.Original))) } - aggrParam := engine.NewAggregateParam(aggr.OpCode, aggr.ColOffset, aggr.Alias) + aggrParam := engine.NewAggregateParam(aggr.OpCode, aggr.ColOffset, aggr.Alias, ctx.VSchema.CollationEnv()) aggrParam.Expr = aggr.Func aggrParam.Original = aggr.Original aggrParam.OrigOpcode = aggr.OriginalOpCode @@ -275,6 +277,7 @@ func transformAggregator(ctx *plancontext.PlanningContext, op *operators.Aggrega WeightStringCol: groupBy.WSOffset, Expr: groupBy.SimplifiedExpr, Type: typ, + CollationEnv: ctx.VSchema.CollationEnv(), }) } @@ -315,6 +318,7 @@ func createMemorySort(ctx *plancontext.PlanningContext, src logicalPlan, orderin WeightStringCol: ordering.WOffset[idx], Desc: order.Inner.Direction == sqlparser.DescOrder, Type: typ, + CollationEnv: ctx.VSchema.CollationEnv(), }) } @@ -542,6 +546,7 @@ func buildRouteLogicalPlan(ctx *plancontext.PlanningContext, op *operators.Route WeightStringCol: order.WOffset, Desc: order.Direction == sqlparser.DescOrder, Type: typ, + CollationEnv: ctx.VSchema.CollationEnv(), }) } if err != nil { @@ -805,19 +810,23 @@ func transformLimit(ctx *plancontext.PlanningContext, op *operators.Limit) (logi return nil, err } - return createLimit(plan, op.AST) + return createLimit(plan, op.AST, ctx.VSchema.CollationEnv()) } -func createLimit(input logicalPlan, limit *sqlparser.Limit) (logicalPlan, error) { +func createLimit(input logicalPlan, limit *sqlparser.Limit, collationEnv *collations.Environment) (logicalPlan, error) { plan := newLimit(input) - pv, err := evalengine.Translate(limit.Rowcount, nil) + cfg := &evalengine.Config{ + Collation: collationEnv.DefaultConnectionCharset(), + CollationEnv: collationEnv, + } + pv, err := evalengine.Translate(limit.Rowcount, cfg) if err != nil { return nil, vterrors.Wrap(err, "unexpected expression in LIMIT") } plan.elimit.Count = pv if limit.Offset != nil { - pv, err = evalengine.Translate(limit.Offset, nil) + pv, err = evalengine.Translate(limit.Offset, cfg) if err != nil { return nil, vterrors.Wrap(err, "unexpected expression in OFFSET") } @@ -862,7 +871,7 @@ func transformHashJoin(ctx *plancontext.PlanningContext, op *operators.HashJoin) fmt.Sprintf("missing type information for [%s]", strings.Join(missingTypes, ", "))) } - comparisonType, err := evalengine.CoerceTypes(ltyp, rtyp) + comparisonType, err := evalengine.CoerceTypes(ltyp, rtyp, ctx.VSchema.CollationEnv()) if err != nil { return nil, err } @@ -878,6 +887,7 @@ func transformHashJoin(ctx *plancontext.PlanningContext, op *operators.HashJoin) ASTPred: op.JoinPredicate(), Collation: comparisonType.Collation(), ComparisonType: comparisonType.Type(), + CollationEnv: ctx.VSchema.CollationEnv(), }, }, nil } diff --git a/go/vt/vtgate/planbuilder/operators/distinct.go b/go/vt/vtgate/planbuilder/operators/distinct.go index 1750846a961..655bf2350cc 100644 --- a/go/vt/vtgate/planbuilder/operators/distinct.go +++ b/go/vt/vtgate/planbuilder/operators/distinct.go @@ -62,9 +62,10 @@ func (d *Distinct) planOffsets(ctx *plancontext.PlanningContext) Operator { } d.Columns = append(d.Columns, engine.CheckCol{ - Col: idx, - WsCol: wsCol, - Type: typ, + Col: idx, + WsCol: wsCol, + Type: typ, + CollationEnv: ctx.VSchema.CollationEnv(), }) } return nil diff --git a/go/vt/vtgate/planbuilder/operators/dml_planning.go b/go/vt/vtgate/planbuilder/operators/dml_planning.go index 3140142858c..b2e59f7b6be 100644 --- a/go/vt/vtgate/planbuilder/operators/dml_planning.go +++ b/go/vt/vtgate/planbuilder/operators/dml_planning.go @@ -80,8 +80,9 @@ func buildChangedVindexesValues( } found = true pv, err := evalengine.Translate(assignment.Expr.EvalExpr, &evalengine.Config{ - ResolveType: ctx.SemTable.TypeForExpr, - Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + Collation: ctx.SemTable.Collation, + CollationEnv: ctx.VSchema.CollationEnv(), }) if err != nil { panic(invalidUpdateExpr(assignment.Name.Name.String(), assignment.Expr.EvalExpr)) diff --git a/go/vt/vtgate/planbuilder/operators/filter.go b/go/vt/vtgate/planbuilder/operators/filter.go index f2171c43a1b..5c4c33f4575 100644 --- a/go/vt/vtgate/planbuilder/operators/filter.go +++ b/go/vt/vtgate/planbuilder/operators/filter.go @@ -119,8 +119,9 @@ func (f *Filter) Compact(*plancontext.PlanningContext) (Operator, *ApplyResult) func (f *Filter) planOffsets(ctx *plancontext.PlanningContext) Operator { cfg := &evalengine.Config{ - ResolveType: ctx.SemTable.TypeForExpr, - Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + Collation: ctx.SemTable.Collation, + CollationEnv: ctx.VSchema.CollationEnv(), } predicate := sqlparser.AndExpressions(f.Predicates...) diff --git a/go/vt/vtgate/planbuilder/operators/hash_join.go b/go/vt/vtgate/planbuilder/operators/hash_join.go index fed0633fbe3..56fe3f61fc8 100644 --- a/go/vt/vtgate/planbuilder/operators/hash_join.go +++ b/go/vt/vtgate/planbuilder/operators/hash_join.go @@ -332,8 +332,9 @@ func (hj *HashJoin) addColumn(ctx *plancontext.PlanningContext, in sqlparser.Exp rewrittenExpr := sqlparser.CopyOnRewrite(in, pre, r.post, ctx.SemTable.CopySemanticInfo).(sqlparser.Expr) cfg := &evalengine.Config{ - ResolveType: ctx.SemTable.TypeForExpr, - Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + Collation: ctx.SemTable.Collation, + CollationEnv: ctx.VSchema.CollationEnv(), } eexpr, err := evalengine.Translate(rewrittenExpr, cfg) if err != nil { diff --git a/go/vt/vtgate/planbuilder/operators/hash_join_test.go b/go/vt/vtgate/planbuilder/operators/hash_join_test.go index 6abe43c05df..038c2d65da0 100644 --- a/go/vt/vtgate/planbuilder/operators/hash_join_test.go +++ b/go/vt/vtgate/planbuilder/operators/hash_join_test.go @@ -22,9 +22,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/test/vschemawrapper" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/vt/vtgate/semantics" + "vitess.io/vitess/go/vt/vtgate/vindexes" ) func TestJoinPredicates(t *testing.T) { @@ -59,7 +61,13 @@ func TestJoinPredicates(t *testing.T) { func TestOffsetPlanning(t *testing.T) { lcol1, lcol2 := sqlparser.NewColName("lhs1"), sqlparser.NewColName("lhs2") rcol1, rcol2 := sqlparser.NewColName("rhs1"), sqlparser.NewColName("rhs2") - ctx := &plancontext.PlanningContext{SemTable: semantics.EmptySemTable()} + ctx := &plancontext.PlanningContext{ + SemTable: semantics.EmptySemTable(), + VSchema: &vschemawrapper.VSchemaWrapper{ + V: &vindexes.VSchema{}, + SysVarEnabled: true, + }, + } lid := semantics.SingleTableSet(0) rid := semantics.SingleTableSet(1) ctx.SemTable.Recursive[lcol1] = lid diff --git a/go/vt/vtgate/planbuilder/operators/info_schema_planning.go b/go/vt/vtgate/planbuilder/operators/info_schema_planning.go index f7de09c4857..fd4f51d2ca4 100644 --- a/go/vt/vtgate/planbuilder/operators/info_schema_planning.go +++ b/go/vt/vtgate/planbuilder/operators/info_schema_planning.go @@ -41,12 +41,13 @@ type InfoSchemaRouting struct { Table *QueryTable } -func (isr *InfoSchemaRouting) UpdateRoutingParams(_ *plancontext.PlanningContext, rp *engine.RoutingParameters) { +func (isr *InfoSchemaRouting) UpdateRoutingParams(ctx *plancontext.PlanningContext, rp *engine.RoutingParameters) { rp.SysTableTableSchema = nil for _, expr := range isr.SysTableTableSchema { eexpr, err := evalengine.Translate(expr, &evalengine.Config{ Collation: collations.SystemCollation.Collation, ResolveColumn: NotImplementedSchemaInfoResolver, + CollationEnv: ctx.VSchema.CollationEnv(), }) if err != nil { panic(err) @@ -59,6 +60,7 @@ func (isr *InfoSchemaRouting) UpdateRoutingParams(_ *plancontext.PlanningContext eexpr, err := evalengine.Translate(expr, &evalengine.Config{ Collation: collations.SystemCollation.Collation, ResolveColumn: NotImplementedSchemaInfoResolver, + CollationEnv: ctx.VSchema.CollationEnv(), }) if err != nil { panic(err) @@ -132,6 +134,7 @@ func extractInfoSchemaRoutingPredicate(ctx *plancontext.PlanningContext, in sqlp _, err := evalengine.Translate(rhs, &evalengine.Config{ Collation: collations.SystemCollation.Collation, ResolveColumn: NotImplementedSchemaInfoResolver, + CollationEnv: ctx.VSchema.CollationEnv(), }) if err != nil { // if we can't translate this to an evalengine expression, diff --git a/go/vt/vtgate/planbuilder/operators/insert.go b/go/vt/vtgate/planbuilder/operators/insert.go index 194ef198772..a47214cb004 100644 --- a/go/vt/vtgate/planbuilder/operators/insert.go +++ b/go/vt/vtgate/planbuilder/operators/insert.go @@ -506,8 +506,9 @@ func insertRowsPlan(ctx *plancontext.PlanningContext, insOp *Insert, ins *sqlpar colNum, _ := findOrAddColumn(ins, col) for rowNum, row := range rows { innerpv, err := evalengine.Translate(row[colNum], &evalengine.Config{ - ResolveType: ctx.SemTable.TypeForExpr, - Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + Collation: ctx.SemTable.Collation, + CollationEnv: ctx.VSchema.CollationEnv(), }) if err != nil { panic(err) @@ -636,8 +637,9 @@ func modifyForAutoinc(ctx *plancontext.PlanningContext, ins *sqlparser.Insert, v } var err error gen.Values, err = evalengine.Translate(autoIncValues, &evalengine.Config{ - ResolveType: ctx.SemTable.TypeForExpr, - Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + Collation: ctx.SemTable.Collation, + CollationEnv: ctx.VSchema.CollationEnv(), }) if err != nil { panic(err) diff --git a/go/vt/vtgate/planbuilder/operators/projection.go b/go/vt/vtgate/planbuilder/operators/projection.go index b416e369ca2..9523578abbc 100644 --- a/go/vt/vtgate/planbuilder/operators/projection.go +++ b/go/vt/vtgate/planbuilder/operators/projection.go @@ -554,8 +554,9 @@ func (p *Projection) planOffsets(ctx *plancontext.PlanningContext) Operator { // for everything else, we'll turn to the evalengine eexpr, err := evalengine.Translate(rewritten, &evalengine.Config{ - ResolveType: ctx.SemTable.TypeForExpr, - Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + Collation: ctx.SemTable.Collation, + CollationEnv: ctx.VSchema.CollationEnv(), }) if err != nil { panic(err) diff --git a/go/vt/vtgate/planbuilder/operators/route.go b/go/vt/vtgate/planbuilder/operators/route.go index d5eee19e5dd..c540ad6791d 100644 --- a/go/vt/vtgate/planbuilder/operators/route.go +++ b/go/vt/vtgate/planbuilder/operators/route.go @@ -118,7 +118,7 @@ func UpdateRoutingLogic(ctx *plancontext.PlanningContext, expr sqlparser.Expr, r } nr := &NoneRouting{keyspace: ks} - if isConstantFalse(expr) { + if isConstantFalse(expr, ctx.VSchema.ConnCollation(), ctx.VSchema.CollationEnv()) { return nr } @@ -162,9 +162,12 @@ func UpdateRoutingLogic(ctx *plancontext.PlanningContext, expr sqlparser.Expr, r // isConstantFalse checks whether this predicate can be evaluated at plan-time. If it returns `false` or `null`, // we know that the query will not return anything, and this can be used to produce better plans -func isConstantFalse(expr sqlparser.Expr) bool { - eenv := evalengine.EmptyExpressionEnv() - eexpr, err := evalengine.Translate(expr, nil) +func isConstantFalse(expr sqlparser.Expr, collation collations.ID, collationEnv *collations.Environment) bool { + eenv := evalengine.EmptyExpressionEnv(collationEnv) + eexpr, err := evalengine.Translate(expr, &evalengine.Config{ + Collation: collation, + CollationEnv: collationEnv, + }) if err != nil { return false } @@ -172,7 +175,7 @@ func isConstantFalse(expr sqlparser.Expr) bool { if err != nil { return false } - if eres.Value(collations.Default()).IsNull() { + if eres.Value(collation).IsNull() { return false } b, err := eres.ToBooleanStrict() diff --git a/go/vt/vtgate/planbuilder/operators/sharded_routing.go b/go/vt/vtgate/planbuilder/operators/sharded_routing.go index 239ae9ce419..0bde10946fb 100644 --- a/go/vt/vtgate/planbuilder/operators/sharded_routing.go +++ b/go/vt/vtgate/planbuilder/operators/sharded_routing.go @@ -608,8 +608,9 @@ func tryMergeJoinShardedRouting( func makeEvalEngineExpr(ctx *plancontext.PlanningContext, n sqlparser.Expr) evalengine.Expr { for _, expr := range ctx.SemTable.GetExprAndEqualities(n) { ee, _ := evalengine.Translate(expr, &evalengine.Config{ - Collation: ctx.SemTable.Collation, - ResolveType: ctx.SemTable.TypeForExpr, + Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + CollationEnv: ctx.VSchema.CollationEnv(), }) if ee != nil { return ee diff --git a/go/vt/vtgate/planbuilder/ordered_aggregate.go b/go/vt/vtgate/planbuilder/ordered_aggregate.go index 34646fa3dea..c6a37c8decb 100644 --- a/go/vt/vtgate/planbuilder/ordered_aggregate.go +++ b/go/vt/vtgate/planbuilder/ordered_aggregate.go @@ -17,6 +17,7 @@ limitations under the License. package planbuilder import ( + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/vtgate/engine" ) @@ -60,6 +61,8 @@ type orderedAggregate struct { groupByKeys []*engine.GroupByParams truncateColumnCount int + + collationEnv *collations.Environment } // Primitive implements the logicalPlan interface @@ -78,6 +81,7 @@ func (oa *orderedAggregate) Primitive() engine.Primitive { GroupByKeys: oa.groupByKeys, TruncateColumnCount: oa.truncateColumnCount, Input: input, + CollationEnv: oa.collationEnv, } } diff --git a/go/vt/vtgate/planbuilder/plancontext/planning_context.go b/go/vt/vtgate/planbuilder/plancontext/planning_context.go index 68ccc95b9fd..e53ce5d5885 100644 --- a/go/vt/vtgate/planbuilder/plancontext/planning_context.go +++ b/go/vt/vtgate/planbuilder/plancontext/planning_context.go @@ -53,7 +53,6 @@ type PlanningContext struct { func CreatePlanningContext(stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, - vschema VSchema, version querypb.ExecuteOptions_PlannerVersion, ) (*PlanningContext, error) { diff --git a/go/vt/vtgate/planbuilder/plancontext/vschema.go b/go/vt/vtgate/planbuilder/plancontext/vschema.go index 286e8d30b67..958a9644a60 100644 --- a/go/vt/vtgate/planbuilder/plancontext/vschema.go +++ b/go/vt/vtgate/planbuilder/plancontext/vschema.go @@ -41,6 +41,7 @@ type VSchema interface { Planner() PlannerVersion SetPlannerVersion(pv PlannerVersion) ConnCollation() collations.ID + CollationEnv() *collations.Environment // ErrorIfShardedF will return an error if the keyspace is sharded, // and produce a warning if the vtgate if configured to do so diff --git a/go/vt/vtgate/planbuilder/predicate_rewrite_test.go b/go/vt/vtgate/planbuilder/predicate_rewrite_test.go index 369a99bf5d3..62931388d70 100644 --- a/go/vt/vtgate/planbuilder/predicate_rewrite_test.go +++ b/go/vt/vtgate/planbuilder/predicate_rewrite_test.go @@ -103,17 +103,19 @@ func TestFuzzRewriting(t *testing.T) { simplified := sqlparser.RewritePredicate(predicate) original, err := evalengine.Translate(predicate, &evalengine.Config{ - Collation: collations.Default(), + Collation: collations.MySQL8().DefaultConnectionCharset(), + CollationEnv: collations.MySQL8(), ResolveColumn: resolveForFuzz, }) require.NoError(t, err) simpler, err := evalengine.Translate(simplified.(sqlparser.Expr), &evalengine.Config{ - Collation: collations.Default(), + Collation: collations.MySQL8().DefaultConnectionCharset(), + CollationEnv: collations.MySQL8(), ResolveColumn: resolveForFuzz, }) require.NoError(t, err) - env := evalengine.EmptyExpressionEnv() + env := evalengine.EmptyExpressionEnv(collations.MySQL8()) env.Row = make([]sqltypes.Value, tc.nodes) for i := range env.Row { env.Row[i] = sqltypes.NewInt32(1) @@ -139,7 +141,7 @@ func testValues(t *testing.T, env *evalengine.ExpressionEnv, i int, original, si require.NoError(t, err) v2, err := env.Evaluate(simpler) require.NoError(t, err) - assert.Equal(t, v1.Value(collations.Default()), v2.Value(collations.Default())) + assert.Equal(t, v1.Value(collations.MySQL8().DefaultConnectionCharset()), v2.Value(collations.MySQL8().DefaultConnectionCharset())) if len(env.Row) > i+1 { testValues(t, env, i+1, original, simpler) } diff --git a/go/vt/vtgate/planbuilder/select.go b/go/vt/vtgate/planbuilder/select.go index 77c883325c5..302215816c5 100644 --- a/go/vt/vtgate/planbuilder/select.go +++ b/go/vt/vtgate/planbuilder/select.go @@ -289,7 +289,10 @@ func handleDualSelects(sel *sqlparser.Select, vschema plancontext.VSchema) (engi if isLFunc { elem := &engine.LockFunc{Typ: expr.Expr.(*sqlparser.LockingFunc)} if lFunc.Name != nil { - n, err := evalengine.Translate(lFunc.Name, nil) + n, err := evalengine.Translate(lFunc.Name, &evalengine.Config{ + Collation: vschema.ConnCollation(), + CollationEnv: vschema.CollationEnv(), + }) if err != nil { return nil, err } @@ -301,7 +304,10 @@ func handleDualSelects(sel *sqlparser.Select, vschema plancontext.VSchema) (engi if len(lockFunctions) > 0 { return nil, vterrors.VT12001(fmt.Sprintf("LOCK function and other expression: [%s] in same select query", sqlparser.String(expr))) } - exprs[i], err = evalengine.Translate(expr.Expr, &evalengine.Config{Collation: vschema.ConnCollation()}) + exprs[i], err = evalengine.Translate(expr.Expr, &evalengine.Config{ + Collation: vschema.ConnCollation(), + CollationEnv: vschema.CollationEnv(), + }) if err != nil { return nil, nil } diff --git a/go/vt/vtgate/planbuilder/set.go b/go/vt/vtgate/planbuilder/set.go index 43d85ee5113..8944b9e8f30 100644 --- a/go/vt/vtgate/planbuilder/set.go +++ b/go/vt/vtgate/planbuilder/set.go @@ -55,7 +55,10 @@ func buildSetPlan(stmt *sqlparser.Set, vschema plancontext.VSchema) (*planResult var setOps []engine.SetOp var err error - ec := new(expressionConverter) + ec := &expressionConverter{ + collationEnv: vschema.CollationEnv(), + collation: vschema.ConnCollation(), + } for _, expr := range stmt.Exprs { // AST struct has been prepared before getting here, so no scope here means that @@ -80,7 +83,7 @@ func buildSetPlan(stmt *sqlparser.Set, vschema plancontext.VSchema) (*planResult } setOps = append(setOps, setOp) case sqlparser.NextTxScope, sqlparser.SessionScope: - planFunc, err := sysvarPlanningFuncs.Get(expr) + planFunc, err := sysvarPlanningFuncs.Get(expr, vschema.CollationEnv()) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/show.go b/go/vt/vtgate/planbuilder/show.go index 2a8b11fb70e..2955e1f0fca 100644 --- a/go/vt/vtgate/planbuilder/show.go +++ b/go/vt/vtgate/planbuilder/show.go @@ -561,10 +561,11 @@ func buildShowVGtidPlan(show *sqlparser.ShowBasic, vschema plancontext.VSchema) } return &engine.OrderedAggregate{ Aggregates: []*engine.AggregateParams{ - engine.NewAggregateParam(popcode.AggregateGtid, 1, "global vgtid_executed"), + engine.NewAggregateParam(popcode.AggregateGtid, 1, "global vgtid_executed", vschema.CollationEnv()), }, TruncateColumnCount: 2, Input: send, + CollationEnv: vschema.CollationEnv(), }, nil } diff --git a/go/vt/vtgate/planbuilder/show_test.go b/go/vt/vtgate/planbuilder/show_test.go index b36133bb1c7..169a916649d 100644 --- a/go/vt/vtgate/planbuilder/show_test.go +++ b/go/vt/vtgate/planbuilder/show_test.go @@ -76,7 +76,7 @@ func TestGenerateCharsetRows(t *testing.T) { append(buildVarCharRow( "utf8mb4", "UTF-8 Unicode", - collations.Local().LookupName(collations.Default())), + collations.MySQL8().LookupName(collations.MySQL8().DefaultConnectionCharset())), sqltypes.NewUint32(4)), } rows2 := [][]sqltypes.Value{ @@ -88,7 +88,7 @@ func TestGenerateCharsetRows(t *testing.T) { append(buildVarCharRow( "utf8mb4", "UTF-8 Unicode", - collations.Local().LookupName(collations.Default())), + collations.MySQL8().LookupName(collations.MySQL8().DefaultConnectionCharset())), sqltypes.NewUint32(4)), } diff --git a/go/vt/vtgate/planbuilder/system_variables.go b/go/vt/vtgate/planbuilder/system_variables.go index eccb263c65a..ff2fbb6f55c 100644 --- a/go/vt/vtgate/planbuilder/system_variables.go +++ b/go/vt/vtgate/planbuilder/system_variables.go @@ -20,6 +20,7 @@ import ( "fmt" "sync" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/sysvars" "vitess.io/vitess/go/vt/vterrors" @@ -27,8 +28,9 @@ import ( ) type sysvarPlanCache struct { - funcs map[string]planFunc - once sync.Once + funcs map[string]planFunc + once sync.Once + collationEnv *collations.Environment } func (pc *sysvarPlanCache) initForSettings(systemVariables []sysvars.SystemVariable, f func(setting) planFunc) { @@ -59,15 +61,19 @@ func (pc *sysvarPlanCache) parseAndBuildDefaultValue(sysvar sysvars.SystemVariab } sel := stmt.(*sqlparser.Select) aliasedExpr := sel.SelectExprs[0].(*sqlparser.AliasedExpr) - def, err := evalengine.Translate(aliasedExpr.Expr, nil) + def, err := evalengine.Translate(aliasedExpr.Expr, &evalengine.Config{ + Collation: pc.collationEnv.DefaultConnectionCharset(), + CollationEnv: pc.collationEnv, + }) if err != nil { panic(fmt.Sprintf("bug in set plan init - default value for %s not able to convert to evalengine.Expr: %s", sysvar.Name, sysvar.Default)) } return def } -func (pc *sysvarPlanCache) init() { +func (pc *sysvarPlanCache) init(collationEnv *collations.Environment) { pc.once.Do(func() { + pc.collationEnv = collationEnv pc.funcs = make(map[string]planFunc) pc.initForSettings(sysvars.ReadOnly, buildSetOpReadOnly) pc.initForSettings(sysvars.IgnoreThese, buildSetOpIgnore) @@ -80,8 +86,8 @@ func (pc *sysvarPlanCache) init() { var sysvarPlanningFuncs sysvarPlanCache -func (pc *sysvarPlanCache) Get(expr *sqlparser.SetExpr) (planFunc, error) { - pc.init() +func (pc *sysvarPlanCache) Get(expr *sqlparser.SetExpr, collationEnv *collations.Environment) (planFunc, error) { + pc.init(collationEnv) pf, ok := pc.funcs[expr.Var.Name.Lowered()] if !ok { return nil, vterrors.VT05006(sqlparser.String(expr)) diff --git a/go/vt/vtgate/planbuilder/vindex_op.go b/go/vt/vtgate/planbuilder/vindex_op.go index c439dec1701..b06606070d7 100644 --- a/go/vt/vtgate/planbuilder/vindex_op.go +++ b/go/vt/vtgate/planbuilder/vindex_op.go @@ -33,8 +33,9 @@ func transformVindexPlan(ctx *plancontext.PlanningContext, op *operators.Vindex) } expr, err := evalengine.Translate(op.Value, &evalengine.Config{ - Collation: ctx.SemTable.Collation, - ResolveType: ctx.SemTable.TypeForExpr, + Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + CollationEnv: ctx.VSchema.CollationEnv(), }) if err != nil { return nil, err diff --git a/go/vt/vtgate/semantics/FakeSI.go b/go/vt/vtgate/semantics/FakeSI.go index 5a91ece816d..94386590814 100644 --- a/go/vt/vtgate/semantics/FakeSI.go +++ b/go/vt/vtgate/semantics/FakeSI.go @@ -47,7 +47,11 @@ func (s *FakeSI) FindTableOrVindex(tablename sqlparser.TableName) (*vindexes.Tab } func (*FakeSI) ConnCollation() collations.ID { - return 45 + return collations.CollationUtf8mb4ID +} + +func (*FakeSI) CollationEnv() *collations.Environment { + return collations.MySQL8() } func (s *FakeSI) ForeignKeyMode(keyspace string) (vschemapb.Keyspace_ForeignKeyMode, error) { diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index d7dab3da078..4959045458f 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -50,7 +50,7 @@ func newAnalyzer(dbName string, si SchemaInformation) *analyzer { a := &analyzer{ scoper: s, tables: newTableCollector(s, si, dbName), - typer: newTyper(), + typer: newTyper(si.CollationEnv()), } s.org = a a.tables.org = a @@ -61,6 +61,7 @@ func newAnalyzer(dbName string, si SchemaInformation) *analyzer { scoper: s, binder: b, expandedColumns: map[sqlparser.TableName][]*sqlparser.ColName{}, + collationEnv: si.CollationEnv(), } s.binder = b return a diff --git a/go/vt/vtgate/semantics/early_rewriter.go b/go/vt/vtgate/semantics/early_rewriter.go index 1a957c8ae60..c71941afdd5 100644 --- a/go/vt/vtgate/semantics/early_rewriter.go +++ b/go/vt/vtgate/semantics/early_rewriter.go @@ -33,6 +33,7 @@ type earlyRewriter struct { clause string warning string expandedColumns map[sqlparser.TableName][]*sqlparser.ColName + collationEnv *collations.Environment } func (r *earlyRewriter) down(cursor *sqlparser.Cursor) error { @@ -46,9 +47,9 @@ func (r *earlyRewriter) down(cursor *sqlparser.Cursor) error { case sqlparser.OrderBy: handleOrderBy(r, cursor, node) case *sqlparser.OrExpr: - rewriteOrExpr(cursor, node) + rewriteOrExpr(cursor, node, r.collationEnv) case *sqlparser.AndExpr: - rewriteAndExpr(cursor, node) + rewriteAndExpr(cursor, node, r.collationEnv) case *sqlparser.NotExpr: rewriteNotExpr(cursor, node) case sqlparser.GroupBy: @@ -176,36 +177,40 @@ func handleOrderBy(r *earlyRewriter, cursor *sqlparser.Cursor, node sqlparser.Or } // rewriteOrExpr rewrites OR expressions when the right side is FALSE. -func rewriteOrExpr(cursor *sqlparser.Cursor, node *sqlparser.OrExpr) { - newNode := rewriteOrFalse(*node) +func rewriteOrExpr(cursor *sqlparser.Cursor, node *sqlparser.OrExpr, collationEnv *collations.Environment) { + newNode := rewriteOrFalse(*node, collationEnv) if newNode != nil { cursor.ReplaceAndRevisit(newNode) } } // rewriteAndExpr rewrites AND expressions when either side is TRUE. -func rewriteAndExpr(cursor *sqlparser.Cursor, node *sqlparser.AndExpr) { - newNode := rewriteAndTrue(*node) +func rewriteAndExpr(cursor *sqlparser.Cursor, node *sqlparser.AndExpr, collationEnv *collations.Environment) { + newNode := rewriteAndTrue(*node, collationEnv) if newNode != nil { cursor.ReplaceAndRevisit(newNode) } } -func rewriteAndTrue(andExpr sqlparser.AndExpr) sqlparser.Expr { +func rewriteAndTrue(andExpr sqlparser.AndExpr, collationEnv *collations.Environment) sqlparser.Expr { // we are looking for the pattern `WHERE c = 1 AND 1 = 1` isTrue := func(subExpr sqlparser.Expr) bool { - evalEnginePred, err := evalengine.Translate(subExpr, nil) + coll := collationEnv.DefaultConnectionCharset() + evalEnginePred, err := evalengine.Translate(subExpr, &evalengine.Config{ + CollationEnv: collationEnv, + Collation: coll, + }) if err != nil { return false } - env := evalengine.EmptyExpressionEnv() + env := evalengine.EmptyExpressionEnv(collationEnv) res, err := env.Evaluate(evalEnginePred) if err != nil { return false } - boolValue, err := res.Value(collations.Default()).ToBool() + boolValue, err := res.Value(coll).ToBool() if err != nil { return false } @@ -419,21 +424,25 @@ func realCloneOfColNames(expr sqlparser.Expr, union bool) sqlparser.Expr { }, nil).(sqlparser.Expr) } -func rewriteOrFalse(orExpr sqlparser.OrExpr) sqlparser.Expr { +func rewriteOrFalse(orExpr sqlparser.OrExpr, collationEnv *collations.Environment) sqlparser.Expr { // we are looking for the pattern `WHERE c = 1 OR 1 = 0` isFalse := func(subExpr sqlparser.Expr) bool { - evalEnginePred, err := evalengine.Translate(subExpr, nil) + coll := collationEnv.DefaultConnectionCharset() + evalEnginePred, err := evalengine.Translate(subExpr, &evalengine.Config{ + CollationEnv: collationEnv, + Collation: coll, + }) if err != nil { return false } - env := evalengine.EmptyExpressionEnv() + env := evalengine.EmptyExpressionEnv(collationEnv) res, err := env.Evaluate(evalEnginePred) if err != nil { return false } - boolValue, err := res.Value(collations.Default()).ToBool() + boolValue, err := res.Value(coll).ToBool() if err != nil { return false } diff --git a/go/vt/vtgate/semantics/info_schema.go b/go/vt/vtgate/semantics/info_schema.go index 76a383b5ac0..b150b5d4141 100644 --- a/go/vt/vtgate/semantics/info_schema.go +++ b/go/vt/vtgate/semantics/info_schema.go @@ -1637,6 +1637,11 @@ func (i *infoSchemaWithColumns) ConnCollation() collations.ID { return i.inner.ConnCollation() } +// CollationEnv implements the SchemaInformation interface +func (i *infoSchemaWithColumns) CollationEnv() *collations.Environment { + return i.inner.CollationEnv() +} + func (i *infoSchemaWithColumns) ForeignKeyMode(keyspace string) (vschemapb.Keyspace_ForeignKeyMode, error) { return i.inner.ForeignKeyMode(keyspace) } diff --git a/go/vt/vtgate/semantics/info_schema_gen_test.go b/go/vt/vtgate/semantics/info_schema_gen_test.go index da06b80ac30..c2c5e07bf82 100644 --- a/go/vt/vtgate/semantics/info_schema_gen_test.go +++ b/go/vt/vtgate/semantics/info_schema_gen_test.go @@ -41,7 +41,7 @@ func TestGenerateInfoSchemaMap(t *testing.T) { require.NoError(t, err) defer db.Close() - collationName := collations.Local().LookupName(collations.SystemCollation.Collation) + collationName := collations.MySQL8().LookupName(collations.SystemCollation.Collation) for _, tbl := range informationSchemaTables80 { result, err := db.Query(fmt.Sprintf("show columns from information_schema.`%s`", tbl)) diff --git a/go/vt/vtgate/semantics/real_table.go b/go/vt/vtgate/semantics/real_table.go index da55d95895f..7aafb697698 100644 --- a/go/vt/vtgate/semantics/real_table.go +++ b/go/vt/vtgate/semantics/real_table.go @@ -19,6 +19,7 @@ package semantics import ( "strings" + "vitess.io/vitess/go/mysql/collations" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -31,6 +32,7 @@ type RealTable struct { ASTNode *sqlparser.AliasedTableExpr Table *vindexes.Table isInfSchema bool + collationEnv *collations.Environment } var _ TableInfo = (*RealTable)(nil) @@ -67,7 +69,7 @@ func (r *RealTable) IsInfSchema() bool { // GetColumns implements the TableInfo interface func (r *RealTable) getColumns() []ColumnInfo { - return vindexTableToColumnInfo(r.Table) + return vindexTableToColumnInfo(r.Table, r.collationEnv) } // GetExpr implements the TableInfo interface @@ -115,7 +117,7 @@ func (r *RealTable) matches(name sqlparser.TableName) bool { return (name.Qualifier.IsEmpty() || name.Qualifier.String() == r.dbName) && r.tableName == name.Name.String() } -func vindexTableToColumnInfo(tbl *vindexes.Table) []ColumnInfo { +func vindexTableToColumnInfo(tbl *vindexes.Table, collationEnv *collations.Environment) []ColumnInfo { if tbl == nil { return nil } @@ -125,7 +127,7 @@ func vindexTableToColumnInfo(tbl *vindexes.Table) []ColumnInfo { cols = append(cols, ColumnInfo{ Name: col.Name.String(), - Type: col.ToEvalengineType(), + Type: col.ToEvalengineType(collationEnv), Invisible: col.Invisible, }) nameMap[col.Name.String()] = nil diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 6668070a575..cf1ff7c2faf 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -150,6 +150,7 @@ type ( SchemaInformation interface { FindTableOrVindex(tablename sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error) ConnCollation() collations.ID + CollationEnv() *collations.Environment // ForeignKeyMode returns the foreign_key flag value ForeignKeyMode(keyspace string) (vschemapb.Keyspace_ForeignKeyMode, error) GetForeignKeyChecksState() *bool diff --git a/go/vt/vtgate/semantics/table_collector.go b/go/vt/vtgate/semantics/table_collector.go index 12fb691874f..3940a19d107 100644 --- a/go/vt/vtgate/semantics/table_collector.go +++ b/go/vt/vtgate/semantics/table_collector.go @@ -231,10 +231,11 @@ func (tc *tableCollector) createTable( vindex vindexes.Vindex, ) TableInfo { table := &RealTable{ - tableName: alias.As.String(), - ASTNode: alias, - Table: tbl, - isInfSchema: isInfSchema, + tableName: alias.As.String(), + ASTNode: alias, + Table: tbl, + isInfSchema: isInfSchema, + collationEnv: tc.si.CollationEnv(), } if alias.As.IsEmpty() { diff --git a/go/vt/vtgate/semantics/typer.go b/go/vt/vtgate/semantics/typer.go index 8b44105d255..a9c783cee18 100644 --- a/go/vt/vtgate/semantics/typer.go +++ b/go/vt/vtgate/semantics/typer.go @@ -27,12 +27,14 @@ import ( // typer is responsible for setting the type for expressions // it does it's work after visiting the children (up), since the children types is often needed to type a node. type typer struct { - m map[sqlparser.Expr]evalengine.Type + m map[sqlparser.Expr]evalengine.Type + collationEnv *collations.Environment } -func newTyper() *typer { +func newTyper(collationEnv *collations.Environment) *typer { return &typer{ - m: map[sqlparser.Expr]evalengine.Type{}, + m: map[sqlparser.Expr]evalengine.Type{}, + collationEnv: collationEnv, } } @@ -43,10 +45,10 @@ func (t *typer) exprType(expr sqlparser.Expr) evalengine.Type { func (t *typer) up(cursor *sqlparser.Cursor) error { switch node := cursor.Node().(type) { case *sqlparser.Literal: - t.m[node] = evalengine.NewType(node.SQLType(), collations.DefaultCollationForType(node.SQLType())) + t.m[node] = evalengine.NewType(node.SQLType(), collations.CollationForType(node.SQLType(), t.collationEnv.DefaultConnectionCharset())) case *sqlparser.Argument: if node.Type >= 0 { - t.m[node] = evalengine.NewType(node.Type, collations.DefaultCollationForType(node.Type)) + t.m[node] = evalengine.NewType(node.Type, collations.CollationForType(node.Type, t.collationEnv.DefaultConnectionCharset())) } case sqlparser.AggrFunc: code, ok := opcode.SupportedAggregates[node.AggrName()] @@ -62,7 +64,7 @@ func (t *typer) up(cursor *sqlparser.Cursor) error { type_ := code.Type(inputType) _, isCount := node.(*sqlparser.Count) _, isCountStart := node.(*sqlparser.CountStar) - t.m[node] = evalengine.NewTypeEx(type_, collations.DefaultCollationForType(type_), !(isCount || isCountStart), 0, 0) + t.m[node] = evalengine.NewTypeEx(type_, collations.CollationForType(type_, t.collationEnv.DefaultConnectionCharset()), !(isCount || isCountStart), 0, 0) } return nil } diff --git a/go/vt/vtgate/simplifier/simplifier_test.go b/go/vt/vtgate/simplifier/simplifier_test.go index e2270a551b5..21c238e1a47 100644 --- a/go/vt/vtgate/simplifier/simplifier_test.go +++ b/go/vt/vtgate/simplifier/simplifier_test.go @@ -120,15 +120,19 @@ func TestSimplifyEvalEngineExpr(t *testing.T) { p0 := plus(p11, p12) expr := SimplifyExpr(p0, func(expr sqlparser.Expr) bool { - local, err := evalengine.Translate(expr, nil) + collationEnv := collations.MySQL8() + local, err := evalengine.Translate(expr, &evalengine.Config{ + CollationEnv: collationEnv, + Collation: collationEnv.DefaultConnectionCharset(), + }) if err != nil { return false } - res, err := evalengine.EmptyExpressionEnv().Evaluate(local) + res, err := evalengine.EmptyExpressionEnv(collationEnv).Evaluate(local) if err != nil { return false } - toInt64, err := res.Value(collations.Default()).ToInt64() + toInt64, err := res.Value(collationEnv.DefaultConnectionCharset()).ToInt64() if err != nil { return false } diff --git a/go/vt/vtgate/tabletgateway.go b/go/vt/vtgate/tabletgateway.go index 7703f47c4fa..d1846168a43 100644 --- a/go/vt/vtgate/tabletgateway.go +++ b/go/vt/vtgate/tabletgateway.go @@ -71,7 +71,7 @@ type TabletGateway struct { srvTopoServer srvtopo.Server localCell string retryCount int - defaultConnCollation uint32 + defaultConnCollation atomic.Uint32 // mu protects the fields of this group. mu sync.Mutex @@ -431,17 +431,17 @@ func (gw *TabletGateway) TabletsHealthyStatus() discovery.TabletsCacheStatusList } func (gw *TabletGateway) updateDefaultConnCollation(tablet *topodatapb.Tablet) { - if atomic.CompareAndSwapUint32(&gw.defaultConnCollation, 0, tablet.DefaultConnCollation) { + if gw.defaultConnCollation.CompareAndSwap(0, tablet.DefaultConnCollation) { return } - if atomic.LoadUint32(&gw.defaultConnCollation) != tablet.DefaultConnCollation { + if gw.defaultConnCollation.Load() != tablet.DefaultConnCollation { log.Warning("this Vitess cluster has tablets with different default connection collations") } } // DefaultConnCollation returns the default connection collation of this TabletGateway func (gw *TabletGateway) DefaultConnCollation() collations.ID { - return collations.ID(atomic.LoadUint32(&gw.defaultConnCollation)) + return collations.ID(gw.defaultConnCollation.Load()) } // NewShardError returns a new error with the shard info amended. diff --git a/go/vt/vtgate/tabletgateway_flaky_test.go b/go/vt/vtgate/tabletgateway_flaky_test.go index 917d931d2ff..19894b0002e 100644 --- a/go/vt/vtgate/tabletgateway_flaky_test.go +++ b/go/vt/vtgate/tabletgateway_flaky_test.go @@ -69,7 +69,7 @@ func TestGatewayBufferingWhenPrimarySwitchesServingState(t *testing.T) { Fields: []*querypb.Field{{ Name: "col1", Type: sqltypes.VarChar, - Charset: uint32(collations.Default()), + Charset: uint32(collations.MySQL8().DefaultConnectionCharset()), }}, RowsAffected: 1, Rows: [][]sqltypes.Value{{ @@ -158,7 +158,7 @@ func TestGatewayBufferingWhileReparenting(t *testing.T) { Fields: []*querypb.Field{{ Name: "col1", Type: sqltypes.VarChar, - Charset: uint32(collations.Default()), + Charset: uint32(collations.MySQL8().DefaultConnectionCharset()), }}, RowsAffected: 1, Rows: [][]sqltypes.Value{{ @@ -287,7 +287,7 @@ func TestInconsistentStateDetectedBuffering(t *testing.T) { Fields: []*querypb.Field{{ Name: "col1", Type: sqltypes.VarChar, - Charset: uint32(collations.Default()), + Charset: uint32(collations.MySQL8().DefaultConnectionCharset()), }}, RowsAffected: 1, Rows: [][]sqltypes.Value{{ diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index 616b1ce8846..aa6c0f347fe 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -104,6 +104,7 @@ type vcursorImpl struct { topoServer *topo.Server logStats *logstats.LogStats collation collations.ID + collationEnv *collations.Environment // fkChecksState stores the state of foreign key checks variable. // This state is meant to be the final fk checks state after consulting the @@ -138,6 +139,7 @@ func newVCursorImpl( serv srvtopo.Server, warnShardedOnly bool, pv plancontext.PlannerVersion, + collationEnv *collations.Environment, ) (*vcursorImpl, error) { keyspace, tabletType, destination, err := parseDestinationTarget(safeSession.TargetString, vschema) if err != nil { @@ -162,7 +164,7 @@ func newVCursorImpl( } } if connCollation == collations.Unknown { - connCollation = collations.Default() + connCollation = collationEnv.DefaultConnectionCharset() } warmingReadsPct := 0 @@ -188,6 +190,7 @@ func newVCursorImpl( pv: pv, warmingReadsPercent: warmingReadsPct, warmingReadsChannel: warmingReadsChan, + collationEnv: collationEnv, }, nil } @@ -206,6 +209,11 @@ func (vc *vcursorImpl) ConnCollation() collations.ID { return vc.collation } +// ConnCollation returns the collation of this session +func (vc *vcursorImpl) CollationEnv() *collations.Environment { + return vc.collationEnv +} + func (vc *vcursorImpl) TimeZone() *time.Location { return vc.safeSession.TimeZone() } @@ -1082,7 +1090,7 @@ func (vc *vcursorImpl) keyForPlan(ctx context.Context, query string, buf io.Stri _, _ = buf.WriteString(vc.keyspace) _, _ = buf.WriteString(vindexes.TabletTypeSuffix[vc.tabletType]) _, _ = buf.WriteString("+Collate:") - _, _ = buf.WriteString(collations.Local().LookupName(vc.collation)) + _, _ = buf.WriteString(vc.collationEnv.LookupName(vc.collation)) if vc.destination != nil { switch vc.destination.(type) { diff --git a/go/vt/vtgate/vcursor_impl_test.go b/go/vt/vtgate/vcursor_impl_test.go index 77be183eacd..37358909641 100644 --- a/go/vt/vtgate/vcursor_impl_test.go +++ b/go/vt/vtgate/vcursor_impl_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/srvtopo" @@ -186,7 +187,7 @@ func TestDestinationKeyspace(t *testing.T) { for i, tc := range tests { t.Run(strconv.Itoa(i)+tc.targetString, func(t *testing.T) { - impl, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: tc.targetString}), sqlparser.MarginComments{}, nil, nil, &fakeVSchemaOperator{vschema: tc.vschema}, tc.vschema, nil, nil, false, querypb.ExecuteOptions_Gen4) + impl, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{TargetString: tc.targetString}), sqlparser.MarginComments{}, nil, nil, &fakeVSchemaOperator{vschema: tc.vschema}, tc.vschema, nil, nil, false, querypb.ExecuteOptions_Gen4, collations.MySQL8()) impl.vschema = tc.vschema dest, keyspace, tabletType, err := impl.TargetDestination(tc.qualifier) if tc.expectedError == "" { @@ -244,7 +245,7 @@ func TestSetTarget(t *testing.T) { for i, tc := range tests { t.Run(fmt.Sprintf("%d#%s", i, tc.targetString), func(t *testing.T) { - vc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{InTransaction: true}), sqlparser.MarginComments{}, nil, nil, &fakeVSchemaOperator{vschema: tc.vschema}, tc.vschema, nil, nil, false, querypb.ExecuteOptions_Gen4) + vc, _ := newVCursorImpl(NewSafeSession(&vtgatepb.Session{InTransaction: true}), sqlparser.MarginComments{}, nil, nil, &fakeVSchemaOperator{vschema: tc.vschema}, tc.vschema, nil, nil, false, querypb.ExecuteOptions_Gen4, collations.MySQL8()) vc.vschema = tc.vschema err := vc.SetTarget(tc.targetString) if tc.expectedError == "" { @@ -294,7 +295,7 @@ func TestKeyForPlan(t *testing.T) { t.Run(fmt.Sprintf("%d#%s", i, tc.targetString), func(t *testing.T) { ss := NewSafeSession(&vtgatepb.Session{InTransaction: false}) ss.SetTargetString(tc.targetString) - vc, err := newVCursorImpl(ss, sqlparser.MarginComments{}, nil, nil, &fakeVSchemaOperator{vschema: tc.vschema}, tc.vschema, srvtopo.NewResolver(&fakeTopoServer{}, nil, ""), nil, false, querypb.ExecuteOptions_Gen4) + vc, err := newVCursorImpl(ss, sqlparser.MarginComments{}, nil, nil, &fakeVSchemaOperator{vschema: tc.vschema}, tc.vschema, srvtopo.NewResolver(&fakeTopoServer{}, nil, ""), nil, false, querypb.ExecuteOptions_Gen4, collations.MySQL8()) require.NoError(t, err) vc.vschema = tc.vschema @@ -316,7 +317,7 @@ func TestFirstSortedKeyspace(t *testing.T) { ks3Schema.Keyspace.Name: ks3Schema, }} - vc, err := newVCursorImpl(NewSafeSession(nil), sqlparser.MarginComments{}, nil, nil, &fakeVSchemaOperator{vschema: vschemaWith2KS}, vschemaWith2KS, srvtopo.NewResolver(&fakeTopoServer{}, nil, ""), nil, false, querypb.ExecuteOptions_Gen4) + vc, err := newVCursorImpl(NewSafeSession(nil), sqlparser.MarginComments{}, nil, nil, &fakeVSchemaOperator{vschema: vschemaWith2KS}, vschemaWith2KS, srvtopo.NewResolver(&fakeTopoServer{}, nil, ""), nil, false, querypb.ExecuteOptions_Gen4, collations.MySQL8()) require.NoError(t, err) ks, err := vc.FirstSortedKeyspace() require.NoError(t, err) diff --git a/go/vt/vtgate/vindexes/consistent_lookup.go b/go/vt/vtgate/vindexes/consistent_lookup.go index 9173ded96e6..cc74966c197 100644 --- a/go/vt/vtgate/vindexes/consistent_lookup.go +++ b/go/vt/vtgate/vindexes/consistent_lookup.go @@ -413,7 +413,7 @@ func (lu *clCommon) Delete(ctx context.Context, vcursor VCursor, rowsColValues [ func (lu *clCommon) Update(ctx context.Context, vcursor VCursor, oldValues []sqltypes.Value, ksid []byte, newValues []sqltypes.Value) error { equal := true for i := range oldValues { - result, err := evalengine.NullsafeCompare(oldValues[i], newValues[i], vcursor.ConnCollation()) + result, err := evalengine.NullsafeCompare(oldValues[i], newValues[i], vcursor.CollationEnv(), vcursor.ConnCollation()) // errors from NullsafeCompare can be ignored. if they are real problems, we'll see them in the Create/Update if err != nil || result != 0 { equal = false diff --git a/go/vt/vtgate/vindexes/consistent_lookup_test.go b/go/vt/vtgate/vindexes/consistent_lookup_test.go index deecc23ebdd..832a16fae9f 100644 --- a/go/vt/vtgate/vindexes/consistent_lookup_test.go +++ b/go/vt/vtgate/vindexes/consistent_lookup_test.go @@ -528,7 +528,11 @@ func (vc *loggingVCursor) InTransactionAndIsDML() bool { } func (vc *loggingVCursor) ConnCollation() collations.ID { - return collations.Default() + return vc.CollationEnv().DefaultConnectionCharset() +} + +func (vc *loggingVCursor) CollationEnv() *collations.Environment { + return collations.MySQL8() } type bv struct { diff --git a/go/vt/vtgate/vindexes/lookup_test.go b/go/vt/vtgate/vindexes/lookup_test.go index b82ab3d4fec..736017f89eb 100644 --- a/go/vt/vtgate/vindexes/lookup_test.go +++ b/go/vt/vtgate/vindexes/lookup_test.go @@ -115,7 +115,11 @@ func (vc *vcursor) execute(query string, bindvars map[string]*querypb.BindVariab } func (vc *vcursor) ConnCollation() collations.ID { - return collations.Default() + return vc.CollationEnv().DefaultConnectionCharset() +} + +func (vc *vcursor) CollationEnv() *collations.Environment { + return collations.MySQL8() } func lookupCreateVindexTestCase( diff --git a/go/vt/vtgate/vindexes/vindex.go b/go/vt/vtgate/vindexes/vindex.go index a5295681248..e9e346c7b89 100644 --- a/go/vt/vtgate/vindexes/vindex.go +++ b/go/vt/vtgate/vindexes/vindex.go @@ -44,6 +44,7 @@ type ( InTransactionAndIsDML() bool LookupRowLockShardSession() vtgatepb.CommitOrder ConnCollation() collations.ID + CollationEnv() *collations.Environment } // Vindex defines the interface required to register a vindex. diff --git a/go/vt/vtgate/vindexes/vschema.go b/go/vt/vtgate/vindexes/vschema.go index e724794b6cb..96c94be87d1 100644 --- a/go/vt/vtgate/vindexes/vschema.go +++ b/go/vt/vtgate/vindexes/vschema.go @@ -218,10 +218,10 @@ func (col *Column) MarshalJSON() ([]byte, error) { return json.Marshal(cj) } -func (col *Column) ToEvalengineType() evalengine.Type { - collation := collations.DefaultCollationForType(col.Type) +func (col *Column) ToEvalengineType(collationEnv *collations.Environment) evalengine.Type { + collation := collations.CollationForType(col.Type, collationEnv.DefaultConnectionCharset()) if sqltypes.IsText(col.Type) { - coll, found := collations.Local().LookupID(col.CollationName) + coll, found := collationEnv.LookupID(col.CollationName) if found { collation = coll } diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index 64260e628f0..435f53e33e0 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -30,6 +30,8 @@ import ( "github.com/spf13/pflag" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/acl" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/stats" @@ -246,6 +248,7 @@ func Init( cell string, tabletTypesToWait []topodatapb.TabletType, pv plancontext.PlannerVersion, + collationEnv *collations.Environment, ) *VTGate { // Build objects from low to high level. // Start with the gateway. If we can't reach the topology service, @@ -319,6 +322,7 @@ func Init( noScatter, pv, warmingReadsPercent, + collationEnv, ) if err := executor.defaultQueryLogger(); err != nil { diff --git a/go/vt/vttablet/endtoend/framework/server.go b/go/vt/vttablet/endtoend/framework/server.go index 7fca349deff..cca4bc90407 100644 --- a/go/vt/vttablet/endtoend/framework/server.go +++ b/go/vt/vttablet/endtoend/framework/server.go @@ -23,6 +23,7 @@ import ( "net/http" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/topo" @@ -77,7 +78,7 @@ func StartCustomServer(ctx context.Context, connParams, connAppDebugParams mysql } TopoServer = memorytopo.NewServer(ctx, "") - Server = tabletserver.NewTabletServer(ctx, "", config, TopoServer, &topodatapb.TabletAlias{}) + Server = tabletserver.NewTabletServer(ctx, "", config, TopoServer, &topodatapb.TabletAlias{}, collations.MySQL8()) Server.Register() err := Server.StartService(Target, dbcfgs, nil /* mysqld */) if err != nil { diff --git a/go/vt/vttablet/onlineddl/executor.go b/go/vt/vttablet/onlineddl/executor.go index bb8431b5199..e376b67ea4c 100644 --- a/go/vt/vttablet/onlineddl/executor.go +++ b/go/vt/vttablet/onlineddl/executor.go @@ -1476,7 +1476,7 @@ func (e *Executor) initVreplicationOriginalMigration(ctx context.Context, online return v, err } - v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, originalShowCreateTable, vreplShowCreateTable, onlineDDL.SQL, onlineDDL.StrategySetting().IsAnalyzeTableFlag()) + v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, originalShowCreateTable, vreplShowCreateTable, onlineDDL.SQL, onlineDDL.StrategySetting().IsAnalyzeTableFlag(), e.env.CollationEnv()) return v, nil } @@ -1530,7 +1530,7 @@ func (e *Executor) initVreplicationRevertMigration(ctx context.Context, onlineDD if err := e.updateArtifacts(ctx, onlineDDL.UUID, vreplTableName); err != nil { return v, err } - v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, "", "", "", false) + v = NewVRepl(onlineDDL.UUID, e.keyspace, e.shard, e.dbName, onlineDDL.Table, vreplTableName, "", "", "", false, e.env.CollationEnv()) v.pos = revertStream.pos return v, nil } diff --git a/go/vt/vttablet/onlineddl/vrepl.go b/go/vt/vttablet/onlineddl/vrepl.go index 5cdb24ae5e4..34eefbbfd0d 100644 --- a/go/vt/vttablet/onlineddl/vrepl.go +++ b/go/vt/vttablet/onlineddl/vrepl.go @@ -136,6 +136,8 @@ type VRepl struct { parser *vrepl.AlterTableParser convertCharset map[string](*binlogdatapb.CharsetConversion) + + collationEnv *collations.Environment } // NewVRepl creates a VReplication handler for Online DDL @@ -149,6 +151,7 @@ func NewVRepl(workflow string, vreplShowCreateTable string, alterQuery string, analyzeTable bool, + collationEnv *collations.Environment, ) *VRepl { return &VRepl{ workflow: workflow, @@ -165,6 +168,7 @@ func NewVRepl(workflow string, enumToTextMap: map[string]string{}, intToEnumMap: map[string]bool{}, convertCharset: map[string](*binlogdatapb.CharsetConversion){}, + collationEnv: collationEnv, } } @@ -553,11 +557,11 @@ func (v *VRepl) generateFilterQuery(ctx context.Context) error { case sourceCol.Type == vrepl.StringColumnType: // Check source and target charset/encoding. If needed, create // a binlogdatapb.CharsetConversion entry (later written to vreplication) - fromCollation := collations.Local().DefaultCollationForCharset(sourceCol.Charset) + fromCollation := v.collationEnv.DefaultCollationForCharset(sourceCol.Charset) if fromCollation == collations.Unknown { return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Character set %s not supported for column %s", sourceCol.Charset, sourceCol.Name) } - toCollation := collations.Local().DefaultCollationForCharset(targetCol.Charset) + toCollation := v.collationEnv.DefaultCollationForCharset(targetCol.Charset) // Let's see if target col is at all textual if targetCol.Type == vrepl.StringColumnType && toCollation == collations.Unknown { return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Character set %s not supported for column %s", targetCol.Charset, targetCol.Name) diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index e606e71a267..1d8021436b1 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -426,7 +426,7 @@ func (tm *TabletManager) getGTIDFromTimestamp(ctx context.Context, pos replicati Port: connParams.Port, } dbCfgs.SetDbParams(*connParams, *connParams, *connParams) - vsClient := vreplication.NewReplicaConnector(connParams) + vsClient := vreplication.NewReplicaConnector(connParams, tm.CollationEnv) filter := &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{{ diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index d65115990f1..143638e994b 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -155,6 +155,7 @@ type TabletManager struct { UpdateStream binlog.UpdateStreamControl VREngine *vreplication.Engine VDiffEngine *vdiff.Engine + CollationEnv *collations.Environment // tmState manages the TabletManager state. tmState *tmState @@ -206,7 +207,7 @@ type TabletManager struct { } // BuildTabletFromInput builds a tablet record from input parameters. -func BuildTabletFromInput(alias *topodatapb.TabletAlias, port, grpcPort int32, db *dbconfigs.DBConfigs) (*topodatapb.Tablet, error) { +func BuildTabletFromInput(alias *topodatapb.TabletAlias, port, grpcPort int32, db *dbconfigs.DBConfigs, collationEnv *collations.Environment) (*topodatapb.Tablet, error) { hostname := tabletHostname if hostname == "" { var err error @@ -244,14 +245,14 @@ func BuildTabletFromInput(alias *topodatapb.TabletAlias, port, grpcPort int32, d return nil, err } - var charset uint8 + var charset collations.ID if db != nil && db.Charset != "" { - charset, err = collations.Local().ParseConnectionCharset(db.Charset) + charset, err = collationEnv.ParseConnectionCharset(db.Charset) if err != nil { return nil, err } } else { - charset = collations.Local().DefaultConnectionCharset() + charset = collationEnv.DefaultConnectionCharset() } return &topodatapb.Tablet{ diff --git a/go/vt/vttablet/tabletmanager/tm_init_test.go b/go/vt/vttablet/tabletmanager/tm_init_test.go index 16dddba7dfd..b0ab9b9a1e2 100644 --- a/go/vt/vttablet/tabletmanager/tm_init_test.go +++ b/go/vt/vttablet/tabletmanager/tm_init_test.go @@ -71,16 +71,16 @@ func TestStartBuildTabletFromInput(t *testing.T) { Type: topodatapb.TabletType_REPLICA, Tags: map[string]string{}, DbNameOverride: "aa", - DefaultConnCollation: uint32(collations.Default()), + DefaultConnCollation: uint32(collations.MySQL8().DefaultConnectionCharset()), } - gotTablet, err := BuildTabletFromInput(alias, port, grpcport, nil) + gotTablet, err := BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) require.NoError(t, err) // Hostname should be resolved. assert.Equal(t, wantTablet, gotTablet) tabletHostname = "" - gotTablet, err = BuildTabletFromInput(alias, port, grpcport, nil) + gotTablet, err = BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) require.NoError(t, err) assert.NotEqual(t, "", gotTablet.Hostname) @@ -92,7 +92,7 @@ func TestStartBuildTabletFromInput(t *testing.T) { Start: []byte(""), End: []byte("\xc0"), } - gotTablet, err = BuildTabletFromInput(alias, port, grpcport, nil) + gotTablet, err = BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) require.NoError(t, err) // KeyRange check is explicit because the next comparison doesn't // show the diff well enough. @@ -102,25 +102,25 @@ func TestStartBuildTabletFromInput(t *testing.T) { // Invalid inputs. initKeyspace = "" initShard = "0" - _, err = BuildTabletFromInput(alias, port, grpcport, nil) + _, err = BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) assert.Contains(t, err.Error(), "init_keyspace and init_shard must be specified") initKeyspace = "test_keyspace" initShard = "" - _, err = BuildTabletFromInput(alias, port, grpcport, nil) + _, err = BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) assert.Contains(t, err.Error(), "init_keyspace and init_shard must be specified") initShard = "x-y" - _, err = BuildTabletFromInput(alias, port, grpcport, nil) + _, err = BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) assert.Contains(t, err.Error(), "cannot validate shard name") initShard = "0" initTabletType = "bad" - _, err = BuildTabletFromInput(alias, port, grpcport, nil) + _, err = BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) assert.Contains(t, err.Error(), "unknown TabletType bad") initTabletType = "primary" - _, err = BuildTabletFromInput(alias, port, grpcport, nil) + _, err = BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) assert.Contains(t, err.Error(), "invalid init_tablet_type PRIMARY") } @@ -153,10 +153,10 @@ func TestBuildTabletFromInputWithBuildTags(t *testing.T) { Type: topodatapb.TabletType_REPLICA, Tags: servenv.AppVersion.ToStringMap(), DbNameOverride: "aa", - DefaultConnCollation: uint32(collations.Default()), + DefaultConnCollation: uint32(collations.MySQL8().DefaultConnectionCharset()), } - gotTablet, err := BuildTabletFromInput(alias, port, grpcport, nil) + gotTablet, err := BuildTabletFromInput(alias, port, grpcport, nil, collations.MySQL8()) require.NoError(t, err) assert.Equal(t, wantTablet, gotTablet) } diff --git a/go/vt/vttablet/tabletmanager/vdiff/controller.go b/go/vt/vttablet/tabletmanager/vdiff/controller.go index 22b1d3f5374..2fd4e3454c0 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/controller.go +++ b/go/vt/vttablet/tabletmanager/vdiff/controller.go @@ -240,7 +240,7 @@ func (ct *controller) start(ctx context.Context, dbClient binlogplayer.DBClient) return err } - wd, err := newWorkflowDiffer(ct, ct.options) + wd, err := newWorkflowDiffer(ct, ct.options, ct.vde.collationEnv) if err != nil { return err } diff --git a/go/vt/vttablet/tabletmanager/vdiff/engine.go b/go/vt/vttablet/tabletmanager/vdiff/engine.go index 1ccf3dc80e6..541b8d018fc 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/engine.go +++ b/go/vt/vttablet/tabletmanager/vdiff/engine.go @@ -24,6 +24,7 @@ import ( "sync" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/vt/proto/tabletmanagerdata" "vitess.io/vitess/go/vt/proto/topodata" @@ -69,14 +70,17 @@ type Engine struct { // modified behavior for that env, e.g. not starting the retry goroutine. This should // NOT be set in production. fortests bool + + collationEnv *collations.Environment } -func NewEngine(config *tabletenv.TabletConfig, ts *topo.Server, tablet *topodata.Tablet) *Engine { +func NewEngine(config *tabletenv.TabletConfig, ts *topo.Server, tablet *topodata.Tablet, collationEnv *collations.Environment) *Engine { vde := &Engine{ controllers: make(map[int64]*controller), ts: ts, thisTablet: tablet, tmClientFactory: func() tmclient.TabletManagerClient { return tmclient.NewTabletManagerClient() }, + collationEnv: collationEnv, } return vde } @@ -94,6 +98,7 @@ func NewTestEngine(ts *topo.Server, tablet *topodata.Tablet, dbn string, dbcf fu dbClientFactoryDba: dbcf, tmClientFactory: tmcf, fortests: true, + collationEnv: collations.MySQL8(), } return vde } diff --git a/go/vt/vttablet/tabletmanager/vdiff/engine_test.go b/go/vt/vttablet/tabletmanager/vdiff/engine_test.go index 0aedeec415b..ca548a9a478 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/engine_test.go +++ b/go/vt/vttablet/tabletmanager/vdiff/engine_test.go @@ -27,7 +27,6 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql/sqlerror" - "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" diff --git a/go/vt/vttablet/tabletmanager/vdiff/table_differ.go b/go/vt/vttablet/tabletmanager/vdiff/table_differ.go index 2d7f88e1055..0420d9499b1 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/table_differ.go +++ b/go/vt/vttablet/tabletmanager/vdiff/table_differ.go @@ -444,20 +444,21 @@ func (td *tableDiffer) setupRowSorters() { for shard, source := range td.wd.ct.sources { sources[shard] = source.shardStreamer } - td.sourcePrimitive = newMergeSorter(sources, td.tablePlan.comparePKs) + td.sourcePrimitive = newMergeSorter(sources, td.tablePlan.comparePKs, td.wd.collationEnv) // Create a merge sorter for the target. targets := make(map[string]*shardStreamer) targets[td.wd.ct.targetShardStreamer.shard] = td.wd.ct.targetShardStreamer - td.targetPrimitive = newMergeSorter(targets, td.tablePlan.comparePKs) + td.targetPrimitive = newMergeSorter(targets, td.tablePlan.comparePKs, td.wd.collationEnv) // If there were aggregate expressions, we have to re-aggregate // the results, which engine.OrderedAggregate can do. if len(td.tablePlan.aggregates) != 0 { td.sourcePrimitive = &engine.OrderedAggregate{ - Aggregates: td.tablePlan.aggregates, - GroupByKeys: pkColsToGroupByParams(td.tablePlan.pkCols), - Input: td.sourcePrimitive, + Aggregates: td.tablePlan.aggregates, + GroupByKeys: pkColsToGroupByParams(td.tablePlan.pkCols, td.wd.collationEnv), + Input: td.sourcePrimitive, + CollationEnv: td.wd.collationEnv, } } } @@ -672,7 +673,7 @@ func (td *tableDiffer) compare(sourceRow, targetRow []sqltypes.Value, cols []com if collationID == collations.Unknown { collationID = collations.CollationBinaryID } - c, err = evalengine.NullsafeCompare(sourceRow[compareIndex], targetRow[compareIndex], collationID) + c, err = evalengine.NullsafeCompare(sourceRow[compareIndex], targetRow[compareIndex], td.wd.collationEnv, collationID) if err != nil { return 0, err } diff --git a/go/vt/vttablet/tabletmanager/vdiff/table_plan.go b/go/vt/vttablet/tabletmanager/vdiff/table_plan.go index f636eea5cae..f71a36a4fe2 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/table_plan.go +++ b/go/vt/vttablet/tabletmanager/vdiff/table_plan.go @@ -59,7 +59,7 @@ type tablePlan struct { aggregates []*engine.AggregateParams } -func (td *tableDiffer) buildTablePlan(dbClient binlogplayer.DBClient, dbName string) (*tablePlan, error) { +func (td *tableDiffer) buildTablePlan(dbClient binlogplayer.DBClient, dbName string, collationEnv *collations.Environment) (*tablePlan, error) { tp := &tablePlan{ table: td.table, dbName: dbName, @@ -112,7 +112,8 @@ func (td *tableDiffer) buildTablePlan(dbClient binlogplayer.DBClient, dbName str aggregates = append(aggregates, engine.NewAggregateParam( /*opcode*/ opcode.AggregateSum, /*offset*/ len(sourceSelect.SelectExprs)-1, - /*alias*/ "")) + /*alias*/ "", collationEnv), + ) } } default: @@ -152,7 +153,7 @@ func (td *tableDiffer) buildTablePlan(dbClient binlogplayer.DBClient, dbName str }, } - err = tp.findPKs(dbClient, targetSelect) + err = tp.findPKs(dbClient, targetSelect, collationEnv) if err != nil { return nil, err } @@ -176,7 +177,7 @@ func (td *tableDiffer) buildTablePlan(dbClient binlogplayer.DBClient, dbName str } // findPKs identifies PKs and removes them from the columns to do data comparison. -func (tp *tablePlan) findPKs(dbClient binlogplayer.DBClient, targetSelect *sqlparser.Select) error { +func (tp *tablePlan) findPKs(dbClient binlogplayer.DBClient, targetSelect *sqlparser.Select, collationEnv *collations.Environment) error { var orderby sqlparser.OrderBy for _, pk := range tp.table.PrimaryKeyColumns { found := false @@ -210,7 +211,7 @@ func (tp *tablePlan) findPKs(dbClient binlogplayer.DBClient, targetSelect *sqlpa Direction: sqlparser.AscOrder, }) } - if err := tp.getPKColumnCollations(dbClient); err != nil { + if err := tp.getPKColumnCollations(dbClient, collationEnv); err != nil { return vterrors.Wrapf(err, "error getting PK column collations for table %s", tp.table.Name) } tp.orderBy = orderby @@ -222,7 +223,7 @@ func (tp *tablePlan) findPKs(dbClient binlogplayer.DBClient, targetSelect *sqlpa // sorting when we do the merge sort and for the comparisons. It then // saves the collations in the tablePlan's comparePKs column info // structs for those subsequent operations. -func (tp *tablePlan) getPKColumnCollations(dbClient binlogplayer.DBClient) error { +func (tp *tablePlan) getPKColumnCollations(dbClient binlogplayer.DBClient, collationEnv *collations.Environment) error { columnList := make([]string, len(tp.comparePKs)) for i := range tp.comparePKs { columnList[i] = tp.comparePKs[i].colName @@ -246,7 +247,6 @@ func (tp *tablePlan) getPKColumnCollations(dbClient binlogplayer.DBClient) error if qr == nil || len(qr.Rows) != len(tp.comparePKs) { return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unexpected result for query %s: %+v", query, qr) } - collationEnv := collations.Local() for _, row := range qr.Named().Rows { columnName := row["column_name"].ToString() collateName := strings.ToLower(row["collation_name"].ToString()) diff --git a/go/vt/vttablet/tabletmanager/vdiff/utils.go b/go/vt/vttablet/tabletmanager/vdiff/utils.go index dc11dbf249c..07e070976a9 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/utils.go +++ b/go/vt/vttablet/tabletmanager/vdiff/utils.go @@ -33,7 +33,7 @@ import ( ) // newMergeSorter creates an engine.MergeSort based on the shard streamers and pk columns -func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compareColInfo) *engine.MergeSort { +func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compareColInfo, collationEnv *collations.Environment) *engine.MergeSort { prims := make([]engine.StreamExecutor, 0, len(participants)) for _, participant := range participants { prims = append(prims, participant) @@ -46,7 +46,7 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compare if cpk.collation != collations.Unknown { collation = cpk.collation } - ob[i] = evalengine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: evalengine.NewType(sqltypes.Unknown, collation)} + ob[i] = evalengine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: evalengine.NewType(sqltypes.Unknown, collation), CollationEnv: collationEnv} } return &engine.MergeSort{ Primitives: prims, @@ -63,10 +63,10 @@ func encodeString(in string) string { return buf.String() } -func pkColsToGroupByParams(pkCols []int) []*engine.GroupByParams { +func pkColsToGroupByParams(pkCols []int, collationEnv *collations.Environment) []*engine.GroupByParams { var res []*engine.GroupByParams for _, col := range pkCols { - res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1}) + res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, CollationEnv: collationEnv}) } return res } diff --git a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ.go b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ.go index 20a1e2dc295..72c8b360ce0 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ.go +++ b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ.go @@ -24,21 +24,20 @@ import ( "google.golang.org/protobuf/encoding/prototext" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/vtgate/vindexes" - - "vitess.io/vitess/go/vt/schema" - "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/schema" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtctl/schematools" "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/vindexes" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" ) @@ -49,13 +48,16 @@ type workflowDiffer struct { tableDiffers map[string]*tableDiffer // key is table name opts *tabletmanagerdatapb.VDiffOptions + + collationEnv *collations.Environment } -func newWorkflowDiffer(ct *controller, opts *tabletmanagerdatapb.VDiffOptions) (*workflowDiffer, error) { +func newWorkflowDiffer(ct *controller, opts *tabletmanagerdatapb.VDiffOptions, collationEnv *collations.Environment) (*workflowDiffer, error) { wd := &workflowDiffer{ ct: ct, opts: opts, tableDiffers: make(map[string]*tableDiffer, 1), + collationEnv: collationEnv, } return wd, nil } @@ -315,7 +317,7 @@ func (wd *workflowDiffer) buildPlan(dbClient binlogplayer.DBClient, filter *binl } td.lastPK = lastpkpb wd.tableDiffers[table.Name] = td - if _, err := td.buildTablePlan(dbClient, wd.ct.vde.dbName); err != nil { + if _, err := td.buildTablePlan(dbClient, wd.ct.vde.dbName, wd.collationEnv); err != nil { return err } } diff --git a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go index 10c6406f046..29c019750f1 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go +++ b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go @@ -67,8 +67,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -87,8 +87,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 where in_keyrange('-80') order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -107,8 +107,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -127,8 +127,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c2, c1 from t1 order by c1 asc", targetQuery: "select c2, c1 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, - comparePKs: []compareColInfo{{1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + comparePKs: []compareColInfo{{1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{1}, selectPks: []int{1}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -147,8 +147,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c0 as c1, c2 from t2 order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -168,8 +168,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["nonpktext"]], sourceQuery: "select c1, textcol from nonpktext order by c1 asc", targetQuery: "select c1, textcol from nonpktext order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "textcol"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "textcol"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -189,8 +189,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["nonpktext"]], sourceQuery: "select textcol, c1 from nonpktext order by c1 asc", targetQuery: "select textcol, c1 from nonpktext order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), false, "textcol"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, - comparePKs: []compareColInfo{{1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "textcol"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + comparePKs: []compareColInfo{{1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{1}, selectPks: []int{1}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -210,8 +210,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["pktext"]], sourceQuery: "select textcol, c2 from pktext order by textcol asc", targetQuery: "select textcol, c2 from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "textcol"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "textcol"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -231,8 +231,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["pktext"]], sourceQuery: "select c2, textcol from pktext order by textcol asc", targetQuery: "select c2, textcol from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, - comparePKs: []compareColInfo{{1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, + comparePKs: []compareColInfo{{1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, pkCols: []int{1}, selectPks: []int{1}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -252,8 +252,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["pktext"]], sourceQuery: "select c2, a + b as textcol from pktext order by textcol asc", targetQuery: "select c2, textcol from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, - comparePKs: []compareColInfo{{1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, + comparePKs: []compareColInfo{{1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "textcol"}}, pkCols: []int{1}, selectPks: []int{1}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -272,8 +272,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["multipk"]], sourceQuery: "select c1, c2 from multipk order by c1 asc, c2 asc", targetQuery: "select c1, c2 from multipk order by c1 asc, c2 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c2"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c2"}}, pkCols: []int{0, 1}, selectPks: []int{0, 1}, orderBy: sqlparser.OrderBy{ @@ -299,8 +299,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 where in_keyrange('-80') order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -321,8 +321,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 where c2 = 2 and in_keyrange('-80') order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -343,8 +343,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 where in_keyrange('-80') and c2 = 2 order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -365,8 +365,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 where c2 = 2 and c1 = 1 and in_keyrange('-80') order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -387,8 +387,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 where c2 = 2 and in_keyrange('-80') order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -408,8 +408,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["t1"]], sourceQuery: "select c1, c2 from t1 group by c1 order by c1 asc", targetQuery: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -429,8 +429,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["aggr"]], sourceQuery: "select c1, c2, count(*) as c3, sum(c4) as c4 from t1 group by c1 order by c1 asc", targetQuery: "select c1, c2, c3, c4 from aggr order by c1 asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c2"}, {2, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c3"}, {3, collations.Local().LookupByName(sqltypes.NULL.String()), false, "c4"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "c1"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c2"}, {2, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c3"}, {3, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "c4"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "c1"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -438,8 +438,8 @@ func TestBuildPlanSuccess(t *testing.T) { Direction: sqlparser.AscOrder, }}, aggregates: []*engine.AggregateParams{ - engine.NewAggregateParam(opcode.AggregateSum, 2, ""), - engine.NewAggregateParam(opcode.AggregateSum, 3, ""), + engine.NewAggregateParam(opcode.AggregateSum, 2, "", collations.MySQL8()), + engine.NewAggregateParam(opcode.AggregateSum, 3, "", collations.MySQL8()), }, }, }, { @@ -454,8 +454,8 @@ func TestBuildPlanSuccess(t *testing.T) { table: testSchema.TableDefinitions[tableDefMap["datze"]], sourceQuery: "select id, dt from datze order by id asc", targetQuery: "select id, convert_tz(dt, 'UTC', 'US/Pacific') as dt from datze order by id asc", - compareCols: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "id"}, {1, collations.Local().LookupByName(sqltypes.NULL.String()), false, "dt"}}, - comparePKs: []compareColInfo{{0, collations.Local().LookupByName(sqltypes.NULL.String()), true, "id"}}, + compareCols: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "id"}, {1, collations.MySQL8().LookupByName(sqltypes.NULL.String()), false, "dt"}}, + comparePKs: []compareColInfo{{0, collations.MySQL8().LookupByName(sqltypes.NULL.String()), true, "id"}}, pkCols: []int{0}, selectPks: []int{0}, orderBy: sqlparser.OrderBy{&sqlparser.Order{ @@ -478,12 +478,12 @@ func TestBuildPlanSuccess(t *testing.T) { dbc := binlogplayer.NewMockDBClient(t) filter := &binlogdatapb.Filter{Rules: []*binlogdatapb.Rule{tcase.input}} vdiffenv.opts.CoreOptions.Tables = tcase.table - wd, err := newWorkflowDiffer(ct, vdiffenv.opts) + wd, err := newWorkflowDiffer(ct, vdiffenv.opts, collations.MySQL8()) require.NoError(t, err) dbc.ExpectRequestRE("select vdt.lastpk as lastpk, vdt.mismatch as mismatch, vdt.report as report", noResults, nil) columnList := make([]string, len(tcase.tablePlan.comparePKs)) collationList := make([]string, len(tcase.tablePlan.comparePKs)) - env := collations.Local() + env := collations.MySQL8() for i := range tcase.tablePlan.comparePKs { columnList[i] = tcase.tablePlan.comparePKs[i].colName if tcase.tablePlan.comparePKs[i].collation != collations.Unknown { @@ -577,7 +577,7 @@ func TestBuildPlanInclude(t *testing.T) { for _, tcase := range testcases { dbc := binlogplayer.NewMockDBClient(t) vdiffenv.opts.CoreOptions.Tables = strings.Join(tcase.tables, ",") - wd, err := newWorkflowDiffer(ct, vdiffenv.opts) + wd, err := newWorkflowDiffer(ct, vdiffenv.opts, collations.MySQL8()) require.NoError(t, err) for _, table := range tcase.tables { query := fmt.Sprintf(`select vdt.lastpk as lastpk, vdt.mismatch as mismatch, vdt.report as report @@ -650,7 +650,7 @@ func TestBuildPlanFailure(t *testing.T) { dbc := binlogplayer.NewMockDBClient(t) filter := &binlogdatapb.Filter{Rules: []*binlogdatapb.Rule{tcase.input}} vdiffenv.opts.CoreOptions.Tables = tcase.input.Match - wd, err := newWorkflowDiffer(ct, vdiffenv.opts) + wd, err := newWorkflowDiffer(ct, vdiffenv.opts, collations.MySQL8()) require.NoError(t, err) dbc.ExpectRequestRE("select vdt.lastpk as lastpk, vdt.mismatch as mismatch, vdt.report as report", noResults, nil) err = wd.buildPlan(dbc, filter, testSchema) diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine.go b/go/vt/vttablet/tabletmanager/vreplication/engine.go index f230ecce045..4a0be0b83ff 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine.go @@ -28,9 +28,8 @@ import ( "time" "vitess.io/vitess/go/constants/sidecar" - + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/sqlerror" - "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/dbconfigs" @@ -111,6 +110,8 @@ type Engine struct { // enabled in NewSimpleTestEngine. This should NOT be used in // production. shortcircuit bool + + collationEnv *collations.Environment } type journalEvent struct { @@ -127,15 +128,16 @@ type PostCopyAction struct { // NewEngine creates a new Engine. // A nil ts means that the Engine is disabled. -func NewEngine(config *tabletenv.TabletConfig, ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, lagThrottler *throttle.Throttler) *Engine { +func NewEngine(config *tabletenv.TabletConfig, ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, lagThrottler *throttle.Throttler, collationEnv *collations.Environment) *Engine { vre := &Engine{ controllers: make(map[int32]*controller), ts: ts, cell: cell, mysqld: mysqld, journaler: make(map[string]*journalEvent), - ec: newExternalConnector(config.ExternalConnections), + ec: newExternalConnector(config.ExternalConnections, collationEnv), throttlerClient: throttle.NewBackgroundClient(lagThrottler, throttlerapp.VReplicationName, throttle.ThrottleCheckPrimaryWrite), + collationEnv: collationEnv, } return vre @@ -167,7 +169,8 @@ func NewTestEngine(ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, db dbClientFactoryDba: dbClientFactoryDba, dbName: dbname, journaler: make(map[string]*journalEvent), - ec: newExternalConnector(externalConfig), + ec: newExternalConnector(externalConfig, collations.MySQL8()), + collationEnv: collations.MySQL8(), } return vre } @@ -184,8 +187,9 @@ func NewSimpleTestEngine(ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaem dbClientFactoryDba: dbClientFactoryDba, dbName: dbname, journaler: make(map[string]*journalEvent), - ec: newExternalConnector(externalConfig), + ec: newExternalConnector(externalConfig, collations.MySQL8()), shortcircuit: true, + collationEnv: collations.MySQL8(), } return vre } diff --git a/go/vt/vttablet/tabletmanager/vreplication/external_connector.go b/go/vt/vttablet/tabletmanager/vreplication/external_connector.go index 1c20e2054be..a0a230dbe28 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/external_connector.go +++ b/go/vt/vttablet/tabletmanager/vreplication/external_connector.go @@ -17,10 +17,10 @@ limitations under the License. package vreplication import ( - "sync" - "context" + "sync" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/grpcclient" @@ -58,15 +58,17 @@ type VStreamerClient interface { } type externalConnector struct { - mu sync.Mutex - dbconfigs map[string]*dbconfigs.DBConfigs - connectors map[string]*mysqlConnector + mu sync.Mutex + dbconfigs map[string]*dbconfigs.DBConfigs + connectors map[string]*mysqlConnector + collationEnv *collations.Environment } -func newExternalConnector(dbcfgs map[string]*dbconfigs.DBConfigs) *externalConnector { +func newExternalConnector(dbcfgs map[string]*dbconfigs.DBConfigs, collationEnv *collations.Environment) *externalConnector { return &externalConnector{ - dbconfigs: dbcfgs, - connectors: make(map[string]*mysqlConnector), + dbconfigs: dbcfgs, + connectors: make(map[string]*mysqlConnector), + collationEnv: collationEnv, } } @@ -91,7 +93,7 @@ func (ec *externalConnector) Get(name string) (*mysqlConnector, error) { return nil, vterrors.Errorf(vtrpcpb.Code_NOT_FOUND, "external mysqlConnector %v not found", name) } c := &mysqlConnector{} - c.env = tabletenv.NewEnv(config, name) + c.env = tabletenv.NewEnv(config, name, ec.collationEnv) c.se = schema.NewEngine(c.env) c.vstreamer = vstreamer.NewEngine(c.env, nil, c.se, nil, "") c.vstreamer.InitDBConfig("", "") diff --git a/go/vt/vttablet/tabletmanager/vreplication/replica_connector.go b/go/vt/vttablet/tabletmanager/vreplication/replica_connector.go index 9c6f427b418..b87b3896f31 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replica_connector.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replica_connector.go @@ -18,6 +18,7 @@ package vreplication import ( "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp" @@ -38,7 +39,7 @@ import ( // This is used by binlog server to make vstream connection // using the vstream connection, it will parse the events from binglog // to fetch the corresponding GTID for required recovery time -func NewReplicaConnector(connParams *mysql.ConnParams) *ReplicaConnector { +func NewReplicaConnector(connParams *mysql.ConnParams, collationEnv *collations.Environment) *ReplicaConnector { // Construct config := tabletenv.NewDefaultConfig() @@ -49,7 +50,7 @@ func NewReplicaConnector(connParams *mysql.ConnParams) *ReplicaConnector { dbCfg.SetDbParams(*connParams, *connParams, *connParams) config.DB = dbCfg c := &ReplicaConnector{conn: connParams} - env := tabletenv.NewEnv(config, "source") + env := tabletenv.NewEnv(config, "source", collationEnv) c.se = schema.NewEngine(env) c.se.SkipMetaCheck = true c.vstreamer = vstreamer.NewEngine(env, nil, c.se, nil, "") diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 9ecf8669d6d..d2590975bb6 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -59,6 +59,7 @@ type ReplicatorPlan struct { ColInfoMap map[string][]*ColumnInfo stats *binlogplayer.Stats Source *binlogdatapb.BinlogSource + collationEnv *collations.Environment } // buildExecution plan uses the field info as input and the partially built @@ -98,11 +99,12 @@ func (rp *ReplicatorPlan) buildExecutionPlan(fieldEvent *binlogdatapb.FieldEvent // requires us to wait for the field info sent by the source. func (rp *ReplicatorPlan) buildFromFields(tableName string, lastpk *sqltypes.Result, fields []*querypb.Field) (*TablePlan, error) { tpb := &tablePlanBuilder{ - name: sqlparser.NewIdentifierCS(tableName), - lastpk: lastpk, - colInfos: rp.ColInfoMap[tableName], - stats: rp.stats, - source: rp.Source, + name: sqlparser.NewIdentifierCS(tableName), + lastpk: lastpk, + colInfos: rp.ColInfoMap[tableName], + stats: rp.stats, + source: rp.Source, + collationEnv: rp.collationEnv, } for _, field := range fields { colName := sqlparser.NewIdentifierCI(field.Name) @@ -217,6 +219,8 @@ type TablePlan struct { PartialInserts map[string]*sqlparser.ParsedQuery // PartialUpdates are same as PartialInserts, but for update statements PartialUpdates map[string]*sqlparser.ParsedQuery + + CollationEnv *collations.Environment } // MarshalJSON performs a custom JSON Marshalling. @@ -299,7 +303,7 @@ func (tp *TablePlan) isOutsidePKRange(bindvars map[string]*querypb.BindVariable, rowVal, _ := sqltypes.BindVariableToValue(bindvar) // TODO(king-11) make collation aware - result, err := evalengine.NullsafeCompare(rowVal, tp.Lastpk.Rows[0][0], collations.Unknown) + result, err := evalengine.NullsafeCompare(rowVal, tp.Lastpk.Rows[0][0], tp.CollationEnv, collations.Unknown) // If rowVal is > last pk, transaction will be a noop, so don't apply this statement if err == nil && result > 0 { tp.Stats.NoopQueryCount.Add(stmtType, 1) @@ -317,7 +321,7 @@ func (tp *TablePlan) isOutsidePKRange(bindvars map[string]*querypb.BindVariable, func (tp *TablePlan) bindFieldVal(field *querypb.Field, val *sqltypes.Value) (*querypb.BindVariable, error) { if conversion, ok := tp.ConvertCharset[field.Name]; ok && !val.IsNull() { // Non-null string value, for which we have a charset conversion instruction - fromCollation := collations.Local().DefaultCollationForCharset(conversion.FromCharset) + fromCollation := tp.CollationEnv.DefaultCollationForCharset(conversion.FromCharset) if fromCollation == collations.Unknown { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Character set %s not supported for column %s", conversion.FromCharset, field.Name) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go index 780b1c0d064..ce8dc61fd38 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/binlog/binlogplayer" "github.com/stretchr/testify/assert" @@ -733,7 +734,7 @@ func TestBuildPlayerPlan(t *testing.T) { } for _, tcase := range testcases { - plan, err := buildReplicatorPlan(getSource(tcase.input), PrimaryKeyInfos, nil, binlogplayer.NewStats()) + plan, err := buildReplicatorPlan(getSource(tcase.input), PrimaryKeyInfos, nil, binlogplayer.NewStats(), collations.MySQL8()) gotPlan, _ := json.Marshal(plan) wantPlan, _ := json.Marshal(tcase.plan) if string(gotPlan) != string(wantPlan) { @@ -747,7 +748,7 @@ func TestBuildPlayerPlan(t *testing.T) { t.Errorf("Filter err(%v): %s, want %v", tcase.input, gotErr, tcase.err) } - plan, err = buildReplicatorPlan(getSource(tcase.input), PrimaryKeyInfos, copyState, binlogplayer.NewStats()) + plan, err = buildReplicatorPlan(getSource(tcase.input), PrimaryKeyInfos, copyState, binlogplayer.NewStats(), collations.MySQL8()) if err != nil { continue } @@ -777,7 +778,7 @@ func TestBuildPlayerPlanNoDup(t *testing.T) { Filter: "select * from t", }}, } - _, err := buildReplicatorPlan(getSource(input), PrimaryKeyInfos, nil, binlogplayer.NewStats()) + _, err := buildReplicatorPlan(getSource(input), PrimaryKeyInfos, nil, binlogplayer.NewStats(), collations.MySQL8()) want := "more than one target for source table t" if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("buildReplicatorPlan err: %v, must contain: %v", err, want) @@ -798,7 +799,7 @@ func TestBuildPlayerPlanExclude(t *testing.T) { Filter: "", }}, } - plan, err := buildReplicatorPlan(getSource(input), PrimaryKeyInfos, nil, binlogplayer.NewStats()) + plan, err := buildReplicatorPlan(getSource(input), PrimaryKeyInfos, nil, binlogplayer.NewStats(), collations.MySQL8()) assert.NoError(t, err) want := &TestReplicatorPlan{ diff --git a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go index 715d87186a6..69e712e9655 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go +++ b/go/vt/vttablet/tabletmanager/vreplication/table_plan_builder.go @@ -22,6 +22,7 @@ import ( "sort" "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/textutil" "vitess.io/vitess/go/vt/binlog/binlogplayer" @@ -60,6 +61,8 @@ type tablePlanBuilder struct { stats *binlogplayer.Stats source *binlogdatapb.BinlogSource pkIndices []bool + + collationEnv *collations.Environment } // colExpr describes the processing to be performed to @@ -129,7 +132,7 @@ const ( // The TablePlan built is a partial plan. The full plan for a table is built // when we receive field information from events or rows sent by the source. // buildExecutionPlan is the function that builds the full plan. -func buildReplicatorPlan(source *binlogdatapb.BinlogSource, colInfoMap map[string][]*ColumnInfo, copyState map[string]*sqltypes.Result, stats *binlogplayer.Stats) (*ReplicatorPlan, error) { +func buildReplicatorPlan(source *binlogdatapb.BinlogSource, colInfoMap map[string][]*ColumnInfo, copyState map[string]*sqltypes.Result, stats *binlogplayer.Stats, collationEnv *collations.Environment) (*ReplicatorPlan, error) { filter := source.Filter plan := &ReplicatorPlan{ VStreamFilter: &binlogdatapb.Filter{FieldEventMode: filter.FieldEventMode}, @@ -138,6 +141,7 @@ func buildReplicatorPlan(source *binlogdatapb.BinlogSource, colInfoMap map[strin ColInfoMap: colInfoMap, stats: stats, Source: source, + collationEnv: collationEnv, } for tableName := range colInfoMap { lastpk, ok := copyState[tableName] @@ -156,7 +160,7 @@ func buildReplicatorPlan(source *binlogdatapb.BinlogSource, colInfoMap map[strin if !ok { return nil, fmt.Errorf("table %s not found in schema", tableName) } - tablePlan, err := buildTablePlan(tableName, rule, colInfos, lastpk, stats, source) + tablePlan, err := buildTablePlan(tableName, rule, colInfos, lastpk, stats, source, collationEnv) if err != nil { return nil, err } @@ -196,7 +200,7 @@ func MatchTable(tableName string, filter *binlogdatapb.Filter) (*binlogdatapb.Ru } func buildTablePlan(tableName string, rule *binlogdatapb.Rule, colInfos []*ColumnInfo, lastpk *sqltypes.Result, - stats *binlogplayer.Stats, source *binlogdatapb.BinlogSource) (*TablePlan, error) { + stats *binlogplayer.Stats, source *binlogdatapb.BinlogSource, collationEnv *collations.Environment) (*TablePlan, error) { filter := rule.Filter query := filter @@ -245,6 +249,7 @@ func buildTablePlan(tableName string, rule *binlogdatapb.Rule, colInfos []*Colum EnumValuesMap: enumValuesMap, ConvertCharset: rule.ConvertCharset, ConvertIntToEnum: rule.ConvertIntToEnum, + CollationEnv: collationEnv, } return tablePlan, nil @@ -256,10 +261,11 @@ func buildTablePlan(tableName string, rule *binlogdatapb.Rule, colInfos []*Colum From: sel.From, Where: sel.Where, }, - lastpk: lastpk, - colInfos: colInfos, - stats: stats, - source: source, + lastpk: lastpk, + colInfos: colInfos, + stats: stats, + source: source, + collationEnv: collationEnv, } if err := tpb.analyzeExprs(sel.SelectExprs); err != nil { @@ -371,6 +377,7 @@ func (tpb *tablePlanBuilder) generate() *TablePlan { TablePlanBuilder: tpb, PartialInserts: make(map[string]*sqlparser.ParsedQuery, 0), PartialUpdates: make(map[string]*sqlparser.ParsedQuery, 0), + CollationEnv: tpb.collationEnv, } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index 196ee6aac86..01b11a26a1e 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -219,7 +219,7 @@ func newVCopierCopyWorker( func (vc *vcopier) initTablesForCopy(ctx context.Context) error { defer vc.vr.dbClient.Rollback() - plan, err := buildReplicatorPlan(vc.vr.source, vc.vr.colInfoMap, nil, vc.vr.stats) + plan, err := buildReplicatorPlan(vc.vr.source, vc.vr.colInfoMap, nil, vc.vr.stats, vc.vr.vre.collationEnv) if err != nil { return err } @@ -385,7 +385,7 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma log.Infof("Copying table %s, lastpk: %v", tableName, copyState[tableName]) - plan, err := buildReplicatorPlan(vc.vr.source, vc.vr.colInfoMap, nil, vc.vr.stats) + plan, err := buildReplicatorPlan(vc.vr.source, vc.vr.colInfoMap, nil, vc.vr.stats, vc.vr.vre.collationEnv) if err != nil { return err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier_atomic.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier_atomic.go index fe92f284ce8..667ea9615f3 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier_atomic.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier_atomic.go @@ -54,7 +54,7 @@ func newCopyAllState(vc *vcopier) (*copyAllState, error) { state := ©AllState{ vc: vc, } - plan, err := buildReplicatorPlan(vc.vr.source, vc.vr.colInfoMap, nil, vc.vr.stats) + plan, err := buildReplicatorPlan(vc.vr.source, vc.vr.colInfoMap, nil, vc.vr.stats, vc.vr.vre.collationEnv) if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index c222bc11781..e15b1f12be4 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -180,7 +180,7 @@ func (vp *vplayer) play(ctx context.Context) error { return nil } - plan, err := buildReplicatorPlan(vp.vr.source, vp.vr.colInfoMap, vp.copyState, vp.vr.stats) + plan, err := buildReplicatorPlan(vp.vr.source, vp.vr.colInfoMap, vp.copyState, vp.vr.stats, vp.vr.vre.collationEnv) if err != nil { vp.vr.stats.ErrorCounts.Add([]string{"Plan"}, 1) return err diff --git a/go/vt/vttablet/tabletserver/connpool/dbconn_test.go b/go/vt/vttablet/tabletserver/connpool/dbconn_test.go index 9717c95d9f7..33c37d2b2c6 100644 --- a/go/vt/vttablet/tabletserver/connpool/dbconn_test.go +++ b/go/vt/vttablet/tabletserver/connpool/dbconn_test.go @@ -27,11 +27,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/pools/smartconnpool" - - "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/dbconfigs" querypb "vitess.io/vitess/go/vt/proto/query" ) @@ -62,11 +62,12 @@ func TestDBConnExec(t *testing.T) { connPool := newPool() mysqlTimings := connPool.env.Stats().MySQLTimings startCounts := mysqlTimings.Counts() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second)) defer cancel() - dbConn, err := newPooledConn(context.Background(), connPool, db.ConnParams()) + dbConn, err := newPooledConn(context.Background(), connPool, params) if dbConn != nil { defer dbConn.Close() } @@ -135,11 +136,12 @@ func TestDBConnExecLost(t *testing.T) { connPool := newPool() mysqlTimings := connPool.env.Stats().MySQLTimings startCounts := mysqlTimings.Counts() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second)) defer cancel() - dbConn, err := newPooledConn(context.Background(), connPool, db.ConnParams()) + dbConn, err := newPooledConn(context.Background(), connPool, params) if dbConn != nil { defer dbConn.Close() } @@ -193,14 +195,15 @@ func TestDBConnDeadline(t *testing.T) { connPool := newPool() mysqlTimings := connPool.env.Stats().MySQLTimings startCounts := mysqlTimings.Counts() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() db.SetConnDelay(100 * time.Millisecond) ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(50*time.Millisecond)) defer cancel() - dbConn, err := newPooledConn(context.Background(), connPool, db.ConnParams()) + dbConn, err := newPooledConn(context.Background(), connPool, params) if dbConn != nil { defer dbConn.Close() } @@ -251,9 +254,10 @@ func TestDBConnKill(t *testing.T) { db := fakesqldb.New(t) defer db.Close() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() - dbConn, err := newPooledConn(context.Background(), connPool, db.ConnParams()) + dbConn, err := newPooledConn(context.Background(), connPool, params) if dbConn != nil { defer dbConn.Close() } @@ -297,9 +301,10 @@ func TestDBConnClose(t *testing.T) { db := fakesqldb.New(t) defer db.Close() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() - dbConn, err := newPooledConn(context.Background(), connPool, db.ConnParams()) + dbConn, err := newPooledConn(context.Background(), connPool, params) require.NoError(t, err) defer dbConn.Close() @@ -322,9 +327,10 @@ func TestDBConnClose(t *testing.T) { func TestDBNoPoolConnKill(t *testing.T) { db := fakesqldb.New(t) connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() - dbConn, err := NewConn(context.Background(), db.ConnParams(), connPool.dbaPool, nil) + dbConn, err := NewConn(context.Background(), params, connPool.dbaPool, nil) if dbConn != nil { defer dbConn.Close() } @@ -376,11 +382,12 @@ func TestDBConnStream(t *testing.T) { } db.AddQuery(sql, expectedResult) connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second)) defer cancel() - dbConn, err := newPooledConn(context.Background(), connPool, db.ConnParams()) + dbConn, err := newPooledConn(context.Background(), connPool, params) if dbConn != nil { defer dbConn.Close() } @@ -436,9 +443,10 @@ func TestDBConnStreamKill(t *testing.T) { } db.AddQuery(sql, expectedResult) connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() - dbConn, err := newPooledConn(context.Background(), connPool, db.ConnParams()) + dbConn, err := newPooledConn(context.Background(), connPool, params) require.NoError(t, err) defer dbConn.Close() @@ -465,10 +473,11 @@ func TestDBConnReconnect(t *testing.T) { defer db.Close() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() - dbConn, err := newPooledConn(context.Background(), connPool, db.ConnParams()) + dbConn, err := newPooledConn(context.Background(), connPool, params) require.NoError(t, err) defer dbConn.Close() @@ -490,11 +499,12 @@ func TestDBConnReApplySetting(t *testing.T) { db.OrderMatters() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() ctx := context.Background() - dbConn, err := newPooledConn(ctx, connPool, db.ConnParams()) + dbConn, err := newPooledConn(ctx, connPool, params) require.NoError(t, err) defer dbConn.Close() diff --git a/go/vt/vttablet/tabletserver/connpool/pool_test.go b/go/vt/vttablet/tabletserver/connpool/pool_test.go index ff43388d12c..f326392b83e 100644 --- a/go/vt/vttablet/tabletserver/connpool/pool_test.go +++ b/go/vt/vttablet/tabletserver/connpool/pool_test.go @@ -24,10 +24,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/pools/smartconnpool" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/callerid" + "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) @@ -35,7 +37,8 @@ func TestConnPoolGet(t *testing.T) { db := fakesqldb.New(t) defer db.Close() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() dbConn, err := connPool.Get(context.Background(), nil) if err != nil { @@ -56,8 +59,9 @@ func TestConnPoolTimeout(t *testing.T) { } cfg.Timeout = time.Second cfg.IdleTimeout = 10 * time.Second - connPool := NewPool(tabletenv.NewEnv(nil, "PoolTest"), "TestPool", cfg) - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + connPool := NewPool(tabletenv.NewEnv(nil, "PoolTest", collations.MySQL8()), "TestPool", cfg) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() dbConn, err := connPool.Get(context.Background(), nil) require.NoError(t, err) @@ -68,10 +72,11 @@ func TestConnPoolTimeout(t *testing.T) { func TestConnPoolGetEmptyDebugConfig(t *testing.T) { db := fakesqldb.New(t) - debugConn := db.ConnParamsWithUname("") + debugConn := dbconfigs.New(db.ConnParamsWithUname("")) defer db.Close() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), debugConn) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, debugConn) im := callerid.NewImmediateCallerID("") ecid := callerid.NewEffectiveCallerID("p", "c", "sc") ctx := context.Background() @@ -89,14 +94,15 @@ func TestConnPoolGetEmptyDebugConfig(t *testing.T) { func TestConnPoolGetAppDebug(t *testing.T) { db := fakesqldb.New(t) - debugConn := db.ConnParamsWithUname("debugUsername") + debugConn := dbconfigs.New(db.ConnParamsWithUname("debugUsername")) ctx := context.Background() im := callerid.NewImmediateCallerID("debugUsername") ecid := callerid.NewEffectiveCallerID("p", "c", "sc") ctx = callerid.NewContext(ctx, ecid, im) defer db.Close() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), debugConn) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, debugConn) defer connPool.Close() dbConn, err := connPool.Get(ctx, nil) if err != nil { @@ -115,7 +121,8 @@ func TestConnPoolSetCapacity(t *testing.T) { db := fakesqldb.New(t) defer db.Close() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() assert.Panics(t, func() { @@ -134,7 +141,8 @@ func TestConnPoolStatJSON(t *testing.T) { if connPool.StatsJSON() != "{}" { t.Fatalf("pool is closed, stats json should be empty; was: %q", connPool.StatsJSON()) } - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() statsJSON := connPool.StatsJSON() if statsJSON == "" || statsJSON == "{}" { @@ -153,7 +161,8 @@ func TestConnPoolStateWhilePoolIsOpen(t *testing.T) { defer db.Close() idleTimeout := 10 * time.Second connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() assert.EqualValues(t, 100, connPool.Capacity(), "pool capacity should be 100") assert.EqualValues(t, 0, connPool.Metrics.WaitTime(), "pool wait time should be 0") @@ -179,7 +188,8 @@ func TestConnPoolStateWithSettings(t *testing.T) { defer db.Close() capacity := 5 connPool := newPoolWithCapacity(capacity) - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() assert.EqualValues(t, 5, connPool.Available(), "pool available connections should be 5") assert.EqualValues(t, 0, connPool.Active(), "pool active connections should be 0") @@ -294,7 +304,8 @@ func TestPoolGetConnTime(t *testing.T) { defer db.Close() connPool := newPool() - connPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + connPool.Open(params, params, params) defer connPool.Close() connPool.getConnTime.Reset() @@ -325,7 +336,7 @@ func newPool() *Pool { } func newPoolWithCapacity(capacity int) *Pool { - return NewPool(tabletenv.NewEnv(nil, "PoolTest"), "TestPool", tabletenv.ConnPoolConfig{ + return NewPool(tabletenv.NewEnv(nil, "PoolTest", collations.MySQL8()), "TestPool", tabletenv.ConnPoolConfig{ Size: capacity, IdleTimeout: 10 * time.Second, }) diff --git a/go/vt/vttablet/tabletserver/fuzz.go b/go/vt/vttablet/tabletserver/fuzz.go index fb14455d3f4..6cb0f60ff93 100644 --- a/go/vt/vttablet/tabletserver/fuzz.go +++ b/go/vt/vttablet/tabletserver/fuzz.go @@ -23,6 +23,7 @@ import ( fuzz "github.com/AdaLogics/go-fuzz-headers" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" @@ -57,7 +58,7 @@ func FuzzGetPlan(data []byte) int { // Set up the environment config := tabletenv.NewDefaultConfig() config.DB = newDBConfigs(db) - env := tabletenv.NewEnv(config, "TabletServerTest") + env := tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8()) se := schema.NewEngine(env) qe := NewQueryEngine(env, se) defer qe.Close() diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go index 3220bd3ffe7..c43e9598d89 100644 --- a/go/vt/vttablet/tabletserver/health_streamer_test.go +++ b/go/vt/vttablet/tabletserver/health_streamer_test.go @@ -30,10 +30,11 @@ import ( "google.golang.org/protobuf/proto" "vitess.io/vitess/go/constants/sidecar" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/dbconfigs" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/sqlparser" @@ -45,7 +46,7 @@ func TestHealthStreamerClosed(t *testing.T) { db := fakesqldb.New(t) defer db.Close() config := newConfig(db) - env := tabletenv.NewEnv(config, "ReplTrackerTest") + env := tabletenv.NewEnv(config, "ReplTrackerTest", collations.MySQL8()) alias := &topodatapb.TabletAlias{ Cell: "cell", Uid: 1, @@ -72,7 +73,7 @@ func TestNotServingPrimaryNoWrite(t *testing.T) { config := newConfig(db) config.SignalWhenSchemaChange = true - env := tabletenv.NewEnv(config, "TestNotServingPrimary") + env := tabletenv.NewEnv(config, "TestNotServingPrimary", collations.MySQL8()) alias := &topodatapb.TabletAlias{ Cell: "cell", Uid: 1, @@ -84,7 +85,7 @@ func TestNotServingPrimaryNoWrite(t *testing.T) { hs.Open() defer hs.Close() target := &querypb.Target{} - hs.InitDBConfig(target, db.ConnParams()) + hs.InitDBConfig(target, dbconfigs.New(db.ConnParams())) // Let's say the tablet goes to a non-serving primary state. hs.MakePrimary(false) @@ -103,7 +104,7 @@ func TestHealthStreamerBroadcast(t *testing.T) { config := newConfig(db) config.SignalWhenSchemaChange = false - env := tabletenv.NewEnv(config, "ReplTrackerTest") + env := tabletenv.NewEnv(config, "ReplTrackerTest", collations.MySQL8()) alias := &topodatapb.TabletAlias{ Cell: "cell", Uid: 1, @@ -114,7 +115,7 @@ func TestHealthStreamerBroadcast(t *testing.T) { hs.Open() defer hs.Close() target := &querypb.Target{} - hs.InitDBConfig(target, db.ConnParams()) + hs.InitDBConfig(target, dbconfigs.New(db.ConnParams())) ch, cancel := testStream(hs) defer cancel() @@ -218,7 +219,7 @@ func TestReloadSchema(t *testing.T) { config.SignalWhenSchemaChange = testcase.enableSchemaChange config.SchemaReloadInterval = 100 * time.Millisecond - env := tabletenv.NewEnv(config, "ReplTrackerTest") + env := tabletenv.NewEnv(config, "ReplTrackerTest", collations.MySQL8()) alias := &topodatapb.TabletAlias{ Cell: "cell", Uid: 1, @@ -336,7 +337,7 @@ func TestReloadView(t *testing.T) { config.SchemaReloadInterval = 100 * time.Millisecond config.EnableViews = true - env := tabletenv.NewEnv(config, "TestReloadView") + env := tabletenv.NewEnv(config, "TestReloadView", collations.MySQL8()) alias := &topodatapb.TabletAlias{Cell: "cell", Uid: 1} se := schema.NewEngine(env) hs := newHealthStreamer(env, alias, se) diff --git a/go/vt/vttablet/tabletserver/messager/engine_test.go b/go/vt/vttablet/tabletserver/messager/engine_test.go index e134a6fbe21..e85abe8b46d 100644 --- a/go/vt/vttablet/tabletserver/messager/engine_test.go +++ b/go/vt/vttablet/tabletserver/messager/engine_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/sqlparser" @@ -67,9 +67,7 @@ var ( ) func TestEngineSchemaChanged(t *testing.T) { - db := fakesqldb.New(t) - defer db.Close() - engine := newTestEngine(db) + engine := newTestEngine() defer engine.Close() engine.schemaChanged(nil, []*schema.Table{meTableT1, tableT2}, nil, nil) @@ -110,9 +108,7 @@ func extractManagerNames(in map[string]*messageManager) map[string]bool { } func TestSubscribe(t *testing.T) { - db := fakesqldb.New(t) - defer db.Close() - engine := newTestEngine(db) + engine := newTestEngine() engine.schemaChanged(nil, []*schema.Table{meTableT1, meTableT2}, nil, nil) f1, ch1 := newEngineReceiver() f2, ch2 := newEngineReceiver() @@ -142,9 +138,7 @@ func TestSubscribe(t *testing.T) { } func TestEngineGenerate(t *testing.T) { - db := fakesqldb.New(t) - defer db.Close() - engine := newTestEngine(db) + engine := newTestEngine() defer engine.Close() engine.schemaChanged(nil, []*schema.Table{meTableT1}, nil, nil) @@ -157,10 +151,10 @@ func TestEngineGenerate(t *testing.T) { } } -func newTestEngine(db *fakesqldb.DB) *Engine { +func newTestEngine() *Engine { config := tabletenv.NewDefaultConfig() tsv := &fakeTabletServer{ - Env: tabletenv.NewEnv(config, "MessagerTest"), + Env: tabletenv.NewEnv(config, "MessagerTest", collations.MySQL8()), } se := schema.NewEngine(tsv) te := NewEngine(tsv, se, newFakeVStreamer()) diff --git a/go/vt/vttablet/tabletserver/messager/message_manager_test.go b/go/vt/vttablet/tabletserver/messager/message_manager_test.go index 5c5ab47d3c8..b4154a21f8d 100644 --- a/go/vt/vttablet/tabletserver/messager/message_manager_test.go +++ b/go/vt/vttablet/tabletserver/messager/message_manager_test.go @@ -31,6 +31,7 @@ import ( "github.com/stretchr/testify/assert" "golang.org/x/sync/semaphore" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/sqlparser" @@ -833,7 +834,7 @@ type fakeTabletServer struct { func newFakeTabletServer() *fakeTabletServer { config := tabletenv.NewDefaultConfig() return &fakeTabletServer{ - Env: tabletenv.NewEnv(config, "MessagerTest"), + Env: tabletenv.NewEnv(config, "MessagerTest", collations.MySQL8()), } } diff --git a/go/vt/vttablet/tabletserver/planbuilder/builder.go b/go/vt/vttablet/tabletserver/planbuilder/builder.go index 4594f6350f6..b8e88916e30 100644 --- a/go/vt/vttablet/tabletserver/planbuilder/builder.go +++ b/go/vt/vttablet/tabletserver/planbuilder/builder.go @@ -19,6 +19,7 @@ package planbuilder import ( "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -27,7 +28,7 @@ import ( vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) -func analyzeSelect(sel *sqlparser.Select, tables map[string]*schema.Table) (plan *Plan, err error) { +func analyzeSelect(sel *sqlparser.Select, tables map[string]*schema.Table, collationEnv *collations.Environment) (plan *Plan, err error) { plan = &Plan{ PlanID: PlanSelect, FullQuery: GenerateLimitQuery(sel), @@ -48,7 +49,10 @@ func analyzeSelect(sel *sqlparser.Select, tables map[string]*schema.Table) (plan return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "%s is not a sequence", sqlparser.ToString(sel.From)) } plan.PlanID = PlanNextval - v, err := evalengine.Translate(nextVal.Expr, nil) + v, err := evalengine.Translate(nextVal.Expr, &evalengine.Config{ + CollationEnv: collationEnv, + Collation: collationEnv.DefaultConnectionCharset(), + }) if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletserver/planbuilder/plan.go b/go/vt/vttablet/tabletserver/planbuilder/plan.go index 6f491692241..97962be553e 100644 --- a/go/vt/vttablet/tabletserver/planbuilder/plan.go +++ b/go/vt/vttablet/tabletserver/planbuilder/plan.go @@ -20,11 +20,11 @@ import ( "encoding/json" "strings" - "vitess.io/vitess/go/vt/vtgate/evalengine" - + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/tableacl" "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -202,7 +202,7 @@ func (plan *Plan) TableNames() (names []string) { } // Build builds a plan based on the schema. -func Build(statement sqlparser.Statement, tables map[string]*schema.Table, dbName string, viewsEnabled bool) (plan *Plan, err error) { +func Build(statement sqlparser.Statement, tables map[string]*schema.Table, dbName string, viewsEnabled bool, collationEnv *collations.Environment) (plan *Plan, err error) { switch stmt := statement.(type) { case *sqlparser.Union: plan, err = &Plan{ @@ -210,7 +210,7 @@ func Build(statement sqlparser.Statement, tables map[string]*schema.Table, dbNam FullQuery: GenerateLimitQuery(stmt), }, nil case *sqlparser.Select: - plan, err = analyzeSelect(stmt, tables) + plan, err = analyzeSelect(stmt, tables, collationEnv) case *sqlparser.Insert: plan, err = analyzeInsert(stmt, tables) case *sqlparser.Update: diff --git a/go/vt/vttablet/tabletserver/planbuilder/plan_test.go b/go/vt/vttablet/tabletserver/planbuilder/plan_test.go index 7c1f364cac8..195a8037210 100644 --- a/go/vt/vttablet/tabletserver/planbuilder/plan_test.go +++ b/go/vt/vttablet/tabletserver/planbuilder/plan_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/tableacl" "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" @@ -83,7 +84,7 @@ func testPlan(t *testing.T, fileName string) { var err error statement, err := sqlparser.Parse(tcase.input) if err == nil { - plan, err = Build(statement, testSchema, "dbName", false) + plan, err = Build(statement, testSchema, "dbName", false, collations.MySQL8()) } PassthroughDMLs = false @@ -120,7 +121,7 @@ func TestPlanInReservedConn(t *testing.T) { var err error statement, err := sqlparser.Parse(tcase.input) if err == nil { - plan, err = Build(statement, testSchema, "dbName", false) + plan, err = Build(statement, testSchema, "dbName", false, collations.MySQL8()) } PassthroughDMLs = false @@ -171,7 +172,7 @@ func TestCustom(t *testing.T) { if err != nil { t.Fatalf("Got error: %v, parsing sql: %v", err.Error(), tcase.input) } - plan, err := Build(statement, schem, "dbName", false) + plan, err := Build(statement, schem, "dbName", false, collations.MySQL8()) var out string if err != nil { out = err.Error() @@ -258,7 +259,7 @@ func TestLockPlan(t *testing.T) { var err error statement, err := sqlparser.Parse(tcase.input) if err == nil { - plan, err = Build(statement, testSchema, "dbName", false) + plan, err = Build(statement, testSchema, "dbName", false, collations.MySQL8()) } var out string diff --git a/go/vt/vttablet/tabletserver/query_engine.go b/go/vt/vttablet/tabletserver/query_engine.go index 0afe76d14ce..a82052a0578 100644 --- a/go/vt/vttablet/tabletserver/query_engine.go +++ b/go/vt/vttablet/tabletserver/query_engine.go @@ -363,7 +363,7 @@ func (qe *QueryEngine) getPlan(curSchema *currentSchema, sql string) (*TabletPla if err != nil { return nil, err } - splan, err := planbuilder.Build(statement, curSchema.tables, qe.env.Config().DB.DBName, qe.env.Config().EnableViews) + splan, err := planbuilder.Build(statement, curSchema.tables, qe.env.Config().DB.DBName, qe.env.Config().EnableViews, qe.env.CollationEnv()) if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletserver/query_engine_test.go b/go/vt/vttablet/tabletserver/query_engine_test.go index f20b5522140..b16539d67f8 100644 --- a/go/vt/vttablet/tabletserver/query_engine_test.go +++ b/go/vt/vttablet/tabletserver/query_engine_test.go @@ -32,6 +32,7 @@ import ( "time" "vitess.io/vitess/go/cache/theine" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/sqlparser" @@ -62,7 +63,7 @@ func TestStrictMode(t *testing.T) { // Test default behavior. config := tabletenv.NewDefaultConfig() config.DB = newDBConfigs(db) - env := tabletenv.NewEnv(config, "TabletServerTest") + env := tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8()) se := schema.NewEngine(env) qe := NewQueryEngine(env, se) qe.se.InitDBConfig(newDBConfigs(db).DbaWithDB()) @@ -355,7 +356,7 @@ func newTestQueryEngine(idleTimeout time.Duration, strict bool, dbcfgs *dbconfig config.OltpReadPool.IdleTimeout = idleTimeout config.OlapReadPool.IdleTimeout = idleTimeout config.TxPool.IdleTimeout = idleTimeout - env := tabletenv.NewEnv(config, "TabletServerTest") + env := tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8()) se := schema.NewEngine(env) qe := NewQueryEngine(env, se) // the integration tests that check cache behavior do not expect a doorkeeper; disable it @@ -455,7 +456,7 @@ func benchmarkPlanCache(b *testing.B, db *fakesqldb.DB, par int) { config := tabletenv.NewDefaultConfig() config.DB = dbcfgs - env := tabletenv.NewEnv(config, "TabletServerTest") + env := tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8()) se := schema.NewEngine(env) qe := NewQueryEngine(env, se) @@ -513,7 +514,7 @@ func TestPlanCachePollution(t *testing.T) { config.DB = dbcfgs // config.LFUQueryCacheSizeBytes = 3 * 1024 * 1024 - env := tabletenv.NewEnv(config, "TabletServerTest") + env := tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8()) se := schema.NewEngine(env) qe := NewQueryEngine(env, se) @@ -829,7 +830,7 @@ func TestAddQueryStats(t *testing.T) { config := tabletenv.NewDefaultConfig() config.DB = newDBConfigs(fakesqldb.New(t)) config.EnablePerWorkloadTableMetrics = testcase.enablePerWorkloadTableMetrics - env := tabletenv.NewEnv(config, "TestAddQueryStats_"+testcase.name) + env := tabletenv.NewEnv(config, "TestAddQueryStats_"+testcase.name, collations.MySQL8()) se := schema.NewEngine(env) qe := NewQueryEngine(env, se) qe.AddStats(testcase.planType, testcase.tableName, testcase.workload, testcase.tabletType, testcase.queryCount, testcase.duration, testcase.mysqlTime, testcase.rowsAffected, testcase.rowsReturned, testcase.errorCount, testcase.errorCode) @@ -870,7 +871,7 @@ func TestPlanPoolUnsafe(t *testing.T) { t.Run(tcase.name, func(t *testing.T) { statement, err := sqlparser.Parse(tcase.query) require.NoError(t, err) - plan, err := planbuilder.Build(statement, map[string]*schema.Table{}, "dbName", false) + plan, err := planbuilder.Build(statement, map[string]*schema.Table{}, "dbName", false, collations.MySQL8()) // Plan building will not fail, but it will mark that reserved connection is needed. // checking plan is valid will fail. require.NoError(t, err) diff --git a/go/vt/vttablet/tabletserver/query_executor.go b/go/vt/vttablet/tabletserver/query_executor.go index 862d41b115d..342755dde1d 100644 --- a/go/vt/vttablet/tabletserver/query_executor.go +++ b/go/vt/vttablet/tabletserver/query_executor.go @@ -28,8 +28,6 @@ import ( "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/pools/smartconnpool" - "vitess.io/vitess/go/mysql/collations" - "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/trace" "vitess.io/vitess/go/vt/callerid" @@ -616,13 +614,13 @@ func (*QueryExecutor) BeginAgain(ctx context.Context, dc *StatefulConnection) er } func (qre *QueryExecutor) execNextval() (*sqltypes.Result, error) { - env := evalengine.NewExpressionEnv(qre.ctx, qre.bindVars, nil) + env := evalengine.NewExpressionEnv(qre.ctx, qre.bindVars, evalengine.NewEmptyVCursor(qre.tsv.collationEnv, time.Local)) result, err := env.Evaluate(qre.plan.NextCount) if err != nil { return nil, err } tableName := qre.plan.TableName() - v := result.Value(collations.Default()) + v := result.Value(qre.tsv.collationEnv.DefaultConnectionCharset()) inc, err := v.ToInt64() if err != nil || inc < 1 { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "invalid increment for sequence %s: %s", tableName, v.String()) diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index 6ea3c90d989..9d83df3ffae 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/sync2" @@ -1487,7 +1488,7 @@ func newTestTabletServer(ctx context.Context, flags executorFlags, db *fakesqldb } dbconfigs := newDBConfigs(db) config.DB = dbconfigs - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) target := &querypb.Target{TabletType: topodatapb.TabletType_PRIMARY} err := tsv.StartService(target, dbconfigs, nil /* mysqld */) if config.TwoPCEnable { diff --git a/go/vt/vttablet/tabletserver/repltracker/reader_test.go b/go/vt/vttablet/tabletserver/repltracker/reader_test.go index 60321cb6164..d9f302062c7 100644 --- a/go/vt/vttablet/tabletserver/repltracker/reader_test.go +++ b/go/vt/vttablet/tabletserver/repltracker/reader_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/test/utils" "github.com/stretchr/testify/assert" @@ -140,12 +141,12 @@ func newReader(db *fakesqldb.DB, frozenTime *time.Time) *heartbeatReader { config := tabletenv.NewDefaultConfig() config.ReplicationTracker.Mode = tabletenv.Heartbeat config.ReplicationTracker.HeartbeatInterval = time.Second - params, _ := db.ConnParams().MysqlParams() + params := db.ConnParams() cp := *params dbc := dbconfigs.NewTestDBConfigs(cp, cp, "") config.DB = dbc - tr := newHeartbeatReader(tabletenv.NewEnv(config, "ReaderTest")) + tr := newHeartbeatReader(tabletenv.NewEnv(config, "ReaderTest", collations.MySQL8())) tr.keyspaceShard = "test:0" if frozenTime != nil { diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker_test.go b/go/vt/vttablet/tabletserver/repltracker/repltracker_test.go index ee74ed52ab5..58a577c7445 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker_test.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/mysqlctl" @@ -38,10 +40,10 @@ func TestReplTracker(t *testing.T) { config := tabletenv.NewDefaultConfig() config.ReplicationTracker.Mode = tabletenv.Heartbeat config.ReplicationTracker.HeartbeatInterval = time.Second - params, _ := db.ConnParams().MysqlParams() + params := db.ConnParams() cp := *params config.DB = dbconfigs.NewTestDBConfigs(cp, cp, "") - env := tabletenv.NewEnv(config, "ReplTrackerTest") + env := tabletenv.NewEnv(config, "ReplTrackerTest", collations.MySQL8()) alias := &topodatapb.TabletAlias{ Cell: "cell", Uid: 1, diff --git a/go/vt/vttablet/tabletserver/repltracker/writer_test.go b/go/vt/vttablet/tabletserver/repltracker/writer_test.go index 664a0464b78..fade8918114 100644 --- a/go/vt/vttablet/tabletserver/repltracker/writer_test.go +++ b/go/vt/vttablet/tabletserver/repltracker/writer_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/dbconfigs" @@ -68,11 +70,11 @@ func newTestWriter(db *fakesqldb.DB, frozenTime *time.Time) *heartbeatWriter { config.ReplicationTracker.Mode = tabletenv.Heartbeat config.ReplicationTracker.HeartbeatInterval = time.Second - params, _ := db.ConnParams().MysqlParams() + params := db.ConnParams() cp := *params dbc := dbconfigs.NewTestDBConfigs(cp, cp, "") - tw := newHeartbeatWriter(tabletenv.NewEnv(config, "WriterTest"), &topodatapb.TabletAlias{Cell: "test", Uid: 1111}) + tw := newHeartbeatWriter(tabletenv.NewEnv(config, "WriterTest", collations.MySQL8()), &topodatapb.TabletAlias{Cell: "test", Uid: 1111}) tw.keyspaceShard = "test:0" if frozenTime != nil { diff --git a/go/vt/vttablet/tabletserver/schema/db_test.go b/go/vt/vttablet/tabletserver/schema/db_test.go index ac6999d309a..88f91b3c99a 100644 --- a/go/vt/vttablet/tabletserver/schema/db_test.go +++ b/go/vt/vttablet/tabletserver/schema/db_test.go @@ -26,9 +26,9 @@ import ( "vitess.io/vitess/go/constants/sidecar" "vitess.io/vitess/go/maps2" - "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/dbconfigs" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" @@ -96,7 +96,7 @@ func TestGenerateFullQuery(t *testing.T) { func TestGetCreateStatement(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) // Success view @@ -131,7 +131,7 @@ func TestGetCreateStatement(t *testing.T) { func TestGetChangedViewNames(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) // Success @@ -164,7 +164,7 @@ func TestGetChangedViewNames(t *testing.T) { func TestGetViewDefinition(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) viewsBV, err := sqltypes.BuildBindVariable([]string{"v1", "lead"}) @@ -336,7 +336,7 @@ func TestGetMismatchedTableNames(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) if tc.dbError != "" { @@ -456,7 +456,7 @@ func TestReloadTablesInDB(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) // Add queries with the expected results and errors. @@ -588,7 +588,7 @@ func TestReloadViewsInDB(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) // Add queries with the expected results and errors. @@ -878,7 +878,7 @@ func TestReloadDataInDB(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) // Add queries with the expected results and errors. diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 9f973324302..a55066a65ec 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -29,6 +29,7 @@ import ( "vitess.io/vitess/go/constants/sidecar" "vitess.io/vitess/go/maps2" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/mysql/sqlerror" @@ -498,7 +499,7 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error { log.V(2).Infof("Reading schema for table: %s", tableName) tableType := row[1].String() - table, err := LoadTable(conn, se.cp.DBName(), tableName, tableType, row[3].ToString()) + table, err := LoadTable(conn, se.cp.DBName(), tableName, tableType, row[3].ToString(), se.env.CollationEnv()) if err != nil { if isView := strings.Contains(tableType, tmutils.TableView); isView { log.Warningf("Failed reading schema for the view: %s, error: %v", tableName, err) @@ -827,6 +828,7 @@ func NewEngineForTests() *Engine { isOpen: true, tables: make(map[string]*Table), historian: newHistorian(false, 0, nil), + env: tabletenv.NewEnv(tabletenv.NewDefaultConfig(), "SchemaEngineForTests", collations.MySQL8()), } return se } @@ -842,6 +844,10 @@ func (se *Engine) GetDBConnector() dbconfigs.Connector { return se.cp } +func (se *Engine) CollationEnv() *collations.Environment { + return se.env.CollationEnv() +} + func extractNamesFromTablesList(tables []*Table) []string { var tableNames []string for _, table := range tables { diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index 2eb117cba36..2dcdeea23cd 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -32,13 +32,12 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/constants/sidecar" - - "vitess.io/vitess/go/mysql/replication" - "vitess.io/vitess/go/mysql/sqlerror" - "vitess.io/vitess/go/event/syslogger" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/test/utils" @@ -581,13 +580,13 @@ func newEngine(reloadTime time.Duration, idleTimeout time.Duration, schemaMaxAge config.OlapReadPool.IdleTimeout = idleTimeout config.TxPool.IdleTimeout = idleTimeout config.SchemaVersionMaxAgeSeconds = schemaMaxAgeSeconds - se := NewEngine(tabletenv.NewEnv(config, "SchemaTest")) + se := NewEngine(tabletenv.NewEnv(config, "SchemaTest", collations.MySQL8())) se.InitDBConfig(newDBConfigs(db).DbaWithDB()) return se } func newDBConfigs(db *fakesqldb.DB) *dbconfigs.DBConfigs { - params, _ := db.ConnParams().MysqlParams() + params := db.ConnParams() cp := *params return dbconfigs.NewTestDBConfigs(cp, cp, "fakesqldb") } @@ -765,7 +764,7 @@ func TestEngineMysqlTime(t *testing.T) { t.Run(tt.name, func(t *testing.T) { se := &Engine{} db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) if tt.timeStampErr != nil { @@ -871,7 +870,7 @@ func TestEnginePopulatePrimaryKeys(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) se := &Engine{} @@ -932,7 +931,7 @@ func TestEngineUpdateInnoDBRowsRead(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) se := &Engine{} se.innoDbReadRowsCounter = stats.NewCounter("TestEngineUpdateInnoDBRowsRead-"+tt.name, "") @@ -959,7 +958,7 @@ func TestEngineUpdateInnoDBRowsRead(t *testing.T) { // TestEngineGetTableData tests the functionality of getTableData function func TestEngineGetTableData(t *testing.T) { db := fakesqldb.New(t) - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) tests := []struct { @@ -1133,7 +1132,7 @@ func TestEngineReload(t *testing.T) { cfg := tabletenv.NewDefaultConfig() cfg.DB = newDBConfigs(db) cfg.SignalWhenSchemaChange = true - conn, err := connpool.NewConn(context.Background(), db.ConnParams(), nil, nil) + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil) require.NoError(t, err) se := newEngine(10*time.Second, 10*time.Second, 0, db) diff --git a/go/vt/vttablet/tabletserver/schema/historian_test.go b/go/vt/vttablet/tabletserver/schema/historian_test.go index f66306966de..1d66ecefd97 100644 --- a/go/vt/vttablet/tabletserver/schema/historian_test.go +++ b/go/vt/vttablet/tabletserver/schema/historian_test.go @@ -39,7 +39,7 @@ func getTable(name string, fieldNames []string, fieldTypes []querypb.Type, pks [ fields := []*querypb.Field{} for i := range fieldNames { typ := fieldTypes[i] - cs := collations.DefaultCollationForType(typ) + cs := collations.CollationForType(typ, collations.MySQL8().DefaultConnectionCharset()) fields = append(fields, &querypb.Field{ Name: fieldNames[i], Type: typ, diff --git a/go/vt/vttablet/tabletserver/schema/load_table.go b/go/vt/vttablet/tabletserver/schema/load_table.go index 687672a4a02..e4e464f3fce 100644 --- a/go/vt/vttablet/tabletserver/schema/load_table.go +++ b/go/vt/vttablet/tabletserver/schema/load_table.go @@ -34,7 +34,7 @@ import ( ) // LoadTable creates a Table from the schema info in the database. -func LoadTable(conn *connpool.PooledConn, databaseName, tableName, tableType string, comment string) (*Table, error) { +func LoadTable(conn *connpool.PooledConn, databaseName, tableName, tableType string, comment string, collationEnv *collations.Environment) (*Table, error) { ta := NewTable(tableName, NoType) sqlTableName := sqlparser.String(ta.Name) if err := fetchColumns(ta, conn, databaseName, sqlTableName); err != nil { @@ -45,7 +45,7 @@ func LoadTable(conn *connpool.PooledConn, databaseName, tableName, tableType str ta.Type = Sequence ta.SequenceInfo = &SequenceInfo{} case strings.Contains(comment, "vitess_message"): - if err := loadMessageInfo(ta, comment); err != nil { + if err := loadMessageInfo(ta, comment, collationEnv); err != nil { return nil, err } ta.Type = Message @@ -68,7 +68,7 @@ func fetchColumns(ta *Table, conn *connpool.PooledConn, databaseName, sqlTableNa return nil } -func loadMessageInfo(ta *Table, comment string) error { +func loadMessageInfo(ta *Table, comment string, collationEnv *collations.Environment) error { ta.MessageInfo = &MessageInfo{} // Extract keyvalues. keyvals := make(map[string]string) @@ -152,7 +152,7 @@ func loadMessageInfo(ta *Table, comment string) error { if specifiedCols[0] != "id" { return fmt.Errorf("vt_message_cols must begin with id: %s", ta.Name.String()) } - ta.MessageInfo.Fields = getSpecifiedMessageFields(ta.Fields, specifiedCols) + ta.MessageInfo.Fields = getSpecifiedMessageFields(ta.Fields, specifiedCols, collationEnv) } else { ta.MessageInfo.Fields = getDefaultMessageFields(ta.Fields, hiddenCols) } @@ -211,11 +211,11 @@ func getDefaultMessageFields(tableFields []*querypb.Field, hiddenCols map[string // we have already validated that all the specified columns exist in the table schema, so we don't need to // check again and possibly return an error here. -func getSpecifiedMessageFields(tableFields []*querypb.Field, specifiedCols []string) []*querypb.Field { +func getSpecifiedMessageFields(tableFields []*querypb.Field, specifiedCols []string, collationEnv *collations.Environment) []*querypb.Field { fields := make([]*querypb.Field, 0, len(specifiedCols)) for _, col := range specifiedCols { for _, field := range tableFields { - if res, _ := evalengine.NullsafeCompare(sqltypes.NewVarChar(field.Name), sqltypes.NewVarChar(strings.TrimSpace(col)), collations.Default()); res == 0 { + if res, _ := evalengine.NullsafeCompare(sqltypes.NewVarChar(field.Name), sqltypes.NewVarChar(strings.TrimSpace(col)), collationEnv, collationEnv.DefaultConnectionCharset()); res == 0 { fields = append(fields, field) break } diff --git a/go/vt/vttablet/tabletserver/schema/load_table_test.go b/go/vt/vttablet/tabletserver/schema/load_table_test.go index 088afac3720..b3da8cf18ab 100644 --- a/go/vt/vttablet/tabletserver/schema/load_table_test.go +++ b/go/vt/vttablet/tabletserver/schema/load_table_test.go @@ -23,7 +23,9 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/test/utils" + "vitess.io/vitess/go/vt/dbconfigs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -227,13 +229,13 @@ func TestLoadTableMessage(t *testing.T) { func newTestLoadTable(tableType string, comment string, db *fakesqldb.DB) (*Table, error) { ctx := context.Background() - appParams := db.ConnParams() - dbaParams := db.ConnParams() + appParams := dbconfigs.New(db.ConnParams()) + dbaParams := dbconfigs.New(db.ConnParams()) cfg := tabletenv.ConnPoolConfig{ Size: 2, IdleTimeout: 10 * time.Second, } - connPool := connpool.NewPool(tabletenv.NewEnv(nil, "SchemaTest"), "", cfg) + connPool := connpool.NewPool(tabletenv.NewEnv(nil, "SchemaTest", collations.MySQL8()), "", cfg) connPool.Open(appParams, dbaParams, appParams) conn, err := connPool.Get(ctx, nil) if err != nil { @@ -241,7 +243,7 @@ func newTestLoadTable(tableType string, comment string, db *fakesqldb.DB) (*Tabl } defer conn.Recycle() - return LoadTable(conn, "fakesqldb", "test_table", tableType, comment) + return LoadTable(conn, "fakesqldb", "test_table", tableType, comment, collations.MySQL8()) } func mockLoadTableQueries(db *fakesqldb.DB) { diff --git a/go/vt/vttablet/tabletserver/schema/tracker_test.go b/go/vt/vttablet/tabletserver/schema/tracker_test.go index 2029235b2e3..017bb941af0 100644 --- a/go/vt/vttablet/tabletserver/schema/tracker_test.go +++ b/go/vt/vttablet/tabletserver/schema/tracker_test.go @@ -17,12 +17,12 @@ limitations under the License. package schema import ( + "context" "testing" "github.com/stretchr/testify/require" - "context" - + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" @@ -78,7 +78,7 @@ func TestTracker(t *testing.T) { } config := se.env.Config() config.TrackSchemaVersions = true - env := tabletenv.NewEnv(config, "TrackerTest") + env := tabletenv.NewEnv(config, "TrackerTest", collations.MySQL8()) initial := env.Stats().ErrorCounters.Counts()["INTERNAL"] tracker := NewTracker(env, vs, se) tracker.Open() @@ -122,7 +122,7 @@ func TestTrackerShouldNotInsertInitialSchema(t *testing.T) { } config := se.env.Config() config.TrackSchemaVersions = true - env := tabletenv.NewEnv(config, "TrackerTest") + env := tabletenv.NewEnv(config, "TrackerTest", collations.MySQL8()) tracker := NewTracker(env, vs, se) tracker.Open() <-vs.done diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go index 23e70a66760..adf53c1d2b1 100644 --- a/go/vt/vttablet/tabletserver/state_manager_test.go +++ b/go/vt/vttablet/tabletserver/state_manager_test.go @@ -24,17 +24,17 @@ import ( "testing" "time" - "google.golang.org/protobuf/proto" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" - "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" - + "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) @@ -704,7 +704,7 @@ func verifySubcomponent(t *testing.T, order int64, component any, state testStat func newTestStateManager(t *testing.T) *stateManager { order.Store(0) config := tabletenv.NewDefaultConfig() - env := tabletenv.NewEnv(config, "StateManagerTest") + env := tabletenv.NewEnv(config, "StateManagerTest", collations.MySQL8()) sm := &stateManager{ statelessql: NewQueryList("stateless"), statefulql: NewQueryList("stateful"), @@ -724,7 +724,7 @@ func newTestStateManager(t *testing.T) *stateManager { tableGC: &testTableGC{}, } sm.Init(env, &querypb.Target{}) - sm.hs.InitDBConfig(&querypb.Target{}, fakesqldb.New(t).ConnParams()) + sm.hs.InitDBConfig(&querypb.Target{}, dbconfigs.New(fakesqldb.New(t).ConnParams())) log.Infof("returning sm: %p", sm) return sm } diff --git a/go/vt/vttablet/tabletserver/stateful_connection_pool_test.go b/go/vt/vttablet/tabletserver/stateful_connection_pool_test.go index b9ea4dfc185..a84052f1d0f 100644 --- a/go/vt/vttablet/tabletserver/stateful_connection_pool_test.go +++ b/go/vt/vttablet/tabletserver/stateful_connection_pool_test.go @@ -25,6 +25,7 @@ import ( "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/dbconfigs" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/vttablet/tabletserver/tx" ) @@ -37,7 +38,8 @@ func TestActivePoolClientRowsFound(t *testing.T) { db.AddQuery("begin", &sqltypes.Result{}) pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) startNormalSize := pool.conns.Available() startFoundRowsSize := pool.foundRowsPool.Available() @@ -63,7 +65,8 @@ func TestActivePoolForAllTxProps(t *testing.T) { db := fakesqldb.New(t) defer db.Close() pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) conn1, err := pool.NewConn(ctx, &querypb.ExecuteOptions{}, nil) require.NoError(t, err) conn1.txProps = &tx.Properties{} @@ -91,7 +94,8 @@ func TestStatefulPoolShutdownNonTx(t *testing.T) { db := fakesqldb.New(t) defer db.Close() pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) // conn1 non-tx, not in use. conn1, err := pool.NewConn(ctx, &querypb.ExecuteOptions{}, nil) @@ -131,7 +135,8 @@ func TestStatefulPoolShutdownAll(t *testing.T) { db := fakesqldb.New(t) defer db.Close() pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) // conn1 not in use conn1, err := pool.NewConn(ctx, &querypb.ExecuteOptions{}, nil) @@ -157,7 +162,8 @@ func TestActivePoolGetConnNonExistentTransaction(t *testing.T) { db := fakesqldb.New(t) defer db.Close() pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) _, err := pool.GetAndLock(12345, "for query") require.EqualError(t, err, "not found") } @@ -167,7 +173,8 @@ func TestExecWithAbortedCtx(t *testing.T) { db := fakesqldb.New(t) defer db.Close() pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) conn, err := pool.NewConn(ctx, &querypb.ExecuteOptions{}, nil) require.NoError(t, err) cancel() @@ -181,7 +188,8 @@ func TestExecWithDbconnClosed(t *testing.T) { db := fakesqldb.New(t) defer db.Close() pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) conn, err := pool.NewConn(ctx, &querypb.ExecuteOptions{}, nil) require.NoError(t, err) conn.Close() @@ -196,7 +204,8 @@ func TestExecWithDbconnClosedHavingTx(t *testing.T) { db := fakesqldb.New(t) defer db.Close() pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) conn, err := pool.NewConn(ctx, &querypb.ExecuteOptions{}, nil) require.NoError(t, err) conn.txProps = &tx.Properties{Conclusion: "foobar"} @@ -212,7 +221,8 @@ func TestFailOnConnectionRegistering(t *testing.T) { db := fakesqldb.New(t) defer db.Close() pool := newActivePool() - pool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + pool.Open(params, params, params) conn, err := pool.NewConn(ctx, &querypb.ExecuteOptions{}, nil) require.NoError(t, err) defer conn.Close() diff --git a/go/vt/vttablet/tabletserver/tabletenv/env.go b/go/vt/vttablet/tabletserver/tabletenv/env.go index 6ae38138922..2d624dfa19b 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/env.go +++ b/go/vt/vttablet/tabletserver/tabletenv/env.go @@ -19,6 +19,7 @@ limitations under the License. package tabletenv import ( + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/tb" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/servenv" @@ -32,29 +33,33 @@ type Env interface { Exporter() *servenv.Exporter Stats() *Stats LogError() + CollationEnv() *collations.Environment } type testEnv struct { - config *TabletConfig - exporter *servenv.Exporter - stats *Stats + config *TabletConfig + exporter *servenv.Exporter + stats *Stats + collationEnv *collations.Environment } // NewEnv creates an Env that can be used for tabletserver subcomponents // without an actual TabletServer. -func NewEnv(config *TabletConfig, exporterName string) Env { +func NewEnv(config *TabletConfig, exporterName string, collationEnv *collations.Environment) Env { exporter := servenv.NewExporter(exporterName, "Tablet") return &testEnv{ - config: config, - exporter: exporter, - stats: NewStats(exporter), + config: config, + exporter: exporter, + stats: NewStats(exporter), + collationEnv: collationEnv, } } -func (*testEnv) CheckMySQL() {} -func (te *testEnv) Config() *TabletConfig { return te.config } -func (te *testEnv) Exporter() *servenv.Exporter { return te.exporter } -func (te *testEnv) Stats() *Stats { return te.stats } +func (*testEnv) CheckMySQL() {} +func (te *testEnv) Config() *TabletConfig { return te.config } +func (te *testEnv) Exporter() *servenv.Exporter { return te.exporter } +func (te *testEnv) Stats() *Stats { return te.stats } +func (te *testEnv) CollationEnv() *collations.Environment { return te.collationEnv } func (te *testEnv) LogError() { if x := recover(); x != nil { diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 308573eb82b..965c11ec366 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -34,6 +34,7 @@ import ( "time" "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/pools/smartconnpool" "vitess.io/vitess/go/sqltypes" @@ -128,6 +129,8 @@ type TabletServer struct { // This field is only stored for testing checkMysqlGaugeFunc *stats.GaugeFunc + + collationEnv *collations.Environment } var _ queryservice.QueryService = (*TabletServer)(nil) @@ -138,8 +141,8 @@ var _ queryservice.QueryService = (*TabletServer)(nil) var RegisterFunctions []func(Controller) // NewServer creates a new TabletServer based on the command line flags. -func NewServer(ctx context.Context, name string, topoServer *topo.Server, alias *topodatapb.TabletAlias) *TabletServer { - return NewTabletServer(ctx, name, tabletenv.NewCurrentConfig(), topoServer, alias) +func NewServer(ctx context.Context, name string, topoServer *topo.Server, alias *topodatapb.TabletAlias, collationEnv *collations.Environment) *TabletServer { + return NewTabletServer(ctx, name, tabletenv.NewCurrentConfig(), topoServer, alias, collationEnv) } var ( @@ -149,7 +152,7 @@ var ( // NewTabletServer creates an instance of TabletServer. Only the first // instance of TabletServer will expose its state variables. -func NewTabletServer(ctx context.Context, name string, config *tabletenv.TabletConfig, topoServer *topo.Server, alias *topodatapb.TabletAlias) *TabletServer { +func NewTabletServer(ctx context.Context, name string, config *tabletenv.TabletConfig, topoServer *topo.Server, alias *topodatapb.TabletAlias, collationEnv *collations.Environment) *TabletServer { exporter := servenv.NewExporter(name, "Tablet") tsv := &TabletServer{ exporter: exporter, @@ -160,6 +163,7 @@ func NewTabletServer(ctx context.Context, name string, config *tabletenv.TabletC enableHotRowProtection: config.HotRowProtection.Mode != tabletenv.Disable, topoServer: topoServer, alias: alias.CloneVT(), + collationEnv: collationEnv, } tsv.QueryTimeout.Store(config.Oltp.QueryTimeout.Nanoseconds()) @@ -336,6 +340,11 @@ func (tsv *TabletServer) Stats() *tabletenv.Stats { return tsv.stats } +// Stats satisfies tabletenv.Env. +func (tsv *TabletServer) CollationEnv() *collations.Environment { + return tsv.collationEnv +} + // LogError satisfies tabletenv.Env. func (tsv *TabletServer) LogError() { if x := recover(); x != nil { diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go index d8595630480..aa8565291be 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_test.go @@ -30,6 +30,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/vt/dbconfigs" vttestpb "vitess.io/vitess/go/vt/proto/vttest" @@ -1562,7 +1563,7 @@ func TestHandleExecUnknownError(t *testing.T) { defer cancel() logStats := tabletenv.NewLogStats(ctx, "TestHandleExecError") config := tabletenv.NewDefaultConfig() - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) defer tsv.handlePanicAndSendLogStats("select * from test_table", nil, logStats) panic("unknown exec error") } @@ -1682,7 +1683,7 @@ func TestHandleExecTabletError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() config := tabletenv.NewDefaultConfig() - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError( @@ -1707,7 +1708,7 @@ func TestTerseErrors(t *testing.T) { config := tabletenv.NewDefaultConfig() config.TerseErrors = true config.SanitizeLogMessages = false - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() @@ -1741,7 +1742,7 @@ func TestSanitizeLogMessages(t *testing.T) { config := tabletenv.NewDefaultConfig() config.TerseErrors = false config.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() @@ -1774,7 +1775,7 @@ func TestTerseErrorsNonSQLError(t *testing.T) { defer cancel() config := tabletenv.NewDefaultConfig() config.TerseErrors = true - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError( @@ -1799,7 +1800,7 @@ func TestSanitizeLogMessagesNonSQLError(t *testing.T) { config := tabletenv.NewDefaultConfig() config.TerseErrors = false config.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError( @@ -1824,7 +1825,7 @@ func TestSanitizeMessagesBindVars(t *testing.T) { config := tabletenv.NewDefaultConfig() config.TerseErrors = true config.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() @@ -1855,7 +1856,7 @@ func TestSanitizeMessagesNoBindVars(t *testing.T) { config := tabletenv.NewDefaultConfig() config.TerseErrors = true config.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError(ctx, "", nil, vterrors.Errorf(vtrpcpb.Code_DEADLINE_EXCEEDED, "sensitive message"), nil) @@ -1873,7 +1874,7 @@ func TestTruncateErrorLen(t *testing.T) { defer cancel() config := tabletenv.NewDefaultConfig() config.TruncateErrorLen = 32 - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError( @@ -1899,7 +1900,7 @@ func TestTruncateMessages(t *testing.T) { config.TerseErrors = false // Sanitize the log messages, which means that the bind vars are omitted config.SanitizeLogMessages = true - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() @@ -1955,7 +1956,7 @@ func TestTerseErrorsIgnoreFailoverInProgress(t *testing.T) { defer cancel() config := tabletenv.NewDefaultConfig() config.TerseErrors = true - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) tl := newTestLogger() defer tl.Close() err := tsv.convertAndLogError(ctx, "select * from test_table where id = :a", @@ -1997,7 +1998,7 @@ func TestACLHUP(t *testing.T) { defer cancel() tableacl.Register("simpleacl", &simpleacl.Factory{}) config := tabletenv.NewDefaultConfig() - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) f, err := os.CreateTemp("", "tableacl") require.NoError(t, err) @@ -2513,7 +2514,7 @@ func setupTabletServerTest(t testing.TB, ctx context.Context, keyspaceName strin func setupTabletServerTestCustom(t testing.TB, ctx context.Context, config *tabletenv.TabletConfig, keyspaceName string) (*fakesqldb.DB, *TabletServer) { db := setupFakeDB(t) sidecardb.AddSchemaInitQueries(db, true) - tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}) + tsv := NewTabletServer(ctx, "TabletServerTest", config, memorytopo.NewServer(ctx, ""), &topodatapb.TabletAlias{}, collations.MySQL8()) require.Equal(t, StateNotConnected, tsv.sm.State()) dbcfgs := newDBConfigs(db) target := &querypb.Target{ diff --git a/go/vt/vttablet/tabletserver/testutils_test.go b/go/vt/vttablet/tabletserver/testutils_test.go index 4760558f6ec..464e84ab47f 100644 --- a/go/vt/vttablet/tabletserver/testutils_test.go +++ b/go/vt/vttablet/tabletserver/testutils_test.go @@ -30,7 +30,7 @@ import ( var errRejected = errors.New("rejected") func newDBConfigs(db *fakesqldb.DB) *dbconfigs.DBConfigs { - params, _ := db.ConnParams().MysqlParams() + params := db.ConnParams() cp := *params return dbconfigs.NewTestDBConfigs(cp, cp, "fakesqldb") } diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index b40f3a0a9cb..58821d29059 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -28,6 +28,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" @@ -108,7 +110,7 @@ func newTestThrottler() *Throttler { s.ThrottleThreshold = &atomic.Uint64{} s.ThrottleThreshold.Store(1) } - env := tabletenv.NewEnv(nil, "TabletServerTest") + env := tabletenv.NewEnv(nil, "TabletServerTest", collations.MySQL8()) throttler := &Throttler{ mysqlClusterProbesChan: make(chan *mysql.ClusterProbes), mysqlClusterThresholds: cache.New(cache.NoExpiration, 0), diff --git a/go/vt/vttablet/tabletserver/tx_engine_test.go b/go/vt/vttablet/tabletserver/tx_engine_test.go index 8b190d675f8..4ed67dc2f66 100644 --- a/go/vt/vttablet/tabletserver/tx_engine_test.go +++ b/go/vt/vttablet/tabletserver/tx_engine_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/vttablet/tabletserver/tx" "github.com/stretchr/testify/assert" @@ -48,7 +49,7 @@ func TestTxEngineClose(t *testing.T) { config.TxPool.Size = 10 config.Oltp.TxTimeout = 100 * time.Millisecond config.GracePeriods.Shutdown = 0 - te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) + te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8())) // Normal close. te.AcceptReadWrite() @@ -151,7 +152,7 @@ func TestTxEngineBegin(t *testing.T) { db.AddQueryPattern(".*", &sqltypes.Result{}) config := tabletenv.NewDefaultConfig() config.DB = newDBConfigs(db) - te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) + te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8())) for _, exec := range []func() (int64, string, error){ func() (int64, string, error) { @@ -197,7 +198,7 @@ func TestTxEngineRenewFails(t *testing.T) { db.AddQueryPattern(".*", &sqltypes.Result{}) config := tabletenv.NewDefaultConfig() config.DB = newDBConfigs(db) - te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) + te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8())) te.AcceptReadOnly() options := &querypb.ExecuteOptions{} connID, _, err := te.ReserveBegin(ctx, options, nil, nil) @@ -535,7 +536,7 @@ func setupTxEngine(db *fakesqldb.DB) *TxEngine { config.TxPool.Size = 10 config.Oltp.TxTimeout = 100 * time.Millisecond config.GracePeriods.Shutdown = 0 - te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) + te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8())) return te } @@ -567,7 +568,7 @@ func TestTxEngineFailReserve(t *testing.T) { db.AddQueryPattern(".*", &sqltypes.Result{}) config := tabletenv.NewDefaultConfig() config.DB = newDBConfigs(db) - te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) + te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8())) options := &querypb.ExecuteOptions{} _, err := te.Reserve(ctx, options, 0, nil) diff --git a/go/vt/vttablet/tabletserver/tx_pool_test.go b/go/vt/vttablet/tabletserver/tx_pool_test.go index 71bba1e128d..1d63e122f30 100644 --- a/go/vt/vttablet/tabletserver/tx_pool_test.go +++ b/go/vt/vttablet/tabletserver/tx_pool_test.go @@ -23,7 +23,9 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/callerid" + "vitess.io/vitess/go/vt/dbconfigs" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" @@ -215,7 +217,8 @@ func primeTxPoolWithConnection(t *testing.T, ctx context.Context) (*fakesqldb.DB txPool, _ := newTxPool() // Set the capacity to 1 to ensure that the db connection is reused. txPool.scp.conns.SetCapacity(1) - txPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + txPool.Open(params, params, params) // Run a query to trigger a database connection. That connection will be // reused by subsequent transactions. @@ -374,7 +377,8 @@ func TestTxPoolGetConnRecentlyRemovedTransaction(t *testing.T) { assertErrorMatch(id, "pool closed") txPool, _ = newTxPool() - txPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + txPool.Open(params, params, params) conn1, _, _, _ = txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil) id = conn1.ReservedID() @@ -389,7 +393,7 @@ func TestTxPoolGetConnRecentlyRemovedTransaction(t *testing.T) { env.Config().SetTxTimeoutForWorkload(1*time.Millisecond, querypb.ExecuteOptions_OLTP) env.Config().SetTxTimeoutForWorkload(1*time.Millisecond, querypb.ExecuteOptions_OLAP) txPool, _ = newTxPoolWithEnv(env) - txPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + txPool.Open(params, params, params) defer txPool.Close() conn1, _, _, err = txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil) @@ -820,7 +824,7 @@ func newEnv(exporterName string) tabletenv.Env { config.OltpReadPool.IdleTimeout = 30 * time.Second config.OlapReadPool.IdleTimeout = 30 * time.Second config.TxPool.IdleTimeout = 30 * time.Second - env := tabletenv.NewEnv(config, exporterName) + env := tabletenv.NewEnv(config, exporterName, collations.MySQL8()) return env } @@ -869,7 +873,8 @@ func setup(t *testing.T) (*fakesqldb.DB, *TxPool, *fakeLimiter, func()) { db.AddQueryPattern(".*", &sqltypes.Result{}) txPool, limiter := newTxPool() - txPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + txPool.Open(params, params, params) return db, txPool, limiter, func() { txPool.Close() @@ -882,7 +887,8 @@ func setupWithEnv(t *testing.T, env tabletenv.Env) (*fakesqldb.DB, *TxPool, *fak db.AddQueryPattern(".*", &sqltypes.Result{}) txPool, limiter := newTxPoolWithEnv(env) - txPool.Open(db.ConnParams(), db.ConnParams(), db.ConnParams()) + params := dbconfigs.New(db.ConnParams()) + txPool.Open(params, params, params) return db, txPool, limiter, func() { txPool.Close() diff --git a/go/vt/vttablet/tabletserver/txlimiter/tx_limiter_test.go b/go/vt/vttablet/tabletserver/txlimiter/tx_limiter_test.go index f41f4a9089c..6257d38718a 100644 --- a/go/vt/vttablet/tabletserver/txlimiter/tx_limiter_test.go +++ b/go/vt/vttablet/tabletserver/txlimiter/tx_limiter_test.go @@ -19,6 +19,7 @@ package txlimiter import ( "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" @@ -47,7 +48,7 @@ func TestTxLimiter_DisabledAllowsAll(t *testing.T) { config.TransactionLimitByPrincipal = false config.TransactionLimitByComponent = false config.TransactionLimitBySubcomponent = false - limiter := New(tabletenv.NewEnv(config, "TabletServerTest")) + limiter := New(tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8())) im, ef := createCallers("", "", "", "") for i := 0; i < 5; i++ { if got, want := limiter.Get(im, ef), true; got != want { @@ -69,7 +70,7 @@ func TestTxLimiter_LimitsOnlyOffendingUser(t *testing.T) { config.TransactionLimitBySubcomponent = false // This should allow 3 slots to all users - newlimiter := New(tabletenv.NewEnv(config, "TabletServerTest")) + newlimiter := New(tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8())) limiter, ok := newlimiter.(*Impl) if !ok { t.Fatalf("New returned limiter of unexpected type: got %T, want %T", newlimiter, limiter) @@ -135,7 +136,7 @@ func TestTxLimiterDryRun(t *testing.T) { config.TransactionLimitBySubcomponent = false // This should allow 3 slots to all users - newlimiter := New(tabletenv.NewEnv(config, "TabletServerTest")) + newlimiter := New(tabletenv.NewEnv(config, "TabletServerTest", collations.MySQL8())) limiter, ok := newlimiter.(*Impl) if !ok { t.Fatalf("New returned limiter of unexpected type: got %T, want %T", newlimiter, limiter) diff --git a/go/vt/vttablet/tabletserver/txserializer/tx_serializer_test.go b/go/vt/vttablet/tabletserver/txserializer/tx_serializer_test.go index d495800e141..773e4f092c1 100644 --- a/go/vt/vttablet/tabletserver/txserializer/tx_serializer_test.go +++ b/go/vt/vttablet/tabletserver/txserializer/tx_serializer_test.go @@ -17,6 +17,7 @@ limitations under the License. package txserializer import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -25,8 +26,7 @@ import ( "testing" "time" - "context" - + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/streamlog" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" @@ -48,7 +48,7 @@ func TestTxSerializer_NoHotRow(t *testing.T) { config.HotRowProtection.MaxQueueSize = 1 config.HotRowProtection.MaxGlobalQueueSize = 1 config.HotRowProtection.MaxConcurrency = 5 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) resetVariables(txs) done, waited, err := txs.Wait(context.Background(), "t1 where1", "t1") @@ -80,7 +80,7 @@ func TestTxSerializerRedactDebugUI(t *testing.T) { config.HotRowProtection.MaxQueueSize = 1 config.HotRowProtection.MaxGlobalQueueSize = 1 config.HotRowProtection.MaxConcurrency = 5 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) resetVariables(txs) done, waited, err := txs.Wait(context.Background(), "t1 where1", "t1") @@ -104,7 +104,7 @@ func TestTxSerializerRedactDebugUI(t *testing.T) { func TestKeySanitization(t *testing.T) { config := tabletenv.NewDefaultConfig() - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) // with a where clause key := "t1 where c1='foo'" want := "t1 ... [REDACTED]" @@ -126,7 +126,7 @@ func TestTxSerializer(t *testing.T) { config.HotRowProtection.MaxQueueSize = 2 config.HotRowProtection.MaxGlobalQueueSize = 3 config.HotRowProtection.MaxConcurrency = 1 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) resetVariables(txs) // tx1. @@ -199,7 +199,7 @@ func TestTxSerializer_ConcurrentTransactions(t *testing.T) { config.HotRowProtection.MaxQueueSize = 3 config.HotRowProtection.MaxGlobalQueueSize = 3 config.HotRowProtection.MaxConcurrency = 2 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) resetVariables(txs) // tx1. @@ -322,7 +322,7 @@ func TestTxSerializerCancel(t *testing.T) { config.HotRowProtection.MaxQueueSize = 4 config.HotRowProtection.MaxGlobalQueueSize = 4 config.HotRowProtection.MaxConcurrency = 2 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) resetVariables(txs) // tx3 and tx4 will record their number once they're done waiting. @@ -423,7 +423,7 @@ func TestTxSerializerDryRun(t *testing.T) { config.HotRowProtection.MaxQueueSize = 1 config.HotRowProtection.MaxGlobalQueueSize = 2 config.HotRowProtection.MaxConcurrency = 1 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) resetVariables(txs) // tx1. @@ -493,7 +493,7 @@ func TestTxSerializerGlobalQueueOverflow(t *testing.T) { config.HotRowProtection.MaxQueueSize = 1 config.HotRowProtection.MaxGlobalQueueSize = 1 config.HotRowProtection.MaxConcurrency = 1 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) // tx1. done1, waited1, err1 := txs.Wait(context.Background(), "t1 where1", "t1") @@ -534,7 +534,7 @@ func TestTxSerializerPending(t *testing.T) { config.HotRowProtection.MaxQueueSize = 1 config.HotRowProtection.MaxGlobalQueueSize = 1 config.HotRowProtection.MaxConcurrency = 1 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) if got, want := txs.Pending("t1 where1"), 0; got != want { t.Errorf("there should be no pending transaction: got = %v, want = %v", got, want) } @@ -545,7 +545,7 @@ func BenchmarkTxSerializer_NoHotRow(b *testing.B) { config.HotRowProtection.MaxQueueSize = 1 config.HotRowProtection.MaxGlobalQueueSize = 1 config.HotRowProtection.MaxConcurrency = 5 - txs := New(tabletenv.NewEnv(config, "TxSerializerTest")) + txs := New(tabletenv.NewEnv(config, "TxSerializerTest", collations.MySQL8())) b.ResetTimer() diff --git a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go index 64c35f6ea95..1f1520fec26 100644 --- a/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go +++ b/go/vt/vttablet/tabletserver/txthrottler/tx_throttler_test.go @@ -28,6 +28,8 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/throttler" "vitess.io/vitess/go/vt/topo" @@ -42,7 +44,7 @@ import ( func TestDisabledThrottler(t *testing.T) { config := tabletenv.NewDefaultConfig() config.EnableTxThrottler = false - env := tabletenv.NewEnv(config, t.Name()) + env := tabletenv.NewEnv(config, t.Name(), collations.MySQL8()) throttler := NewTxThrottler(env, nil) throttler.InitDBConfig(&querypb.Target{ Keyspace: "keyspace", @@ -106,7 +108,7 @@ func TestEnabledThrottler(t *testing.T) { config.EnableTxThrottler = true config.TxThrottlerTabletTypes = &topoproto.TabletTypeListFlag{topodatapb.TabletType_REPLICA} - env := tabletenv.NewEnv(config, t.Name()) + env := tabletenv.NewEnv(config, t.Name(), collations.MySQL8()) throttler := NewTxThrottler(env, ts) throttlerImpl, _ := throttler.(*txThrottler) assert.NotNil(t, throttlerImpl) @@ -169,7 +171,7 @@ func TestFetchKnownCells(t *testing.T) { func TestDryRunThrottler(t *testing.T) { config := tabletenv.NewDefaultConfig() - env := tabletenv.NewEnv(config, t.Name()) + env := tabletenv.NewEnv(config, t.Name(), collations.MySQL8()) testCases := []struct { Name string diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go index 36bcc8f181a..35bea172cd0 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine_test.go @@ -243,7 +243,7 @@ func TestVStreamerWaitForMySQL(t *testing.T) { testDB.AddQuery(replicaLagQuery, sbmres) for _, tt := range tests { - tt.fields.cp = testDB.ConnParams() + tt.fields.cp = dbconfigs.New(testDB.ConnParams()) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() t.Run(tt.name, func(t *testing.T) { diff --git a/go/vt/vttablet/tabletserver/vstreamer/main_flaky_test.go b/go/vt/vttablet/tabletserver/vstreamer/main_flaky_test.go index f3743c6de46..9d978a6e5f2 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/main_flaky_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/main_flaky_test.go @@ -26,6 +26,7 @@ import ( _flag "vitess.io/vitess/go/internal/flag" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/testenv" @@ -93,7 +94,7 @@ func customEngine(t *testing.T, modifier func(mysql.ConnParams) mysql.ConnParams config := env.TabletEnv.Config().Clone() config.DB = dbconfigs.NewTestDBConfigs(modified, modified, modified.DbName) - engine := NewEngine(tabletenv.NewEnv(config, "VStreamerTest"), env.SrvTopo, env.SchemaEngine, nil, env.Cells[0]) + engine := NewEngine(tabletenv.NewEnv(config, "VStreamerTest", collations.MySQL8()), env.SrvTopo, env.SchemaEngine, nil, env.Cells[0]) engine.InitDBConfig(env.KeyspaceName, env.ShardName) engine.Open() return engine diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index fc9408d050e..b60c0bd6754 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -56,6 +56,8 @@ type Plan struct { // Filters is the list of filters to be applied to the columns // of the table. Filters []Filter + + collationEnv *collations.Environment } // Opcode enumerates the operators supported in a where clause @@ -162,14 +164,14 @@ func getOpcode(comparison *sqlparser.ComparisonExpr) (Opcode, error) { } // compare returns true after applying the comparison specified in the Filter to the actual data in the column -func compare(comparison Opcode, columnValue, filterValue sqltypes.Value, charset collations.ID) (bool, error) { +func compare(comparison Opcode, columnValue, filterValue sqltypes.Value, collationEnv *collations.Environment, charset collations.ID) (bool, error) { // use null semantics: return false if either value is null if columnValue.IsNull() || filterValue.IsNull() { return false, nil } // at this point neither values can be null // NullsafeCompare returns 0 if values match, -1 if columnValue < filterValue, 1 if columnValue > filterValue - result, err := evalengine.NullsafeCompare(columnValue, filterValue, charset) + result, err := evalengine.NullsafeCompare(columnValue, filterValue, collationEnv, charset) if err != nil { return false, err } @@ -228,7 +230,7 @@ func (plan *Plan) filter(values, result []sqltypes.Value, charsets []collations. return false, nil } default: - match, err := compare(filter.Opcode, values[filter.ColNum], filter.Value, charsets[filter.ColNum]) + match, err := compare(filter.Opcode, values[filter.ColNum], filter.Value, plan.collationEnv, charsets[filter.ColNum]) if err != nil { return false, err } @@ -344,7 +346,7 @@ func tableMatches(table sqlparser.TableName, dbname string, filter *binlogdatapb return ruleMatches(table.Name.String(), filter) } -func buildPlan(ti *Table, vschema *localVSchema, filter *binlogdatapb.Filter) (*Plan, error) { +func buildPlan(ti *Table, vschema *localVSchema, filter *binlogdatapb.Filter, collationEnv *collations.Environment) (*Plan, error) { for _, rule := range filter.Rules { switch { case strings.HasPrefix(rule.Match, "/"): @@ -356,9 +358,9 @@ func buildPlan(ti *Table, vschema *localVSchema, filter *binlogdatapb.Filter) (* if !result { continue } - return buildREPlan(ti, vschema, rule.Filter) + return buildREPlan(ti, vschema, rule.Filter, collationEnv) case rule.Match == ti.Name: - return buildTablePlan(ti, vschema, rule.Filter) + return buildTablePlan(ti, vschema, rule.Filter, collationEnv) } } return nil, nil @@ -366,9 +368,10 @@ func buildPlan(ti *Table, vschema *localVSchema, filter *binlogdatapb.Filter) (* // buildREPlan handles cases where Match has a regular expression. // If so, the Filter can be an empty string or a keyrange, like "-80". -func buildREPlan(ti *Table, vschema *localVSchema, filter string) (*Plan, error) { +func buildREPlan(ti *Table, vschema *localVSchema, filter string, collationEnv *collations.Environment) (*Plan, error) { plan := &Plan{ - Table: ti, + Table: ti, + collationEnv: collationEnv, } plan.ColExprs = make([]ColExpr, len(ti.Fields)) for i, col := range ti.Fields { @@ -409,7 +412,7 @@ func buildREPlan(ti *Table, vschema *localVSchema, filter string) (*Plan, error) // BuildTablePlan handles cases where a specific table name is specified. // The filter must be a select statement. -func buildTablePlan(ti *Table, vschema *localVSchema, query string) (*Plan, error) { +func buildTablePlan(ti *Table, vschema *localVSchema, query string, collationEnv *collations.Environment) (*Plan, error) { sel, fromTable, err := analyzeSelect(query) if err != nil { log.Errorf("%s", err.Error()) @@ -421,7 +424,8 @@ func buildTablePlan(ti *Table, vschema *localVSchema, query string) (*Plan, erro } plan := &Plan{ - Table: ti, + Table: ti, + collationEnv: collationEnv, } if err := plan.analyzeWhere(vschema, sel.Where); err != nil { log.Errorf("%s", err.Error()) @@ -532,11 +536,14 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er if val.Type != sqlparser.IntVal && val.Type != sqlparser.StrVal { return fmt.Errorf("unexpected: %v", sqlparser.String(expr)) } - pv, err := evalengine.Translate(val, nil) + pv, err := evalengine.Translate(val, &evalengine.Config{ + Collation: plan.collationEnv.DefaultConnectionCharset(), + CollationEnv: plan.collationEnv, + }) if err != nil { return err } - env := evalengine.EmptyExpressionEnv() + env := evalengine.EmptyExpressionEnv(plan.collationEnv) resolved, err := env.Evaluate(pv) if err != nil { return err @@ -544,7 +551,7 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er plan.Filters = append(plan.Filters, Filter{ Opcode: opcode, ColNum: colnum, - Value: resolved.Value(collations.Default()), + Value: resolved.Value(plan.collationEnv.DefaultConnectionCharset()), }) case *sqlparser.FuncExpr: if !expr.Name.EqualString("in_keyrange") { diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index 03001362073..174c3b793f0 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -259,6 +259,7 @@ func TestPlanBuilder(t *testing.T) { Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -289,6 +290,7 @@ func TestPlanBuilder(t *testing.T) { VindexColumns: []int{0}, KeyRange: nil, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -311,6 +313,7 @@ func TestPlanBuilder(t *testing.T) { Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -333,6 +336,7 @@ func TestPlanBuilder(t *testing.T) { Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -355,6 +359,7 @@ func TestPlanBuilder(t *testing.T) { Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -385,6 +390,7 @@ func TestPlanBuilder(t *testing.T) { VindexColumns: []int{0}, KeyRange: nil, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -415,6 +421,7 @@ func TestPlanBuilder(t *testing.T) { VindexColumns: []int{0}, KeyRange: nil, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -445,6 +452,7 @@ func TestPlanBuilder(t *testing.T) { VindexColumns: nil, KeyRange: nil, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t2, @@ -478,6 +486,7 @@ func TestPlanBuilder(t *testing.T) { VindexColumns: []int{0, 1}, KeyRange: nil, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -501,6 +510,7 @@ func TestPlanBuilder(t *testing.T) { }, }}, convertUsingUTF8Columns: map[string]bool{"val": true}, + collationEnv: collations.MySQL8(), }, }, { inTable: regional, @@ -524,6 +534,7 @@ func TestPlanBuilder(t *testing.T) { Vindex: testLocalVSchema.vschema.Keyspaces["ks"].Vindexes["region_vdx"], VindexColumns: []int{0, 1}, }}, + collationEnv: collations.MySQL8(), }, }, { inTable: t1, @@ -636,7 +647,7 @@ func TestPlanBuilder(t *testing.T) { t.Run(tcase.inRule.String(), func(t *testing.T) { plan, err := buildPlan(tcase.inTable, testLocalVSchema, &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{tcase.inRule}, - }) + }, collations.MySQL8()) if tcase.outErr != "" { assert.Nil(t, plan) @@ -733,7 +744,7 @@ func TestPlanBuilderFilterComparison(t *testing.T) { t.Run(tcase.name, func(t *testing.T) { plan, err := buildPlan(t1, testLocalVSchema, &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{{Match: "t1", Filter: tcase.inFilter}}, - }) + }, collations.MySQL8()) if tcase.outErr != "" { assert.Nil(t, plan) @@ -775,7 +786,7 @@ func TestCompare(t *testing.T) { } for _, tc := range testcases { t.Run("", func(t *testing.T) { - got, err := compare(tc.opcode, tc.columnValue, tc.filterValue, collations.CollationUtf8mb4ID) + got, err := compare(tc.opcode, tc.columnValue, tc.filterValue, collations.MySQL8(), collations.CollationUtf8mb4ID) require.NoError(t, err) require.Equal(t, tc.want, got) }) diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go index 99ebbbdfaa5..e3883ba05cd 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go @@ -188,7 +188,7 @@ func (rs *rowStreamer) buildPlan() error { // This is because the row format of a read is identical // to the row format of a binlog event. So, the same // filtering will work. - rs.plan, err = buildTablePlan(ti, rs.vschema, rs.query) + rs.plan, err = buildTablePlan(ti, rs.vschema, rs.query, rs.se.CollationEnv()) if err != nil { log.Errorf("%s", err.Error()) return err diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index 9828481397b..7322e781cc8 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -23,12 +23,12 @@ import ( "testing" "time" - "vitess.io/vitess/go/vt/log" - "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/log" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) @@ -206,7 +206,7 @@ func TestStreamRowsUnicode(t *testing.T) { engine = savedEngine }() engine = customEngine(t, func(in mysql.ConnParams) mysql.ConnParams { - in.Charset = "latin1" + in.Charset = collations.CollationLatin1Swedish return in }) defer engine.Close() diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index a05dc3b2c05..b3899c5f25d 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -26,6 +26,7 @@ import ( "strings" "vitess.io/vitess/go/json2" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/srvtopo" @@ -108,7 +109,7 @@ func Init(ctx context.Context) (*Env, error) { te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) config := tabletenv.NewDefaultConfig() config.DB = te.Dbcfgs - te.TabletEnv = tabletenv.NewEnv(config, "VStreamerTest") + te.TabletEnv = tabletenv.NewEnv(config, "VStreamerTest", collations.MySQL8()) te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) pos, _ := te.Mysqld.PrimaryPosition() if strings.HasPrefix(strings.ToLower(pos.GTIDSet.Flavor()), string(mysqlctl.FlavorMariaDB)) { diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index d8a364d1aef..4816a042e78 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -682,7 +682,7 @@ func (vs *vstreamer) buildJournalPlan(id uint64, tm *mysql.TableMap) error { // Build a normal table plan, which means, return all rows // and columns as is. Special handling is done when we actually // receive the row event. We'll build a JOURNAL event instead. - plan, err := buildREPlan(table, nil, "") + plan, err := buildREPlan(table, nil, "", vs.se.CollationEnv()) if err != nil { return err } @@ -716,7 +716,7 @@ func (vs *vstreamer) buildVersionPlan(id uint64, tm *mysql.TableMap) error { // Build a normal table plan, which means, return all rows // and columns as is. Special handling is done when we actually // receive the row event. We'll build a JOURNAL event instead. - plan, err := buildREPlan(table, nil, "") + plan, err := buildREPlan(table, nil, "", vs.se.CollationEnv()) if err != nil { return err } @@ -738,7 +738,7 @@ func (vs *vstreamer) buildTablePlan(id uint64, tm *mysql.TableMap) (*binlogdatap Name: tm.Name, Fields: cols, } - plan, err := buildPlan(table, vs.vschema, vs.filter) + plan, err := buildPlan(table, vs.vschema, vs.filter, vs.se.CollationEnv()) if err != nil { return nil, err } @@ -768,11 +768,12 @@ func (vs *vstreamer) buildTableColumns(tm *mysql.TableMap) ([]*querypb.Field, er if err != nil { return nil, fmt.Errorf("unsupported type: %d, position: %d", typ, i) } + coll := collations.CollationForType(t, vs.se.CollationEnv().DefaultConnectionCharset()) fields = append(fields, &querypb.Field{ Name: fmt.Sprintf("@%d", i+1), Type: t, - Charset: uint32(collations.DefaultCollationForType(t)), - Flags: mysql.FlagsForColumn(t, collations.DefaultCollationForType(t)), + Charset: uint32(coll), + Flags: mysql.FlagsForColumn(t, coll), }) } st, err := vs.se.GetTableForPos(sqlparser.NewIdentifierCS(tm.Name), replication.EncodePosition(vs.pos)) @@ -956,7 +957,7 @@ func (vs *vstreamer) rebuildPlans() error { // cause that to change. continue } - newPlan, err := buildPlan(plan.Table, vs.vschema, vs.filter) + newPlan, err := buildPlan(plan.Table, vs.vschema, vs.filter, vs.se.CollationEnv()) if err != nil { return err } diff --git a/go/vt/vttest/local_cluster.go b/go/vt/vttest/local_cluster.go index 5ef0ea2c314..6134e68f71c 100644 --- a/go/vt/vttest/local_cluster.go +++ b/go/vt/vttest/local_cluster.go @@ -36,13 +36,12 @@ import ( "google.golang.org/protobuf/proto" "vitess.io/vitess/go/constants/sidecar" - - "vitess.io/vitess/go/vt/sidecardb" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/proto/logutil" + "vitess.io/vitess/go/vt/sidecardb" "vitess.io/vitess/go/vt/vtctl/vtctlclient" vschemapb "vitess.io/vitess/go/vt/proto/vschema" @@ -280,7 +279,11 @@ type LocalCluster struct { // cluster access should be performed through the vtgate port. func (db *LocalCluster) MySQLConnParams() mysql.ConnParams { connParams := db.mysql.Params(db.DbName()) - connParams.Charset = db.Config.Charset + ch, err := collations.MySQL8().ParseConnectionCharset(db.Config.Charset) + if err != nil { + panic(err) + } + connParams.Charset = ch return connParams } @@ -301,7 +304,11 @@ func (db *LocalCluster) MySQLCleanConnParams() mysql.ConnParams { mysqlctl = toxiproxy.mysqlctl } connParams := mysqlctl.Params(db.DbName()) - connParams.Charset = db.Config.Charset + ch, err := collations.MySQL8().ParseConnectionCharset(db.Config.Charset) + if err != nil { + panic(err) + } + connParams.Charset = ch return connParams } diff --git a/go/vt/wrangler/external_cluster_test.go b/go/vt/wrangler/external_cluster_test.go index 3c878411b6b..952738ba8b5 100644 --- a/go/vt/wrangler/external_cluster_test.go +++ b/go/vt/wrangler/external_cluster_test.go @@ -4,10 +4,10 @@ import ( "context" "testing" - "vitess.io/vitess/go/test/utils" - "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -18,7 +18,7 @@ func TestVitessCluster(t *testing.T) { defer cancel() ts := memorytopo.NewServer(ctx, "zone1") tmc := newTestWranglerTMClient() - wr := New(logutil.NewConsoleLogger(), ts, tmc) + wr := New(logutil.NewConsoleLogger(), ts, tmc, collations.MySQL8()) name, topoType, topoServer, topoRoot := "c1", "x", "y", "z" t.Run("Zero clusters to start", func(t *testing.T) { diff --git a/go/vt/wrangler/fake_tablet_test.go b/go/vt/wrangler/fake_tablet_test.go index cae4e8ffc41..7eec43f0fd1 100644 --- a/go/vt/wrangler/fake_tablet_test.go +++ b/go/vt/wrangler/fake_tablet_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" vdiff2 "vitess.io/vitess/go/vt/vttablet/tabletmanager/vdiff" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" @@ -199,7 +200,8 @@ func (ft *fakeTablet) StartActionLoop(t *testing.T, wr *Wrangler) { MysqlDaemon: ft.FakeMysqlDaemon, DBConfigs: &dbconfigs.DBConfigs{}, QueryServiceControl: tabletservermock.NewController(), - VDiffEngine: vdiff2.NewEngine(config, wr.TopoServer(), ft.Tablet), + VDiffEngine: vdiff2.NewEngine(config, wr.TopoServer(), ft.Tablet, collations.MySQL8()), + CollationEnv: collations.MySQL8(), } if err := ft.TM.Start(ft.Tablet, nil); err != nil { t.Fatal(err) diff --git a/go/vt/wrangler/materializer_env_test.go b/go/vt/wrangler/materializer_env_test.go index b98621ffa1b..d01e9c7ab32 100644 --- a/go/vt/wrangler/materializer_env_test.go +++ b/go/vt/wrangler/materializer_env_test.go @@ -30,6 +30,7 @@ import ( "go.uber.org/goleak" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -129,7 +130,7 @@ func newTestMaterializerEnv(t *testing.T, ctx context.Context, ms *vtctldatapb.M cell: "cell", tmc: newTestMaterializerTMClient(), } - env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc) + env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc, collations.MySQL8()) tabletID := 100 for _, shard := range sources { _ = env.addTablet(tabletID, env.ms.SourceKeyspace, shard, topodatapb.TabletType_PRIMARY) diff --git a/go/vt/wrangler/materializer_test.go b/go/vt/wrangler/materializer_test.go index 242bca31e49..7d46be48397 100644 --- a/go/vt/wrangler/materializer_test.go +++ b/go/vt/wrangler/materializer_test.go @@ -30,6 +30,8 @@ import ( "google.golang.org/protobuf/encoding/prototext" "google.golang.org/protobuf/proto" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" @@ -1541,7 +1543,7 @@ func TestCreateLookupVindexFailures(t *testing.T) { defer cancel() topoServ := memorytopo.NewServer(ctx, "cell") - wr := New(logutil.NewConsoleLogger(), topoServ, nil) + wr := New(logutil.NewConsoleLogger(), topoServ, nil, collations.MySQL8()) unique := map[string]*vschemapb.Vindex{ "v": { @@ -2541,7 +2543,7 @@ func TestMaterializerNoSourcePrimary(t *testing.T) { cell: "cell", tmc: newTestMaterializerTMClient(), } - env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc) + env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc, collations.MySQL8()) defer env.close() tabletID := 100 diff --git a/go/vt/wrangler/resharder_env_test.go b/go/vt/wrangler/resharder_env_test.go index ee39c7e5eaa..867f8670bc7 100644 --- a/go/vt/wrangler/resharder_env_test.go +++ b/go/vt/wrangler/resharder_env_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/logutil" @@ -93,7 +94,7 @@ func newTestResharderEnv(t *testing.T, ctx context.Context, sources, targets []s cell: "cell", tmc: newTestResharderTMClient(), } - env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc) + env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc, collations.MySQL8()) initTopo(t, env.topoServ, "ks", sources, targets, []string{"cell"}) tabletID := 100 for _, shard := range sources { diff --git a/go/vt/wrangler/tablet_test.go b/go/vt/wrangler/tablet_test.go index 1350b6b574c..9f2c869fa0a 100644 --- a/go/vt/wrangler/tablet_test.go +++ b/go/vt/wrangler/tablet_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/logutil" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo" @@ -36,7 +37,7 @@ func TestInitTabletShardConversion(t *testing.T) { cell := "cell1" ts := memorytopo.NewServer(ctx, cell) - wr := New(logutil.NewConsoleLogger(), ts, nil) + wr := New(logutil.NewConsoleLogger(), ts, nil, collations.MySQL8()) tablet := &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -70,7 +71,7 @@ func TestDeleteTabletBasic(t *testing.T) { cell := "cell1" ts := memorytopo.NewServer(ctx, cell) - wr := New(logutil.NewConsoleLogger(), ts, nil) + wr := New(logutil.NewConsoleLogger(), ts, nil, collations.MySQL8()) tablet := &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -102,7 +103,7 @@ func TestDeleteTabletTruePrimary(t *testing.T) { cell := "cell1" ts := memorytopo.NewServer(ctx, cell) - wr := New(logutil.NewConsoleLogger(), ts, nil) + wr := New(logutil.NewConsoleLogger(), ts, nil, collations.MySQL8()) tablet := &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -149,7 +150,7 @@ func TestDeleteTabletFalsePrimary(t *testing.T) { cell := "cell1" ts := memorytopo.NewServer(ctx, cell) - wr := New(logutil.NewConsoleLogger(), ts, nil) + wr := New(logutil.NewConsoleLogger(), ts, nil, collations.MySQL8()) tablet1 := &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ @@ -201,7 +202,7 @@ func TestDeleteTabletShardNonExisting(t *testing.T) { cell := "cell1" ts := memorytopo.NewServer(ctx, cell) - wr := New(logutil.NewConsoleLogger(), ts, nil) + wr := New(logutil.NewConsoleLogger(), ts, nil, collations.MySQL8()) tablet := &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index 0ba8adc9a06..c2ef5d8e675 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -27,13 +27,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/mysql/replication" - - "vitess.io/vitess/go/vt/discovery" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" @@ -92,7 +91,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { db := fakesqldb.New(t) defer db.Close() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -344,7 +343,7 @@ func TestBackupRestoreLagged(t *testing.T) { db := fakesqldb.New(t) defer db.Close() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -563,7 +562,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) { db := fakesqldb.New(t) defer db.Close() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -738,7 +737,7 @@ func TestDisableActiveReparents(t *testing.T) { db := fakesqldb.New(t) defer db.Close() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() diff --git a/go/vt/wrangler/testlib/copy_schema_shard_test.go b/go/vt/wrangler/testlib/copy_schema_shard_test.go index 866ec2fe931..ebee27b1c0e 100644 --- a/go/vt/wrangler/testlib/copy_schema_shard_test.go +++ b/go/vt/wrangler/testlib/copy_schema_shard_test.go @@ -22,10 +22,10 @@ import ( "testing" "time" - "vitess.io/vitess/go/vt/discovery" - + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/tmutils" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -56,7 +56,7 @@ func copySchema(t *testing.T, useShardAsSource bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 99cc1839186..dbd7a9f978a 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -25,9 +25,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/mysql/replication" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/sets" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/logutil" @@ -50,7 +50,7 @@ func TestEmergencyReparentShard(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -204,7 +204,7 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) { discovery.SetTabletPickerRetryDelay(5 * time.Millisecond) ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // Create a primary, a couple good replicas oldPrimary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, nil) diff --git a/go/vt/wrangler/testlib/external_reparent_test.go b/go/vt/wrangler/testlib/external_reparent_test.go index c0152de3cf3..4ebaa69bb4d 100644 --- a/go/vt/wrangler/testlib/external_reparent_test.go +++ b/go/vt/wrangler/testlib/external_reparent_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/discovery" "github.com/stretchr/testify/assert" @@ -50,7 +51,7 @@ func TestTabletExternallyReparentedBasic(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -143,7 +144,7 @@ func TestTabletExternallyReparentedToReplica(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // Create an old primary, a new primary, two good replicas, one bad replica oldPrimary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, nil) @@ -226,7 +227,7 @@ func TestTabletExternallyReparentedWithDifferentMysqlPort(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // Create an old primary, a new primary, two good replicas, one bad replica oldPrimary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, nil) @@ -319,7 +320,7 @@ func TestTabletExternallyReparentedContinueOnUnexpectedPrimary(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // Create an old primary, a new primary, two good replicas, one bad replica oldPrimary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, nil) @@ -405,7 +406,7 @@ func TestTabletExternallyReparentedRerun(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // Create an old primary, a new primary, and a good replica. oldPrimary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, nil) @@ -509,7 +510,7 @@ func TestRPCTabletExternallyReparentedDemotesPrimaryToConfiguredTabletType(t *te ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // Create an old primary and a new primary oldPrimary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_SPARE, nil) diff --git a/go/vt/wrangler/testlib/fake_tablet.go b/go/vt/wrangler/testlib/fake_tablet.go index 9c511185769..56f1596dca9 100644 --- a/go/vt/wrangler/testlib/fake_tablet.go +++ b/go/vt/wrangler/testlib/fake_tablet.go @@ -29,6 +29,7 @@ import ( "google.golang.org/grpc" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/dbconfigs" @@ -209,6 +210,7 @@ func (ft *FakeTablet) StartActionLoop(t *testing.T, wr *wrangler.Wrangler) { DBConfigs: &dbconfigs.DBConfigs{}, QueryServiceControl: tabletservermock.NewController(), VREngine: vreplication.NewTestEngine(wr.TopoServer(), ft.Tablet.Alias.Cell, ft.FakeMysqlDaemon, binlogplayer.NewFakeDBClient, binlogplayer.NewFakeDBClient, topoproto.TabletDbName(ft.Tablet), nil), + CollationEnv: collations.MySQL8(), } if err := ft.TM.Start(ft.Tablet, nil); err != nil { t.Fatalf("Error in tablet - %v, err - %v", topoproto.TabletAliasString(ft.Tablet.Alias), err.Error()) diff --git a/go/vt/wrangler/testlib/find_tablet_test.go b/go/vt/wrangler/testlib/find_tablet_test.go index 5b6f26f7056..41e308a98ee 100644 --- a/go/vt/wrangler/testlib/find_tablet_test.go +++ b/go/vt/wrangler/testlib/find_tablet_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -36,7 +37,7 @@ func TestFindTablet(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // Create an old primary, two good replicas oldPrimary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, nil) diff --git a/go/vt/wrangler/testlib/permissions_test.go b/go/vt/wrangler/testlib/permissions_test.go index 4a0e71512f3..55f436e9c24 100644 --- a/go/vt/wrangler/testlib/permissions_test.go +++ b/go/vt/wrangler/testlib/permissions_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/topo/topoproto" @@ -47,7 +48,7 @@ func TestPermissions(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 0125e69cac0..c38de353966 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/vt/mysqlctl" @@ -51,7 +52,7 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -167,7 +168,7 @@ func TestPlannedReparentShardNoError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -303,7 +304,7 @@ func TestPlannedReparentInitialization(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -389,7 +390,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -497,7 +498,7 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -603,7 +604,7 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -683,7 +684,7 @@ func TestPlannedReparentShardRelayLogErrorStartReplication(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -768,7 +769,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() @@ -908,7 +909,7 @@ func TestPlannedReparentShardSamePrimary(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index 0d1d84e89f5..c7c5f06e4ac 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -24,17 +24,15 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/replication" - - "vitess.io/vitess/go/vt/vtctl/reparentutil/reparenttestutil" - "vitess.io/vitess/go/vt/discovery" - "vitess.io/vitess/go/vt/vtctl/reparentutil" - "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl/reparentutil" + "vitess.io/vitess/go/vt/vtctl/reparentutil/reparenttestutil" "vitess.io/vitess/go/vt/vttablet/tmclient" "vitess.io/vitess/go/vt/wrangler" @@ -51,7 +49,7 @@ func TestShardReplicationStatuses(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // create shard and tablets if _, err := ts.GetOrCreateShard(ctx, "test_keyspace", "0"); err != nil { @@ -135,7 +133,7 @@ func TestReparentTablet(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // create shard and tablets if _, err := ts.GetOrCreateShard(ctx, "test_keyspace", "0"); err != nil { @@ -192,7 +190,7 @@ func TestSetReplicationSource(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) // create shard and tablets _, err := ts.GetOrCreateShard(ctx, "test_keyspace", "0") diff --git a/go/vt/wrangler/testlib/shard_test.go b/go/vt/wrangler/testlib/shard_test.go index a0b1b0a3562..d5af9599d7a 100644 --- a/go/vt/wrangler/testlib/shard_test.go +++ b/go/vt/wrangler/testlib/shard_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -35,7 +36,7 @@ func TestDeleteShardCleanup(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() diff --git a/go/vt/wrangler/testlib/version_test.go b/go/vt/wrangler/testlib/version_test.go index 102bcdfe6e5..fb6cdbc0209 100644 --- a/go/vt/wrangler/testlib/version_test.go +++ b/go/vt/wrangler/testlib/version_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/discovery" "vitess.io/vitess/go/vt/topo/topoproto" @@ -70,7 +71,7 @@ func TestVersion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ts := memorytopo.NewServer(ctx, "cell1", "cell2") - wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient()) + wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) vp := NewVtctlPipe(t, ts) defer vp.Close() diff --git a/go/vt/wrangler/testlib/vtctl_pipe.go b/go/vt/wrangler/testlib/vtctl_pipe.go index 448f248821a..3ec62ffdc82 100644 --- a/go/vt/wrangler/testlib/vtctl_pipe.go +++ b/go/vt/wrangler/testlib/vtctl_pipe.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/topo" @@ -76,7 +77,7 @@ func NewVtctlPipe(t *testing.T, ts *topo.Server) *VtctlPipe { // Create a gRPC server and listen on the port server := grpc.NewServer() - grpcvtctlserver.StartServer(server, ts) + grpcvtctlserver.StartServer(server, ts, collations.MySQL8()) go server.Serve(listener) // Create a VtctlClient gRPC client to talk to the fake server diff --git a/go/vt/wrangler/traffic_switcher.go b/go/vt/wrangler/traffic_switcher.go index 498e83a2a3a..ba1309f0f8f 100644 --- a/go/vt/wrangler/traffic_switcher.go +++ b/go/vt/wrangler/traffic_switcher.go @@ -28,6 +28,8 @@ import ( "golang.org/x/sync/errgroup" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/json2" "vitess.io/vitess/go/maps2" "vitess.io/vitess/go/sqlescape" @@ -136,6 +138,7 @@ type trafficSwitcher struct { func (ts *trafficSwitcher) TopoServer() *topo.Server { return ts.wr.ts } func (ts *trafficSwitcher) TabletManagerClient() tmclient.TabletManagerClient { return ts.wr.tmc } +func (ts *trafficSwitcher) CollationEnv() *collations.Environment { return ts.wr.collationEnv } func (ts *trafficSwitcher) Logger() logutil.Logger { return ts.wr.logger } func (ts *trafficSwitcher) VReplicationExec(ctx context.Context, alias *topodatapb.TabletAlias, query string) (*querypb.QueryResult, error) { return ts.wr.VReplicationExec(ctx, alias, query) diff --git a/go/vt/wrangler/traffic_switcher_env_test.go b/go/vt/wrangler/traffic_switcher_env_test.go index 572b2b4a9e6..d0ca09d6624 100644 --- a/go/vt/wrangler/traffic_switcher_env_test.go +++ b/go/vt/wrangler/traffic_switcher_env_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/sync/semaphore" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/sqlescape" @@ -118,7 +119,7 @@ func newTestTableMigrater(ctx context.Context, t *testing.T) *testMigraterEnv { func newTestTableMigraterCustom(ctx context.Context, t *testing.T, sourceShards, targetShards []string, fmtQuery string) *testMigraterEnv { tme := &testMigraterEnv{} tme.ts = memorytopo.NewServer(ctx, "cell1", "cell2") - tme.wr = New(logutil.NewConsoleLogger(), tme.ts, tmclient.NewTabletManagerClient()) + tme.wr = New(logutil.NewConsoleLogger(), tme.ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) tme.wr.sem = semaphore.NewWeighted(1) tme.sourceShards = sourceShards tme.targetShards = targetShards @@ -382,7 +383,7 @@ func newTestTablePartialMigrater(ctx context.Context, t *testing.T, shards, shar require.Greater(t, len(shards), 1, "shard by shard migrations can only be done on sharded keyspaces") tme := &testMigraterEnv{} tme.ts = memorytopo.NewServer(ctx, "cell1", "cell2") - tme.wr = New(logutil.NewConsoleLogger(), tme.ts, tmclient.NewTabletManagerClient()) + tme.wr = New(logutil.NewConsoleLogger(), tme.ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) tme.wr.sem = semaphore.NewWeighted(1) tme.sourceShards = shards tme.targetShards = shards @@ -538,7 +539,7 @@ func newTestTablePartialMigrater(ctx context.Context, t *testing.T, shards, shar func newTestShardMigrater(ctx context.Context, t *testing.T, sourceShards, targetShards []string) *testShardMigraterEnv { tme := &testShardMigraterEnv{} tme.ts = memorytopo.NewServer(ctx, "cell1", "cell2") - tme.wr = New(logutil.NewConsoleLogger(), tme.ts, tmclient.NewTabletManagerClient()) + tme.wr = New(logutil.NewConsoleLogger(), tme.ts, tmclient.NewTabletManagerClient(), collations.MySQL8()) tme.sourceShards = sourceShards tme.targetShards = targetShards tme.tmeDB = fakesqldb.New(t) diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 5de1a4b1d3f..5e4e064e238 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -110,6 +110,8 @@ type vdiff struct { tables []string sourceTimeZone string targetTimeZone string + + collationEnv *collations.Environment } // compareColInfo contains the metadata for a column of the table being diffed @@ -142,6 +144,8 @@ type tableDiffer struct { // source Primitive and targetPrimitive are used for streaming sourcePrimitive engine.Primitive targetPrimitive engine.Primitive + + collationEnv *collations.Environment } // shardStreamer streams rows from one shard. This works for @@ -218,6 +222,7 @@ func (wr *Wrangler) VDiff(ctx context.Context, targetKeyspace, workflowName, sou tables: includeTables, sourceTimeZone: ts.sourceTimeZone, targetTimeZone: ts.targetTimeZone, + collationEnv: wr.collationEnv, } for shard, source := range ts.Sources() { df.sources[shard] = &shardStreamer{ @@ -485,8 +490,8 @@ func (df *vdiff) buildVDiffPlan(ctx context.Context, filter *binlogdatapb.Filter // findPKs identifies PKs, determines any collations to be used for // them, and removes them from the columns used for data comparison. -func findPKs(table *tabletmanagerdatapb.TableDefinition, targetSelect *sqlparser.Select, td *tableDiffer) (sqlparser.OrderBy, error) { - columnCollations, err := getColumnCollations(table) +func findPKs(table *tabletmanagerdatapb.TableDefinition, targetSelect *sqlparser.Select, td *tableDiffer, collationEnv *collations.Environment) (sqlparser.OrderBy, error) { + columnCollations, err := getColumnCollations(table, collationEnv) if err != nil { return nil, err } @@ -530,8 +535,7 @@ func findPKs(table *tabletmanagerdatapb.TableDefinition, targetSelect *sqlparser // getColumnCollations determines the proper collation to use for each // column in the table definition leveraging MySQL's collation inheritance // rules. -func getColumnCollations(table *tabletmanagerdatapb.TableDefinition) (map[string]collations.ID, error) { - collationEnv := collations.Local() +func getColumnCollations(table *tabletmanagerdatapb.TableDefinition, collationEnv *collations.Environment) (map[string]collations.ID, error) { createstmt, err := sqlparser.Parse(table.Schema) if err != nil { return nil, err @@ -569,7 +573,7 @@ func getColumnCollations(table *tabletmanagerdatapb.TableDefinition) (map[string } // The table is using the global default charset and collation and // we inherit that. - return collations.Default() + return collationEnv.DefaultConnectionCharset() } columnCollations := make(map[string]collations.ID) @@ -655,7 +659,8 @@ func (df *vdiff) buildTablePlan(table *tabletmanagerdatapb.TableDefinition, quer return nil, fmt.Errorf("unexpected: %v", sqlparser.String(statement)) } td := &tableDiffer{ - targetTable: table.Name, + targetTable: table.Name, + collationEnv: df.collationEnv, } sourceSelect := &sqlparser.Select{} targetSelect := &sqlparser.Select{} @@ -696,7 +701,7 @@ func (df *vdiff) buildTablePlan(table *tabletmanagerdatapb.TableDefinition, quer aggregates = append(aggregates, engine.NewAggregateParam( /*opcode*/ opcode.AggregateSum, /*offset*/ len(sourceSelect.SelectExprs)-1, - /*alias*/ "")) + /*alias*/ "", df.collationEnv)) } } default: @@ -735,7 +740,7 @@ func (df *vdiff) buildTablePlan(table *tabletmanagerdatapb.TableDefinition, quer }, } - orderby, err := findPKs(table, targetSelect, td) + orderby, err := findPKs(table, targetSelect, td, df.collationEnv) if err != nil { return nil, err } @@ -751,31 +756,32 @@ func (df *vdiff) buildTablePlan(table *tabletmanagerdatapb.TableDefinition, quer td.sourceExpression = sqlparser.String(sourceSelect) td.targetExpression = sqlparser.String(targetSelect) - td.sourcePrimitive = newMergeSorter(df.sources, td.comparePKs) - td.targetPrimitive = newMergeSorter(df.targets, td.comparePKs) + td.sourcePrimitive = newMergeSorter(df.sources, td.comparePKs, df.collationEnv) + td.targetPrimitive = newMergeSorter(df.targets, td.comparePKs, df.collationEnv) // If there were aggregate expressions, we have to re-aggregate // the results, which engine.OrderedAggregate can do. if len(aggregates) != 0 { td.sourcePrimitive = &engine.OrderedAggregate{ - Aggregates: aggregates, - GroupByKeys: pkColsToGroupByParams(td.pkCols), - Input: td.sourcePrimitive, + Aggregates: aggregates, + GroupByKeys: pkColsToGroupByParams(td.pkCols, td.collationEnv), + Input: td.sourcePrimitive, + CollationEnv: df.collationEnv, } } return td, nil } -func pkColsToGroupByParams(pkCols []int) []*engine.GroupByParams { +func pkColsToGroupByParams(pkCols []int, collationEnv *collations.Environment) []*engine.GroupByParams { var res []*engine.GroupByParams for _, col := range pkCols { - res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, Type: evalengine.Type{}}) + res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, Type: evalengine.Type{}, CollationEnv: collationEnv}) } return res } // newMergeSorter creates an engine.MergeSort based on the shard streamers and pk columns. -func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compareColInfo) *engine.MergeSort { +func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compareColInfo, collationEnv *collations.Environment) *engine.MergeSort { prims := make([]engine.StreamExecutor, 0, len(participants)) for _, participant := range participants { prims = append(prims, participant) @@ -788,7 +794,7 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compare if cpk.collation != collations.Unknown { collation = cpk.collation } - ob = append(ob, evalengine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: evalengine.NewType(sqltypes.Unknown, collation)}) + ob = append(ob, evalengine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: evalengine.NewType(sqltypes.Unknown, collation), CollationEnv: collationEnv}) } return &engine.MergeSort{ Primitives: prims, @@ -1309,7 +1315,7 @@ func (td *tableDiffer) compare(sourceRow, targetRow []sqltypes.Value, cols []com if col.collation == collations.Unknown { collationID = collations.CollationBinaryID } - c, err = evalengine.NullsafeCompare(sourceRow[compareIndex], targetRow[compareIndex], collationID) + c, err = evalengine.NullsafeCompare(sourceRow[compareIndex], targetRow[compareIndex], td.collationEnv, collationID) if err != nil { return 0, err } diff --git a/go/vt/wrangler/vdiff_env_test.go b/go/vt/wrangler/vdiff_env_test.go index 01f3a3a0f9e..5f1c3f5072e 100644 --- a/go/vt/wrangler/vdiff_env_test.go +++ b/go/vt/wrangler/vdiff_env_test.go @@ -23,8 +23,8 @@ import ( "sync" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/fakesqldb" - "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/logutil" @@ -78,7 +78,7 @@ func newTestVDiffEnv(t testing.TB, ctx context.Context, sourceShards, targetShar tabletType: topodatapb.TabletType_REPLICA, tmc: newTestVDiffTMClient(), } - env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc) + env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc, collations.MySQL8()) // Generate a unique dialer name. dialerName := fmt.Sprintf("VDiffTest-%s-%d", t.Name(), rand.Intn(1000000000)) diff --git a/go/vt/wrangler/vdiff_test.go b/go/vt/wrangler/vdiff_test.go index 72e9e215c17..1d25f00c830 100644 --- a/go/vt/wrangler/vdiff_test.go +++ b/go/vt/wrangler/vdiff_test.go @@ -37,6 +37,7 @@ import ( ) func TestVDiffPlanSuccess(t *testing.T) { + collationEnv := collations.MySQL8() schm := &tabletmanagerdatapb.SchemaDefinition{ TableDefinitions: []*tabletmanagerdatapb.TableDefinition{{ Name: "t1", @@ -95,8 +96,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { input: &binlogdatapb.Rule{ @@ -112,8 +114,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { input: &binlogdatapb.Rule{ @@ -129,8 +132,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { input: &binlogdatapb.Rule{ @@ -146,8 +150,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{1, collations.Unknown, true}}, pkCols: []int{1}, selectPks: []int{1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { input: &binlogdatapb.Rule{ @@ -163,8 +168,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { // non-pk text column. @@ -181,8 +187,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { // non-pk text column, different order. @@ -199,8 +206,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{1, collations.Unknown, true}}, pkCols: []int{1}, selectPks: []int{1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { // pk text column. @@ -213,12 +221,13 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "pktext", sourceExpression: "select textcol, c2 from pktext order by textcol asc", targetExpression: "select textcol, c2 from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Default(), true}, {1, collations.Unknown, false}}, - comparePKs: []compareColInfo{{0, collations.Default(), true}}, + compareCols: []compareColInfo{{0, collationEnv.DefaultConnectionCharset(), true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collationEnv.DefaultConnectionCharset(), true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Default(), false}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Default(), false}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collationEnv.DefaultConnectionCharset(), false}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collationEnv.DefaultConnectionCharset(), false}}, collationEnv), + collationEnv: collationEnv, }, }, { // pk text column, different order. @@ -231,12 +240,13 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "pktext", sourceExpression: "select c2, textcol from pktext order by textcol asc", targetExpression: "select c2, textcol from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collations.Default(), true}}, - comparePKs: []compareColInfo{{1, collations.Default(), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collationEnv.DefaultConnectionCharset(), true}}, + comparePKs: []compareColInfo{{1, collationEnv.DefaultConnectionCharset(), true}}, pkCols: []int{1}, selectPks: []int{1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default(), false}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default(), false}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collationEnv.DefaultConnectionCharset(), false}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collationEnv.DefaultConnectionCharset(), false}}, collationEnv), + collationEnv: collationEnv, }, }, { // text column as expression. @@ -249,12 +259,13 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "pktext", sourceExpression: "select c2, a + b as textcol from pktext order by textcol asc", targetExpression: "select c2, textcol from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collations.Default(), true}}, - comparePKs: []compareColInfo{{1, collations.Default(), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collationEnv.DefaultConnectionCharset(), true}}, + comparePKs: []compareColInfo{{1, collationEnv.DefaultConnectionCharset(), true}}, pkCols: []int{1}, selectPks: []int{1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default(), false}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default(), false}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collationEnv.DefaultConnectionCharset(), false}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collationEnv.DefaultConnectionCharset(), false}}, collationEnv), + collationEnv: collationEnv, }, }, { input: &binlogdatapb.Rule{ @@ -269,8 +280,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}, pkCols: []int{0, 1}, selectPks: []int{0, 1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { // in_keyrange @@ -287,8 +299,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { // in_keyrange on RHS of AND. @@ -306,8 +319,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collationEnv, }, }, { // in_keyrange on LHS of AND. @@ -325,8 +339,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collations.MySQL8(), }, }, { // in_keyrange on cascaded AND expression @@ -344,8 +359,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collations.MySQL8(), }, }, { // in_keyrange parenthesized @@ -363,8 +379,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collations.MySQL8(), }, }, { // group by @@ -381,8 +398,9 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collations.MySQL8(), }, }, { // aggregations @@ -401,13 +419,15 @@ func TestVDiffPlanSuccess(t *testing.T) { selectPks: []int{0}, sourcePrimitive: &engine.OrderedAggregate{ Aggregates: []*engine.AggregateParams{ - engine.NewAggregateParam(opcode.AggregateSum, 2, ""), - engine.NewAggregateParam(opcode.AggregateSum, 3, ""), + engine.NewAggregateParam(opcode.AggregateSum, 2, "", collationEnv), + engine.NewAggregateParam(opcode.AggregateSum, 3, "", collationEnv), }, - GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1}}, - Input: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1, CollationEnv: collations.MySQL8()}}, + Input: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + CollationEnv: collations.MySQL8(), }, - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collations.MySQL8(), }, }, { input: &binlogdatapb.Rule{ @@ -423,15 +443,16 @@ func TestVDiffPlanSuccess(t *testing.T) { comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}, collationEnv), + collationEnv: collations.MySQL8(), }, }} for _, tcase := range testcases { t.Run(tcase.input.Filter, func(t *testing.T) { filter := &binlogdatapb.Filter{Rules: []*binlogdatapb.Rule{tcase.input}} - df := &vdiff{sourceTimeZone: tcase.sourceTimeZone, targetTimeZone: "UTC"} + df := &vdiff{sourceTimeZone: tcase.sourceTimeZone, targetTimeZone: "UTC", collationEnv: collations.MySQL8()} err := df.buildVDiffPlan(context.Background(), filter, schm, nil) require.NoError(t, err, tcase.input) require.Equal(t, 1, len(df.differs), tcase.input) @@ -486,7 +507,7 @@ func TestVDiffPlanFailure(t *testing.T) { }} for _, tcase := range testcases { filter := &binlogdatapb.Filter{Rules: []*binlogdatapb.Rule{tcase.input}} - df := &vdiff{} + df := &vdiff{collationEnv: collations.MySQL8()} err := df.buildVDiffPlan(context.Background(), filter, schm, nil) assert.EqualError(t, err, tcase.err, tcase.input) } @@ -1079,7 +1100,7 @@ func TestVDiffFindPKs(t *testing.T) { for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - _, err := findPKs(tc.table, tc.targetSelect, tc.tdIn) + _, err := findPKs(tc.table, tc.targetSelect, tc.tdIn, collations.MySQL8()) require.NoError(t, err) require.EqualValues(t, tc.tdOut, tc.tdIn) }) @@ -1139,7 +1160,7 @@ func TestVDiffPlanInclude(t *testing.T) { } func TestGetColumnCollations(t *testing.T) { - collationEnv := collations.Local() + collationEnv := collations.MySQL8() tests := []struct { name string table *tabletmanagerdatapb.TableDefinition @@ -1160,7 +1181,7 @@ func TestGetColumnCollations(t *testing.T) { }, want: map[string]collations.ID{ "c1": collations.Unknown, - "name": collations.Default(), + "name": collationEnv.DefaultConnectionCharset(), }, }, { @@ -1169,8 +1190,8 @@ func TestGetColumnCollations(t *testing.T) { Schema: "create table t1 (c1 varchar(10), name varchar(10), primary key(c1))", }, want: map[string]collations.ID{ - "c1": collations.Default(), - "name": collations.Default(), + "c1": collationEnv.DefaultConnectionCharset(), + "name": collationEnv.DefaultConnectionCharset(), }, }, { @@ -1180,7 +1201,7 @@ func TestGetColumnCollations(t *testing.T) { }, want: map[string]collations.ID{ "c1": collations.Unknown, - "name": collations.Default(), + "name": collationEnv.DefaultConnectionCharset(), }, }, { @@ -1237,7 +1258,7 @@ func TestGetColumnCollations(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := getColumnCollations(tt.table) + got, err := getColumnCollations(tt.table, collationEnv) if (err != nil) != tt.wantErr { t.Errorf("getColumnCollations() error = %v, wantErr = %t", err, tt.wantErr) return diff --git a/go/vt/wrangler/vexec_test.go b/go/vt/wrangler/vexec_test.go index ead2be6a56f..10289740144 100644 --- a/go/vt/wrangler/vexec_test.go +++ b/go/vt/wrangler/vexec_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/textutil" "vitess.io/vitess/go/vt/logutil" @@ -44,7 +45,7 @@ func TestVExec(t *testing.T) { env := newWranglerTestEnv(t, ctx, []string{"0"}, []string{"-80", "80-"}, "", nil, time.Now().Unix()) defer env.close() var logger = logutil.NewMemoryLogger() - wr := New(logger, env.topoServ, env.tmc) + wr := New(logger, env.topoServ, env.tmc, collations.MySQL8()) vx := newVExec(ctx, workflow, keyspace, query, wr) err := vx.getPrimaries() @@ -189,7 +190,7 @@ func TestWorkflowListStreams(t *testing.T) { env := newWranglerTestEnv(t, ctx, []string{"0"}, []string{"-80", "80-"}, "", nil, 1234) defer env.close() logger := logutil.NewMemoryLogger() - wr := New(logger, env.topoServ, env.tmc) + wr := New(logger, env.topoServ, env.tmc, collations.MySQL8()) _, err := wr.WorkflowAction(ctx, workflow, keyspace, "listall", false, nil) require.NoError(t, err) @@ -365,7 +366,7 @@ func TestWorkflowListAll(t *testing.T) { env := newWranglerTestEnv(t, ctx, []string{"0"}, []string{"-80", "80-"}, "", nil, 0) defer env.close() logger := logutil.NewMemoryLogger() - wr := New(logger, env.topoServ, env.tmc) + wr := New(logger, env.topoServ, env.tmc, collations.MySQL8()) workflows, err := wr.ListAllWorkflows(ctx, keyspace, true) require.Nil(t, err) @@ -386,7 +387,7 @@ func TestVExecValidations(t *testing.T) { env := newWranglerTestEnv(t, ctx, []string{"0"}, []string{"-80", "80-"}, "", nil, 0) defer env.close() - wr := New(logutil.NewConsoleLogger(), env.topoServ, env.tmc) + wr := New(logutil.NewConsoleLogger(), env.topoServ, env.tmc, collations.MySQL8()) vx := newVExec(ctx, workflow, keyspace, query, wr) @@ -472,7 +473,7 @@ func TestWorkflowUpdate(t *testing.T) { env := newWranglerTestEnv(t, ctx, []string{"0"}, []string{"-80", "80-"}, "", nil, 1234) defer env.close() logger := logutil.NewMemoryLogger() - wr := New(logger, env.topoServ, env.tmc) + wr := New(logger, env.topoServ, env.tmc, collations.MySQL8()) nullSlice := textutil.SimulatedNullStringSlice // Used to represent a non-provided value nullOnDDL := binlogdatapb.OnDDLAction(textutil.SimulatedNullInt) // Used to represent a non-provided value tests := []struct { diff --git a/go/vt/wrangler/wrangler.go b/go/vt/wrangler/wrangler.go index dbb046a36b3..59ab836b1f8 100644 --- a/go/vt/wrangler/wrangler.go +++ b/go/vt/wrangler/wrangler.go @@ -23,6 +23,7 @@ import ( "golang.org/x/sync/semaphore" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/topo" @@ -57,16 +58,19 @@ type Wrangler struct { VExecFunc func(ctx context.Context, workflow, keyspace, query string, dryRun bool) (map[*topo.TabletInfo]*sqltypes.Result, error) // Limt the number of concurrent background goroutines if needed. sem *semaphore.Weighted + + collationEnv *collations.Environment } // New creates a new Wrangler object. -func New(logger logutil.Logger, ts *topo.Server, tmc tmclient.TabletManagerClient) *Wrangler { +func New(logger logutil.Logger, ts *topo.Server, tmc tmclient.TabletManagerClient, collationEnv *collations.Environment) *Wrangler { return &Wrangler{ - logger: logger, - ts: ts, - tmc: tmc, - vtctld: grpcvtctldserver.NewVtctldServer(ts), - sourceTs: ts, + logger: logger, + ts: ts, + tmc: tmc, + vtctld: grpcvtctldserver.NewVtctldServer(ts), + sourceTs: ts, + collationEnv: collationEnv, } } @@ -74,11 +78,12 @@ func New(logger logutil.Logger, ts *topo.Server, tmc tmclient.TabletManagerClien // in production. func NewTestWrangler(logger logutil.Logger, ts *topo.Server, tmc tmclient.TabletManagerClient) *Wrangler { return &Wrangler{ - logger: logger, - ts: ts, - tmc: tmc, - vtctld: grpcvtctldserver.NewTestVtctldServer(ts, tmc), - sourceTs: ts, + logger: logger, + ts: ts, + tmc: tmc, + vtctld: grpcvtctldserver.NewTestVtctldServer(ts, tmc), + sourceTs: ts, + collationEnv: collations.MySQL8(), } } diff --git a/go/vt/wrangler/wrangler_env_test.go b/go/vt/wrangler/wrangler_env_test.go index 4dd5e342c35..46846784918 100644 --- a/go/vt/wrangler/wrangler_env_test.go +++ b/go/vt/wrangler/wrangler_env_test.go @@ -23,6 +23,7 @@ import ( "sync" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/log" @@ -68,7 +69,7 @@ func newWranglerTestEnv(t testing.TB, ctx context.Context, sourceShards, targetS tabletType: topodatapb.TabletType_REPLICA, tmc: newTestWranglerTMClient(), } - env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc) + env.wr = New(logutil.NewConsoleLogger(), env.topoServ, env.tmc, collations.MySQL8()) env.tmc.tablets = make(map[int]*testWranglerTablet) // Generate a unique dialer name.