Add retry option when receiving kBusy on first row after snapshot

Summary: When there is contention on rows that causes kBusy errors if this occurs on the first row since we created a snapshot we can release the snapshot (and iterator) and retry to avoid returning too many deadlock errors.

Test Plan: MTR

Reviewers: spetrunia, yoshinorim, hermanlee4

Reviewed By: hermanlee4

Subscribers: spetrunia, webscalesql-eng

Differential Revision: https://reviews.facebook.net/D54231

mysql-test/suite/rocksdb/include/rocksdb_concurrent_delete.inc

@@ -0,0 +1,53 @@
+# Usage:
+#
+# let $order = ASC;  # or DESC
+# let $comment = "rev:cf2";  # or ""
+# --source suite/rocksdb/include/rocksdb_concurrent_delete.inc
+
+let $first_row = -1; # Error this should never happen
+if ($order == 'ASC')
+{
+  let $first_row = 1;
+}
+if ($order == 'DESC')
+{
+  let $first_row = 3;
+}
+
+connect (con, localhost, root,,);
+connection default;
+
+--disable_warnings
+SET debug_sync='RESET';
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+eval CREATE TABLE t1 (pk INT PRIMARY KEY COMMENT $comment, a INT);
+INSERT INTO t1 VALUES(1,1), (2,2), (3,3);
+
+# This will cause the SELECT to block after finding the first row, but
+# before locking and reading it.
+connection con;
+SET debug_sync='rocksdb_concurrent_delete SIGNAL parked WAIT_FOR go';
+send_eval SELECT * FROM t1 order by t1.pk $order FOR UPDATE;
+
+# While that connection is waiting, delete the first row (the one con
+# is about to lock and read
+connection default;
+SET debug_sync='now WAIT_FOR parked';
+eval DELETE FROM t1 WHERE pk = $first_row;
+
+# Signal the waiting select to continue
+SET debug_sync='now SIGNAL go';
+
+# Now get the results from the select.  The first entry (1,1) (or (3,3) when
+# using reverse ordering) should be missing.  Prior to the fix the SELECT
+# would have returned: "1815: Internal error: NotFound:"
+connection con;
+reap;
+
+# Cleanup
+connection default;
+disconnect con;
+set debug_sync='RESET';
+drop table t1;

mysql-test/suite/rocksdb/r/deadlock.result

@@ -6,6 +6,7 @@ DROP DATABASE IF EXISTS mysqlslap;
 CREATE DATABASE mysqlslap;
 USE mysqlslap;
 CREATE TABLE t1(id1 BIGINT, id2 BIGINT, count INT, PRIMARY KEY(id1, id2), KEY(id2)) ENGINE=rocksdb;
+CREATE TABLE t1rev(id1 BIGINT, id2 BIGINT, count INT, PRIMARY KEY(id1, id2) COMMENT "rev:cf2", KEY(id2) COMMENT "rev:cf2") ENGINE=rocksdb;
 SET @save = @@global.rocksdb_lock_wait_timeout;
 SET GLOBAL rocksdb_lock_wait_timeout = 60;
 SELECT count from t1;
@@ -14,5 +15,23 @@ count
 SELECT count from t1;
 count
 100000
+SELECT count from t1;
+count
+150000
+SELECT count from t1;
+count
+200000
+SELECT count from t1rev;
+count
+50000
+SELECT count from t1rev;
+count
+100000
+SELECT count from t1rev;
+count
+150000
+SELECT count from t1rev;
+count
+200000
 SET GLOBAL rocksdb_lock_wait_timeout = @save;
 DROP DATABASE mysqlslap;

mysql-test/suite/rocksdb/r/delete_before_lock.result

@@ -7,10 +7,9 @@ update t1 set value=100 where id1=1;
 set debug_sync='now WAIT_FOR parked';
 delete from t1 where id1=1 and id2=1;
 set debug_sync='now SIGNAL go';
-ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 select * from t1 where id1=1 for update;
 id1	id2	value
-1	2	1
-1	3	1
+1	2	100
+1	3	100
 set debug_sync='RESET';
 drop table t1;

mysql-test/suite/rocksdb/r/rocksdb_concurrent_delete.result

@@ -1,12 +1,56 @@
+SET debug_sync='RESET';
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY COMMENT "", a INT);
+INSERT INTO t1 VALUES(1,1), (2,2), (3,3);
+SET debug_sync='rocksdb_concurrent_delete SIGNAL parked WAIT_FOR go';
+SELECT * FROM t1 order by t1.pk ASC FOR UPDATE;
+SET debug_sync='now WAIT_FOR parked';
+DELETE FROM t1 WHERE pk = 1;
+SET debug_sync='now SIGNAL go';
+pk	a
+2	2
+3	3
 set debug_sync='RESET';
-drop table if exists t1;
-create table t1 (pk int primary key, a int);
-insert into t1 values(1,1), (2,2), (3,3);
-set debug_sync='rocksdb_concurrent_delete SIGNAL parked WAIT_FOR go';
-select * from t1 for update;
-set debug_sync='now WAIT_FOR parked';
-delete from t1 where pk = 1;
-set debug_sync='now SIGNAL go';
-ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+drop table t1;
+SET debug_sync='RESET';
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY COMMENT "", a INT);
+INSERT INTO t1 VALUES(1,1), (2,2), (3,3);
+SET debug_sync='rocksdb_concurrent_delete SIGNAL parked WAIT_FOR go';
+SELECT * FROM t1 order by t1.pk DESC FOR UPDATE;
+SET debug_sync='now WAIT_FOR parked';
+DELETE FROM t1 WHERE pk = 3;
+SET debug_sync='now SIGNAL go';
+pk	a
+2	2
+1	1
+set debug_sync='RESET';
+drop table t1;
+SET debug_sync='RESET';
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY COMMENT "rev:cf2", a INT);
+INSERT INTO t1 VALUES(1,1), (2,2), (3,3);
+SET debug_sync='rocksdb_concurrent_delete SIGNAL parked WAIT_FOR go';
+SELECT * FROM t1 order by t1.pk ASC FOR UPDATE;
+SET debug_sync='now WAIT_FOR parked';
+DELETE FROM t1 WHERE pk = 1;
+SET debug_sync='now SIGNAL go';
+pk	a
+2	2
+3	3
+set debug_sync='RESET';
+drop table t1;
+SET debug_sync='RESET';
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (pk INT PRIMARY KEY COMMENT "rev:cf2", a INT);
+INSERT INTO t1 VALUES(1,1), (2,2), (3,3);
+SET debug_sync='rocksdb_concurrent_delete SIGNAL parked WAIT_FOR go';
+SELECT * FROM t1 order by t1.pk DESC FOR UPDATE;
+SET debug_sync='now WAIT_FOR parked';
+DELETE FROM t1 WHERE pk = 3;
+SET debug_sync='now SIGNAL go';
+pk	a
+2	2
+1	1
 set debug_sync='RESET';
 drop table t1;

mysql-test/suite/rocksdb/t/deadlock.test

@@ -14,6 +14,7 @@ DROP DATABASE IF EXISTS mysqlslap;
 CREATE DATABASE mysqlslap;
 USE mysqlslap;
 CREATE TABLE t1(id1 BIGINT, id2 BIGINT, count INT, PRIMARY KEY(id1, id2), KEY(id2)) ENGINE=rocksdb;
+CREATE TABLE t1rev(id1 BIGINT, id2 BIGINT, count INT, PRIMARY KEY(id1, id2) COMMENT "rev:cf2", KEY(id2) COMMENT "rev:cf2") ENGINE=rocksdb;
 
 SET @save = @@global.rocksdb_lock_wait_timeout;
 SET GLOBAL rocksdb_lock_wait_timeout = 60;
@@ -22,12 +23,20 @@ SET GLOBAL rocksdb_lock_wait_timeout = 60;
 SELECT count from t1;
 --exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="UPDATE t1 SET count=count+1 WHERE id1=1 AND id2=1"
 SELECT count from t1;
-## Disable these for now
-## TODO(jkedgar) add retry ability so these tests can be enabled again
-##--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="UPDATE t1 SET count=count+1 WHERE id2=1"
-##SELECT count from t1;
-##--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="UPDATE t1 SET count=count+1"
-##SELECT count from t1;
+--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="UPDATE t1 SET count=count+1 WHERE id2=1"
+SELECT count from t1;
+--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="UPDATE t1 SET count=count+1"
+SELECT count from t1;
+
+# Same tests on a table with reverse orderings
+--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="INSERT INTO t1rev VALUES(1, 1, 1) ON DUPLICATE KEY UPDATE count=count+1"
+SELECT count from t1rev;
+--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="UPDATE t1rev SET count=count+1 WHERE id1=1 AND id2=1"
+SELECT count from t1rev;
+--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="UPDATE t1rev SET count=count+1 WHERE id2=1"
+SELECT count from t1rev;
+--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=50000 --query="UPDATE t1rev SET count=count+1"
+SELECT count from t1rev;
 
 SET GLOBAL rocksdb_lock_wait_timeout = @save;
 

mysql-test/suite/rocksdb/t/delete_before_lock.test

@@ -26,7 +26,6 @@ delete from t1 where id1=1 and id2=1;
 set debug_sync='now SIGNAL go';
 
 connection con;
---error ER_LOCK_DEADLOCK
 reap;
 select * from t1 where id1=1 for update;
 

mysql-test/suite/rocksdb/t/rocksdb_concurrent_delete.test

@@ -7,44 +7,18 @@
 # deleted before the GetForUpdate() call could occur.  When this happened
 # a nearly useless error was being returned.
 
-connect (con, localhost, root,,);
-connection default;
+let $order=ASC;
+let $comment="";
+--source suite/rocksdb/include/rocksdb_concurrent_delete.inc
 
---disable_warnings
-set debug_sync='RESET';
-drop table if exists t1;
---enable_warnings
+let $order=DESC;
+let $comment="";
+--source suite/rocksdb/include/rocksdb_concurrent_delete.inc
 
-create table t1 (pk int primary key, a int);
-insert into t1 values(1,1), (2,2), (3,3);
+let $order=ASC;
+let $comment="rev:cf2";
+--source suite/rocksdb/include/rocksdb_concurrent_delete.inc
 
-# This will cause the select to block after finding the first row, but
-# before locking and reading it.
-connection con;
-set debug_sync='rocksdb_concurrent_delete SIGNAL parked WAIT_FOR go';
-send select * from t1 for update;
-
-# While that connection is waiting, delete the first row (the one con
-# is about to lock and read
-connection default;
-set debug_sync='now WAIT_FOR parked';
-delete from t1 where pk = 1;
-
-# Signal the waiting select to continue
-set debug_sync='now SIGNAL go';
-
-# Now get the results from the select.  The first entry (1,1) should
-# be missing.  Prior to the fix the select would have returned:
-# "1815: Internal error: NotFound:"
-connection con;
-# As of https://reviews.facebook.net/D54129 this will get a deadlock error
-# TODO(jkedgar): Once retrying is available this should no longer return
-# an error
---error ER_LOCK_DEADLOCK
-reap;
-
-# Cleanup
-connection default;
-disconnect con;
-set debug_sync='RESET';
-drop table t1;
+let $order=DESC;
+let $comment="rev:cf2";
+--source suite/rocksdb/include/rocksdb_concurrent_delete.inc