Retry capybara tests on specific errors

test/support/retry_on_flaky_tests.rb

@@ -0,0 +1,39 @@
+# Include in `test_helper.rb` like this:
+#
+# class ActiveSupport::TestCase
+#   prepend RetryOnFlakyTests[FlakyError, AnotherFlakyError, max_tries: 3]
+# end
+
+module RetryOnFlakyTests
+  def self.[](*error_classes, max_tries: 3)
+    Module.new do
+      define_method :max_tries do
+        max_tries
+      end
+
+      define_method :error_classes do
+        error_classes
+      end
+
+      def run_one_method(klass, method_name, reporter)
+        report_result = nil
+        max_tries.times do
+          result = Minitest.run_one_method(klass, method_name)
+          report_result ||= result
+          (report_result = result) and break if result.passed?
+
+          break unless retryable_failure?(result)
+        end
+        reporter.record(report_result)
+      end
+
+      def retryable_failure?(result)
+        result.failures.map do |failure|
+          failure.error.to_s
+        end.any? do |failure_msg|
+          error_classes.first {|error_class| failure_msg =~ error_class.name }
+        end
+      end
+    end
+  end
+end

test/test_helper.rb

@@ -70,6 +70,16 @@
 class ActiveSupport::TestCase
   include CustomAssertions
 
+  extend RetryOnFlakyTests[
+    # randomly happening on CI
+    Ferrum::PendingConnectionsError,
+    # race condition when trying to move mouse to element, can happen e.g. after fade-in/out of modal dialog
+    Ferrum::CoordinatesNotFoundError,
+    # race condition when trying to click element, can happen e.g. after fade-in/out of modal dialog
+    Capybara::Cuprite::MouseEventFailed,
+    max_tries: 3
+  ]
+
   # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.
   #
   # Note: You'll currently still have to declare fixtures explicitly in integration tests