Merge branch 'main' into dependabot-bundler-distrib-core-rubocop-rspe…

…c-3.0.1

.github/workflows/rspec-distrib-tests.yml

@@ -21,9 +21,9 @@ jobs:
           working-directory: rspec-distrib
 
       - name: Run specs
-        run: bundle exec rspec spec/
+        run: bundle exec rspec spec
         working-directory: rspec-distrib
 
       - name: Run features
-        run: bundle exec rspec features/
+        run: bundle exec rspec features
         working-directory: rspec-distrib

README.md

@@ -2,9 +2,24 @@
 
 ## What's *-distrib?
 
-This is a collection of gems for running test in parallel on
+This is a collection of gems for running tests in parallel on
 multiple machines/processes.
 
-* [distrib-core](./distrib-core/README.md)
-* [features-parser](./features-parser/README.md)
-* [rspec-distrib](./rspec-distrib/README.md)
+This is a monorepo. Please follow to each gem's directory to get more insights.
+
+## distrib-core
+
+[distrib-core](./distrib-core/README.md) gem defines core classes. It can be used as a building block for various *-distrib runners.
+
+## rspec-distrib
+
+[rspec-distrib](./rspec-distrib/README.md) gem is an implementation of distrib runner for [RSpec](https://rspec.info/) tests.
+
+## cucumber-distrib
+
+cucumber-distrib gem is an implementation of distrib runner for [Cucumber](https://cucumber.io/).
+It will be open-sourced down the road. Stay tuned!
+
+## features-parser
+
+[features-parser](./features-parser/README.md) gem is a helper for cucumber-distrib runner.

distrib-core/.rubocop.yml

@@ -4,6 +4,7 @@ require:
 AllCops:
   DisplayCopNames: true
   NewCops: enable
+  SuggestExtensions: false
   Exclude:
     - coverage/**/*
     - bundle/**/*

distrib-core/Gemfile.lock

@@ -66,7 +66,7 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     strscan (3.1.0)
-    timecop (0.9.9)
+    timecop (0.9.10)
     unicode-display_width (2.5.0)
     yard (0.9.36)
 

distrib-core/README.md

@@ -1,14 +1,16 @@
 # distrib-core
 
-Is a common core module for [rspec-distrib](../rspec-distrib).
+This is a common core module for *-distrib runners.
+At this point for [rspec-distrib](../rspec-distrib).
 
 ## Installation
 
 Add the gem to the application's Gemfile:
 
 ```ruby
-gem 'distrib-core', git: '[email protected]:toptal/test-distrib.git',
-  glob: 'distrib-core/*.gemspec'
+git '[email protected]:toptal/test-distrib.git', branch: 'main' do
+  gem 'distrib-core', require: false, group: [:test]
+end
 ```
 
 ## Getting started

distrib-core/lib/distrib-core.rb

@@ -1,2 +1,17 @@
-# So we can require it as 'distrib-core'.
-require 'distrib_core'
+require 'distrib_core/core_ext/drb_tcp_socket'
+require 'distrib_core/logger_broadcaster'
+require 'distrib_core/leader'
+require 'distrib_core/configuration'
+require 'distrib_core/distrib'
+require 'distrib_core/drb_helper'
+require 'distrib_core/metrics'
+require 'distrib_core/received_signals'
+require 'distrib_core/worker'
+
+# A core module. Has a quick alias to configuration.
+module DistribCore
+  # Alias to {DistribCore::Configuration.current}
+  def self.configuration
+    Configuration.current
+  end
+end

distrib-core/lib/distrib_core/configuration.rb

@@ -4,12 +4,12 @@
 require 'distrib_core/logger_broadcaster'
 
 module DistribCore
-  # This module contains shared attrs instantiated by specific configuration classes.
+  # This module contains shared attributes instantiated by specific configuration classes.
   #
   # @see DistribCore::Distrib#configure
   module Configuration
     class << self
-      # Set global configuration. Can be set only one time
+      # Set global configuration. Can be set only once.
       #
       # @param configuration [DistribCore::Configuration]
       def current=(configuration)
@@ -26,6 +26,8 @@ def current
 
     TIMEOUT_STRATEGIES = %i[repush release].freeze
 
+    # A test provider. It should be a callable object that returns a list of tests to execute.
+    #
     # @example Override default list of the tests:
     #     ...configure do |config|
     #       config.tests_provider = -> {
@@ -34,6 +36,9 @@ def current
     #     end
     attr_writer :tests_provider
 
+    # An object to handle errors. Defines strategy of managing retries of tests.
+    # It should be an instance of {DistribCore::Leader::ErrorHandler}.
+    #
     # @example Specify object to process exceptions during execution
     #     ...configure do |config|
     #       config.error_handler = MyErrorHandler.new
@@ -42,6 +47,8 @@ def current
 
     attr_writer :logger
 
+    # Set timeout for tests. It can be a number of seconds or a callable object that returns a number of seconds.
+    #
     # @example Set equal timeout for all tests to 30 seconds:
     #     ...configure do |config|
     #       config.test_timeout = 30 # seconds
@@ -55,41 +62,71 @@ def current
     #     end
     attr_accessor :test_timeout
 
-    # @example Set how long leader will wait before first test processed by workers. Leader will exit if no tests picked in this period
+    # Specify how long leader will wait before first test processed by workers. Leader will exit if no tests picked in this period.
+    # Value is in seconds.
+    #
+    # @example Set timeout to 10 minutes
     #     ...configure do |config|
     #       config.first_test_picked_timeout = 10*60 # 10 minutes
     #     end
     attr_accessor :first_test_picked_timeout
 
-    # @example Specify custom options for DRb service. Defaults are `{ safe_level: 1 }`. @see `DRb::DRbServer.new` for complete list
+    # Specify custom options for DRb service. Defaults are `{ safe_level: 1 }`.
+    # @see `DRb::DRbServer.new` for complete list.
+    #
+    # @example
     #     ...configure do |config|
     #       config.drb = {safe_level: 0, verbose: true}
     #     end
     attr_accessor :drb
 
-    # @example Specify custom block to pre-process examples before reporting them to the leader. Useful to add additional information about workers.
+    # Specify custom block to pre-process examples before reporting them to the leader. Useful to add additional information about workers.
+    #
+    # @example
     #     ...configure do |config|
     #       config.before_test_report = -> (file_name, example_groups) do
     #         example_groups.each { |eg| eg.metadata[:custom] = 'foo' }
     #       end
     #     end
     attr_accessor :before_test_report
 
-    # @example Specify custom block which will be called on leader after run.
+    # Specify custom block which will be called on leader after run.
+    #
+    # @example
     #     ...configure do |config|
     #       config.on_finish = -> () do
     #         'Whatever logic before leader exit'
     #       end
     #     end
     attr_accessor :on_finish
 
+    # Specify a debug logger.
+    #
     # @example Disable (mute) debug logger
     #     ...configure do |config|
     #       config.debug_logger = Logger.new(nil)
     #     end
     attr_writer :debug_logger
 
-    attr_accessor :tests_processing_stopped_timeout, :drb_tcp_socket_connection_timeout, :leader_connection_attempts
+    # Specify how long leader will wait for test to be picked by a worker. Leader will exit if no test is picked in this period.
+    # Value is in seconds.
+    attr_accessor :tests_processing_stopped_timeout
+
+    # Specify a connection timeout on DRb Socket.
+    # Value is in seconds.
+    # @see {DRb::DRbTCPSocket} monkey patch.
+    attr_accessor :drb_tcp_socket_connection_timeout
+
+    # Specify how many times worker will try to connect to the leader.
+    attr_accessor :leader_connection_attempts
+
+    # Specify a strategy for handling timed out tests.
+    # Can be one of `:repush` or `:release`.
+    #
+    # @example Change strategy to `:release`
+    #     ...configure do |config|
+    #       config.timeout_strategy = :release
+    #     end
     attr_reader :timeout_strategy
 
     # Initialize configuration with default values and set it to {DistribCore::Configuration.current}
@@ -100,24 +137,24 @@ def initialize
       @first_test_picked_timeout = 10 * 60 # 10 minutes
       @tests_processing_stopped_timeout = 5 * 60 # 5 minutes
       @drb = { safe_level: 1 }
-      @drb_tcp_socket_connection_timeout = 5 # 5 seconds
+      @drb_tcp_socket_connection_timeout = 5 # in seconds
       @leader_connection_attempts = 200
       self.timeout_strategy = :repush
     end
 
-    # Provider for tests to execute
+    # Provider for tests to execute.
     #
     # @return [Proc, Object#call] an object which responds to `#call`
     def tests_provider
       @tests_provider || raise(NotImplementedError)
     end
 
-    # Object to handle errors from workers
+    # Object to handle errors from workers.
     def error_handler
       @error_handler || raise(NotImplementedError)
     end
 
-    # Gives a timeout for a particular test based on `#test_timeout`
+    # Gives a timeout for a particular test based on `#test_timeout`.
     #
     # @see #test_timeout
     #

distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb

@@ -1,10 +1,12 @@
-# A patch to reduce connection timeout on DRb Socket.
-# @see https://rubydoc.info/stdlib/drb/DRb
 module DRb
+  # A monkey patch to reduce connection timeout on DRb Socket.
   # The following monkey-patch sets much lower value for connection timeout
   # By default it is over 2 minutes and it is causing a major worker shutdown
   # delay when the leader has finished already.
+  #
+  # @see https://rubydoc.info/stdlib/drb/DRb
   # @see https://rubydoc.info/stdlib/drb/DRb/DRbTCPSocket
+  # @see https://github.com/ruby/drb/blob/master/lib/drb/drb.rb
   class DRbTCPSocket
     # @param uri [String]
     # @param config [Hash]

distrib-core/lib/distrib_core/distrib.rb

@@ -8,7 +8,7 @@ def configure(...)
       configuration.instance_eval(...)
     end
 
-    # Set kind of the current instance
+    # Set kind of the current instance.
     #
     # @param kind [Symbol] `:leader` or `:worker` only
     def kind=(kind)

distrib-core/lib/distrib_core/leader/error_handler.rb

@@ -17,6 +17,9 @@ def initialize(exception_extractor)
         @exception_extractor = exception_extractor
       end
 
+      # Decides if the test should be retried.
+      #
+      # @return [TrueClass, FalseClass]
       def retry_test?(test, results, exception) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
         return false if retries_per_test[test] >= retry_attempts
 
@@ -43,6 +46,9 @@ def retry_test?(test, results, exception) # rubocop:disable Metrics/AbcSize, Met
         retried
       end
 
+      # Decides if the exception should be ignored.
+      #
+      # @return [TrueClass, FalseClass]
       def ignore_worker_failure?(exception)
         self.failed_workers_count += 1
 

distrib-core/lib/distrib_core/leader/queue_builder.rb

@@ -1,8 +1,7 @@
 module DistribCore
   module Leader
-    # Helper that builds a list of the test files to execute sorted by average
-    # execution time descending. The order strategy is backed by
-    # https://en.wikipedia.org/wiki/Queueing_theory
+    # Helper that builds a list of the test files to execute.
+    # The order of files is controlled by used tests provider.
     module QueueBuilder
       # @return [Array<String>] list of test files in the order they should be enqueued
       def self.tests

distrib-core/lib/distrib_core/leader/queue_with_lease.rb

@@ -3,7 +3,7 @@
 
 module DistribCore
   module Leader
-    # Generic queue with lease.
+    # Generic queue-like, thread-safe container with lease.
     #
     # Additionally it keeps the time of the lease, allowing watchdog to return
     # (repush) timed out entries back to the queue.

distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb

@@ -1,6 +1,7 @@
 module DistribCore
   module Leader
     # Only retry if the error is different.
+    # This is a specialized strategy to manage retries of tests.
     class RetryOnDifferentErrorHandler < ErrorHandler
       def initialize(exception_extractor, retry_limit: 2, repeated_error_limit: 1)
         super(exception_extractor)

distrib-core/lib/distrib_core/leader/watchdog.rb

@@ -1,5 +1,3 @@
-require 'rainbow/refinement'
-
 module DistribCore
   module Leader
     # A watchdog to observe the state of queue.
@@ -8,8 +6,6 @@ module Leader
     # A thread watching over presence of the entries on the queue and lease
     # timeouts. Stops the {Leader} by stopping its DRb exposed service.
     class Watchdog # rubocop:disable Metrics/ClassLength
-      using Rainbow
-
       def initialize(queue)
         @queue = queue
         @failed = false

distrib-core/lib/distrib_core/metrics.rb

@@ -2,7 +2,7 @@ module DistribCore
   # Collect metrics from Leader and Workers.
   module Metrics
     class << self
-      # Stores metrics
+      # Stores metrics.
       def report
         @report ||= {
           queue_exposed_at: nil,
@@ -12,17 +12,17 @@ def report
         }
       end
 
-      # Records Leader is ready to serve tests
+      # Records when Leader was ready to serve tests.
       def queue_exposed
         report[:queue_exposed_at] = Time.now.to_i
       end
 
-      # Records first test was taken by a worker
+      # Records when first test was taken by a worker
       def test_taken
         report[:first_test_taken_at] ||= Time.now.to_i
       end
 
-      # Records when watchdog repushes files back to queue because of timeout
+      # Records when watchdog repushes files back to queue because of timeout.
       #
       # @param test [String]
       # @param timeout_in_seconds [Float] timeout which was exceeded