From 16b5fa524104b82b205025872d85e83433d83758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20K=2E=20Lema=C5=84ski?= Date: Mon, 3 Jun 2024 16:45:26 +0200 Subject: [PATCH] Initial commit for open-sourcing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Evgenii Pecherkin <1036610+epecherkin@users.noreply.github.com> Co-Authored-By: Phil Pirozhkov <6916+pirj@users.noreply.github.com> Co-Authored-By: Ale ∴ <8848152+alexvko@users.noreply.github.com> Co-Authored-By: Aleksandr Kariakin Co-Authored-By: Vasyl Melnychuk <154947+sqrel@users.noreply.github.com> Co-Authored-By: Diego Guerra <442283+dgsuarez@users.noreply.github.com> Co-Authored-By: Jaimerson Araújo <2944985+jaimerson@users.noreply.github.com> Co-Authored-By: Michał Młoźniak <7219+ronin@users.noreply.github.com> Co-Authored-By: Ebeagu Samuel Co-Authored-By: Sasha Alexandrov <95298+juike@users.noreply.github.com> Co-Authored-By: Velichko Stoev <8302188+velichkostoev@users.noreply.github.com> Co-Authored-By: Danilo Resende Co-Authored-By: Danil Nurgaliev Co-Authored-By: Oleg Polivannyi <28644461+oleg-polivannyi@users.noreply.github.com> Co-Authored-By: Achilles Charmpilas Co-Authored-By: Oleh Adam Dubnytskyy Co-Authored-By: Alex Rodionov <665846+p0deje@users.noreply.github.com> Co-Authored-By: Bartek Bułat <151912+barthez@users.noreply.github.com> Co-Authored-By: Bartek Wilczek <12280141+bwilczek@users.noreply.github.com> Co-Authored-By: Denis Usanov <481078+dck@users.noreply.github.com> Co-Authored-By: Ilya Denisov --- .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/bug_report.yml | 38 +++ .github/ISSUE_TEMPLATE/feature_request.yml | 31 ++ .github/ISSUE_TEMPLATE/question.yml | 17 + .github/dependabot.yml | 69 ++++ .github/workflows/distrib-core-linters.yml | 32 ++ .github/workflows/distrib-core-specs.yml | 29 ++ .github/workflows/features-parser-specs.yml | 26 ++ .github/workflows/rspec-distrib-linters.yml | 32 ++ .github/workflows/rspec-distrib-tests.yml | 29 ++ .gitignore | 4 + README.md | 10 + distrib-core/.gitignore | 12 + distrib-core/.rspec | 3 + distrib-core/.rubocop.yml | 47 +++ distrib-core/.ruby-version | 1 + distrib-core/CODE_OF_CONDUCT.md | 14 + distrib-core/Gemfile | 12 + distrib-core/Gemfile.lock | 96 ++++++ distrib-core/LICENSE.txt | 21 ++ distrib-core/README.md | 28 ++ distrib-core/distrib-core.gemspec | 18 + distrib-core/lib/distrib-core.rb | 2 + distrib-core/lib/distrib_core.rb | 17 + .../lib/distrib_core/configuration.rb | 161 +++++++++ .../distrib_core/core_ext/drb_tcp_socket.rb | 20 ++ distrib-core/lib/distrib_core/distrib.rb | 41 +++ distrib-core/lib/distrib_core/drb_helper.rb | 119 +++++++ distrib-core/lib/distrib_core/leader.rb | 30 ++ .../lib/distrib_core/leader/drb_callable.rb | 36 ++ .../lib/distrib_core/leader/error_handler.rb | 96 ++++++ .../lib/distrib_core/leader/queue_builder.rb | 13 + .../distrib_core/leader/queue_with_lease.rb | 139 ++++++++ .../retry_on_different_error_handler.rb | 46 +++ .../lib/distrib_core/leader/watchdog.rb | 149 +++++++++ .../lib/distrib_core/logger_broadcaster.rb | 32 ++ distrib-core/lib/distrib_core/metrics.rb | 35 ++ .../lib/distrib_core/received_signals.rb | 60 ++++ .../lib/distrib_core/spec/configuration.rb | 78 +++++ distrib-core/lib/distrib_core/spec/distrib.rb | 60 ++++ distrib-core/lib/distrib_core/worker.rb | 40 +++ .../spec/distrib_core/configuration_spec.rb | 21 ++ .../spec/distrib_core/distrib_spec.rb | 19 ++ .../spec/distrib_core/drb_helper_spec.rb | 71 ++++ .../distrib_core/leader/drb_callable_spec.rb | 50 +++ .../distrib_core/leader/error_handler_spec.rb | 157 +++++++++ .../distrib_core/leader/queue_builder_spec.rb | 16 + .../leader/queue_with_lease_spec.rb | 166 +++++++++ .../retry_on_different_error_handler_spec.rb | 105 ++++++ .../spec/distrib_core/leader/watchdog_spec.rb | 279 ++++++++++++++++ .../distrib_core/logger_broadcaster_spec.rb | 40 +++ .../spec/distrib_core/metrics_spec.rb | 41 +++ distrib-core/spec/spec_helper.rb | 29 ++ features-parser/.gitignore | 12 + features-parser/.rspec | 3 + features-parser/.rubocop.yml | 22 ++ features-parser/.ruby-version | 1 + features-parser/CODE_OF_CONDUCT.md | 14 + features-parser/Gemfile | 8 + features-parser/Gemfile.lock | 87 +++++ features-parser/LICENSE.txt | 21 ++ features-parser/README.md | 22 ++ features-parser/bin/console | 11 + features-parser/bin/setup | 8 + features-parser/features-parser.gemspec | 27 ++ features-parser/lib/features-parser.rb | 18 + .../lib/features_parser/catalog.rb | 77 +++++ .../lib/features_parser/example.rb | 40 +++ .../lib/features_parser/feature.rb | 27 ++ .../lib/features_parser/name_normalizer.rb | 30 ++ .../lib/features_parser/name_provider.rb | 78 +++++ .../lib/features_parser/outline.rb | 13 + .../lib/features_parser/scenario.rb | 37 ++ .../lib/features_parser/scenario_parser.rb | 86 +++++ .../lib/features_parser/version.rb | 5 + .../spec/features_parser/catalog_spec.rb | 72 ++++ .../spec/features_parser/example_spec.rb | 43 +++ .../spec/features_parser/feature_spec.rb | 40 +++ .../features_parser/name_normalizer_spec.rb | 41 +++ .../features_parser/name_provider_spec.rb | 44 +++ .../spec/features_parser/outline_spec.rb | 46 +++ .../features_parser/scenario_parser_spec.rb | 57 ++++ .../spec/features_parser/scenario_spec.rb | 46 +++ features-parser/spec/features_parser_spec.rb | 7 + features-parser/spec/spec_helper.rb | 15 + .../spec/support/parse-error.feature | 17 + features-parser/spec/support/some.feature | 43 +++ rspec-distrib/.gitignore | 11 + rspec-distrib/.rspec | 4 + rspec-distrib/.rspec-distrib | 6 + rspec-distrib/.rubocop.yml | 53 +++ rspec-distrib/.rubocop_todo.yml | 45 +++ rspec-distrib/.ruby-version | 1 + rspec-distrib/CODE_OF_CONDUCT.md | 14 + rspec-distrib/Gemfile | 16 + rspec-distrib/Gemfile.lock | 101 ++++++ rspec-distrib/LICENSE.txt | 21 ++ rspec-distrib/README.md | 315 ++++++++++++++++++ rspec-distrib/docs/startup.png | Bin 0 -> 26969 bytes rspec-distrib/docs/watchdog.png | Bin 0 -> 16942 bytes rspec-distrib/docs/worker.png | Bin 0 -> 31627 bytes rspec-distrib/exe/rspec-distrib | 40 +++ .../features/aborting_worker_spec.rb | 21 ++ .../features/configuration_issue_spec.rb | 20 ++ .../features/failing_after_all_spec.rb | 20 ++ .../features/failing_after_suite_spec.rb | 22 ++ .../features/failing_before_all_spec.rb | 20 ++ .../features/failing_before_suite_spec.rb | 23 ++ .../features/failing_inside_examples_spec.rb | 20 ++ .../failing_multiple_exceptions_spec.rb | 24 ++ .../features/failing_outside_examples_spec.rb | 21 ++ rspec-distrib/features/feature_helper.rb | 7 + rspec-distrib/features/fixtures/specs/.rspec | 3 + .../features/fixtures/specs/.rspec-distrib | 35 ++ .../specs/failing_after_all/fail_spec.rb | 7 + .../specs/failing_after_all/pass_spec.rb | 3 + .../specs/failing_before_all/bar_spec.rb | 7 + .../specs/failing_before_all/foo_spec.rb | 3 + .../specs/failing_inside_examples/bar_spec.rb | 4 + .../specs/failing_inside_examples/foo_spec.rb | 3 + .../1_pass_spec.rb | 3 + .../2_fail_spec.rb | 7 + .../3_pass_spec.rb | 3 + .../failing_outside_examples/bar_spec.rb | 7 + .../failing_outside_examples/baz_spec.rb | 11 + .../failing_outside_examples/foo_spec.rb | 3 + .../failing_outside_examples/fur_spec.rb | 5 + .../fixtures/specs/features_formatter.rb | 28 ++ .../fixtures/specs/flaky_retries/foo_spec.rb | 24 ++ .../fixtures/specs/flaky_retries/zap_spec.rb | 5 + .../fixtures/specs/passing/bar_spec.rb | 4 + .../fixtures/specs/passing/foo_spec.rb | 3 + .../fixtures/specs/prevent_eval/foo_spec.rb | 11 + .../specs/signals_handling/foo_spec.rb | 6 + .../features/fixtures/specs/spec_helper.rb | 32 ++ .../fixtures/specs/syntax_error/bar_spec.rb | 5 + .../fixtures/specs/syntax_error/foo_spec.rb | 3 + .../specs/timeout_of_spec/bar_spec.rb | 5 + .../specs/timeout_of_spec/foo_spec.rb | 3 + .../timeout_processing_stopped/bar_spec.rb | 2 + .../timeout_processing_stopped/foo_spec.rb | 3 + rspec-distrib/features/flaky_retries_spec.rb | 22 ++ rspec-distrib/features/passing_spec.rb | 21 ++ rspec-distrib/features/prevent_eval_spec.rb | 13 + .../features/signals_handling_spec.rb | 79 +++++ .../support/shared_contexts/base_pipeline.rb | 132 ++++++++ rspec-distrib/features/syntax_error_spec.rb | 20 ++ .../features/timeout_no_spec_picked_spec.rb | 26 ++ rspec-distrib/features/timeout_of_spec.rb | 19 ++ .../timeout_processing_stopped_spec.rb | 26 ++ rspec-distrib/lib/rspec/distrib.rb | 20 ++ .../lib/rspec/distrib/configuration.rb | 124 +++++++ .../lib/rspec/distrib/example_group.rb | 224 +++++++++++++ rspec-distrib/lib/rspec/distrib/leader.rb | 199 +++++++++++ .../lib/rspec/distrib/leader/reporter.rb | 91 +++++ .../lib/rspec/distrib/leader/rspec_helper.rb | 35 ++ .../rspec/distrib/leader/tests_provider.rb | 20 ++ rspec-distrib/lib/rspec/distrib/worker.rb | 19 ++ .../lib/rspec/distrib/worker/configuration.rb | 28 ++ .../rspec/distrib/worker/leader_reporter.rb | 35 ++ .../lib/rspec/distrib/worker/rspec_runner.rb | 205 ++++++++++++ rspec-distrib/rspec-distrib.gemspec | 22 ++ .../spec/rspec/distrib/configuration_spec.rb | 50 +++ .../spec/rspec/distrib/example_group_spec.rb | 65 ++++ .../execution_results/exception_spec.rb | 40 +++ .../rspec/distrib/leader/reporter_spec.rb | 154 +++++++++ .../rspec/distrib/leader/rspec_helper_spec.rb | 47 +++ .../distrib/leader/tests_provider_spec.rb | 10 + .../spec/rspec/distrib/leader_spec.rb | 222 ++++++++++++ .../distrib/worker/configuration_spec.rb | 13 + .../distrib/worker/leader_reporter_spec.rb | 16 + .../rspec/distrib/worker/rspec_runner_spec.rb | 94 ++++++ .../spec/rspec/distrib/worker_spec.rb | 36 ++ rspec-distrib/spec/rspec/distrib_spec.rb | 18 + rspec-distrib/spec/spec_helper.rb | 32 ++ 175 files changed, 7096 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/ISSUE_TEMPLATE/question.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/distrib-core-linters.yml create mode 100644 .github/workflows/distrib-core-specs.yml create mode 100644 .github/workflows/features-parser-specs.yml create mode 100644 .github/workflows/rspec-distrib-linters.yml create mode 100644 .github/workflows/rspec-distrib-tests.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 distrib-core/.gitignore create mode 100644 distrib-core/.rspec create mode 100644 distrib-core/.rubocop.yml create mode 100644 distrib-core/.ruby-version create mode 100644 distrib-core/CODE_OF_CONDUCT.md create mode 100644 distrib-core/Gemfile create mode 100644 distrib-core/Gemfile.lock create mode 100644 distrib-core/LICENSE.txt create mode 100644 distrib-core/README.md create mode 100644 distrib-core/distrib-core.gemspec create mode 100644 distrib-core/lib/distrib-core.rb create mode 100644 distrib-core/lib/distrib_core.rb create mode 100644 distrib-core/lib/distrib_core/configuration.rb create mode 100644 distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb create mode 100644 distrib-core/lib/distrib_core/distrib.rb create mode 100644 distrib-core/lib/distrib_core/drb_helper.rb create mode 100644 distrib-core/lib/distrib_core/leader.rb create mode 100644 distrib-core/lib/distrib_core/leader/drb_callable.rb create mode 100644 distrib-core/lib/distrib_core/leader/error_handler.rb create mode 100644 distrib-core/lib/distrib_core/leader/queue_builder.rb create mode 100644 distrib-core/lib/distrib_core/leader/queue_with_lease.rb create mode 100644 distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb create mode 100644 distrib-core/lib/distrib_core/leader/watchdog.rb create mode 100644 distrib-core/lib/distrib_core/logger_broadcaster.rb create mode 100644 distrib-core/lib/distrib_core/metrics.rb create mode 100644 distrib-core/lib/distrib_core/received_signals.rb create mode 100644 distrib-core/lib/distrib_core/spec/configuration.rb create mode 100644 distrib-core/lib/distrib_core/spec/distrib.rb create mode 100644 distrib-core/lib/distrib_core/worker.rb create mode 100644 distrib-core/spec/distrib_core/configuration_spec.rb create mode 100644 distrib-core/spec/distrib_core/distrib_spec.rb create mode 100644 distrib-core/spec/distrib_core/drb_helper_spec.rb create mode 100644 distrib-core/spec/distrib_core/leader/drb_callable_spec.rb create mode 100644 distrib-core/spec/distrib_core/leader/error_handler_spec.rb create mode 100644 distrib-core/spec/distrib_core/leader/queue_builder_spec.rb create mode 100644 distrib-core/spec/distrib_core/leader/queue_with_lease_spec.rb create mode 100644 distrib-core/spec/distrib_core/leader/retry_on_different_error_handler_spec.rb create mode 100644 distrib-core/spec/distrib_core/leader/watchdog_spec.rb create mode 100644 distrib-core/spec/distrib_core/logger_broadcaster_spec.rb create mode 100644 distrib-core/spec/distrib_core/metrics_spec.rb create mode 100644 distrib-core/spec/spec_helper.rb create mode 100644 features-parser/.gitignore create mode 100644 features-parser/.rspec create mode 100644 features-parser/.rubocop.yml create mode 100644 features-parser/.ruby-version create mode 100644 features-parser/CODE_OF_CONDUCT.md create mode 100644 features-parser/Gemfile create mode 100644 features-parser/Gemfile.lock create mode 100644 features-parser/LICENSE.txt create mode 100644 features-parser/README.md create mode 100755 features-parser/bin/console create mode 100755 features-parser/bin/setup create mode 100644 features-parser/features-parser.gemspec create mode 100644 features-parser/lib/features-parser.rb create mode 100644 features-parser/lib/features_parser/catalog.rb create mode 100644 features-parser/lib/features_parser/example.rb create mode 100644 features-parser/lib/features_parser/feature.rb create mode 100644 features-parser/lib/features_parser/name_normalizer.rb create mode 100644 features-parser/lib/features_parser/name_provider.rb create mode 100644 features-parser/lib/features_parser/outline.rb create mode 100644 features-parser/lib/features_parser/scenario.rb create mode 100644 features-parser/lib/features_parser/scenario_parser.rb create mode 100644 features-parser/lib/features_parser/version.rb create mode 100644 features-parser/spec/features_parser/catalog_spec.rb create mode 100644 features-parser/spec/features_parser/example_spec.rb create mode 100644 features-parser/spec/features_parser/feature_spec.rb create mode 100644 features-parser/spec/features_parser/name_normalizer_spec.rb create mode 100644 features-parser/spec/features_parser/name_provider_spec.rb create mode 100644 features-parser/spec/features_parser/outline_spec.rb create mode 100644 features-parser/spec/features_parser/scenario_parser_spec.rb create mode 100644 features-parser/spec/features_parser/scenario_spec.rb create mode 100644 features-parser/spec/features_parser_spec.rb create mode 100644 features-parser/spec/spec_helper.rb create mode 100644 features-parser/spec/support/parse-error.feature create mode 100644 features-parser/spec/support/some.feature create mode 100644 rspec-distrib/.gitignore create mode 100644 rspec-distrib/.rspec create mode 100644 rspec-distrib/.rspec-distrib create mode 100644 rspec-distrib/.rubocop.yml create mode 100644 rspec-distrib/.rubocop_todo.yml create mode 100644 rspec-distrib/.ruby-version create mode 100644 rspec-distrib/CODE_OF_CONDUCT.md create mode 100644 rspec-distrib/Gemfile create mode 100644 rspec-distrib/Gemfile.lock create mode 100644 rspec-distrib/LICENSE.txt create mode 100644 rspec-distrib/README.md create mode 100644 rspec-distrib/docs/startup.png create mode 100644 rspec-distrib/docs/watchdog.png create mode 100644 rspec-distrib/docs/worker.png create mode 100755 rspec-distrib/exe/rspec-distrib create mode 100644 rspec-distrib/features/aborting_worker_spec.rb create mode 100644 rspec-distrib/features/configuration_issue_spec.rb create mode 100644 rspec-distrib/features/failing_after_all_spec.rb create mode 100644 rspec-distrib/features/failing_after_suite_spec.rb create mode 100644 rspec-distrib/features/failing_before_all_spec.rb create mode 100644 rspec-distrib/features/failing_before_suite_spec.rb create mode 100644 rspec-distrib/features/failing_inside_examples_spec.rb create mode 100644 rspec-distrib/features/failing_multiple_exceptions_spec.rb create mode 100644 rspec-distrib/features/failing_outside_examples_spec.rb create mode 100644 rspec-distrib/features/feature_helper.rb create mode 100644 rspec-distrib/features/fixtures/specs/.rspec create mode 100644 rspec-distrib/features/fixtures/specs/.rspec-distrib create mode 100644 rspec-distrib/features/fixtures/specs/failing_after_all/fail_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_after_all/pass_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_before_all/bar_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_before_all/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_inside_examples/bar_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_inside_examples/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/1_pass_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/2_fail_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/3_pass_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_outside_examples/bar_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_outside_examples/baz_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_outside_examples/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/failing_outside_examples/fur_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/features_formatter.rb create mode 100644 rspec-distrib/features/fixtures/specs/flaky_retries/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/flaky_retries/zap_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/passing/bar_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/passing/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/prevent_eval/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/signals_handling/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/spec_helper.rb create mode 100644 rspec-distrib/features/fixtures/specs/syntax_error/bar_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/syntax_error/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/timeout_of_spec/bar_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/timeout_of_spec/foo_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/timeout_processing_stopped/bar_spec.rb create mode 100644 rspec-distrib/features/fixtures/specs/timeout_processing_stopped/foo_spec.rb create mode 100644 rspec-distrib/features/flaky_retries_spec.rb create mode 100644 rspec-distrib/features/passing_spec.rb create mode 100644 rspec-distrib/features/prevent_eval_spec.rb create mode 100644 rspec-distrib/features/signals_handling_spec.rb create mode 100644 rspec-distrib/features/support/shared_contexts/base_pipeline.rb create mode 100644 rspec-distrib/features/syntax_error_spec.rb create mode 100644 rspec-distrib/features/timeout_no_spec_picked_spec.rb create mode 100644 rspec-distrib/features/timeout_of_spec.rb create mode 100644 rspec-distrib/features/timeout_processing_stopped_spec.rb create mode 100644 rspec-distrib/lib/rspec/distrib.rb create mode 100644 rspec-distrib/lib/rspec/distrib/configuration.rb create mode 100644 rspec-distrib/lib/rspec/distrib/example_group.rb create mode 100644 rspec-distrib/lib/rspec/distrib/leader.rb create mode 100644 rspec-distrib/lib/rspec/distrib/leader/reporter.rb create mode 100644 rspec-distrib/lib/rspec/distrib/leader/rspec_helper.rb create mode 100644 rspec-distrib/lib/rspec/distrib/leader/tests_provider.rb create mode 100644 rspec-distrib/lib/rspec/distrib/worker.rb create mode 100644 rspec-distrib/lib/rspec/distrib/worker/configuration.rb create mode 100644 rspec-distrib/lib/rspec/distrib/worker/leader_reporter.rb create mode 100644 rspec-distrib/lib/rspec/distrib/worker/rspec_runner.rb create mode 100644 rspec-distrib/rspec-distrib.gemspec create mode 100644 rspec-distrib/spec/rspec/distrib/configuration_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/example_group_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/execution_results/exception_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/leader/reporter_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/leader/rspec_helper_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/leader/tests_provider_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/leader_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/worker/configuration_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/worker/leader_reporter_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/worker/rspec_runner_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib/worker_spec.rb create mode 100644 rspec-distrib/spec/rspec/distrib_spec.rb create mode 100644 rspec-distrib/spec/spec_helper.rb diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..f202e48 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @toptal/devx diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..da97bb7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,38 @@ +name: Bug report +description: Report a bug to help us improve +title: "[BUG] Brief description of the issue" +labels: [bug] +body: + - type: markdown + attributes: + value: | + **Describe the bug** + A clear and concise description of what the bug is. + + - type: textarea + id: steps-to-reproduce + attributes: + label: "Steps to reproduce" + description: "Steps to reproduce the behavior" + placeholder: "1. Go to '...'\n2. Run '....'\n3. See error" + + - type: textarea + id: expected-behavior + attributes: + label: "Expected behavior" + description: "A clear and concise description of what you expected to happen" + placeholder: "Expected behavior" + + - type: textarea + id: screenshots + attributes: + label: "Screenshots" + description: "If applicable, add screenshots to help explain your problem" + placeholder: "Add screenshots here" + + - type: textarea + id: additional-context + attributes: + label: "Additional context" + description: "Add any other context about the problem here" + placeholder: "Additional context" diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..aec91e9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,31 @@ +name: Feature request +description: Suggest an idea for this project +title: "[FEATURE] Brief description of the feature" +labels: [enhancement] +body: + - type: markdown + attributes: + value: | + **Is your feature request related to a problem? Please describe.** + A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + + - type: textarea + id: solution + attributes: + label: "Describe the solution you'd like" + description: "A clear and concise description of what you want to happen" + placeholder: "Describe the solution" + + - type: textarea + id: alternatives + attributes: + label: "Describe alternatives you've considered" + description: "A clear and concise description of any alternative solutions or features you've considered" + placeholder: "Describe alternatives" + + - type: textarea + id: additional-context + attributes: + label: "Additional context" + description: "Add any other context or screenshots about the feature request here" + placeholder: "Additional context" diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 0000000..c4ed20a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,17 @@ +name: Question +description: Ask a question or start a discussion +title: "[QUESTION] Brief description of the question" +labels: [question] +body: + - type: markdown + attributes: + value: | + **Your question** + Describe your question or topic for discussion. + + - type: textarea + id: additional-context + attributes: + label: "Additional context" + description: "Add any other context or information that might be useful to answer your question" + placeholder: "Additional context" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..401cdd6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,69 @@ +version: 2 + +updates: + - package-ecosystem: bundler + directory: "/distrib-core" + schedule: + interval: "weekly" + day: "wednesday" + time: "07:00" + pull-request-branch-name: + separator: "-" + labels: + - "no-jira" + - "ruby" + - "dependencies" + reviewers: + - "toptal/devx" + insecure-external-code-execution: allow + open-pull-requests-limit: 2 + + - package-ecosystem: bundler + directory: "/rspec-distrib" + schedule: + interval: "weekly" + day: "wednesday" + time: "07:00" + pull-request-branch-name: + separator: "-" + labels: + - "no-jira" + - "ruby" + - "dependencies" + reviewers: + - "toptal/devx" + insecure-external-code-execution: allow + open-pull-requests-limit: 2 + + - package-ecosystem: bundler + directory: "/features-parser" + schedule: + interval: "weekly" + day: "wednesday" + time: "07:00" + pull-request-branch-name: + separator: "-" + labels: + - "no-jira" + - "ruby" + - "dependencies" + reviewers: + - "toptal/devx" + insecure-external-code-execution: allow + open-pull-requests-limit: 2 + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "wednesday" + time: "07:00" + pull-request-branch-name: + separator: "-" + labels: + - "no-jira" + - "dependencies" + - "gha" + reviewers: + - "toptal/devx" + open-pull-requests-limit: 2 diff --git a/.github/workflows/distrib-core-linters.yml b/.github/workflows/distrib-core-linters.yml new file mode 100644 index 0000000..c43a06f --- /dev/null +++ b/.github/workflows/distrib-core-linters.yml @@ -0,0 +1,32 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +env: + BUNDLE_FROZEN: true + +jobs: + rubocop: + name: distrib-core - Linters + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + working-directory: distrib-core + + - name: Run RuboCop + run: bundle exec rubocop + working-directory: distrib-core + + - name: Run yardoc + run: bundle exec yardoc --fail-on-warning + working-directory: distrib-core diff --git a/.github/workflows/distrib-core-specs.yml b/.github/workflows/distrib-core-specs.yml new file mode 100644 index 0000000..7439d0d --- /dev/null +++ b/.github/workflows/distrib-core-specs.yml @@ -0,0 +1,29 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +permissions: + contents: read + +jobs: + test: + name: distrib-core - Specs + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + working-directory: distrib-core + + - name: Run tests + run: bundle exec rspec + working-directory: distrib-core diff --git a/.github/workflows/features-parser-specs.yml b/.github/workflows/features-parser-specs.yml new file mode 100644 index 0000000..e39a6d0 --- /dev/null +++ b/.github/workflows/features-parser-specs.yml @@ -0,0 +1,26 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + name: features-parser - Specs + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + working-directory: features-parser + + - name: Run specs + run: bundle exec rspec + working-directory: features-parser diff --git a/.github/workflows/rspec-distrib-linters.yml b/.github/workflows/rspec-distrib-linters.yml new file mode 100644 index 0000000..431374d --- /dev/null +++ b/.github/workflows/rspec-distrib-linters.yml @@ -0,0 +1,32 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +env: + BUNDLE_ONLY: linters + +jobs: + linters: + name: rspec-distrib - Linters + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + working-directory: rspec-distrib + + - name: Run RuboCop + run: bundle exec rubocop + working-directory: rspec-distrib + + - name: Run yardoc + run: bundle exec yardoc --fail-on-warning + working-directory: rspec-distrib diff --git a/.github/workflows/rspec-distrib-tests.yml b/.github/workflows/rspec-distrib-tests.yml new file mode 100644 index 0000000..2e5e3a8 --- /dev/null +++ b/.github/workflows/rspec-distrib-tests.yml @@ -0,0 +1,29 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + tests: + name: rspec-distrib - Tests + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + working-directory: rspec-distrib + + - name: Run specs + run: bundle exec rspec spec/ + working-directory: rspec-distrib + + - name: Run features + run: bundle exec rspec features/ + working-directory: rspec-distrib diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0ad0580 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +# Ignore generated credentials from google-github-actions/auth +gha-creds-*.json +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..4e3d762 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# test-distrib + +## What's *-distrib? + +This is a collection of gems for running test in parallel on +multiple machines/processes. + +* [distrib-core](./distrib-core/README.md) +* [features-parser](./features-parser/README.md) +* [rspec-distrib](./rspec-distrib/README.md) diff --git a/distrib-core/.gitignore b/distrib-core/.gitignore new file mode 100644 index 0000000..02ca693 --- /dev/null +++ b/distrib-core/.gitignore @@ -0,0 +1,12 @@ +*.gem +# for a library or gem, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# Gemfile.lock +.ruby-gemset +.bundle +distrib.log +/bundle +/coverage/ +/.yardoc/ +/doc/ +/.idea/* diff --git a/distrib-core/.rspec b/distrib-core/.rspec new file mode 100644 index 0000000..ddc1538 --- /dev/null +++ b/distrib-core/.rspec @@ -0,0 +1,3 @@ +--color +--require spec_helper +--require pry diff --git a/distrib-core/.rubocop.yml b/distrib-core/.rubocop.yml new file mode 100644 index 0000000..7cc356b --- /dev/null +++ b/distrib-core/.rubocop.yml @@ -0,0 +1,47 @@ +require: + - rubocop-rspec + +AllCops: + DisplayCopNames: true + NewCops: enable + Exclude: + - coverage/**/* + - bundle/**/* + - vendor/**/* + +Layout/LineLength: + Max: 120 + Exclude: + - lib/distrib_core/configuration.rb + +Metrics/BlockLength: + Exclude: + - spec/**/* + - lib/distrib_core/spec/**/* + +RSpec/MessageSpies: + Enabled: false + +RSpec/MultipleExpectations: + Enabled: false + +RSpec/ExampleLength: + Enabled: false + +RSpec/NestedGroups: + Max: 4 + +RSpec/StubbedMock: + Enabled: false + +RSpec/MultipleMemoizedHelpers: + Enabled: false + +Naming/FileName: + Exclude: + - lib/distrib-core.rb + +Style/FrozenStringLiteralComment: + Enabled: true + Include: + - spec/**/* diff --git a/distrib-core/.ruby-version b/distrib-core/.ruby-version new file mode 100644 index 0000000..351227f --- /dev/null +++ b/distrib-core/.ruby-version @@ -0,0 +1 @@ +3.2.4 diff --git a/distrib-core/CODE_OF_CONDUCT.md b/distrib-core/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d6f70c9 --- /dev/null +++ b/distrib-core/CODE_OF_CONDUCT.md @@ -0,0 +1,14 @@ +# The Community Code of Conduct + +**Note:** We have picked the following code of conduct based on [Ruby's own code of conduct](https://www.ruby-lang.org/en/conduct/). + +This document provides a few simple community guidelines for a safe, respectful, +productive, and collaborative place for any person who is willing to contribute +to the community. It applies to all "collaborative spaces", which are +defined as community communications channels (such as mailing lists, submitted +patches, commit comments, etc.). + +* Participants will be tolerant of opposing views. +* Participants must ensure that their language and actions are free of personal attacks and disparaging personal remarks. +* When interpreting the words and actions of others, participants should always assume good intentions. +* Behaviour which can be reasonably considered harassment will not be tolerated. diff --git a/distrib-core/Gemfile b/distrib-core/Gemfile new file mode 100644 index 0000000..b73df93 --- /dev/null +++ b/distrib-core/Gemfile @@ -0,0 +1,12 @@ +source 'https://rubygems.org' + +gemspec + +gem 'pry' +gem 'pry-byebug' +gem 'rspec', '~> 3.13' +gem 'rubocop' +gem 'rubocop-rspec' +gem 'simplecov' +gem 'timecop' +gem 'yard' diff --git a/distrib-core/Gemfile.lock b/distrib-core/Gemfile.lock new file mode 100644 index 0000000..0b38d5c --- /dev/null +++ b/distrib-core/Gemfile.lock @@ -0,0 +1,96 @@ +PATH + remote: . + specs: + distrib-core (0.0.1) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + byebug (11.1.3) + coderay (1.1.3) + diff-lcs (1.5.1) + docile (1.4.0) + json (2.7.2) + language_server-protocol (3.17.0.3) + method_source (1.1.0) + parallel (1.24.0) + parser (3.3.1.0) + ast (~> 2.4.1) + racc + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + pry-byebug (3.10.1) + byebug (~> 11.0) + pry (>= 0.13, < 0.15) + racc (1.8.0) + rainbow (3.1.1) + regexp_parser (2.9.2) + rexml (3.2.8) + strscan (>= 3.0.9) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) + rubocop (1.64.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.31.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) + rubocop-capybara (2.20.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.25.1) + rubocop (~> 1.41) + rubocop-rspec (2.29.2) + rubocop (~> 1.40) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + rubocop-rspec_rails (~> 2.28) + rubocop-rspec_rails (2.28.3) + rubocop (~> 1.40) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + strscan (3.1.0) + timecop (0.9.8) + unicode-display_width (2.5.0) + yard (0.9.36) + +PLATFORMS + ruby + +DEPENDENCIES + distrib-core! + pry + pry-byebug + rspec (~> 3.13) + rubocop + rubocop-rspec + simplecov + timecop + yard + +BUNDLED WITH + 2.2.30 diff --git a/distrib-core/LICENSE.txt b/distrib-core/LICENSE.txt new file mode 100644 index 0000000..08b3961 --- /dev/null +++ b/distrib-core/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Toptal + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/distrib-core/README.md b/distrib-core/README.md new file mode 100644 index 0000000..47909c8 --- /dev/null +++ b/distrib-core/README.md @@ -0,0 +1,28 @@ +# distrib-core + +Is a common core module for [rspec-distrib](../rspec-distrib). + +## Installation + +Add the gem to the application's Gemfile: + +```ruby +gem 'distrib-core', git: 'git@github.com:toptal/test-distrib.git', + glob: 'distrib-core/*.gemspec' +``` + +## Getting started + +```shell +bundle install +bundle exec rspec +bundle exec rubocop +``` + +## Contributing + +Bug reports and pull requests are welcome [on GitHub](https://github.com/toptal/test-distrib/issues). + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/distrib-core/distrib-core.gemspec b/distrib-core/distrib-core.gemspec new file mode 100644 index 0000000..94ede11 --- /dev/null +++ b/distrib-core/distrib-core.gemspec @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +Gem::Specification.new do |s| + s.name = 'distrib-core' + s.version = '0.0.1' + s.authors = ['Toptal, LLC'] + s.email = ['open-source@toptal.com'] + s.license = 'MIT' + + s.summary = 'Core classes for rspec-distrib and cucumber-distrib' + s.description = '' + s.homepage = 'https://github.com/toptal/test-distrib' + s.required_ruby_version = '>= 3.2.4' + + s.files = Dir['lib/**/*.rb'] + + s.metadata['rubygems_mfa_required'] = 'true' +end diff --git a/distrib-core/lib/distrib-core.rb b/distrib-core/lib/distrib-core.rb new file mode 100644 index 0000000..3402411 --- /dev/null +++ b/distrib-core/lib/distrib-core.rb @@ -0,0 +1,2 @@ +# So we can require it as 'distrib-core'. +require 'distrib_core' diff --git a/distrib-core/lib/distrib_core.rb b/distrib-core/lib/distrib_core.rb new file mode 100644 index 0000000..f42f845 --- /dev/null +++ b/distrib-core/lib/distrib_core.rb @@ -0,0 +1,17 @@ +require 'distrib_core/core_ext/drb_tcp_socket' +require 'distrib_core/logger_broadcaster' +require 'distrib_core/leader' +require 'distrib_core/configuration' +require 'distrib_core/distrib' +require 'distrib_core/drb_helper' +require 'distrib_core/metrics' +require 'distrib_core/received_signals' +require 'distrib_core/worker' + +# A core module. Has a quick alias to configuration. +module DistribCore + # Alias to {DistribCore::Configuration.current} + def self.configuration + Configuration.current + end +end diff --git a/distrib-core/lib/distrib_core/configuration.rb b/distrib-core/lib/distrib_core/configuration.rb new file mode 100644 index 0000000..6137fcb --- /dev/null +++ b/distrib-core/lib/distrib_core/configuration.rb @@ -0,0 +1,161 @@ +require 'logger' +require 'distrib_core/leader/error_handler' +require 'distrib_core/leader/retry_on_different_error_handler' +require 'distrib_core/logger_broadcaster' + +module DistribCore + # This module contains shared attrs instantiated by specific configuration classes. + # + # @see DistribCore::Distrib#configure + module Configuration + class << self + # Set global configuration. Can be set only one time + # + # @param configuration [DistribCore::Configuration] + def current=(configuration) + raise('Configuration is already set') if @current && @current != configuration + + @current = configuration + end + + # @return [DistribCore::Configuration] global configuration + def current + @current || raise('Configuration is not set') + end + end + + TIMEOUT_STRATEGIES = %i[repush release].freeze + + # @example Override default list of the tests: + # ...configure do |config| + # config.tests_provider = -> { + # Dir.glob(['features/**/*_feature.rb', 'engines/**/*_feature.rb']) + # } + # end + attr_writer :tests_provider + + # @example Specify object to process exceptions during execution + # ...configure do |config| + # config.error_handler = MyErrorHandler.new + # end + attr_writer :error_handler + + attr_writer :logger + + # @example Set equal timeout for all tests to 30 seconds: + # ...configure do |config| + # config.test_timeout = 30 # seconds + # end + # + # @example Or you can specify timeout per test. An object that responds to `call` and receives the test as an argument. The proc returns the timeout in seconds. + # ...configure do |config| + # config.test_timeout = ->(test) do + # 10 + 2 * average_execution_in_seconds(test) + # end + # end + attr_accessor :test_timeout + + # @example Set how long leader will wait before first test processed by workers. Leader will exit if no tests picked in this period + # ...configure do |config| + # config.first_test_picked_timeout = 10*60 # 10 minutes + # end + attr_accessor :first_test_picked_timeout + + # @example Specify custom options for DRb service. Defaults are `{ safe_level: 1 }`. @see `DRb::DRbServer.new` for complete list + # ...configure do |config| + # config.drb = {safe_level: 0, verbose: true} + # end + attr_accessor :drb + + # @example Specify custom block to pre-process examples before reporting them to the leader. Useful to add additional information about workers. + # ...configure do |config| + # config.before_test_report = -> (file_name, example_groups) do + # example_groups.each { |eg| eg.metadata[:custom] = 'foo' } + # end + # end + attr_accessor :before_test_report + + # @example Specify custom block which will be called on leader after run. + # ...configure do |config| + # config.on_finish = -> () do + # 'Whatever logic before leader exit' + # end + # end + attr_accessor :on_finish + + # @example Disable (mute) debug logger + # ...configure do |config| + # config.debug_logger = Logger.new(nil) + # end + attr_writer :debug_logger + + attr_accessor :tests_processing_stopped_timeout, :drb_tcp_socket_connection_timeout, :leader_connection_attempts + attr_reader :timeout_strategy + + # Initialize configuration with default values and set it to {DistribCore::Configuration.current} + def initialize + DistribCore::Configuration.current = self + + @test_timeout = 60 # 1 minute + @first_test_picked_timeout = 10 * 60 # 10 minutes + @tests_processing_stopped_timeout = 5 * 60 # 5 minutes + @drb = { safe_level: 1 } + @drb_tcp_socket_connection_timeout = 5 # 5 seconds + @leader_connection_attempts = 200 + self.timeout_strategy = :repush + end + + # Provider for tests to execute + # + # @return [Proc, Object#call] an object which responds to `#call` + def tests_provider + @tests_provider || raise(NotImplementedError) + end + + # Object to handle errors from workers + def error_handler + @error_handler || raise(NotImplementedError) + end + + # Gives a timeout for a particular test based on `#test_timeout` + # + # @see #test_timeout + # + # @param test [String] a test + # @return [Float] timeout in seconds + def timeout_for(test) + test_timeout.respond_to?(:call) ? test_timeout.call(test) : test_timeout + end + + # @return [Logger] + def logger + @logger ||= Logger.new($stdout, level: :info) + end + + # Set how Watchdog will handle timed out test. + def timeout_strategy=(value) + unless TIMEOUT_STRATEGIES.include?(value) + raise "Invalid Timeout Strategy. Given: #{value.inspect}. Expected one of: #{TIMEOUT_STRATEGIES.inspect}" + end + + @timeout_strategy = value + end + + # Main logging interface used by distrib. + # + # @return [LoggerBroadcaster] + # @api private + def broadcaster + @broadcaster ||= LoggerBroadcaster.new([logger, debug_logger]) + end + + # Debugging logger. However user configures `logger`, + # this one collects messages logged at all levels. + # + # @return [Logger] + # @api private + def debug_logger + @debug_logger ||= Logger.new('distrib.log', level: :debug) + end + end +end diff --git a/distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb b/distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb new file mode 100644 index 0000000..32d2a9c --- /dev/null +++ b/distrib-core/lib/distrib_core/core_ext/drb_tcp_socket.rb @@ -0,0 +1,20 @@ +# A patch to reduce connection timeout on DRb Socket. +# @see https://rubydoc.info/stdlib/drb/DRb +module DRb + # The following monkey-patch sets much lower value for connection timeout + # By default it is over 2 minutes and it is causing a major worker shutdown + # delay when the leader has finished already. + # @see https://rubydoc.info/stdlib/drb/DRb/DRbTCPSocket + class DRbTCPSocket + # @param uri [String] + # @param config [Hash] + def self.open(uri, config) + host, port, = parse_uri(uri) + # Original line was: + # soc = TCPSocket.open(host, port) + timeout = DistribCore.configuration.drb_tcp_socket_connection_timeout + soc = Socket.tcp(host, port, connect_timeout: timeout) + new(uri, soc, config) + end + end +end diff --git a/distrib-core/lib/distrib_core/distrib.rb b/distrib-core/lib/distrib_core/distrib.rb new file mode 100644 index 0000000..0bdf880 --- /dev/null +++ b/distrib-core/lib/distrib_core/distrib.rb @@ -0,0 +1,41 @@ +module DistribCore + # This module is used to define common methods on root classes. + module Distrib + # Call to prepare configuration. + # + # @see DistribCore::Configuration + def configure(...) + configuration.instance_eval(...) + end + + # Set kind of the current instance + # + # @param kind [Symbol] `:leader` or `:worker` only + def kind=(kind) + raise("Mode is already set: #{kind}") if @kind + + kind = kind&.to_sym + + raise(ArgumentError, 'Invalid kind, should be `leader` or `worker`') unless %i[leader worker].include?(kind) + + @kind = kind + end + + # @return kind of current instance. `:leader` or `:worker` + # + # @raise [RuntimeError] if kind is not set + def kind + @kind || raise('kind is not set') + end + + # @return [TrueClass, FalseClass] `true` when `kind` is `:leader` + def leader? + kind == :leader + end + + # @return [TrueClass, FalseClass] true when `kind` is `:worker` + def worker? + kind == :worker + end + end +end diff --git a/distrib-core/lib/distrib_core/drb_helper.rb b/distrib-core/lib/distrib_core/drb_helper.rb new file mode 100644 index 0000000..1e6f303 --- /dev/null +++ b/distrib-core/lib/distrib_core/drb_helper.rb @@ -0,0 +1,119 @@ +require 'drb' + +module DistribCore + # Helper that handles some problematic cases with DRb. + module DRbHelper + class << self + # Checks if any of the passed object is a `DRbUnknown`. + # It tries to load such objects and logs explained error if fails. + # Used on Leader when receiving report from workers. + # + # @param objects [Array] + # @return [TrueClass, FalseClass] `true` if failed to load one of the passed objects + def drb_unknown?(*objects) + got_drb_unknown = false + + objects.each do |object| + next unless object.is_a?(DRb::DRbUnknown) + + error = error_of_marshal_load(object) + + logger.error 'Parse error:' + logger.error error + logger.debug "Can't parse: #{object.inspect}" + + got_drb_unknown = true + end + + got_drb_unknown + end + + # Checks if error was caused by `Marshal#dump`. + # Recursively explores object and tries to `Marshal.dump` it. + # If dump failed - calls the same function for its instance variables. + # Logs path to the un-dumpable object. + # Used on Worker to find objects(or parts) which can't be sent to Leader. + # + # @param error [Exception] + # @param objects [Object] + # @return [TrueClass, FalseClass] + def dump_failed?(error, objects) + return false unless marshal_error?(error) + + dig_dump(objects) + true + end + + private + + # Checks if error was caused by `Marshal#dump`. + # + # @param error [Exception] + # @return [Boolean] + def marshal_error?(error) + while error + if error.is_a?(TypeError) && error.message.include?('no _dump_data is defined for') + logger.error 'Marshal dump error:' + logger.error error + return true + end + + error = error.cause + end + false + end + + # Recursively explores object and tries to `Marshal.dump` it. + # If dump failed - calls the same function for its instance variables. + # Logs path to the un-dumpable object. + # Used on Worker to find objects(or parts) which can't be sent to Leader. + # + # @param obj [Object] object to explore + # @param parent_objects [Array] list of parent objects we already digging in + # @param vars [Array] list of instance variables we already digging in + def dig_dump(obj, parent_objects = [], vars = []) # rubocop:disable Metrics/ + objs = if obj.is_a?(Array) + obj.flatten + elsif obj.is_a?(Hash) + obj.to_a.flatten + else + [obj] + end + + objs.each do |o| + Marshal.dump(o) + rescue TypeError + if o.instance_variables.none? + path = [parent_objects, vars].transpose.map(&:join).join(' ') + logger.debug "Cant serialize #{o} in path #{path}" + else + o.instance_variables.each do |var| + val = o.instance_variable_get(var) + next unless val + + dig_dump(val, parent_objects + [o.class], vars + [var]) + end + end + end + + nil + end + + def error_of_marshal_load(object) + error = nil + + begin + Marshal.load(object.buf) # rubocop:disable Security/MarshalLoad + rescue StandardError => e + error = e + end + + error + end + + def logger + DistribCore.configuration.broadcaster + end + end + end +end diff --git a/distrib-core/lib/distrib_core/leader.rb b/distrib-core/lib/distrib_core/leader.rb new file mode 100644 index 0000000..eb41890 --- /dev/null +++ b/distrib-core/lib/distrib_core/leader.rb @@ -0,0 +1,30 @@ +require 'distrib_core/leader/drb_callable' +require 'distrib_core/leader/queue_builder' +require 'distrib_core/leader/queue_with_lease' +require 'distrib_core/leader/watchdog' + +module DistribCore + # Stores common methods for Leader (basic module for Leader). + module Leader + # @param klass [Class] + def self.included(klass) + klass.extend(ClassMethods) + klass.extend(DRbCallable) + end + + private + + # Methods to define on class-level + module ClassMethods + private + + def logger + DistribCore.configuration.broadcaster + end + end + + def logger + DistribCore.configuration.broadcaster + end + end +end diff --git a/distrib-core/lib/distrib_core/leader/drb_callable.rb b/distrib-core/lib/distrib_core/leader/drb_callable.rb new file mode 100644 index 0000000..eb8b7f5 --- /dev/null +++ b/distrib-core/lib/distrib_core/leader/drb_callable.rb @@ -0,0 +1,36 @@ +require 'distrib_core/drb_helper' + +module DistribCore + module Leader + # A wrapper for methods available through DRb. + # Use this module to wrap DRb exposed methods to handle and log any error. + module DRbCallable + # Wraps the method. If it raises exception - logs it and calls `handle_non_example_exception` + # + # @example + # drb_callable def ping + # puts 'pong' + # end + # + # @param method_name [Symbol] + # @return NilClass + def drb_callable(method_name) # rubocop:disable Metrics/MethodLength: + alias_method "drb_callable_#{method_name}", method_name + + define_method method_name do |*args| # rubocop:disable Metrics/MethodLength: + if DistribCore::DRbHelper.drb_unknown?(*args) + handle_non_example_exception + nil + else + public_send("drb_callable_#{method_name}", *args) + end + rescue StandardError => e + logger.error "Failed to call #{method_name}" + logger.error e + handle_non_example_exception + nil + end + end + end + end +end diff --git a/distrib-core/lib/distrib_core/leader/error_handler.rb b/distrib-core/lib/distrib_core/leader/error_handler.rb new file mode 100644 index 0000000..6e03c05 --- /dev/null +++ b/distrib-core/lib/distrib_core/leader/error_handler.rb @@ -0,0 +1,96 @@ +module DistribCore + module Leader + # Default strategy to manage retries of tests. + class ErrorHandler + attr_accessor :retryable_exceptions, :retry_attempts, :fatal_worker_failures, :failed_workers_threshold + attr_reader :failed_workers_count + + def initialize(exception_extractor) + @retryable_exceptions = [] + @retry_attempts = 0 + @retries_per_test = Hash.new(0) + + @fatal_worker_failures = [] + @failed_workers_threshold = 0 + @failed_workers_count = 0 + + @exception_extractor = exception_extractor + end + + def retry_test?(test, results, exception) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity + return false if retries_per_test[test] >= retry_attempts + + exceptions = exception_extractor.failures_of(results) + exceptions.push(exception) if exception + + failures_causes = exception_extractor.unpack_causes(exceptions) + + return false if failures_causes.empty? + + if retryable_exceptions.empty? + retries_per_test[test] += 1 + return true + end + + retried = failures_causes.all? do |causes| + causes.any? do |cause| + retryable_exceptions.include?(cause.original_class) + end + end + + retries_per_test[test] += 1 if retried + + retried + end + + def ignore_worker_failure?(exception) + self.failed_workers_count += 1 + + return false if missing_exception?(exception) || exceeded_failures_threshold? || fatal_failure?(exception) + + true + end + + private + + attr_reader :exception_extractor, :retries_per_test + attr_writer :failed_workers_count + + def missing_exception?(exception) + return false if exception + + logger.debug 'Exception missing' + true + end + + def exceeded_failures_threshold? + if failed_workers_count > failed_workers_threshold + logger.debug "#{failed_workers_count} failure(s) reported, " \ + "which exceeds the threshold of #{failed_workers_threshold}" + return true + end + + false + end + + def fatal_failure?(exception) + failure_causes = exception_extractor.unpack_causes([exception]).first + + cause_class = failure_causes.find do |cause| + fatal_worker_failures.include?(cause.original_class) + end&.original_class + + if cause_class + logger.debug "Fatal failure found: #{cause_class}" + return true + end + + false + end + + def logger + DistribCore.configuration.broadcaster + end + end + end +end diff --git a/distrib-core/lib/distrib_core/leader/queue_builder.rb b/distrib-core/lib/distrib_core/leader/queue_builder.rb new file mode 100644 index 0000000..f6f82f6 --- /dev/null +++ b/distrib-core/lib/distrib_core/leader/queue_builder.rb @@ -0,0 +1,13 @@ +module DistribCore + module Leader + # Helper that builds a list of the test files to execute sorted by average + # execution time descending. The order strategy is backed by + # https://en.wikipedia.org/wiki/Queueing_theory + module QueueBuilder + # @return [Array] list of test files in the order they should be enqueued + def self.tests + ::DistribCore.configuration.tests_provider.call + end + end + end +end diff --git a/distrib-core/lib/distrib_core/leader/queue_with_lease.rb b/distrib-core/lib/distrib_core/leader/queue_with_lease.rb new file mode 100644 index 0000000..6005d75 --- /dev/null +++ b/distrib-core/lib/distrib_core/leader/queue_with_lease.rb @@ -0,0 +1,139 @@ +require 'timeout' +require 'monitor' + +module DistribCore + module Leader + # Generic queue with lease. + # + # Additionally it keeps the time of the lease, allowing watchdog to return + # (repush) timed out entries back to the queue. + # + # Lifecycle of an entry in the queue: + # [QUEUED]--(lease)-->[LEASED] -(release)-> out of the queue + # ^---(repush)--/ + # + class QueueWithLease + include MonitorMixin + + SYNC_TIMEOUT_SEC = 60 + + attr_reader :initialized_at, :last_activity_at + + # @param entries [Array] the entries to enqueue + def initialize(entries = []) + # To initialize [MonitorMixin](https://ruby-doc.org/3.2.4/exts/monitor/MonitorMixin.html) + super() + @entries = entries.dup + @leased = {} + @completed = Set.new + @initialized_at = Time.now + end + + # @return [Object] the next entry in the queue + def lease + loop do + sleep 0.1 + + entry = synchronize_with_timeout { entries.pop } + next unless entry + + next if completed?(entry) + + record_lease(entry) + return entry + end + end + + # It's only necessary to remove the entry from the list of leased ones, and + # this have to be done atomically with pushing to the queue to avoid race + # conditions when the entry is released by another thread, or there's an + # attempt to lease it and we release it immediately after. + # + # @param entry [String] + def repush(entry) + synchronize_with_timeout do + leased.delete(entry) + + # We want to insert an entry before the last one, so it won't be leased again with the same worker + # If there is no last entry, we just push it to the end + entries.insert(entries.empty? ? -1 : -2, entry) + end + end + + # @param entry [String] + # @return [NilClass, Set] `nil` if was already completed + def release(entry) + return if completed?(entry) + + synchronize_with_timeout do + leased.delete(entry) + completed.add(entry) + end + end + + # @param entry [String] + # @return [TrueClass, FalseClass] `true` if `entry` was already completed + def completed?(entry) + synchronize_with_timeout { completed.include?(entry) } + end + + # @return [TrueClass, FalseClass] `true` if there is no more enqueued or leased entries + def empty? + size.zero? + end + + # @return [Integer] amount of not completed entries + def size + synchronize_with_timeout { leased.size + entries.size } + end + + # @return [Integer] amount of completed entries + def completed_size + completed.size + end + + # @api private + # @return [Integer] amount of leased entries + def leased_size + leased.size + end + + # @api private + # Iterate over leased entries + def select_leased(...) + synchronize_with_timeout { leased.dup.select(...) } + end + + # @return [Array] Lists of tests in the queue + def entries_list + synchronize_with_timeout { entries.dup } + end + + # @return [TrueClass, FalseClass] `true` if there was already some activity in the queue + def visited? + @last_activity_at != nil + end + + private + + attr_reader :entries, :completed, :leased + + def record_lease(entry) + synchronize_with_timeout do + leased[entry] = Time.now + @last_activity_at = Time.now + end + end + + def synchronize_with_timeout(&block) + Timeout.timeout(SYNC_TIMEOUT_SEC) do + synchronize do + yield block + end + end + rescue Timeout::Error + raise 'Timeout while waiting for synchronization (deadlock)!' + end + end + end +end diff --git a/distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb b/distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb new file mode 100644 index 0000000..78d3487 --- /dev/null +++ b/distrib-core/lib/distrib_core/leader/retry_on_different_error_handler.rb @@ -0,0 +1,46 @@ +module DistribCore + module Leader + # Only retry if the error is different. + class RetryOnDifferentErrorHandler < ErrorHandler + def initialize(exception_extractor, retry_limit: 2, repeated_error_limit: 1) + super(exception_extractor) + @exceptions_per_test = Hash.new([]) + @retry_limit = retry_limit + @repeated_error_limit = repeated_error_limit + end + + def retry_test?(test, results, exception) + return false if retries_per_test[test] >= retry_limit + + failures_causes = aggregate_failure_causes(exception, results) + + return false if failures_causes.empty? || repeated_error_limit_exceeded?(test, failures_causes) + + exceptions_per_test[test] += failures_causes + retries_per_test[test] += 1 + + true + end + + private + + attr_reader :exceptions_per_test, :retry_limit, :repeated_error_limit + + def aggregate_failure_causes(exception, results) + exceptions = exception_extractor.failures_of(results) + exceptions.push(exception) if exception + exception_extractor.unpack_causes(exceptions).flatten + end + + def repeated_error_limit_exceeded?(test, failures_causes) + failures_causes.any? do |new| + failures_with_same_exception = exceptions_per_test[test].select do |old| + old.original_class == new.original_class && old.message == new.message + end + + failures_with_same_exception.count >= repeated_error_limit + end + end + end + end +end diff --git a/distrib-core/lib/distrib_core/leader/watchdog.rb b/distrib-core/lib/distrib_core/leader/watchdog.rb new file mode 100644 index 0000000..f354cc2 --- /dev/null +++ b/distrib-core/lib/distrib_core/leader/watchdog.rb @@ -0,0 +1,149 @@ +require 'rainbow/refinement' + +module DistribCore + module Leader + # A watchdog to observe the state of queue. + # A thread running on the background that keeps checking if there are tests to run. + # It closes the connection between Leader and Workers when the queues are empty. + # A thread watching over presence of the entries on the queue and lease + # timeouts. Stops the {Leader} by stopping its DRb exposed service. + class Watchdog # rubocop:disable Metrics/ClassLength + using Rainbow + + def initialize(queue) + @queue = queue + @failed = false + @logger = DistribCore.configuration.broadcaster + end + + def start # rubocop:disable Metrics/MethodLength, Metrics/AbcSize + Thread.new do + loop do + if ::DistribCore::ReceivedSignals.any? + logger.warn ::DistribCore::ReceivedSignals.message + DRb.current_server.stop_service + break + end + + handle_timed_out_tests + + if queue.empty? # no more tests left + logger.info 'Queue is empty. Stopping.' + DRb.current_server.stop_service + break + end + + timeout_error = handle_workers_timeout + + if timeout_error + logger.error timeout_error + logger.info not_executed_tests_after_timeout_report + @failed = true + DRb.current_server.stop_service + break + end + + Kernel.sleep(1) + end + end + end + + # @return [TrueClass, FalseClass] `true` when watchdog encountered an error + def failed? + @failed + end + + private + + attr_reader :queue, :logger + + def config + DistribCore.configuration + end + + def timed_out_tests + queue.select_leased do |test, start_time| + start_time < Time.now - config.timeout_for(test) + end.keys + end + + def repush(test) + queue.repush(test) + Metrics.watchdog_repushed(test, config.timeout_for(test)) + end + + def handle_timed_out_tests + timed_out_tests.each do |test| + timeout = config.timeout_for(test) + logger.warn "#{test} (Timeout: #{formatted_timeout(timeout)}) #{strategy_warn}" + + release_on_timeout? ? queue.release(test) : repush(test) + end + end + + def release_on_timeout? + config.timeout_strategy == :release + end + + def strategy_warn + return 'will NOT be pushed back to the queue - marking as completed.' if release_on_timeout? + + 'but will be pushed back to the queue.' + end + + def not_executed_tests_after_timeout_report + tests = queue.entries_list + tests_to_show = [tests.length, 10].min + + <<~TEXT + #{tests.length} tests not executed, showing #{tests_to_show}: + #{tests.take(tests_to_show).join("\n")} + TEXT + end + + def handle_workers_timeout + if queue.visited? + tests_picked_timeout_error + else + workers_failed_to_start_error + end + end + + def tests_picked_timeout_error + return unless (Time.now - queue.last_activity_at) > config.tests_processing_stopped_timeout + + <<~ERROR_MESSAGE.strip + Workers did not pick tests for too long! + After Workers processed #{queue.completed_size} test(s), Leader will abort as it waited for over + #{formatted_timeout(config.tests_processing_stopped_timeout)} which is the configured time to wait for + Workers to pick up tests. + Aborting... + ERROR_MESSAGE + end + + def workers_failed_to_start_error + return if queue.leased_size.nonzero? + + return unless (Time.now - queue.initialized_at) > config.first_test_picked_timeout + + <<~ERROR_MESSAGE.strip + Leader has reached the time limit of #{formatted_timeout(config.first_test_picked_timeout)} for the first test being picked from the queue. + This probably means that all workers have failed to be initialized or took too long to start. + Leader will now abort. + Aborting... + ERROR_MESSAGE + end + + def formatted_timeout(time) + minutes = time.to_i / 60 + seconds = time.to_i % 60 + + formatted_text = [] + formatted_text << "#{minutes} minute(s)" if minutes.positive? + formatted_text << "#{seconds} second(s)" if seconds.positive? + + formatted_text.join(' ') + end + end + end +end diff --git a/distrib-core/lib/distrib_core/logger_broadcaster.rb b/distrib-core/lib/distrib_core/logger_broadcaster.rb new file mode 100644 index 0000000..fde7c42 --- /dev/null +++ b/distrib-core/lib/distrib_core/logger_broadcaster.rb @@ -0,0 +1,32 @@ +require 'logger' + +module DistribCore + # Broadcasts logs to multiple loggers. + class LoggerBroadcaster < Logger + private :level, :level=, :progname, :progname=, :datetime_format, :datetime_format=, + :formatter, :formatter= + + def initialize(loggers) + super(nil) + @loggers = loggers + end + + def add(severity, message = nil, progname = nil) + @loggers.each do |target| + target.add(severity, message, progname) + end + end + + def <<(message) + @loggers.each { |logger| logger << message } + end + + def close + @loggers.each(&:close) + end + + def reopen(_logdev = nil) + @loggers.each(&:reopen) + end + end +end diff --git a/distrib-core/lib/distrib_core/metrics.rb b/distrib-core/lib/distrib_core/metrics.rb new file mode 100644 index 0000000..ced5025 --- /dev/null +++ b/distrib-core/lib/distrib_core/metrics.rb @@ -0,0 +1,35 @@ +module DistribCore + # Collect metrics from Leader and Workers. + module Metrics + class << self + # Stores metrics + def report + @report ||= { + queue_exposed_at: nil, + first_test_taken_at: nil, + watchdog_repush_count: 0, + repushed_files: Hash.new { |h, k| h[k] = [] } + } + end + + # Records Leader is ready to serve tests + def queue_exposed + report[:queue_exposed_at] = Time.now.to_i + end + + # Records first test was taken by a worker + def test_taken + report[:first_test_taken_at] ||= Time.now.to_i + end + + # Records when watchdog repushes files back to queue because of timeout + # + # @param test [String] + # @param timeout_in_seconds [Float] timeout which was exceeded + def watchdog_repushed(test, timeout_in_seconds) + report[:watchdog_repush_count] += 1 + report[:repushed_files][test] << timeout_in_seconds + end + end + end +end diff --git a/distrib-core/lib/distrib_core/received_signals.rb b/distrib-core/lib/distrib_core/received_signals.rb new file mode 100644 index 0000000..c8cff10 --- /dev/null +++ b/distrib-core/lib/distrib_core/received_signals.rb @@ -0,0 +1,60 @@ +module DistribCore + # A handler for signal interruptions (like INT and TERM). + # Stores information about received signals. + module ReceivedSignals + class << self + # Defines trap for singlas and collects them. + # Exits after second 'INT' + # + # @param sig [String] signal to trap + # @example + # ::DistribCore::ReceivedSignals.trap('INT') + # + def trap(sig) + Signal.trap(sig) do + # Second SIGINT finishes the process immediately + if sig == 'INT' && signals.member?('INT') + @force_int = true + puts 'Received second SIGINT. Exiting...' + Kernel.exit(2) # 2 is exit code for SIGINT + end + puts "Received #{sig}" + signals.add(sig) + end + end + + # @return [TrueClass, FalseClass] `true` when received any signal + def any? + signals.any? + end + + # @param sig [String] + # @return [TrueClass, FalseClass] `true` if signal `sig` was recieved + def received?(sig) + signals.member?(sig) + end + + # @return [TrueClass, FalseClass] `true` if 'INT' was sent twice + def force_int? + @force_int + end + + # @return [String] human-readable message about received signals + def message + "RECEIVED SIGNAL #{signals.to_a.join(', ')}." if any? + end + + # @return [Integer] proper exit code based on received signal + def exit_code + return 0 if signals.empty? + + Signal.list[signals.first] + end + + # @return [Set] list of received signals + def signals + @signals ||= Set.new + end + end + end +end diff --git a/distrib-core/lib/distrib_core/spec/configuration.rb b/distrib-core/lib/distrib_core/spec/configuration.rb new file mode 100644 index 0000000..448d9a5 --- /dev/null +++ b/distrib-core/lib/distrib_core/spec/configuration.rb @@ -0,0 +1,78 @@ +# Shared examples to test configuration. +RSpec.shared_examples 'DistribCore configuration' do + around do |example| + config = DistribCore::Configuration.instance_variable_get(:@current) + DistribCore::Configuration.instance_variable_set(:@current, nil) + example.run + DistribCore::Configuration.instance_variable_set(:@current, config) + end + + describe '.current' do + it 'raise error when configuration is missing' do + expect { DistribCore::Configuration.current }.to raise_error(RuntimeError) + end + end + + describe '.current =' do + it 'raise error if sets it more than once' do + some_config = Object.new + DistribCore::Configuration.current = some_config + expect(DistribCore::Configuration.current).to eq(some_config) + another_config = Object.new + expect { DistribCore::Configuration.current = another_config }.to raise_error(RuntimeError) + expect { DistribCore::Configuration.current = some_config }.not_to raise_error + end + end + + it 'initialization sets global value' do + configuration + expect(DistribCore.configuration).to eq configuration + end + + it 'has default options' do + expect(configuration.test_timeout).to be_positive + expect(configuration.first_test_picked_timeout).to be_positive + expect(configuration.tests_processing_stopped_timeout).to be_positive + expect(configuration.drb[:safe_level]).to be 1 + end + + it 'has default logger' do + expect(configuration.logger).not_to be_nil + expect(configuration.logger.level).to eq(Logger::INFO) + end + + it 'has debug logger' do + expect(configuration.debug_logger).not_to be_nil + expect(configuration.debug_logger.level).to eq(Logger::DEBUG) + end + + it 'has broadcaster' do + expect(configuration.broadcaster).to be_instance_of(DistribCore::LoggerBroadcaster) + expect(configuration).not_to respond_to(:broadcaster=) + end + + describe '#timeout_for' do + it 'returns test_timeout for any file' do + expect(configuration.timeout_for('any_test')).to eq configuration.test_timeout + end + + it 'calls test_timeout if it is Proc' do + handler = ->(test) { "Value for #{test}" } + configuration.test_timeout = handler + expect(configuration.timeout_for('foo')).to eq('Value for foo') + end + end + + describe '#timeout_strategy=' do + specify do + expect(configuration.timeout_strategy).to eq(:repush) + + configuration.timeout_strategy = :release + expect(configuration.timeout_strategy).to eq(:release) + + expect do + configuration.timeout_strategy = :invalid + end.to raise_error(RuntimeError, /Invalid Timeout Strategy/) + end + end +end diff --git a/distrib-core/lib/distrib_core/spec/distrib.rb b/distrib-core/lib/distrib_core/spec/distrib.rb new file mode 100644 index 0000000..38d33ce --- /dev/null +++ b/distrib-core/lib/distrib_core/spec/distrib.rb @@ -0,0 +1,60 @@ +RSpec.shared_examples 'DistribCore root module' do + around do |example| + config = DistribCore::Configuration.instance_variable_get(:@current) + DistribCore::Configuration.instance_variable_set(:@current, nil) + example.run + DistribCore::Configuration.instance_variable_set(:@current, config) + end + + def configure(...) + root.configure(...) + end + + def configuration + DistribCore.configuration + end + + it 'can change tests provider' do + configure do |config| + config.tests_provider = :foo + end + + expect(configuration.tests_provider).to eq(:foo) + end + + describe '#test_timeout' do + it 'can change the test timeout using Integer' do + configure do |config| + config.test_timeout = 30 + end + + expect(configuration.test_timeout).to eq(30) + end + + it 'can change the test timeout using Proc' do + callable = ->(_file) { 30 } + + configure do |config| + config.test_timeout = callable + end + + expect(configuration.test_timeout).to eq(callable) + end + end + + it 'can change first_test_picked_timeout' do + configure do |config| + config.first_test_picked_timeout = 60 + end + + expect(configuration.first_test_picked_timeout).to eq(60) + end + + it 'can change tests_processing_stopped_timeout' do + configure do |config| + config.tests_processing_stopped_timeout = 60 + end + + expect(configuration.tests_processing_stopped_timeout).to eq(60) + end +end diff --git a/distrib-core/lib/distrib_core/worker.rb b/distrib-core/lib/distrib_core/worker.rb new file mode 100644 index 0000000..4826ca8 --- /dev/null +++ b/distrib-core/lib/distrib_core/worker.rb @@ -0,0 +1,40 @@ +module DistribCore + # Stores common methods for Workers (basic module for workers). + module Worker + private + + def connect_to_leader_with_timeout + tries = 0 + max_tries = DistribCore::Configuration.current.leader_connection_attempts + + begin + yield + rescue DRb::DRbConnError + tries += 1 + sleep 1 + retry if tries < max_tries + raise + end + end + + def received_any_signal? + ::DistribCore::ReceivedSignals.any? + end + + def received_int? + ::DistribCore::ReceivedSignals.received?('INT') + end + + def received_force_int? + ::DistribCore::ReceivedSignals.force_int? + end + + def received_term? + ::DistribCore::ReceivedSignals.received?('TERM') + end + + def logger + DistribCore.configuration.logger + end + end +end diff --git a/distrib-core/spec/distrib_core/configuration_spec.rb b/distrib-core/spec/distrib_core/configuration_spec.rb new file mode 100644 index 0000000..55fa0ee --- /dev/null +++ b/distrib-core/spec/distrib_core/configuration_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'distrib_core/spec/configuration' + +RSpec.describe DistribCore::Configuration do + subject(:configuration) do + Class.new do + include DistribCore::Configuration + end.new + end + + it '#tests_provider' do + expect { configuration.tests_provider }.to raise_error(NotImplementedError) + end + + it '#error_handler' do + expect { configuration.error_handler }.to raise_error(NotImplementedError) + end + + include_examples 'DistribCore configuration' +end diff --git a/distrib-core/spec/distrib_core/distrib_spec.rb b/distrib-core/spec/distrib_core/distrib_spec.rb new file mode 100644 index 0000000..b9486b5 --- /dev/null +++ b/distrib-core/spec/distrib_core/distrib_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'distrib_core/spec/distrib' + +RSpec.describe DistribCore::Distrib do + subject(:root) do + Class.new do + extend DistribCore::Distrib + + def self.configuration + @configuration ||= Class.new do + include DistribCore::Configuration + end.new + end + end + end + + include_examples 'DistribCore root module' +end diff --git a/distrib-core/spec/distrib_core/drb_helper_spec.rb b/distrib-core/spec/distrib_core/drb_helper_spec.rb new file mode 100644 index 0000000..84baa6e --- /dev/null +++ b/distrib-core/spec/distrib_core/drb_helper_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +RSpec.describe DistribCore::DRbHelper do # rubocop:disable RSpec/FilePath, RSpec/SpecFilePathFormat + let(:broadcaster) { instance_double(DistribCore::LoggerBroadcaster) } + + before do + configuration = instance_double(DistribCore::Configuration, broadcaster:) + allow(DistribCore::Configuration).to receive(:current).and_return(configuration) + end + + describe '.drb_unknown?' do + it 'returns false for regular object' do + expect(described_class.drb_unknown?(Object.new, 1)).to be false + expect(described_class.drb_unknown?('asd', StandardError.new('asd'))).to be false + end + + it 'returns true and logs error if meet DRb::DRbUnknown' do + object = nil + A = Class.new # rubocop:disable Lint/ConstantDefinitionInBlock, RSpec/LeakyConstantDeclaration: + data = Marshal.dump(A) + Object.send(:remove_const, :A) # rubocop:disable RSpec/RemoveConst + + begin + Marshal.load(data) # rubocop:disable Security/MarshalLoad + rescue StandardError => e + object = DRb::DRbUnknown.new(e, data) + end + + expect(broadcaster).to receive(:error).with('Parse error:') + expect(broadcaster).to receive(:error).with(an_instance_of(ArgumentError)) + expect(broadcaster).to receive(:debug).with(/Can't parse:.*DRbUnknown.*A.*/) + expect(described_class.drb_unknown?('asd', object, 1)).to be true + end + end + + describe '.dump_failed?' do + it 'returns false if error is not related to unsuccessful dump' do + expect(described_class.dump_failed?( + TypeError.new('asd'), + ['any'] + )).to be false + end + + it 'returns true and prints info when meet unsuccessful dump' do + # Failed dump of Proc works fine + stub_const('A', Class.new do + attr_accessor :a + end) + + stub_const('B', Class.new do + attr_accessor :b + end) + + object = A.new.tap { |a| a.a = B.new.tap { |b| b.b = -> {} } } + + begin + error = Marshal.dump(object) + rescue StandardError => e + error = e + end + + expect(broadcaster).to receive(:error).with('Marshal dump error:') + expect(broadcaster).to receive(:error).with(error) + expect(broadcaster).to receive(:debug).with(/Cant serialize.*Proc.*in path A@a B@b/) + expect(described_class.dump_failed?( + error, + object + )).to be true + end + end +end diff --git a/distrib-core/spec/distrib_core/leader/drb_callable_spec.rb b/distrib-core/spec/distrib_core/leader/drb_callable_spec.rb new file mode 100644 index 0000000..c1d9e7c --- /dev/null +++ b/distrib-core/spec/distrib_core/leader/drb_callable_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +RSpec.describe DistribCore::Leader::DRbCallable do # rubocop:disable RSpec/FilePath, RSpec/SpecFilePathFormat + subject(:object) do + Class.new do + extend DistribCore::Leader::DRbCallable + + def initialize(handler, logger) + @handler = handler + @logger = logger + end + + attr_reader :handler, :logger + + drb_callable def wrapped_function(*args) + raise args.first if args.first.is_a?(Exception) + + puts args + end + + def handle_non_example_exception + handler.handle_non_example_exception + end + end.new(handler, logger) + end + + let(:handler) { double } + let(:logger) { instance_double(Logger) } + + it 'passes all arguments to original function and returns nil' do + expect($stdout).to receive(:puts).with(['asd', 1]) + expect(object.wrapped_function('asd', 1)).to be_nil + end + + it 'catches and records any raised error' do + error = StandardError.new('asd') + expect(logger).to receive(:error).with('Failed to call wrapped_function') + expect(logger).to receive(:error).with(error) + expect(handler).to receive(:handle_non_example_exception) + expect($stdout).not_to receive(:puts) + expect { object.wrapped_function(error) }.not_to raise_error + end + + it 'prevents call if args has a DRbUnknown' do + allow(DistribCore::DRbHelper).to receive(:drb_unknown?).with(1, 'asd').and_return(true) + expect(handler).to receive(:handle_non_example_exception) + expect($stdout).not_to receive(:puts) + object.wrapped_function(1, 'asd') + end +end diff --git a/distrib-core/spec/distrib_core/leader/error_handler_spec.rb b/distrib-core/spec/distrib_core/leader/error_handler_spec.rb new file mode 100644 index 0000000..2c6ac9d --- /dev/null +++ b/distrib-core/spec/distrib_core/leader/error_handler_spec.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +# rubocop:disable RSpec/VerifiedDoubles +RSpec.describe DistribCore::Leader::ErrorHandler do + let(:instance) { described_class.new(exception_extractor) } + let(:exception_extractor) { double } + let(:failure) { nil } + + before do + allow(exception_extractor).to receive(:unpack_causes).and_return([[failure]]) + end + + describe '#retry_test?' do + subject(:should_retry) { instance.retry_test?(test, results, exception) } + + let(:test) { 'foo_spec.rb' } + let(:results) { [double] } + let(:failure) { double(original_class: 'FooError', cause: nil, message: 'foo', backtrace: []) } + let(:exception) { nil } + + shared_examples 'checks' do + it 'returns false' do + expect(should_retry).to be(false) + end + + context 'when retries configured without list of exceptions' do + before do + instance.retry_attempts = 1 + end + + it 'returns true' do + expect(should_retry).to be(true) + end + end + + context 'when list of exceptions configured without retries' do + before do + instance.retryable_exceptions = ['FooError'] + end + + it 'returns false' do + expect(should_retry).to be(false) + end + end + + context 'when retries configured but got another error' do + before do + instance.retry_attempts = 1 + instance.retryable_exceptions = ['BarError'] + end + + it 'returns false' do + expect(should_retry).to be(false) + end + end + + context 'when should retry' do + before do + instance.retry_attempts = 1 + instance.retryable_exceptions = ['FooError'] + end + + it 'returns true' do + expect(should_retry).to be(true) + end + + it 'returns false if retries depleted' do + instance.retry_test?(test, results, exception) + expect(should_retry).to be(false) + end + end + end + + context 'when failed example' do + before { allow(exception_extractor).to receive(:failures_of).and_return([failure]) } + + include_examples 'checks' + end + + context 'when failed outside of example' do + let(:exception) { failure } + + before { allow(exception_extractor).to receive(:failures_of).and_return([]) } + + include_examples 'checks' + end + end + + describe '#ignore_worker_failure?' do + subject(:should_ignore) { instance.ignore_worker_failure?(exception) } + + let(:exception) { double(original_class: 'FooError', cause: nil, message: 'foo', backtrace: []) } + let(:failure) { exception } + let(:broadcaster) { instance_double(DistribCore::LoggerBroadcaster) } + + before do + configuration = instance_double(DistribCore::Configuration) + allow(configuration).to receive(:broadcaster).and_return(broadcaster) + allow(DistribCore::Configuration).to receive(:current).and_return(configuration) + end + + context 'with threshold set, but without fatal failures set' do + before do + instance.failed_workers_threshold = 1 + end + + it 'returns true' do + expect(should_ignore).to be(true) + end + end + + context 'when non-fatal error occurs' do + before do + instance.failed_workers_threshold = 1 + instance.fatal_worker_failures = ['BarError'] + end + + it 'returns true' do + expect(should_ignore).to be(true) + end + end + + context 'when there is a missing exception' do + let(:exception) { nil } + + it 'returns false' do + expect(broadcaster).to receive(:debug).with('Exception missing') + expect(should_ignore).to be(false) + end + end + + context 'when the threshold is exceeded' do + before do + instance.failed_workers_threshold = 1 + instance.ignore_worker_failure?(exception) + end + + it 'returns false' do + expect(broadcaster).to receive(:debug).with('2 failure(s) reported, which exceeds the threshold of 1') + expect(should_ignore).to be(false) + end + end + + context 'when fatal failure occurs' do + before do + instance.failed_workers_threshold = 1 + instance.fatal_worker_failures = ['FooError'] + end + + it 'returns false' do + expect(broadcaster).to receive(:debug).with('Fatal failure found: FooError') + expect(should_ignore).to be(false) + end + end + end +end +# rubocop:enable RSpec/VerifiedDoubles diff --git a/distrib-core/spec/distrib_core/leader/queue_builder_spec.rb b/distrib-core/spec/distrib_core/leader/queue_builder_spec.rb new file mode 100644 index 0000000..2df10ac --- /dev/null +++ b/distrib-core/spec/distrib_core/leader/queue_builder_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +RSpec.describe DistribCore::Leader::QueueBuilder do + describe '.tests' do + let(:config) { instance_double(DistribCore::Configuration) } + let(:provider) { double } + + it 'calls the configured files provider' do + expect(DistribCore::Configuration).to receive(:current).and_return(config) + expect(config).to receive(:tests_provider).and_return(provider) + expect(provider).to receive(:call).and_return([1, 2, 3]) + + expect(described_class.tests).to eq [1, 2, 3] + end + end +end diff --git a/distrib-core/spec/distrib_core/leader/queue_with_lease_spec.rb b/distrib-core/spec/distrib_core/leader/queue_with_lease_spec.rb new file mode 100644 index 0000000..4b5546a --- /dev/null +++ b/distrib-core/spec/distrib_core/leader/queue_with_lease_spec.rb @@ -0,0 +1,166 @@ +# frozen_string_literal: true + +RSpec.describe DistribCore::Leader::QueueWithLease do + describe '#lease' do + it 'leases all entries in FIFO order' do + queue_with_lease = described_class.new(%i[a b c]) + expect(queue_with_lease.lease).to eq(:c) + expect(queue_with_lease.lease).to eq(:b) + expect(queue_with_lease.lease).to eq(:a) + end + + it 'blocks when nothing left in the queue' do + entries = [] + queue_with_lease = described_class.new(entries) + expect(entries).not_to receive(:pop) + waiting_for_lease = Thread.new do + queue_with_lease.lease + end + sleep(0.5) + waiting_for_lease.kill + end + + it 'gets the next one if the element has completed already' do + queue_with_lease = described_class.new(%i[a b c]) + allow(queue_with_lease).to receive(:completed).and_return(%i[b]) + expect(queue_with_lease.lease).to eq(:c) + expect(queue_with_lease.lease).to eq(:a) + end + end + + describe '#repush' do + it 'pushes the entry back to the queue' do + queue_with_lease = described_class.new([:a]) + entry = queue_with_lease.lease + queue_with_lease.repush(entry) + expect(queue_with_lease.lease).to eq(:a) + end + end + + describe '#release' do + it 'returns true' do + queue_with_lease = described_class.new([:a]) + entry = queue_with_lease.lease + expect(queue_with_lease.release(entry)).to be_truthy + end + + context 'when already released' do + it 'returns nil' do + queue_with_lease = described_class.new([:a]) + entry = queue_with_lease.lease + queue_with_lease.release(entry) + expect(queue_with_lease.release(entry)).to be_nil + end + end + end + + describe '#completed?' do + it 'returns false for not released entry' do + queue_with_lease = described_class.new([:a]) + expect(queue_with_lease.completed?(:a)).to be false + end + + it 'returns true for released entry' do + queue_with_lease = described_class.new([:a]) + queue_with_lease.release(:a) + expect(queue_with_lease.completed?(:a)).to be true + end + end + + describe '#empty?' do + context 'when initially empty' do + it 'is empty' do + queue_with_lease = described_class.new([]) + expect(queue_with_lease).to be_empty + end + end + + context 'with some data' do + it 'is not empty' do + queue_with_lease = described_class.new([:a]) + expect(queue_with_lease).not_to be_empty + end + end + + context 'when leased' do + it 'is not empty' do + queue_with_lease = described_class.new([:a]) + _entry = queue_with_lease.lease + expect(queue_with_lease).not_to be_empty + end + end + + context 'when leased and released' do + it 'is empty' do + queue_with_lease = described_class.new([:a]) + entry = queue_with_lease.lease + queue_with_lease.release(entry) + expect(queue_with_lease).to be_empty + end + end + + context 'when something is left on the queue' do + it 'is not empty' do + queue_with_lease = described_class.new(%i[a b]) + entry = queue_with_lease.lease + queue_with_lease.release(entry) + expect(queue_with_lease).not_to be_empty + end + end + end + + describe '#completed_size' do + it do + queue_with_lease = described_class.new(%i[a b c]) + expect(queue_with_lease.completed_size).to eq 0 + end + + it do + queue_with_lease = described_class.new(%i[a b c]) + queue_with_lease.release(:a) + queue_with_lease.release(:b) + expect(queue_with_lease.completed_size).to eq 2 + end + end + + describe 'leased' do + subject(:queue_with_lease) { described_class.new(%i[a b]) } + + before do + allow(Time).to receive(:now).and_return(1) + queue_with_lease.lease + allow(Time).to receive(:now).and_return(2) + queue_with_lease.lease + end + + describe '#select_leased' do + it 'returns a hash of entries with times the block returns true' do + expect(queue_with_lease.select_leased { true }).to eq(b: 1, a: 2) + end + + it 'returns nothing when the block returns false' do + expect(queue_with_lease.select_leased { false }).to eq({}) + end + + it 'yields its elements' do + expect { |b| queue_with_lease.select_leased(&b) }.to yield_successive_args([:b, 1], [:a, 2]) + end + + it 'works on a copy of the original object' do + selected_before = queue_with_lease.select_leased + count_before = selected_before.count + + queue_with_lease.repush(:c) + queue_with_lease.lease + + expect(selected_before.count).to eq(count_before) + end + end + + describe '#leased_size' do + it 'returns the count of leased entries' do + expect(queue_with_lease.leased_size).to eq(2) + end + end + end +end diff --git a/distrib-core/spec/distrib_core/leader/retry_on_different_error_handler_spec.rb b/distrib-core/spec/distrib_core/leader/retry_on_different_error_handler_spec.rb new file mode 100644 index 0000000..de1982e --- /dev/null +++ b/distrib-core/spec/distrib_core/leader/retry_on_different_error_handler_spec.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +# rubocop:disable RSpec/VerifiedDoubles +RSpec.describe DistribCore::Leader::RetryOnDifferentErrorHandler do + subject(:handler) do + described_class.new(exception_extractor, + retry_limit:, + repeated_error_limit:) + end + + before do + stub_failure(failure) + end + + def stub_failure(failure) + allow(exception_extractor).to receive_messages(failures_of: [failure], unpack_causes: [[failure]]) + end + + describe '#retry_test?' do + subject(:should_retry) { handler.retry_test?(test, results, exception) } + + let(:test) { 'foo_spec.rb' } + let(:results) { [double] } + let(:failure) { double(original_class: 'FooError', cause: nil, message: 'foo', backtrace: []) } + let(:other_failure) { double(original_class: 'OtherFooError', cause: nil, message: 'foo', backtrace: []) } + let(:exception) { nil } + let(:exception_extractor) { double } + let(:retry_limit) { 3 } + let(:repeated_error_limit) { 2 } + + context 'when it is the first time failing' do + it { is_expected.to be_truthy } + end + + context "when repeated error limit isn't set" do + let(:handler) do + described_class.new(exception_extractor, + retry_limit:) + end + + it { is_expected.to be_truthy } + end + + context 'when it fails under retry limit and under repeated error limit with the same exception' do + before do + (repeated_error_limit - 1).times do + stub_failure(failure) + + handler.retry_test?(test, results, exception) + end + end + + it { is_expected.to be_truthy } + end + + context 'when it fails under retry limit and over repeated error limit with the same exception' do + before do + repeated_error_limit.times do + stub_failure(failure) + + handler.retry_test?(test, results, exception) + end + end + + it { is_expected.to be_falsey } + end + + context 'when it fails with a different exception under retry limit' do + before do + handler.retry_test?(test, results, exception) + + stub_failure(other_failure) + end + + it { is_expected.to be_truthy } + end + + context 'when it fails with a different message under retry limit' do + let(:other_failure) { double(original_class: 'FooError', cause: nil, message: 'OTHER', backtrace: []) } + + before do + handler.retry_test?(test, results, exception) + + stub_failure(other_failure) + end + + it { is_expected.to be_truthy } + end + + context 'when it fails over retry_limit with a different exception' do + before do + retry_limit.times do |i| + stub_failure(double(original_class: "FooError#{i}", cause: nil, message: 'error', backtrace: [])) + + handler.retry_test?(test, results, exception) + end + + stub_failure(failure) + end + + it { is_expected.to be_falsy } + end + end +end +# rubocop:enable RSpec/VerifiedDoubles diff --git a/distrib-core/spec/distrib_core/leader/watchdog_spec.rb b/distrib-core/spec/distrib_core/leader/watchdog_spec.rb new file mode 100644 index 0000000..dad2157 --- /dev/null +++ b/distrib-core/spec/distrib_core/leader/watchdog_spec.rb @@ -0,0 +1,279 @@ +# frozen_string_literal: true + +RSpec.describe DistribCore::Leader::Watchdog do + # Runs watchdog in the current thread + def start_watchdog_in_the_same_thread + allow(Thread).to receive(:new).and_wrap_original do |*, &block| + block.call + end + + watchdog.start + end + + subject(:watchdog) { described_class.new(queue) } + + before do + allow(Kernel).to receive(:sleep) + allow(DRb).to receive_message_chain(:current_server, :stop_service) # rubocop:disable RSpec/MessageChain + end + + let!(:configuration) do + config = Class.new do |klass| + klass.include DistribCore::Configuration + end.new + + config.instance_variable_set(:@broadcaster, broadcaster) + + config + end + + let(:broadcaster) { Logger.new(nil) } + + after do + DistribCore::Configuration.instance_variable_set(:@current, nil) + end + + def stub_leased(leased) + allow(queue).to receive_messages(select_leased: leased, leased_size: leased.size) + end + + describe 'pulling back of timed out entries' do + let(:queue) { DistribCore::Leader::QueueWithLease.new(%i[one two]) } + + before do + allow(queue).to receive(:empty?).and_return(true) + allow(broadcaster).to receive(:info) + allow(broadcaster).to receive(:warn) + end + + context 'with no timed out entries' do + it 'does not pull anything' do + stub_leased({}) + expect(queue).not_to receive(:repush) + + start_watchdog_in_the_same_thread + + expect(broadcaster).not_to have_received(:warn) + end + end + + context 'with timed out entries' do + it 'pulls back timed out entries' do + configuration.test_timeout = 172 + stub_leased(one: 1, two: 2) + + expect(queue).to receive(:repush).with(:one).once.ordered + expect(queue).to receive(:repush).with(:two).once.ordered + expect(queue).not_to receive(:repush) + + start_watchdog_in_the_same_thread + + expect(broadcaster).to have_received(:warn).with('one (Timeout: 2 minute(s) 52 second(s)) ' \ + 'but will be pushed back to the queue.') + expect(broadcaster).to have_received(:warn).with('two (Timeout: 2 minute(s) 52 second(s)) ' \ + 'but will be pushed back to the queue.') + end + end + + describe 'timeout calculation' do + context 'when none of the entries are timed out' do + it 'does not pull back anything' do + Timecop.freeze(Time.now - 59) do + 2.times { queue.lease } + end + + expect(queue).not_to receive(:repush) + + start_watchdog_in_the_same_thread + end + end + + context 'when one of the entries is timed out' do + it 'pulls back one entry' do + Timecop.freeze(Time.now - 61) { queue.lease } + Timecop.freeze(Time.now - 20) { queue.lease } + expect(queue).to receive(:repush).with(:two).once + expect(queue).not_to receive(:repush).with(:one) + + start_watchdog_in_the_same_thread + end + + it 'reports entries to metrics' do + Timecop.freeze(Time.now - 61) { queue.lease } + expect(DistribCore::Metrics).to receive(:watchdog_repushed).with(:two, 60).once + + start_watchdog_in_the_same_thread + end + end + + context 'when both of the entries are timed out' do + it 'pulls back all entries' do + Timecop.freeze(Time.now - 61) { queue.lease } + Timecop.freeze(Time.now - 62) { queue.lease } + expect(queue).to receive(:repush).with(:one).once + expect(queue).to receive(:repush).with(:two).once + + start_watchdog_in_the_same_thread + end + end + + context 'when strategy is set to release timed out test' do + it 'releases test' do + configuration.timeout_strategy = :release + Timecop.freeze(Time.now - 61) { queue.lease } + expect(queue).to receive(:release).with(:two).once + + start_watchdog_in_the_same_thread + end + end + end + end + + describe 'shut down' do + let(:queue) { instance_double(DistribCore::Leader::QueueWithLease, select_leased: {}, leased_size: 0) } + + context 'when the queue is empty' do + it 'shuts down immediately' do + allow(queue).to receive(:empty?).and_return(true) + start_time = Time.now.to_f + start_watchdog_in_the_same_thread + expect(Time.now.to_f).to be_within(1.0).of(start_time) + end + end + + context 'when the queue eventually becomes empty' do + it 'shuts down in several seconds' do + allow(queue).to receive(:empty?).and_return(false, false, false, false, true) + allow(queue).to receive(:visited?) + allow(queue).to receive_messages(last_activity_at: Time.now, initialized_at: Time.now - 10, + entries_list: []) + expect(Kernel).to receive(:sleep).with(1).exactly(4).times + + start_watchdog_in_the_same_thread + end + end + + context 'when first test was not picked in configured time frame' do + let(:first_test_picked_timeout) { 60 } + + before do + configuration.first_test_picked_timeout = first_test_picked_timeout + allow(broadcaster).to receive(:error) + allow(broadcaster).to receive(:info) + allow(queue).to receive_messages( + empty?: false, + visited?: false, + initialized_at: Time.now - (2 * first_test_picked_timeout), + entries_list: [] + ) + end + + it 'shuts down on timeout' do + service = double + allow(DRb).to receive(:current_server).and_return(service) + expect(service).to receive(:stop_service) + + start_watchdog_in_the_same_thread + end + + it 'logs the correct error message' do + expected_message = <<~EXPECTED.strip + Leader has reached the time limit of 1 minute(s) for the first test being picked from the queue. + This probably means that all workers have failed to be initialized or took too long to start. + Leader will now abort. + Aborting... + EXPECTED + + start_watchdog_in_the_same_thread + + expect(broadcaster).to have_received(:error).with(expected_message) + end + + context 'when the queue has had activity' do + before do + allow(queue).to receive(:last_activity_at).and_return(Time.now - 1) + end + + it 'does not timeout' do + allow(queue).to receive(:empty?).and_return(false, true) + allow(queue).to receive(:visited?).and_return(true) + expect(broadcaster).not_to receive(:error) + + start_watchdog_in_the_same_thread + end + end + end + + context 'when workers stopped processing tests for configured time frame' do + let(:tests_processing_stopped_timeout) { 60 } + + before do + configuration.tests_processing_stopped_timeout = tests_processing_stopped_timeout + allow(broadcaster).to receive(:error) + allow(broadcaster).to receive(:info) + allow(queue).to receive_messages( + visited?: true, + empty?: false, + initialized_at: Time.now - (2 * tests_processing_stopped_timeout), + completed_size: 10, + last_activity_at: Time.now - tests_processing_stopped_timeout - 1, + entries_list: [] + ) + end + + it 'shuts down on timeout' do + start_watchdog_in_the_same_thread + + expect(broadcaster).to have_received(:error).with(/Workers did not pick tests for too long!/) + end + + it 'logs the correct error message' do + expected_message = <<~EXPECTED.strip + Workers did not pick tests for too long! + After Workers processed 10 test(s), Leader will abort as it waited for over + 1 minute(s) which is the configured time to wait for + Workers to pick up tests. + Aborting... + EXPECTED + + start_watchdog_in_the_same_thread + + expect(broadcaster).to have_received(:error).with(expected_message) + end + end + + context 'when there are some test still in the queue after leader has aborted' do + let(:queue) { DistribCore::Leader::QueueWithLease.new(%i[one two]) } + + before do + configuration.tests_processing_stopped_timeout = 60 + allow(broadcaster).to receive(:error) + allow(broadcaster).to receive(:info) + allow(broadcaster).to receive(:warn) + end + + it 'logs the stuck tests to STDOUT' do + Timecop.freeze(Time.now - 61) do + 2.times { queue.lease } + end + expected_message = <<~EXPECTED + 2 tests not executed, showing 2: + one + two + EXPECTED + start_watchdog_in_the_same_thread + + expect(broadcaster).to have_received(:info).with(expected_message) + end + end + + describe 'at shutdown' do + it 'prints results and stops service' do + allow(queue).to receive(:empty?).and_return(true) + expect(DRb).to receive_message_chain(:current_server, :stop_service) # rubocop:disable RSpec/MessageChain + + start_watchdog_in_the_same_thread + end + end + end +end diff --git a/distrib-core/spec/distrib_core/logger_broadcaster_spec.rb b/distrib-core/spec/distrib_core/logger_broadcaster_spec.rb new file mode 100644 index 0000000..1c6ee57 --- /dev/null +++ b/distrib-core/spec/distrib_core/logger_broadcaster_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +RSpec.describe DistribCore::LoggerBroadcaster do + subject(:broadcaster) do + described_class.new(loggers) + end + + let(:loggers) { [info_logger, debug_logger] } + let(:info_logger) { Logger.new(nil, level: Logger::INFO) } + let(:debug_logger) { Logger.new(nil, level: Logger::DEBUG) } + + describe '#add' do + # Role of broadcaster is to call each logger. + # They decide on themselves if message should be printed or not. + it 'all loggers are called' do + expect(loggers).to all(receive(:add).with(Logger::DEBUG, nil, 'hello')) + + # debug, info, etc - all call `add` + broadcaster.debug 'hello' + end + end + + describe '#<<' do + it 'all loggers are called' do + expect(loggers).to all(receive(:<<).with('hello')) + + broadcaster << 'hello' + end + end + + %i[close reopen].each do |method| + describe "##{method}" do + it 'all loggers are called' do + expect(loggers).to all(receive(method)) + + broadcaster.public_send(method) + end + end + end +end diff --git a/distrib-core/spec/distrib_core/metrics_spec.rb b/distrib-core/spec/distrib_core/metrics_spec.rb new file mode 100644 index 0000000..405bddf --- /dev/null +++ b/distrib-core/spec/distrib_core/metrics_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +RSpec.describe DistribCore::Metrics do + let(:now) { Time.now } + + before do + allow(Time).to receive(:now).and_return(now) + described_class.instance_variable_set(:@report, nil) + end + + describe '#queue_exposed' do + it 'register the current time' do + described_class.queue_exposed + expect(described_class.report[:queue_exposed_at]).to eq(now.to_i) + end + end + + describe '#test_taken' do + it 'register the current time for the first spec taken' do + described_class.test_taken + allow(Time).to receive(:now).and_return(now + 60) + described_class.test_taken + expect(described_class.report[:first_test_taken_at]).to eq(now.to_i) + end + end + + describe '#watchdog_repushed' do + before do + described_class.watchdog_repushed('foo.rb', 10.0) + described_class.watchdog_repushed('foo.rb', 11.0) + end + + it 'increase the counter' do + expect(described_class.report[:watchdog_repush_count]).to eq(2) + end + + it 'count timeouts per file' do + expect(described_class.report[:repushed_files]).to eq('foo.rb' => [10.0, 11.0]) + end + end +end diff --git a/distrib-core/spec/spec_helper.rb b/distrib-core/spec/spec_helper.rb new file mode 100644 index 0000000..df71bc4 --- /dev/null +++ b/distrib-core/spec/spec_helper.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'bundler/setup' + +if ENV['DISABLE_SIMPLECOV'] != '1' + require 'simplecov' + + SimpleCov.start do + add_filter '_spec.rb' + command_name "distrib-core-#{Process.pid}" + end + + SimpleCov.at_exit { SimpleCov.instance_variable_set('@result', nil) } +end + +require 'English' +require 'distrib-core' +require 'timecop' + +RSpec.configure do |config| + if ENV['DISABLE_SIMPLECOV'] != '1' + config.after(:suite) do + SimpleCov.result.format! + end + end + + config.order = :random + Kernel.srand config.seed +end diff --git a/features-parser/.gitignore b/features-parser/.gitignore new file mode 100644 index 0000000..805a512 --- /dev/null +++ b/features-parser/.gitignore @@ -0,0 +1,12 @@ +/.bundle/ +/.idea/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/features-parser/.rspec b/features-parser/.rspec new file mode 100644 index 0000000..34c5164 --- /dev/null +++ b/features-parser/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/features-parser/.rubocop.yml b/features-parser/.rubocop.yml new file mode 100644 index 0000000..51510db --- /dev/null +++ b/features-parser/.rubocop.yml @@ -0,0 +1,22 @@ +AllCops: + DisplayCopNames: true + NewCops: enable + +Style/StringLiterals: + Enabled: true + EnforcedStyle: single_quotes + +Style/StringLiteralsInInterpolation: + Enabled: true + EnforcedStyle: single_quotes + +Layout/LineLength: + Max: 120 + +Metrics/BlockLength: + Exclude: + - spec/**/* + +Naming/FileName: + Exclude: + - lib/features-parser.rb diff --git a/features-parser/.ruby-version b/features-parser/.ruby-version new file mode 100644 index 0000000..9b7a431 --- /dev/null +++ b/features-parser/.ruby-version @@ -0,0 +1 @@ +3.2.4 \ No newline at end of file diff --git a/features-parser/CODE_OF_CONDUCT.md b/features-parser/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d6f70c9 --- /dev/null +++ b/features-parser/CODE_OF_CONDUCT.md @@ -0,0 +1,14 @@ +# The Community Code of Conduct + +**Note:** We have picked the following code of conduct based on [Ruby's own code of conduct](https://www.ruby-lang.org/en/conduct/). + +This document provides a few simple community guidelines for a safe, respectful, +productive, and collaborative place for any person who is willing to contribute +to the community. It applies to all "collaborative spaces", which are +defined as community communications channels (such as mailing lists, submitted +patches, commit comments, etc.). + +* Participants will be tolerant of opposing views. +* Participants must ensure that their language and actions are free of personal attacks and disparaging personal remarks. +* When interpreting the words and actions of others, participants should always assume good intentions. +* Behaviour which can be reasonably considered harassment will not be tolerated. diff --git a/features-parser/Gemfile b/features-parser/Gemfile new file mode 100644 index 0000000..1b75802 --- /dev/null +++ b/features-parser/Gemfile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +gemspec + +gem 'rspec', '~> 3.13' +gem 'rubocop' diff --git a/features-parser/Gemfile.lock b/features-parser/Gemfile.lock new file mode 100644 index 0000000..f86ba05 --- /dev/null +++ b/features-parser/Gemfile.lock @@ -0,0 +1,87 @@ +PATH + remote: . + specs: + features-parser (0.0.1) + activesupport + cucumber-gherkin + +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.1.3.3) + base64 + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + minitest (>= 5.1) + mutex_m + tzinfo (~> 2.0) + ast (2.4.2) + base64 (0.2.0) + bigdecimal (3.1.8) + concurrent-ruby (1.2.3) + connection_pool (2.4.1) + cucumber-gherkin (28.0.0) + cucumber-messages (>= 19.1.4, < 24) + cucumber-messages (23.0.0) + diff-lcs (1.5.1) + drb (2.2.1) + i18n (1.14.5) + concurrent-ruby (~> 1.0) + json (2.7.2) + language_server-protocol (3.17.0.3) + minitest (5.23.1) + mutex_m (0.2.0) + parallel (1.24.0) + parser (3.3.1.0) + ast (~> 2.4.1) + racc + racc (1.8.0) + rainbow (3.1.1) + regexp_parser (2.9.2) + rexml (3.2.8) + strscan (>= 3.0.9) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) + rubocop (1.64.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.31.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) + ruby-progressbar (1.13.0) + strscan (3.1.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.5.0) + +PLATFORMS + x86_64-linux + +DEPENDENCIES + features-parser! + rspec (~> 3.13) + rubocop + +BUNDLED WITH + 2.4.14 diff --git a/features-parser/LICENSE.txt b/features-parser/LICENSE.txt new file mode 100644 index 0000000..08b3961 --- /dev/null +++ b/features-parser/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Toptal + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/features-parser/README.md b/features-parser/README.md new file mode 100644 index 0000000..8fa4dcf --- /dev/null +++ b/features-parser/README.md @@ -0,0 +1,22 @@ +# FeaturesParser + +## Installation + +Add the gem to the application's Gemfile: + +```ruby +gem 'features-parser', git: 'git@github.com:toptal/test-distrib.git', + glob: 'features-parser/*.gemspec' +``` + +## Development + +After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. + +## Contributing + +Bug reports and pull requests are welcome [on GitHub](https://github.com/toptal/test-distrib/issues). + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/features-parser/bin/console b/features-parser/bin/console new file mode 100755 index 0000000..c1520b6 --- /dev/null +++ b/features-parser/bin/console @@ -0,0 +1,11 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'bundler/setup' +require 'features-parser' + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +require 'irb' +IRB.start(__FILE__) diff --git a/features-parser/bin/setup b/features-parser/bin/setup new file mode 100755 index 0000000..dce67d8 --- /dev/null +++ b/features-parser/bin/setup @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +set -vx + +bundle install + +# Do any other automated setup that you need to do here diff --git a/features-parser/features-parser.gemspec b/features-parser/features-parser.gemspec new file mode 100644 index 0000000..bfe0fca --- /dev/null +++ b/features-parser/features-parser.gemspec @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require_relative 'lib/features_parser/version' + +Gem::Specification.new do |spec| + spec.name = 'features-parser' + spec.version = FeaturesParser::VERSION + spec.authors = ['Toptal, LLC'] + spec.email = ['open-source@toptal.com'] + spec.license = 'MIT' + + spec.summary = 'Utility to parse features.' + spec.description = 'Utility to parse features.' + spec.homepage = 'https://github.com/toptal/test-distrib' + spec.required_ruby_version = '>= 3.2.4' + + spec.files = Dir['lib/**/*.rb'] + spec.require_paths = ['lib'] + + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = spec.homepage + + spec.add_dependency 'activesupport' + spec.add_dependency 'cucumber-gherkin' + + spec.metadata['rubygems_mfa_required'] = 'true' +end diff --git a/features-parser/lib/features-parser.rb b/features-parser/lib/features-parser.rb new file mode 100644 index 0000000..1a48099 --- /dev/null +++ b/features-parser/lib/features-parser.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'active_support/core_ext/object/blank' + +# Main module for the gem. +module FeaturesParser +end + +require_relative 'features_parser/version' + +require_relative 'features_parser/catalog' +require_relative 'features_parser/example' +require_relative 'features_parser/feature' +require_relative 'features_parser/name_normalizer' +require_relative 'features_parser/name_provider' +require_relative 'features_parser/outline' +require_relative 'features_parser/scenario_parser' +require_relative 'features_parser/scenario' diff --git a/features-parser/lib/features_parser/catalog.rb b/features-parser/lib/features_parser/catalog.rb new file mode 100644 index 0000000..ca9519a --- /dev/null +++ b/features-parser/lib/features_parser/catalog.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module FeaturesParser + # Registry with Scenario and ScenarioOutline examples. + # + # It contains a map of normalized names of a Scenario (or + # ScenarioOutline example) to {Scenario} objects. + # Guarantees uniqueness of Scenario's name and ScenarioOutline + # example's values inside its Feature. + # Primarily used for mapping a subset of scenarios to their + # executable format (:) + class Catalog + def initialize + @catalog = {} + @parser = ScenarioParser.new(catalog: self) + end + + def parse(files) + @parser.parse(files) + + self + end + + def register(scenario) + validate_uniqueness!(scenario) + @catalog[scenario.normalized_name] = scenario + end + + def reset + @catalog = {} + end + + def names + @catalog.keys + end + + def executable_paths + @catalog.values.map(&:executable_path) + end + + def empty? + @catalog.empty? + end + + # @param [Array] passed_names scenario names + def executable_paths_for(passed_names) + unknown_names = passed_names - names + raise KeyError, "Unknown scenarios passed: #{unknown_names}." if unknown_names.any? + + @catalog.values_at(*passed_names).map(&:executable_path) + end + + # @param [Array] passed_paths executable paths + def names_for(passed_paths) + passed_paths.map do |path| + @catalog.find { |_name, feature| feature.executable_path == path }.first + end.compact + end + + private + + # Validates uniqueness of Scenario name across the Feature + # + # @param [Scenario] scenario + # @raise [KeyError] + def validate_uniqueness!(scenario) + existing_scenario = @catalog[scenario.normalized_name] + return unless existing_scenario + return if scenario.executable_path == existing_scenario.executable_path + + message = "Trying to add #{scenario.normalized_name} from #{scenario.executable_path}, " \ + "but it is already defined in #{@catalog[scenario.normalized_name].executable_path}" + + raise KeyError, message if @catalog[scenario.normalized_name] + end + end +end diff --git a/features-parser/lib/features_parser/example.rb b/features-parser/lib/features_parser/example.rb new file mode 100644 index 0000000..f309e27 --- /dev/null +++ b/features-parser/lib/features_parser/example.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative 'name_normalizer' + +module FeaturesParser + # A value object representing Gherkin's ScenarioOutline Example. + # + # @!attribute [r] outline + # @return [Outline] ScenarioOutline object + # + # @!attribute [r] executable_path + # @return [String] path to the file with this feature + class Example + attr_reader :outline, :line + + # @param [Outline] outline ScenarioOutline object + # @param [Hash] ast AST tree provided by gherkin gem + def initialize(outline, ast) + raise "Incorrect node supplied: #{ast.class}" unless ast.is_a?(Cucumber::Messages::TableRow) + + @outline = outline + @line = ast.location.line + @cells = normalize_cells(ast.cells) + end + + def normalized_name + @normalized_name ||= [outline.normalized_name, @cells.join('|')].join('/') + end + + def executable_path + @executable_path ||= outline.executable_path.gsub(/:\d+$/, ":#{line}") + end + + private + + def normalize_cells(ast) + ast.map { |cell| NameNormalizer.normalize(cell.value) } + end + end +end diff --git a/features-parser/lib/features_parser/feature.rb b/features-parser/lib/features_parser/feature.rb new file mode 100644 index 0000000..99adbc4 --- /dev/null +++ b/features-parser/lib/features_parser/feature.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require_relative 'name_normalizer' + +module FeaturesParser + # A value object representing Gherkin's Feature. + # + # @attr_reader [String] name name of the Feature + # @attr_reader [String] executable_path path to the file + # with this feature + class Feature + attr_reader :name, :executable_path + + # @param [Hash] ast AST tree provided by gherkin gem + # @param [String] path full path to a file containing feature + def initialize(ast, path) + raise "Incorrect node supplied: #{ast.class}" unless ast.is_a?(Cucumber::Messages::Feature) + + @name = ast.name + @executable_path = path + end + + def normalized_name + @normalized_name ||= NameNormalizer.normalize(name) + end + end +end diff --git a/features-parser/lib/features_parser/name_normalizer.rb b/features-parser/lib/features_parser/name_normalizer.rb new file mode 100644 index 0000000..4dd9588 --- /dev/null +++ b/features-parser/lib/features_parser/name_normalizer.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'active_support/core_ext/object/blank' + +module FeaturesParser + # Normalizes and cleans passed string. + # + # Replaces special characters in a string, + # so that it may be used as part of a 'pretty' identifier. + # + # "Donald E. Knuth" becomes "donald-e-knuth" + module NameNormalizer + def self.normalize(string, sep = '-') + normalized_string = string.dup + + # Turn unwanted chars into the separator + normalized_string.gsub!(/[^\w-]+/i, sep) + + if sep.present? + re_sep = Regexp.escape(sep) + + # No more than one of the separator in a row + # plus remove leading/trailing separator + normalized_string = normalized_string.split(/(?:#{re_sep})+/).reject(&:empty?).join(sep) + end + + normalized_string.downcase + end + end +end diff --git a/features-parser/lib/features_parser/name_provider.rb b/features-parser/lib/features_parser/name_provider.rb new file mode 100644 index 0000000..4f2a196 --- /dev/null +++ b/features-parser/lib/features_parser/name_provider.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require_relative 'example' +require_relative 'outline' +require_relative 'feature' +require_relative 'scenario' + +module FeaturesParser + # Provides normalized name for given Cucumber object. + # + # Used primarily to keep report keys' naming scheme + # in sync with logic of {FeaturesParser::ScenarioParser} + # + # @attr_reader [Cucumber::RunningTestCase::ScenarioOutlineExample] object + class NameProvider + attr_reader :object + + # @param [Cucumber::RunningTestCase::ScenarioOutlineExample, + # Cucumber::RunningTestCase::Scenario] object a Cucumber object representing + # Scenario or ScenarioOutline Example + def initialize(object) + @object = object + end + + def normalized_name + scenario_or_example = object.outline? ? example : scenario + scenario_or_example.normalized_name + end + + private + + def example + cells = object.cell_values.map { |value| Cucumber::Messages::TableCell.new(value:) } + + table_row = Cucumber::Messages::TableRow.new( + location: object.location, + cells: + ) + + Example.new(outline, table_row) + end + + def outline + outline = Cucumber::Messages::Scenario.new( + name: clean_outline_name(object.name), + location: object.location, + keyword: 'Scenario Outline' + ) + + Outline.new(feature, outline) + end + + def feature + feature = Cucumber::Messages::Feature.new( + name: object.feature.name, + location: object.feature.location + ) + + Feature.new(feature, object.feature.location.file) + end + + def scenario + scenario = Cucumber::Messages::Scenario.new( + name: object.name, + location: object.location, + keyword: 'Scenario' + ) + + Scenario.new(feature, scenario) + end + + def clean_outline_name(name) + return name unless /\(#\d+\)$/.match?(name) + + name[/(.*),/, 1] + end + end +end diff --git a/features-parser/lib/features_parser/outline.rb b/features-parser/lib/features_parser/outline.rb new file mode 100644 index 0000000..69f931c --- /dev/null +++ b/features-parser/lib/features_parser/outline.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require_relative 'scenario' + +module FeaturesParser + # A value object representing Gherkin's Scenario Outline. + class Outline < Scenario + def check_input_ast!(ast) + raise "Incorrect node supplied: #{ast.class}" unless ast.is_a?(Cucumber::Messages::Scenario) + raise "Incorrect node supplied: #{ast.keyword}" if ast.keyword != 'Scenario Outline' + end + end +end diff --git a/features-parser/lib/features_parser/scenario.rb b/features-parser/lib/features_parser/scenario.rb new file mode 100644 index 0000000..24c32d1 --- /dev/null +++ b/features-parser/lib/features_parser/scenario.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require_relative 'name_normalizer' + +module FeaturesParser + # A value object representing Gherkin's Scenario. + # + # @attr_reader [String] name name of the Scenario + # @attr_reader [String] line executable line number + # of the Scenario + class Scenario + attr_reader :name, :line + + # @param [Feature] feature + # @param [Hash] scenario_ast AST tree provided by gherkin gem + def initialize(feature, scenario_ast) + check_input_ast!(scenario_ast) + + @name = scenario_ast.name + @line = scenario_ast.location.line + @feature = feature + end + + def check_input_ast!(ast) + raise "Incorrect node supplied: #{ast.class}" unless ast.is_a?(Cucumber::Messages::Scenario) + raise "Incorrect node supplied: #{ast.keyword}" if ast.keyword != 'Scenario' + end + + def normalized_name + @normalized_name ||= [@feature.normalized_name, NameNormalizer.normalize(name)].join('/') + end + + def executable_path + "#{@feature.executable_path}:#{line}" + end + end +end diff --git a/features-parser/lib/features_parser/scenario_parser.rb b/features-parser/lib/features_parser/scenario_parser.rb new file mode 100644 index 0000000..9665715 --- /dev/null +++ b/features-parser/lib/features_parser/scenario_parser.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require_relative 'feature' +require_relative 'scenario' +require_relative 'outline' +require_relative 'example' +require_relative 'catalog' +require 'gherkin' + +module FeaturesParser + # Transforms an array of `*.feature` paths, reads them, + # and returns normalized representation of Scenario and + # ScenarioOutline example stanzas in given files. + # + # Parsed items will be automatically registered in {Catalog} class. + # + # We need this information so we can: + # 1) filter out non-existing scenarios from report + # 2) add unknown (according to report) scenarios for leftover distributor + # 3) map further parsed items to their executable representation ":" + # Requirements: gem 'gherkin' + class ScenarioParser + ParserError = Class.new(StandardError) + + def initialize(catalog: Catalog.new) + @parser = Gherkin::Parser.new + @catalog = catalog + end + + def parse(files) + files.flat_map do |file| + parse_file(file) + end + end + + private + + attr_reader :parser, :catalog + + def parse_file(file) + ast = create_ast(file) + + feature = Feature.new(ast.feature, file) + scenarios = parse_scenarios(ast.feature, feature) + examples = parse_examples(ast.feature, feature) + + results = scenarios + examples + + register(results) + + results.map(&:normalized_name) + end + + def create_ast(file) + parser.parse(File.read(file)) + rescue StandardError => e + raise ParserError, "Filename: #{file}\n#{e.message}" + end + + def parse_scenarios(ast, feature) + filter_by_keyword(ast, 'Scenario').map do |scenario| + Scenario.new(feature, scenario) + end + end + + def parse_examples(ast, feature) + filter_by_keyword(ast, 'Scenario Outline').flat_map do |scenario| + outline = Outline.new(feature, scenario) + + scenario.examples.flat_map do |examples| + examples.table_body.map do |example| + Example.new(outline, example) + end + end + end + end + + def filter_by_keyword(ast, keyword) + ast.children.filter_map { |child| child.scenario if child.scenario&.keyword == keyword } + end + + def register(items) + items.each { |item| catalog.register(item) } + end + end +end diff --git a/features-parser/lib/features_parser/version.rb b/features-parser/lib/features_parser/version.rb new file mode 100644 index 0000000..404f66e --- /dev/null +++ b/features-parser/lib/features_parser/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module FeaturesParser + VERSION = '0.0.1' +end diff --git a/features-parser/spec/features_parser/catalog_spec.rb b/features-parser/spec/features_parser/catalog_spec.rb new file mode 100644 index 0000000..6617828 --- /dev/null +++ b/features-parser/spec/features_parser/catalog_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser::Catalog do + subject(:catalog) { described_class.new } + + let(:scenario1) do + instance_double( + FeaturesParser::Scenario, + normalized_name: 'some-feature/scenario1', + executable_path: 'some-feature.feature:5' + ) + end + + let(:scenario2) do + instance_double( + FeaturesParser::Scenario, + normalized_name: 'another-feature/scenario2', + executable_path: 'another-feature.feature:7' + ) + end + + let(:scenarios) { [scenario1, scenario2] } + + before do + catalog.reset + + catalog.register(scenario1) + catalog.register(scenario2) + end + + describe 'validate_uniqueness' do + context 'when file is the same' do + it 'allows same scenario to be registered twice' do + catalog.register(scenario1) + end + end + + context 'when file is different' do + it 'fails if scenario with the same name is registered twice' do + same_name_different_file = instance_double( + FeaturesParser::Scenario, + normalized_name: scenario1.normalized_name, + executable_path: 'different-feature.feature:10' + ) + expect { catalog.register(same_name_different_file) }.to raise_error KeyError + end + end + end + + it 'returns names of all registered scenarios' do + scenario_names = scenarios.map(&:normalized_name) + expect(catalog.names).to eq scenario_names + end + + describe 'executable paths' do + it 'returns executable paths' do + scenario_paths = scenarios.map(&:executable_path) + expect(catalog.executable_paths).to eq scenario_paths + end + + it 'filters executable paths' do + names = [scenario1.normalized_name] + expect(catalog.executable_paths_for(names)).to eq [scenario1.executable_path] + end + + it 'fails on non-registered name' do + unknown_scenario = 'pants' + expect { catalog.executable_paths_for([unknown_scenario]) }.to raise_error KeyError + expect { catalog.executable_paths_for([scenario1, unknown_scenario]) }.to raise_error KeyError + end + end +end diff --git a/features-parser/spec/features_parser/example_spec.rb b/features-parser/spec/features_parser/example_spec.rb new file mode 100644 index 0000000..c8d42e6 --- /dev/null +++ b/features-parser/spec/features_parser/example_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser::Example do + subject(:example) { described_class.new(outline, ast) } + + let(:ast) do + Cucumber::Messages::TableRow.new( + location: Cucumber::Messages::Location.new(line: 26, column: 7), + cells: [ + Cucumber::Messages::TableCell.new( + location: Cucumber::Messages::Location.new(line: 26, column: 9), + value: 'user' + ), + Cucumber::Messages::TableCell.new( + location: Cucumber::Messages::Location.new(line: 26, column: 21), + value: 'john@doe.com' + ) + ] + ) + end + + let(:outline) do + instance_double( + FeaturesParser::Outline, + normalized_name: 'outline-name', + executable_path: 'some.feature:22' + ) + end + + it 'throws exception on incorrect type' do + expect { described_class.new(outline, type: :unknown) }.to raise_error RuntimeError, /Incorrect node supplied/ + end + + it 'uses cell values in normalized name' do + allow(FeaturesParser::NameNormalizer).to receive(:normalize) { |input| "#{input}-pants" } + expect(example.normalized_name).to eq 'outline-name/user-pants|john@doe.com-pants' + end + + it 'parses line number and has executable path' do + expect(example.line).to eq 26 + expect(example.executable_path).to eq 'some.feature:26' + end +end diff --git a/features-parser/spec/features_parser/feature_spec.rb b/features-parser/spec/features_parser/feature_spec.rb new file mode 100644 index 0000000..47db003 --- /dev/null +++ b/features-parser/spec/features_parser/feature_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser::Feature do + subject(:feature) { described_class.new(ast, path) } + + let(:ast) do + Cucumber::Messages::Feature.new(name: 'My feature') + end + + let(:path) { 'some.feature' } + + it "has original feature's name and executable_path" do + expect(feature.name).to eq 'My feature' + expect(feature.executable_path).to eq path + end + + context 'when incorrect type' do + let(:ast) { Cucumber::Messages::Scenario.new } + + it 'complains about incorrect type' do + expect { feature }.to raise_error('Incorrect node supplied: Cucumber::Messages::Scenario') + end + end + + describe '#normalized_name' do + before do + allow(FeaturesParser::NameNormalizer).to receive(:normalize).with('My feature').and_return('pants') + end + + it 'delegates normalization to NameNormalizer' do + expect(feature.normalized_name).to eq('pants') + end + + it 'memoizes value from NameNormalizer' do + expect(FeaturesParser::NameNormalizer).to receive(:normalize).once + + 2.times { feature.normalized_name } + end + end +end diff --git a/features-parser/spec/features_parser/name_normalizer_spec.rb b/features-parser/spec/features_parser/name_normalizer_spec.rb new file mode 100644 index 0000000..8086d09 --- /dev/null +++ b/features-parser/spec/features_parser/name_normalizer_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser::NameNormalizer do + subject(:normalizer) { described_class } + + it 'uses dash as default separator' do + expect(normalizer.normalize('some string')).to eq 'some-string' + end + + it 'does not modify input string' do + input = 'some input' + normalizer.normalize(input) + + expect(input).to eq 'some input' + end + + it 'supports custom separator' do + expect(normalizer.normalize('some string', '/')).to eq 'some/string' + end + + it 'converts non-letters and non-digits to separator' do + expect(normalizer.normalize('underscored_text $#% dashed-text 123')).to eq 'underscored_text-dashed-text-123' + end + + it 'squashes several separators into one' do + expect(normalizer.normalize('several separators')).to eq 'several-separators' + end + + it 'removes separators in the beginning and in the end' do + expect(normalizer.normalize('- some text -')).to eq 'some-text' + end + + it 'downcases string' do + expect(normalizer.normalize('PaNcAkEs')).to eq 'pancakes' + end + + it 'passes combined test' do + expect(normalizer.normalize(' !#$ PaNCakes % WILL conqu3r... tHe world! -- ')) + .to eq 'pancakes-will-conqu3r-the-world' + end +end diff --git a/features-parser/spec/features_parser/name_provider_spec.rb b/features-parser/spec/features_parser/name_provider_spec.rb new file mode 100644 index 0000000..5748d0f --- /dev/null +++ b/features-parser/spec/features_parser/name_provider_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser::NameProvider do + subject(:provider) { described_class.new(cucumber_object) } + + let(:cucumber_feature) do + # No original class loaded to verify + double(name: 'Some Feature', location: double(file: 'some.feature')) + end + + context 'when the object is a scenario' do + let(:cucumber_object) do + # No original class loaded to verify + double( + outline?: false, + name: 'Some scenario', + location: double(line: 5), + feature: cucumber_feature + ) + end + + it 'returns normalized name' do + expect(provider.normalized_name).to eq 'some-feature/some-scenario' + end + end + + context 'when the object is an example' do + let(:cucumber_object) do + # No original class loaded to verify + double( + outline?: true, + name: 'Some, example, that may contain commas, Scenarios Section Title (#5)', + location: double(line: 5), + cell_values: %w[admin 7% $1,200], + feature: cucumber_feature + ) + end + + it 'returns normalized name' do + expected_name = 'some-feature/some-example-that-may-contain-commas/admin|7|1-200' + expect(provider.normalized_name).to eq expected_name + end + end +end diff --git a/features-parser/spec/features_parser/outline_spec.rb b/features-parser/spec/features_parser/outline_spec.rb new file mode 100644 index 0000000..7e985a1 --- /dev/null +++ b/features-parser/spec/features_parser/outline_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser::Outline do + subject(:scenario) { described_class.new(feature, outline) } + + let(:feature) do + instance_double( + FeaturesParser::Feature, + normalized_name: 'normalized-feature-name', + executable_path: 'some.feature' + ) + end + + let(:outline) do + Cucumber::Messages::Scenario.new( + name: 'My outline', + keyword: 'Scenario Outline', + location: Cucumber::Messages::Location.new(line: 3) + ) + end + + it "returns original outline's basic info" do + expect(scenario.name).to eq 'My outline' + expect(scenario.line).to eq 3 + expect(scenario.executable_path).to eq 'some.feature:3' + end + + describe '#normalized_name' do + def mock_name_normalizer + allow(FeaturesParser::NameNormalizer).to receive(:normalize).with('My outline').and_return('pants') + end + + before { mock_name_normalizer } + + it 'includes feature name and outline name' do + expect(scenario.normalized_name).to eq 'normalized-feature-name/pants' + end + + it 'memoizes normalized name' do + expect(feature).to receive(:normalized_name).once + expect(FeaturesParser::NameNormalizer).to receive(:normalize).once + + 2.times { scenario.normalized_name } + end + end +end diff --git a/features-parser/spec/features_parser/scenario_parser_spec.rb b/features-parser/spec/features_parser/scenario_parser_spec.rb new file mode 100644 index 0000000..bd9aaca --- /dev/null +++ b/features-parser/spec/features_parser/scenario_parser_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser::ScenarioParser do + subject(:parsed_names) { described_class.new(catalog:).parse([file]) } + + let(:catalog) { FeaturesParser::Catalog.new } + let(:file) { 'spec/support/some.feature' } + + let(:scenario) do + 'user-does-random-things/sending-as-a-guest-user' + end + + let(:outline_examples) do + %w[ + user-does-random-things/staff-sends-feedback/user|john-doe-com + user-does-random-things/staff-sends-feedback/moderator|agent-smith-com + user-does-random-things/staff-sends-feedback/admin|neo-matrix-com + ] + end + + let(:scenarios_examples) do + %w[ + user-does-random-things/client-gets-discount/organic|5|15 + user-does-random-things/client-gets-discount/ad-campaign|10|5 + user-does-random-things/client-gets-discount/email|5|5 + user-does-random-things/client-gets-discount/social-media|10|10 + ] + end + + it 'parses and returns normalized names' do + expect(parsed_names).to eq([scenario] + outline_examples + scenarios_examples) + end + + context 'when Catalog' do + it 'registers parsed scenarios and outline examples in catalog' do + lines = %w[10 26 27 28 37 38 42 43] + expected_paths = lines.map { |line| [file, line].join(':') } + all_names = [scenario] + outline_examples + scenarios_examples + + parsed_names + + expect(catalog.names).to eq all_names + expect(catalog.executable_paths).to eq expected_paths + end + end + + describe 'parsing errors' do + let(:file) { 'spec/support/parse-error.feature' } + + it 'provides filename with parse error' do + expect { parsed_names }.to raise_error do |error| + expect(error).to be_a(FeaturesParser::ScenarioParser::ParserError) + expect(error.message).to match(/Filename: #{file}.+? got 'Background:'/m) + end + end + end +end diff --git a/features-parser/spec/features_parser/scenario_spec.rb b/features-parser/spec/features_parser/scenario_spec.rb new file mode 100644 index 0000000..ce026a0 --- /dev/null +++ b/features-parser/spec/features_parser/scenario_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser::Scenario do + subject(:scenario) { described_class.new(feature, ast) } + + let(:feature) do + instance_double( + FeaturesParser::Feature, + normalized_name: 'normalized-feature-name', + executable_path: 'some.feature' + ) + end + + let(:ast) do + Cucumber::Messages::Scenario.new( + name: 'My scenario', + location: Cucumber::Messages::Location.new(line: 3), + keyword: 'Scenario' + ) + end + + it "returns original scenario's basic info" do + expect(scenario.name).to eq 'My scenario' + expect(scenario.line).to eq 3 + expect(scenario.executable_path).to eq 'some.feature:3' + end + + describe '#normalized_name' do + def mock_name_normalizer + allow(FeaturesParser::NameNormalizer).to receive(:normalize).with('My scenario').and_return('pants') + end + + before { mock_name_normalizer } + + it 'includes feature name and scenario name' do + expect(scenario.normalized_name).to eq 'normalized-feature-name/pants' + end + + it 'memoizes normalized name' do + expect(feature).to receive(:normalized_name).once + expect(FeaturesParser::NameNormalizer).to receive(:normalize).once + + 2.times { scenario.normalized_name } + end + end +end diff --git a/features-parser/spec/features_parser_spec.rb b/features-parser/spec/features_parser_spec.rb new file mode 100644 index 0000000..cbeded4 --- /dev/null +++ b/features-parser/spec/features_parser_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +RSpec.describe FeaturesParser do + it 'has a version number' do + expect(FeaturesParser::VERSION).not_to be nil + end +end diff --git a/features-parser/spec/spec_helper.rb b/features-parser/spec/spec_helper.rb new file mode 100644 index 0000000..4b1197b --- /dev/null +++ b/features-parser/spec/spec_helper.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'features-parser' + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = '.rspec_status' + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/features-parser/spec/support/parse-error.feature b/features-parser/spec/support/parse-error.feature new file mode 100644 index 0000000..1be11db --- /dev/null +++ b/features-parser/spec/support/parse-error.feature @@ -0,0 +1,17 @@ +Feature: User does random things + In order to provide great service for our customers + So as a user + I should be able to do whatever I want + + @guest + Scenario: Sending as a guest user + When I am not authorized + And I complete contact form + Then email should be sent: + | to | subject | + | user | We have received your application | + Background: + | role | email | + | user | john@doe.com | + | moderator | agent@smith.com | + | admin | neo@matrix.com | diff --git a/features-parser/spec/support/some.feature b/features-parser/spec/support/some.feature new file mode 100644 index 0000000..4ff9ae6 --- /dev/null +++ b/features-parser/spec/support/some.feature @@ -0,0 +1,43 @@ +Feature: User does random things + In order to provide great service for our customers + So as a user + I should be able to do whatever I want + + Background: + Given I open homepage + + @guest + Scenario: Sending as a guest user + When I am not authorized + And I complete contact form + Then email should be sent: + | to | subject | + | user | We have received your application | + + @logged_in + Scenario Outline: Staff sends feedback + When I am authorized as + And I complete contact form + Then email should be sent: + | to | subject | + | | Knock, knock, | + Examples: + | role | email | + | user | john@doe.com | + | moderator | agent@smith.com | + | admin | neo@matrix.com | + + Scenario Outline: Client gets discount + When my referral status is + And I put of goods into the cart + Then I get discount + + Scenarios: First time customer + | status | amount | discount | + | organic | 5 | 15% | + | ad campaign | 10 | 5% | + + Scenarios: Returning customer + | status | amount | discount | + | email | 5 | 5% | + | social media | 10 | 10% | diff --git a/rspec-distrib/.gitignore b/rspec-distrib/.gitignore new file mode 100644 index 0000000..7c72775 --- /dev/null +++ b/rspec-distrib/.gitignore @@ -0,0 +1,11 @@ +*.gem +# for a library or gem, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# Gemfile.lock +.ruby-gemset +.bundle +/bundle/ +/coverage/ +/.yardoc/ +/doc/ +distrib.log diff --git a/rspec-distrib/.rspec b/rspec-distrib/.rspec new file mode 100644 index 0000000..38fa261 --- /dev/null +++ b/rspec-distrib/.rspec @@ -0,0 +1,4 @@ +--color +--require spec_helper +--require pry +--exclude-pattern 'features/fixtures/**/*_spec.rb' diff --git a/rspec-distrib/.rspec-distrib b/rspec-distrib/.rspec-distrib new file mode 100644 index 0000000..b72b32b --- /dev/null +++ b/rspec-distrib/.rspec-distrib @@ -0,0 +1,6 @@ +RSpec::Distrib.configure do |config| + config.error_handler = DistribCore::Leader::RetryOnDifferentErrorHandler.new( + ::RSpec::Distrib::Leader::RSpecHelper, + retry_limit: 3 + ) +end diff --git a/rspec-distrib/.rubocop.yml b/rspec-distrib/.rubocop.yml new file mode 100644 index 0000000..d5c877b --- /dev/null +++ b/rspec-distrib/.rubocop.yml @@ -0,0 +1,53 @@ +inherit_from: .rubocop_todo.yml + +require: + - rubocop-rspec + +AllCops: + DisplayCopNames: true + NewCops: enable + Exclude: + - features/fixtures/**/* + - coverage/**/* + - bundle/**/* + - vendor/**/* + +RSpec/StubbedMock: + Enabled: false + +RSpec/MultipleMemoizedHelpers: + Enabled: false + +Layout/LineLength: + Max: 120 + +Metrics/BlockLength: + Exclude: + - spec/**/* + - features/**/* + +RSpec/MessageSpies: + Enabled: false + +RSpec/MultipleExpectations: + Enabled: false + +RSpec/ExampleLength: + Enabled: false + +RSpec/NestedGroups: + Max: 4 + +RSpec/DescribeClass: + Exclude: + - features/**/* + +RSpec/ExpectActual: + Exclude: + - features/fixtures/**/* + +Style/FrozenStringLiteralComment: + Enabled: true + Include: + - spec/**/* + - features/**/* diff --git a/rspec-distrib/.rubocop_todo.yml b/rspec-distrib/.rubocop_todo.yml new file mode 100644 index 0000000..5f3117f --- /dev/null +++ b/rspec-distrib/.rubocop_todo.yml @@ -0,0 +1,45 @@ +# This configuration was generated by +# `rubocop --auto-gen-config --auto-gen-only-exclude --exclude-limit 2000` +# on 2024-05-21 13:51:43 UTC using RuboCop version 1.63.5. +# The point is for the user to remove these configuration records +# one by one as the offenses are removed from the code base. +# Note that changes in the inspected code, or installation of new +# versions of RuboCop, may require this file to be generated again. + +# Offense count: 1 +# Configuration parameters: AllowedParentClasses. +Lint/MissingSuper: + Exclude: + - 'lib/rspec/distrib/worker/rspec_runner.rb' + +# Offense count: 1 +# Configuration parameters: AllowComments, AllowNil. +Lint/SuppressedException: + Exclude: + - 'lib/rspec/distrib/worker/rspec_runner.rb' + +# Offense count: 1 +# Configuration parameters: AllowedMethods, AllowedPatterns, Max. +Metrics/CyclomaticComplexity: + Exclude: + - 'features/support/shared_contexts/base_pipeline.rb' + +# Offense count: 1 +# Configuration parameters: AllowedMethods, AllowedPatterns, Max. +Metrics/PerceivedComplexity: + Exclude: + - 'features/support/shared_contexts/base_pipeline.rb' + +# Offense count: 1 +# Configuration parameters: EnforcedStyle, CheckMethodNames, CheckSymbols, AllowedIdentifiers, AllowedPatterns. +# SupportedStyles: snake_case, normalcase, non_integer +# AllowedIdentifiers: capture3, iso8601, rfc1123_date, rfc822, rfc2822, rfc3339, x86_64 +Naming/VariableNumber: + Exclude: + - 'spec/rspec/distrib/example_group_spec.rb' + +# Offense count: 1 +# This cop supports unsafe autocorrection (--autocorrect-all). +Style/GlobalStdStream: + Exclude: + - 'spec/rspec/distrib/leader_spec.rb' diff --git a/rspec-distrib/.ruby-version b/rspec-distrib/.ruby-version new file mode 100644 index 0000000..9b7a431 --- /dev/null +++ b/rspec-distrib/.ruby-version @@ -0,0 +1 @@ +3.2.4 \ No newline at end of file diff --git a/rspec-distrib/CODE_OF_CONDUCT.md b/rspec-distrib/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d6f70c9 --- /dev/null +++ b/rspec-distrib/CODE_OF_CONDUCT.md @@ -0,0 +1,14 @@ +# The Community Code of Conduct + +**Note:** We have picked the following code of conduct based on [Ruby's own code of conduct](https://www.ruby-lang.org/en/conduct/). + +This document provides a few simple community guidelines for a safe, respectful, +productive, and collaborative place for any person who is willing to contribute +to the community. It applies to all "collaborative spaces", which are +defined as community communications channels (such as mailing lists, submitted +patches, commit comments, etc.). + +* Participants will be tolerant of opposing views. +* Participants must ensure that their language and actions are free of personal attacks and disparaging personal remarks. +* When interpreting the words and actions of others, participants should always assume good intentions. +* Behaviour which can be reasonably considered harassment will not be tolerated. diff --git a/rspec-distrib/Gemfile b/rspec-distrib/Gemfile new file mode 100644 index 0000000..3acdaf7 --- /dev/null +++ b/rspec-distrib/Gemfile @@ -0,0 +1,16 @@ +source 'https://rubygems.org' + +gemspec + +gem 'distrib-core', path: '../distrib-core' + +gem 'pry' +gem 'pry-byebug' +gem 'rspec', '~> 3.13.0' +gem 'simplecov' + +group :linters do + gem 'rubocop' + gem 'rubocop-rspec' + gem 'yard' +end diff --git a/rspec-distrib/Gemfile.lock b/rspec-distrib/Gemfile.lock new file mode 100644 index 0000000..dac9c75 --- /dev/null +++ b/rspec-distrib/Gemfile.lock @@ -0,0 +1,101 @@ +PATH + remote: ../distrib-core + specs: + distrib-core (0.0.1) + +PATH + remote: . + specs: + rspec-distrib (0.0.1) + rspec-core (~> 3.12) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + byebug (11.1.3) + coderay (1.1.3) + diff-lcs (1.5.1) + docile (1.4.0) + json (2.7.2) + language_server-protocol (3.17.0.3) + method_source (1.1.0) + parallel (1.24.0) + parser (3.3.1.0) + ast (~> 2.4.1) + racc + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + pry-byebug (3.10.1) + byebug (~> 11.0) + pry (>= 0.13, < 0.15) + racc (1.8.0) + rainbow (3.1.1) + regexp_parser (2.9.2) + rexml (3.2.8) + strscan (>= 3.0.9) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) + rubocop (1.64.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.31.1, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.31.3) + parser (>= 3.3.1.0) + rubocop-capybara (2.20.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.25.1) + rubocop (~> 1.41) + rubocop-rspec (2.29.2) + rubocop (~> 1.40) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + rubocop-rspec_rails (~> 2.28) + rubocop-rspec_rails (2.28.3) + rubocop (~> 1.40) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + strscan (3.1.0) + unicode-display_width (2.5.0) + yard (0.9.36) + +PLATFORMS + ruby + +DEPENDENCIES + distrib-core! + pry + pry-byebug + rspec (~> 3.13.0) + rspec-distrib! + rubocop + rubocop-rspec + simplecov + yard + +BUNDLED WITH + 2.4.20 diff --git a/rspec-distrib/LICENSE.txt b/rspec-distrib/LICENSE.txt new file mode 100644 index 0000000..08b3961 --- /dev/null +++ b/rspec-distrib/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Toptal + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/rspec-distrib/README.md b/rspec-distrib/README.md new file mode 100644 index 0000000..c7942bf --- /dev/null +++ b/rspec-distrib/README.md @@ -0,0 +1,315 @@ +# rspec-distrib + +A command-line tool to build a queue of specs and run them in parallel on +multiple machines/processes with RSpec. + +## Rationale + +A really large spec suite (hours) takes significantly less to execute when +spread across multiple processes. + +Naive approach is to split spec files into batches, and run a process passing it +a batch of files, but due to uneven distribution spikes in total wall clock time +happen. Distrib's queue feature allows to dynamically +serve spec files one by one over the network, rather than pre-splitting them +into batches. + +In the cloud the machines are either ephemeral, e.g. short-lived and can be +preempted at any moment of time, or significantly more expensive, making it +either impossible or expensive to run specs on a single machine. The goal is to +reduce total build time to minutes, and even running on a beefiest machine +doesn't help much to achieve that. + +## Overview + +rspec-distrib is a relatively simple client-server (Leader-Worker) wrapper on +top of RSpec, which dynamically serves spec file names to clients (workers) that +load and execute them one by one, and aggregates the results in the same format +a regular `rspec` would do. + +It is possible to run rspec-distrib on a local machine, or run the Leader +locally, and Workers remotely, or run both Leader and Workers on CI, depending +on your needs. There are no tangible limitations on the number of Workers. +Workers can run side-by-side using [parallel_tests], sharing the external +services like database, Redis, Memcached, ElasticSearch, that your integration +tests might need. + +The queue is fault-tolerant, e.g. when a worker machine goes down or experiences +a network partition, the spec file being executed is returned back to the queue +on timeout, and later passed to another worker. + +Spec files are served from the slowest (basing on previous builds results) to +fastest to reduce worker idle time, and reduce the risk of waiting for a long +spec file to execute in the end of the build. + +## Installation + +Add to the Gemfile: + +```ruby +gem 'distrib-core', git: 'git@github.com:toptal/test-distrib.git', + glob: 'distrib-core/*.gemspec' +gem 'rspec-distrib', git: 'git@github.com:toptal/test-distrib.git', + glob: 'rspec-distrib/*.gemspec' +``` + +## Running + +There is not much difference between running on local machine, or across the +network. + +```shell +$ rspec-distrib start | $ rspec-distrib join localhost +4386 files have been enqueued | .................*......F..._ +Using seed 27792 | +``` + +You may have to prefix the command with `bundle exec`. + +## Leader + +The Leader purpose is to serve spec file names one by one to workers, and +aggregate the results of running those specs with RSpec. + +The following command: +- builds a queue of spec files +- starts a watchdog thread (see more about watchdog in the 3nd stage) +- exposes a Leader [DRb] server on all the network interfaces + +```shell +rspec-distrib start +``` + +You can specify seed, which will be used to randomise order of examples on workers. Or it will be generated automatically. + +```shell +rspec-distrib start 12345 +``` + +![Startup](docs/startup.png) + +Once there are no more spec files left, the Leader drops all connections, and +reports. + +![Watchdog](docs/watchdog.png) + +## Worker + +Worker connects to the Leader, receives spec file names from it, and report back +the results. + +The following command runs a worker: + +```shell +rspec-distrib join leader_address +``` + +where `leader_address` is either an IP address, or a domain name. + +Worker requests spec file names from the Leader, `next_test_to_run` and reports +the execution results `report_file` back to the leader. + +![Worker workflow](docs/worker.png) + +## Reports + +Workers are sending the example reports to the Leader immediately after running +specs from that spec file. This means that if a worker dies, the Leader has kept +all the previous reports. + +## Configuration + +`rspec-distrib` expects to find configuration in `.rspec-distrib` file +which is loaded if it exists. Configuration is expected to be a Ruby file. + +### Spec files to execute + +Override default list of the spec files: + +```ruby +RSpec::Distrib.configure do |config| + config.tests_provider = -> { + Dir.glob(['spec/**/*_spec.rb', 'engines/**/*_spec.rb']) + } +end +``` + +### Handling failures on workers + +There are several types of issue may occur during execution of specs on workers. + +1. Expectation errors (aka legit failures of spec) +2. Failures in between of spec executions (`before`/`after` blocks) +3. Worker failed to start completely + +`rspec-distrib` provides an ability to handle such errors. + +#### Spec retries + +`rspec-distrib` can re-run specs by certain exceptions: + +```ruby +RSpec::Distrib.configure do |config| + config.error_handler.retryable_exceptions = ['Elasticsearch::Transport::Transport::Errors::ServiceUnavailable'] + config.error_handler.retry_attempts = 2 +end +``` + +It means that any spec which failed because of `Elasticsearch::Transport::Transport::Errors::ServiceUnavailable` +will be retried up to two times. + +Leaving `retryable_exceptions` empty means ANY. + +#### Handling workers which failed on start or between spec executions + +`rspec-distrib` can be configured to fail leader if failures which occur in workers on startup or in `before`/`after` blocks. + +```ruby +RSpec::Distrib.configure do |config| + config.error_handler.fatal_worker_failures = ['NameError'] + config.error_handler.failed_workers_threshold = 2 +end +``` + +It means that if worker failed with DB error outside of spec - leader will not stop and continue run anyway on other workers. + +Leaving `fatal_worker_failures` empty means no error will fail leader. + +#### Custom logic to handle failures + +You can specify your own object to handle failures. Here is the interface for it: + +```ruby +class MyErrorHandlingStrategy + def retry_test?(file, example_groups, exception) + # return true to retry the file + end + + def ignore_worker_failure?(exception, context_description) + # return true to ingore the exception + end +end +``` + +And use it like this: + +```ruby +RSpec::Distrib.configure do |config| + config.error_handler = MyErrorHandlingStrategy.new +end +``` + +### Timeouts + +Set equal timeout for all spec files to 30 seconds: + +```ruby +RSpec::Distrib.configure do |config| + config.test_timeout = 30 # seconds +end +``` + +Specify timeout per spec file. An object that responds to `call` and receives +the spec file path as an argument. The proc returns the timeout in seconds. + +```ruby +RSpec::Distrib.configure do |config| + config.timeout_proc = ->(spec_file) do + 10 + 2 * average_execution_in_seconds(spec_file) + end +end +``` + +If both `timeout_proc` and `test_timeout` are provided, `timeout_proc` +will take precedence, unless it returns a falsey value, in which case +it will fallback to `test_timeout`. +This is useful for cases where some specs have a timeout strategy and some +don't. + +### See lib/rspec/distrib/configuration.rb for the full list of options + +## RSpec configuration + +All RSpec configuration should be in `spec_helper.rb` or `rails_helper.rb`. +You should require `spec_helper.rb`/`rails_helper.rb` in `.rspec` file. +If you require `spec_helper.rb`/`rails_helper.rb` in spec file - configuration may not apply properly! + +## FAQ + +> Is it simple to use it in my project? + +Yes, it's simple if you are using RSpec already. rspec-distrib is almost a +drop-in replacement. If you plan to run several Workers side-by-side on a single +machine, check [parallel_tests] documentation how to set your project up. + +> How is timeout defined? + +Timeout is configurable in a configuration file. +It can be configured for all of the spec files, or, if you have a +storage where you keep previous execution times per spec file, we encourage +you to use this average of a couple of last builds to calculate the timeout. +Using double the average execution time plus ten seconds is a good strategy +that prevents spec files from returning to the queue while still being executed. +It mitigates two cases: + +1) spec execution time doubled + +2) spec was fast (milliseconds), and then a spec file or its dependencies change +and it execution time changes, and it takes seconds. + +> What happens if there's a really slow spec? + +Spec file is picked up by Worker #1, times out, is returned to the queue, and is +picked up by the next worker, Worker #2. The first to submit the results wins. + +> What if there's nothing left in the queue, but some spec files are being +processed? + +The idle workers wait in the queue just in case there's a timed out spec file. + +> Is it secure? + +[DRb], the transport used, is not secure. Executing of arbitrary code on the +Leader machine is possible. Make sure no one outside of the test environment can +communicate with it. + +> Who can access the Leader machine? + +Anyone who has access to it over the network. Make sure it's not exposed to the +Internet. + +> What is the default port? + +It's port 8787, default for [DRb]. + +> Is it thread-safe? + +Yes, thread-safe data structures are used in the implementation, and access to +non-thread-safe data structures is synchronized. + +> Are the workers using the same seed the execute the specs? + +Yes. + +> Is it fault-tolerant? + +Yes. Any number of workers can crash, the build results are not affected, but +the total build time will. Make sure to restart or spawn additional workers when +they crash. + +> How the worker knows when there are no more specs to run? + +The Leader drops the connection when there's nothing left, and Workers shut down +gracefully. + +[DRb]: https://ruby-doc.org/stdlib-2.5.3/libdoc/drb/rdoc/DRb.html +[knapsack]: https://github.com/ArturT/knapsack +[parallel_tests]: https://github.com/grosser/parallel_tests + +## Contributing + +Bug reports and pull requests are welcome [on GitHub](https://github.com/toptal/test-distrib/issues). + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/rspec-distrib/docs/startup.png b/rspec-distrib/docs/startup.png new file mode 100644 index 0000000000000000000000000000000000000000..d6f7e03953c25fa515c77de540bd1df7239fba1f GIT binary patch literal 26969 zcmeFZXEYpc+b=rCV6+%5gq(1hnM6e6$5uokLv>@}?gsI4D*}cwIi{6r zC7*fzA?d~X~YdVxmo zGjJ)do6;q(5nHxj^i-_S{l+t^pt+@C7CYXTXescNm2Oj zm-mY|s&tu{aQd$YzP?BI;{A(RdO>mLaxyplHm0WmH`+6ARNP=wbQH$R-Q!=AlDeh4 z>`ic0-1G^G&q!c67gjPLoN_ppzSNlAuWZJ=<6;vws4fB3mpnsTJ;{${bxRIzV0b0L zPw^>eZw&qet}5)`ic7Vk6ZasvmZB~2dyIYlkE>T$lvb<_d89QHScC2EQe?!UUs;!C zDM071N4SoUglS%}KmPq&li8T|rC8VJuomL+q8`@oEN8oT_Pc{F<*(w#pbJtzn@tL# zYko~#V}_#Sbxo{V&P*9*$Bjh7_N1@vJ93DMq@vvM-Y<0=PkBEyqH<9*W=RsF&!QBG zrWMW7PTgQ#ecBVf&tujP!}WfCZ^P=LVza80B}x4CBZ$8%AGV_~Rg|vmt9I#TYOD(^uDjzUa{CJXD>5{LL#zmGw6Pwh+ zP!PrIFQ`*ZaQCBHxswZ9yY`4&xaNy8KBljiEfQN&*^O!x@uw**(^0kLK1A<&;n!w) zJAOP4wf4_H&lpS2MBLzp4!QVD+Qb~uu}F;F{J4D-{=Vxu+p@xoL5o6Oz&UO?Qj%QCi|Kyr|o)bukN*EwphIPwuR}%{SbkgV)$qVAKxdoocFujPs>Nl zRap8h9z=bj{X`ak3we}dWI$yoF>W(@8o#o3xZ*Bp9Jtdybt`RQ?Vgb~dZAqG2EFT#gL-W?qi80BA1$7Ees=L^ zBSNaP<6)l;D&~89r1*0j0$s^6hS!G`wD1kuT-PwES3SRBXWgQD)~)5mUAE5^{Zr>yUctH#%gIH>FB#Z*04;@a&{kh*O8!NX?L zEwKOR+!I%|F4`}O?=$gzG@^nn?P_&)wX)A%*iq3LI6ZVP8X9x&WY4+Hb1OEmq(Xjb z@PW+IA?0M-9kXvuN8xs1)!{=Dx9|-!wB473;N#;73`Z1)ydl?k&~x!EA5GrezB>xd z;Z(EnvAu=U6q4-yW2I24UVT!CPa`nW$|U{)^chF&c}D;##Sdl0mun8omJ`1+B~My( zM`VAlt~Mp0X$O_v?o-#_`236U*T!VcEAc`V)`^W?Nbo(i6y2xKymFI=lki62u-CRG z&lYae_b4{`bg$st?`oOxicKg@#W`!_3_2_0w@_aYB^hk*qRRQO66isq* z%JO{j0&S;f`Gtb04SoMoIYBitJKMSMV{-8b|RJM&F<8^eO>?Ky- zt7sS+g5$P(to1sTdNp44ur?G331d)x{bWfr$>igC^kkBW%_&565c9V*TKkZ`9nzW*`6YGIP&d}xHgu@PaFY2U zHP$Muxicf--Sc;RQNHGb;@m<$bL@vpzADcItq#s?^unlSU>BMFO94+czteZUHAAlH z$-%5!b!ep8JL8hKyXJI!>BjuvN~NhpEqAj$7=M&bpqP*<i6Qj)xf|P{u>GLeTq(M|+fLkm_XilMg+~W!q2l60ki9i1|_ei&OJ1>-w9p8?( z;l(W$HBBn=CV4w}IH?)}1s{=IMQQc@9K2{8TA_R7`N(r#WNWv@HS1%uft9k?;MvR6 z!@40GLR0mkw$r2Ccn;EBHO&AbRU8g9o){aC(_&jt$k&@|xlp&D7_(zNYGj2eAO5rill* zt4n2jlIr39o%};R08#jnAqZ`FF0UsI*Ic`rb`S5*%4O6@SYr}*?6aUYw8g#=JL7Ou za0%w`if-&^_}DU;R5rIi4@Kb^!@qWV2X@*kdzB@CfSzfN_B384+*4E3{%1iBDkQAf zNxBdIzcUS&3DY>MI!+^%?LqcQ9FPW_(Ww~1I9z1bzVe8fN7vVIQW@Nmj#wQ_xWWK+ zJUFKDx->XuOoBiVIxqT_-e0ON>f=ad+pa>{<>4eU+FOYf&}Q3Sn!m(ECquE6K*H8` z2$(PmXOh)_zi(>--*3?z>)|pEs|fg(M8u&P-dkw>T`*e;EJ!5uI`V&O8lc}tQms-( z*yBUQW-gYWsyGEi2xtb!WjjAvXyEvy`(_Y&a-SvDSm|@bK5p-?~T23`L7H3NF3R`74?bMjp+hH@B6s zd>CeQny#|k`6S2ip)Zj$kwdzAmSec|p^)&|-e{ifWSPSo6X8F+V+Fj+T-t5sVI20n ze^$gZIoiHHQ5t{6=!s;l9Z95$dBTAeOC79y z1yK;nENvZq&q}VU;>&wHe6y9hP4m}}Pr1uolq!2ok0<1?5>%)SSXb6Ai9Ov*X(VKO z`calvF?4`P9|P`ke<<^AphNJ!23DdXv66v{!77XJ^ELt7iC$^4yRS8I(h@oG99oX7 zY{1pZ(#%Bx1D7hXV+UD*6v+)D5ly~Z&4)(Q%|3PNc-GowRMzwF8lGO9EF?dB_RK)w zg9kWAuf`u~Rrzxc$;GCv7S8HNL)tSwX^FOY+|7IHt8-1BW$Bha*ws_9t6%zO_3iVM zt?$Yvc`am}C70+#dZUEqugZ?v#RF=H{ZZ{{tfcf3!hX&+X#8y263@F^pl8Y!E6;4v z5k#n7Zq&?^_9z{@Y#4UgXcp|Dm61*#UfkewT~Yiqn%!yh?WDr@!HmwwnVUepVhMnJ zx(lcNp#`SxaNbA7$b0q1-?0I>uTS|pK9LpU1-iN z-feB{gNL8h*sC@TmAf~#S}$7c&4bApxC05uYBRE~6+wW&D+41WqIGlXG^ovF-`&tB zcCs4>g7eISl5x09y>SO{f>AoSx5Bj3&HdSgDh>yN6HCb1`+o>210}>LWc;kw3qoh% zS-OKuKG6<9{Lqs!=0w7dISTv}c`1HmB^~KgGV&PT9rDw_X~tgcC>VRONEpYy9Mu2{ zCi}^@CY7k`A&?aA3kvXaAh4|rYPUHD5xUyiEXMw4Zx=5Vzi?@2|L={_2tY#8p}2HIODSQmpkx_aG``M+Y3G)!Vx?3h_g$J)qCkTpr!A zqV_hZP0Yok*YE}`cq0HVfR&}2|7B^_Zbu-l-YdU1Y9(cI`|NV?cu*UUW_gPeb^rpJ zc+3=TMGg%-q!G+tEU2icnCVtfPF*Y2d2#{N3*Nq2&Gb$&Xq(hFV|T z4JJLu)a33T{+z>eUsPTJ)9*Q7ovunnSU+H~q96xqY5I?kQTLbmbW_Si3;ynS?Tyls zLuW84G{APQ@onT)-W>hIGTq9KCA~(sA}07tao@(<_A{D;!r!k;nNUfH)d=$d(qitf z7n@0Cb51|I)fL6qZhQUkG907DxE)OEz;i1pDXEVsM$F8k1~nq&b70r-vvIxOr(VYg zD8I|6dsjYui;>_ic1Bn?+?^tBwvYEc(ZG&X$LCj!tao%KLT9)iwwaO z($1KtT~q6}8O=`>z&y%uPr9jB)djf`Kv_W;OUq*tkgDE21oTgFPCHG&Ht9o5Rj7jB z`O$a$y;_I4$7r%RCFUATsAV#@;ZmZG5$|I7InMUXTiZwZdA4gzeI6TAYV$r@cSo~; zxfP~a4MNMgEW2BQTO3mUPfd$J=u$(;pG%84Ee2rNWcrjP&fGpDrW#xy={2}may*91 zpRBzpFD@Z05u+oipcQc|3`3yQ;ft{PFSSwT(ZbzRLDgh9b`VL3 zcxPB!_K%ig35@2P3Gpnk(U4gm!C3|FLm;k0F%I1}4&*_d{Eo~Va9Ekt(9$SItOpaC zf6|Fw94peubC_$i+RRQAar=RsYjl5bcG6D3kib89N1CTJGRXhYGjtmYLLpSNvCwL) zUt=9tjH}UP|G9QxN=}iHsUyX2m_H}!QI>CQE~5(EjakR&5wP1X68~mz{s?$DXBp2e zvo-fG9nV=PLIhlOHA42E1mv-1KItfl`Ut~7WJ^9tYm#N-MO9{&@BB28f%4Kl14d?wg z4KHDnTx76HRQPsGcx|>B-B?HQ;o$w!-b}rsT$hOkfypH`o&ZF?I*uDe+HR>AhPi{n z=Xov=D1V0mal|Q&L4Wp_-bS%;w#ZMLc6XzNZJD z9n%L`+AX|oEE!<3t>b&+g%xWlh7%0PllNH-pkD5^>1-Zau&d{viK_z7(e23`gN?H+Vms@XP_|8 z$ez+g{h>I%>#C5H3HSKm{AD?Rk0_}Rl5moH2f6}lR8Y|jk4h@~&!SYMMkmbdpAQnG zO_pz$Ibuk*^pK^kdLoYm)l8207AORRbrFSmTN(>uH_2MH=m#S88>@^l2IH;B$5wJP zMp&gJ2TI9gNy?~NWR9{((twwYZi;w$eiZ775Bj9V&y?CLMklrx6f`ULF6nm3!oxUUgxSvNwtkrka*V*Mr(-{rdl)@FD1{>W>J z=d0U*L9O>@ON>B(?xP=jf@DqmJ|E=ZdpSmai8nwXBm^EEr|t6DSPul`GyARR6V+LaCUV?@mO*jz-s@a?O&+B< z;u|d8al8SNXk>ILA*YCq4dAJ{rc5=~Yqw%q5g3QgI}gnP@y&N5pYIu-{coL8yx%i^ z3_bmlCM1L7OwTY#t=RhcisvRuBAZ0+9a1DDHPGl>hURBE)9>RAOu&*w(Z|9_q+La>9+ta$_Q>vkK3ptnQ-# zwvYj)JEt_?DZajxT-DbgeCu(bNy`xXyv7p+WbIrLYRzM z-&LW&MM0|5MPD?>1g}{$>l(V_%O^ipsqBUD?mA$HxEHW-4O1;5)VAE{$rrp)2?Nh@ z5WPH3yf~<9<+?ciE>Z1`PrOenXs64_P&U!gIjP_e0(iWAE#=jN*R3IJ#Pn+tzuU>w z4?Fd@MBfr?cm@P15)zz#6`#&JU1E*ivKip$_vb@EH#xL2w*tw;dCV#Oq&u8%{%$xw zULP2Dtna+yl#~f^gowzTK)eIc0rR!?pVhy8H@-Md7OuZ$PF()&+?4q=I3S;8IGkz` zT4CBeH?CuBPxRIFxwWAZBDBdLCk7$}xza5VKqlm>d$?tmcDc!$JWl%DtzP;y`M0Z^ zX77-jGjBhhGK_xG?!(87<}2}k-^)vul&N<8KDZLS1OlOJO~dp|7?c>Qd-_r5&}km~|UfqQ2Z^?tRI#7_}3ZS&>ao*0q;#3FA5E zp6qppUhIjgc8M;d|LHdd4Lw%nmuz(leoaT7BCwfCPCDpT{kKaX;E5_C(DDA_r&BU6 zC8g;E{C7(XlT44{5K6xaN$0P+WHf?je7DgC5Y~iX(%;yhL8!3_WaI;#O>@Uu}vcppJX8{6BK4= zUqLWtkstqwJ}224*L|_tY0iC8n6g{eK^AHs+I3SF`(njl9Va%g(dGg_{rxRw`E|T< z)$$D)mu-k##~3Z@rhF&~bf!2EE#o|r%m2jb>noe_KsHkP#Z`oLPnjFMrXO%jBC$uW z+1Eo{?GjP(xx6HPg^}klP44YIV2+0V*mi_9CG=uDL_*%#*;(P%Msy8*D%-`m%H>9{G?y+VBY`B|wQLbpM@L_jeFTx}YIRy-R>7b=%Z7%4ZUdIaxsh@~FmLENYj=#rRoMplf-j_N7853+22A-Lz|Kb1QQ{ z&{z7|#rSRt6Q)TC)1J3H1(8`A|KjP))n{~RDTC)9jnB>ODpARPWbf}tBQL4qKUusm z8m$_@t9t6nCH%^=XwQ1JS-ku}rc`X=ni>BC)pgRq;XuUIpyGoyX4uRqyG{cf)1W?M zDjEr!S)c><{6f#~cPE3D*6(gw#{{eu&|^f%b)2s0wDKKQ@LxlQFI|D)#w-=}kQA^E~2v57GiAPtL)cyTCU;2^+-3Rj~{r;3l1W zcFRHRv*b*&ngONhjx7x{Yz%`BJ2&a;y>MwIVx77f28n7Ya_&uhX?;RUD^GLVT9uLF z_veN4iI@j9psSfVL1-V5_v}Ds#2hK0 zouiqE!UrcbrD6Oq?a5yc9UAWaZ1_(9kd{ZaXK!^>>GhyKt@S2R**j%B@B%FgOfAi5 z@o*qHI@|KUb+IdP5zjX{*p2vD2Hs$)w8#=#X^t$)&|K4!TxmW!({DYhH()awaR_Ub zf;-D7SnzixaN`*e%GhM-Aav&pn^#e@-s`0P-G~V*9sg(g|IbL*mezbQDQsEg;iF!=Nj}}k z%Fac@`r6ApE_#4sr`Z%w%}v;TFbCpGe~w^;1#bnq5mPkwzY<7-f)td>!Rjb0LK5Uz zm_T6h*6>hCXpT8U#Jvin9NpQ_@RDgV6>hfpka>uC|a}B^+B0jGRC~nHHIsSty{@qCa|KlKP5EKWKJ;OJ`j=w|#7RtM_`GR)9F! zqi5r~vIJ$?zcc!GsfD}~=_ICw5csV|8J%M9FH!8XUy zh^uAiKn0FW|3?V6!6zJkC%k3Ucsp`&E1hGPz1 z?suJ7?rfj(@smcbn&kiHL}!J3H~*J)FwOz^Y~KzIA*@~b@<)`HqjX=ziv?L6Q8F1q~pxc5N0%oGXAWw>zy_3SSIVt;Kf~p3n}D-TUz7T8D_#NVz*{Z9&5$0|BR@Etuvt-?f{c|y+#o_Z7re~r zpVAqA#Iw|K)@?tZ{Ho^vkH8$k3`2?Rff$v*Oo%h8460-3p6IqdF`z~Or;J`A5~~K+ zIlElXiTV^U_v`DUM}Gp6W^Qba61Rv^9zNUTSq?%lVMLRjjT} z!|u#`cK5F64c(`>+bNW}A4vO^?~KGWShC*W2*a z>98UN8<{4Md! zTeL~sNjPL;>~9~ByOd~@5l#6P+8#VCt1%Njh@sli;QUV_(?BJDFXDQAd42_2nUN@~ zf{D1*qN{v&U$ZCJ_BkOjq{&2Mw5~f;Y^OrL6dISC9=^xGX6dO&#m-56pGq?npXCKA z&oa+tvBTAJD`M#WbaciGvAws~&oZ_?CM^>$MMSMzwk_;ELoyEkLJY2BX^@2?#};XQ z46&6>e0AyJKjn^Z2_QF>^+ZYPDTqN6WIB!}F|zR6kSE`CSNSXoYiPVH zdx?j_t_$CMcAnu9-aFfXdXOA$@>obC>*EWTA5-~N)y}xWd-I=P$WzHnn%djAe6Xx? zAMdI18dAQt?JpuZ{8QR6cOhptEG#Audm13{fB5mzf8ED1USLWUX$=n|#3oJs&v&x~ zOH7mQa5{}D3Ec-@8M>>{Dqe-09_X&R%v>8;Df{Y{=JB@d(-NTD8J5Ev5*+nrKY*T#f?a{#a zqB-Lb(G=aSOj?CEpjK3n=5_D)i+^DO{!6u8a4aeuR7bMuBt9<*}I6~M%0|BPA&;e2FBpsW!KCKTj|{CBF5CWBj`U?}{ni}LQi4zZ%imF>?9Ei#R@>nJW2 zI*uyS0U>fU#6XfOZaFL5Zh=GnO`6Rjk-}niZ0rdBzQtyvN*-i64neNS{Mx7~y2|{s z_w;y9*xp6!qp=N|T5y=JLMnkV&;QqlxdH1~19gMG-64q~foJCps`g4;4{NFHVC@&p zEL=Po^O*DTx0R8;s~T~JqW2UoMNqxI4a(J%pfoZp9QJeSGfy#{wwp<3r z;iguNK`BFFsI=zuNr@L<1{xm(ntl&0IajpUU2$2oozZ;#;7Xjt+*y_qqu$^#f4a1S z<;UzVJL9un>W%j4dMvuF8joz8`iiPHQ?FHh30fIBzjFAXCz@^-Me!RH%pXKt=7L_6 z^NT)NTk8me|Nh{yDAcmfvmCRZ<#*$zPTSE72aiS3$}hmVJd^mn!6KDynD;duJnYBn zOr^8c^{Uda8N3+CZqdEtI3c>k*@p-UDBS@=~gyI~CA1zVp)jPdfwEIBJU5}=J zx-EGP6zy^)BKZ>a-aV<*3w8n39bzOUGWZAgB1Kz+{b#{cP{$*=erNcmZEr%pN*4Ea zkMQ@GxlrinCL_%68--N%oMX-G5)*7i_EAqM0!sJ53RKPKR#$2h3{#vQwvM`JNp4%% zICZz}biQg8M~=KM>Mh$uMTnPtsraGJzB&Cs`=-Io54TzqI=baPs0D`f4tkaMrFle+ zL6J2(v{1E$cV4>|Qx&FG_i-ppNaH4>Xob?2Q7K=q3%k6Y)~q#YhgRk@tAHBd>O^U} z*!k~oELFko>AxkCl65Uh;9o&>V>we}gMV8^{K$fGAJn&yp+6Qyx85;vYL~=J*(9diefjHk>SM?eLPk;=Hv@%!kkmpcGG;L zmy)KDrh>2hFV6Ap$mm2YZnx_j&d!ngnhKfx1_fM!IIF4H`wyqKK@BwZv;uc{P(FLq zG3s<|ySY&L;6Eaucr9J!k3rw|he<8jRKr%C#|-d79nSx~tfIKQVZ ziXlhx&(SXtm}I(7=;d?Cl$1C96!AJBDU#^DTekS=s{eW<_$vx2oN7u*m0$r@3jZ!>X{YP}e) z!bwj<3h7cgq_kM2-OckjJ0AJ5$8T}idgVjfx5sayA^PXLq7>HS5o|ONqEBKnI5;L} z$NO)1=pqa0^4)O+dP9QP=#dCje(Pc6uD-iQFL1>#FieU9OvUg*=n^Ehf~69aarxYS zsJ7!%PX!*tGLUyZ;BM$@<3Lg@hO92}ns=x4(wI;kAYH<&8FRwqLsd}{CYNK0+GL+os!u|v5cNu3;0lXy02uJIWA*7xIUsSK|0Z$T@P3kK zkK_Zn@5)S+QcpMPSsW&#L_MEUlENUhca%Mzw&>Mv&EF#RIZX0>Bat{hV0#Y=6bNRf zK&={LbDtB(yMmAak#NZkBFMaflw~Gaw3I{&%9s?_vA|)vubRPoE`|Rl`CO+Py9u0} zTV|1GIqU}#YDtlxa%R%V61*;>yXyCfGfKiTSgf)&1*s~8HQV3xLb*t=AO$V3tT4gPk zSy3lsDY+(!9Y+|Ym=~50bmTkfn=OX&wW`Xr&Uzsx2t+9ReVI-~dq)jhj`on)ZX|{e zjX+j7F0`=pJ!jXIIRV$TKsg7LuOShZJQ-=3PEUz*g5?90+b*O00+P(OAS2vCwC#fePe=x&r&)O$!Uj|-{% zfE(A{hX@VL!V90!E9?b6!WB0MA}>Q82g$J}_=Y1_xiMAQIfWTu43#x|14X4j_cE-__#&X0?&WU|?TL42#s$Xo3g_uuP0-;SVTo;^bFY zA-qOULf)ku<5`<2p=Qtw zL#rlwoj(w6Lxnx>qXmEk)==7VCkJrbtMa5k->NA7L=Bo$rl_;)lCuI&H`Y}86SkuV z@4h}8+KL&YY3uW=K2eRyS(uGhOL}93)Aa4V3MnDBVDYc~00yDUy}odMAXKd?QcM`} zrP7tRjJ=r06ocI8N#kQKvF-eIU2Pin!bWl0w@_}}(fRE9^YL5Qbx!^x&wxPyl$(Xg zk%3KuDb%*X>1K#~;+3-fj!oj#h+n=*oZD+4oX_2;YrEH;b+Ml94AC zv+SA|M3{xQuL~$4a<-!OBZaC?)aZ1B`@iLS01TVtW)|lS#VJav(V;t!F3Cm0Qo#nR zb;xE!M&l+}-v`u({)1XlCP>_m-xV@CU^}g#)T&j|Mpfc|<;@U_Nbz7}>t!A1qg1DA zyfTr_SMMcRK(?d6k9fptf-dvVOaC&(zrXZx8DkMwd)syz)C+h_B^S59bmG}=mjv0? zmEeZ>3Vu>^$D*)!UAqAEQg#+2o}vQZlk7qkBVho=a8<`|a`iH_CyG{Q*(w+q;8fdY zEI!p$j_c>+_tYl!bkOuW2U`n{{dGTs-VabtBwD3dz4t8#0X5HL^sAdYxVN3PCc{`W z*&f!~#Rr7V4bkSFH zLIOC%WFY$KLXJsz#Fyo_=9@`XamujC8P_4MXlke;w?X}Iz>Vz3-I3Sw53wj7fN0cx zm+2tqjtKQ}7P6Z{;^LF=?0+FR?1fH2+@wP;3ABan1U!KJcyZ%Fe@4R(g0gG4uc1)q zD-IyLkq5l?4u}W|bFu?;K;+g|DFx8Kp3Cfi*y?IYLR_TtC(*|AA8$%DoEe7CSMjuL zj9pO|0f#Gu+>%+s;>+r@4zS^xHh?xfU*Vnv4#fa~wfkG+@|Du9kVKGR>umQ+Oen{I z_W+c3?RO_#Z7+d5%=%05Ek)snv4o*8-LKKwoLU4JYsv$9*M43W&SQ*=bTCO=BBu@w zx{HgVp0gIfFIc<@KZ7{Xeka#H!G58o`Q63&NjEoh09#)asEeZ)!PD{M+q-BRJw?12 zP;!=;$3FEbg}e5+b{2gVKTH&9u;z7uQX)5qTHUnaFGbvo8#o1h=TOG~by9dEpV z&>;2&Q1MRN4{&VW*DfZst?mX=X4gixP8z2wlzsE=D zmZ@;*SS(Z(XKZA01gA^8 z+{k`n)0Du$yQ5JW!itJkrVOCUCh#claKCM?{WqJA8<&`PT!Q8{>3D`#{Khw45-=BX3;-6d0d6IM ze~7w`LG^baTXGm8<-yzPx_j{cGBq|~IObLFpe!9nEHS)GHoQBE_QA&_E;{{W|LILB zB!DXER0%X^_$yTQ=n_Xfv>TaRS&^a!AV1C81$@Tg7H#2y`|$pvS-bu`XhQ%xIe*!b zw+WGw{Iz2AKU!`6;(QPwK<|v|WZ|C=0L~|2t!;$`q08c9kwMk~ z;dgQs3vxwZt3A@yR|z%6@Zp&74C&EGXe4Zn29CvWm@s1}US`6mUBwM*V|&8hYR?S7 zMgIjul>|ki|MSzo0C4}i)ZpW+Fz>;_kw}--a@v5+UYatG(J~_)$cAbG7rjP9{TENe zu}?72SJDRh3M>~|d|iHTe`&j+U;ireX=PK=%lY%z8ao42`XN;x!TP$wI!oS2!Df5g z=khWfE|2ljhdMNZc5M4=wyEjh(QT_}k~D}BNC z*8spr;&GtW^YQ@gsgcJzzv*-FG_#AbJtw)(u>BUGMe95pG&^Tg*VfEXxAa)!fXh8* zzy;sae|4)+?-?!`rWz)CsJ}@xpk|jsUB=sI4~=97|n?9GWN+Z1p_OB)yw|h$%!|_0%2kF%GouTqP9xBoCUO{!s6Z*`H>(Y0$=Jr4hOXx}remTWLk! zI5)L&E;-ny5OmiR&fL8n4O-Y9{QV{RI<}z;EeLGOG}YG=>>$-Ygn96M)EWkMDF(bM z_8^PGcD}`-qp_VAIsozxb#0j2IG_)CBtQ^krHaD^utpfaQ+k(3YBM9Tf0 zwA{;O7YzEH;Gh9AvSQa-gu0jYpGh`$73E-3J$90^P!i~iL-zbLY3i;WpcMv=6_D&v zYd-2;xql`(M^+4hNq4Z56mP9ycVhnglavc(dLH;F1!%;ybTdr>Z%6MBt;?R$b8d)& zI~{<7&y0pO0B*tm`6+hp{R-%(3Jn0l55E(}0tk!D!7AE&UTHjR|5-lh(@U3nnguvV z{r7N1PBc-Vy%hZx2*~))SI*!xx_^IhUQ)%f>93JMEC4A&KmSWrj+e4bU^UfJpa^e$ ztgnN^M8aH1{tSPn2E-|{tRxX^kwraB<2?E}d1$9v1gr$)ijgN+*)q8heVzJ`iidx@ zYLW$3+Wy<>R3!*TLDm0i6jG={A3Nzam?S%Bp;L%{`OloO+t zhV9>bVO43#+l74vYc z% zE4k?FsfqH;%7<^1CvuH37JNs__wRqe18s?0I!SK@q>!Cm#~Eww9VMn_WbV~$)Z2zQ zrQ|}T9!EbR+EARKLZ%4ih?wI)+yHNuHBslUGBN~t`q=H`%G3r8 z>{IUaW#nxB`p35WF$LV+k;WdpHB+8B_(<)_dT2frTsIXE${>v#dW|JIU!z$+gI#gs zt#VQU)bFn(EEgPU?tLt(}1 zXY`c>&i}}xa?2y*cbrb^rF9dxZr(Jwg&KG_*-c}X-L?h@d>)oNN372v^w-3E zUGYEYNr%3F|9yAgRP_i84v1m)-J^g#KkF3*rN{Unf zoMH5VuASF8Uu907C4P$rOE}hh-?Ro|!2yRNbdsmi+5U0xmd2wV7t~iJ{7wgDP|h_M z0@dAc&WaCcz$|hPQ*=I)qL^i0!b9BR%yD&`f9$33Ow(Vpu9wj|`H!(r^-~@szPuD<=fh{Of6ooAjS>fW_gF z&{Lr0KjLG<-#G8Steb$jv=^H79Gj-4gEr(nX|IVd$$yQh-|C?E*H^$^(fgGYU2(|V z4Ql%xLp&-i6c~s|VGU$L&5dEBq43a$HdfKV2CT^cK%|2uSz*CHDhvcSu{*o2GS((4 z{dEs^-aBWfSOeql-{dt&3H>I)eU-4ElA)(L{jW=8Alt~|2Ui~43pWFpVXEdNrt~?I zOR`YcrR##`|2XuAIeZ%PfP#^&Z|20lUn7jXs5fEeQF(g((BU!G<-Ye7tMDHL#}OIj z3~wOBsO{-T+j;Qc0b9knPDOJ4l`y-oK1HuwaJTa^kKLD`egFL1v$!^~U(T~lMStSI z!(VOZ!hf7!Iok>gwg*?ZYIedUUDxSXs*|wPFoGrEIZA@~}y`V|e0#qVh0Mc-EBrgI0 zLG%FjI+l1EH6EP)U5p76=ThloK1UL5fL;V+!_#lu?7RVwt_jx#(WJcx7=SmR1CX?> zP0%WP5oqjwtQGS|)L7;~Yg8*}{q@*9+aX;4mEQ)Q^Fm8~3=IKo01aboHI!*D3rgo`SM#!k_;X0BVL> z^t8WE_A{5HMR@^gkh(bDaOK00dx!Ifkf3#eAcPVAjpAfW^$I*OM%k8!+G=5 zBNr@yVXYa!VGj9#yBupR756>)ypV*2Rq184IxdJ~(G2yKoVLr?`0BBiVUCTiozDSZ zjvMI&o`iKm_A_o~;FwH>k53in0eS*-La#?FXNV|`O^9Lxjxxpbht1yZd}pR513gDp zWGDlGr{x>^lC8mw+-^Df84nQY@7WU`)(u~yjVk_$=X7K#4gSj zF6{lVT}8~;Hmd$D0IeuE-7AnNRZIGmEBCQKm0!cFtYL-53-pdvD;xnd;YW}MbJzhS zRt9~`rhdk?cBwC&ItW(|B`%JJb~+i(d9-xAQvn!5lHU8rtCG!)xh5SNfyc}a8-4m- z!6d+(#s#B;!#2q%vG&y3c%goY876Yb3>smqZmN}`XOW1w8c>8zX~_V{kR8CQ^aW7s z_c$6a_e*p+x#fTKpaFDB_qSqkkzN39-@m2J>;V32f(zhr!457(`MVv3BAxO~Y|}dX zunazW0rb6F?&Nn~UULK`aIdpB}$5XS+cKX8<8cHEXl5v z$S&D(&(Hh5_ufD5{pWr@_YeR0&7AX`bDrNh=Xt*0=Q}tRaZo&BUo$QY7QgshE?amA zRMaGVZNB|1Hj@?Gx1FXA4omhsCqkv9`TO5>NR3*8y?b99;ab9V>8Uspi@;^Va7jy5 zQws$-QL!7}R80-Y;bg#_!T5ciK7tMSl`m7yod?M8V*#LGF{KRW%fE84-*g}XaSrFv z+3uhuiZc+M;Wd26BORZvF!<``wQB+pX(L0~CnezEyS8d9bjO_A;*c=P9ioEU!Mx{1 zlkpcpLY|gYGq+Fmz!;EsZU_PpA&X_Dl+%a3?9pVI6p7~ zY_vTAxPmtZ=F5pK zK@k+%>%{@!wd?T#tn+w~q`4WWX2)YQT$1j^=k{m8^dR@V@pkR2z%CF-n)twC#Bh#HoU zVx7njUL(H4*ND{=f$Qs4-ZG}Xszd0*g6kQVqI=P&&?5z{=cV0Q%wO{kv1GXP$_rn6 zSn0FnP;nVK4|2fqEUJ4_v!v}_k4C|R-yNzwK9Fz(6*b++Xy_5ObgW}Wnsb+EzMUlk zm&)h25#z<;r3y$~n8CD+5Bx~K3W4-_ud7W}y6wzqhSiV@Dnl-~G*l20&OmlhdGP+M zCcQS>&6y@jH$HF7*Sq$-fWW}UYoLss-y!t1fzUN+Zc zh;jyE19@Hyihis%MGjU+za5*Y+f8iRd-J8^J*H`QeEEeMIzNo65GL0xi^G(K=K=K$ z(cKUH{oU=+DT~hQT>32FexBGMBW4D)C{mx8RdxfBpWP?O@5Y#fFf7eoiuTU4?(Y&q z)bIil4^vdA?v;3twV7xeo@0?Af&`Kk(K|)fKE>E?b8s$iW7g!pkqIqN zB$^wu^-_w!p>??)>>FoYRMgR`j=de)dO|avxQM~Y4P}_;(FP|cQCJkYOxA8{a|4tG zm1p|>N(PiZ86jR-XV}V$a+q6)oJaq~z%B@(*&A3o;e+QjXXN^;o|0X5U2fkx1)`q! zDlDA24$AT!|Iy01fHeh1TWwWvH;hG8>OdEN_B#v0zUhT6&$!@Db8JIyFjx%oMTcK3 z;InxMK9RH}p+()xZQm+{q-S6$bs9O7x)4cGn5HOeHzJY4~6KPM5 zC<=j!EyB77(B3)0CAHZA$X-uR%@4e;ji)GF6TX9a^}vrtMo3AO&;R$dK|%`gh&F}L z0*47_$IGarqe{8XS?GjP=c@w#ocym5b8$7I-mhB^@eO6&y}$oT6G1^yhB5+IQc7tv z;$Hz$A6D4oOBQV<6+uC8FA$;im}SWgbQ&2f&S`$hV;T*Zjl{ly-FAC9{Ba1>|5|e> z~Q<)`rOys&*sXV9;n%EG*^nFqQZ5XtR(gmkPFsH_>0pG%RZC4 zCO=l@Hkr<0O4s=VhtHVhl-<@Sm9KL(>)0mze^-w?by9vPff=EB9$$C<`!y|qNN z3u!r_iFL2za*GL6bB!>&8*@4&yh;Pzy7s#eHH-k+U+m_s7KihTA6iYlBwwur5!Iz( ztn?uQ3N@r@jqA*&be^Tp&Y)3EvyTxC704<|VplVn`km=ZP}&)<^_U|$%ovcM-Z|3+ za^;Ip_-)-;SNPEq`sMcrN>kbH=lXU;`?TbZ1-Fs3PT4eG8zJ9L;zJ&jihKzocVig% ztL^Ut9AT(P<5*&;;E5kXK$yq)b*(1w#umQ`u#ml~y2+QOc27K1e`!?JYQ$6;PmUyhy94HyclW$06_*(!X3o^{lW?h2` zHX5vOiVPK|Ikj^Mr#w^?bCQ?;<82QFW$GLdA{28y#-V7UnKO`Ec(;KWDuB`Gb8;NZ z;hCsFEtZ*doUOxG=b0+H0s@_jj_6R--C_m~DQdVGw?IaM{ZQUF2>$IjJ#_>Y-~{&_ z2kcmvuQSZ#y;t!8oHVOm;2=48RfgMS#L~N}#WV>!U@=!aHry|gXh`*F$8fM)}CqA7bw9qBAn=_l52F~6W zZSk5RoH~tIHEU7V}seWgy730 zm8$GhEnmTi{!Rm@N)D7QYN70?eGC@0#4$2rj`YQ zc2cJ-e%>&OW=2NMWTzboP~0p)XnQS~9dZc4mP+`g=MFs6K)Ajj>?!u_5{J!m5`yN6uIztvqpd8xKakO$ChqqPG9oeQGp;Stu z^2ud%NY$_&vwDsNs+iI!mg~!e$2Ym)th$x?!Nu08>Whgny>Vb%_du#1%p@!GcCn?&ig*Ez?#Co+G)hT}UYp%cMoQW5d5}d?+^>nB23ZXeBSY zA0g-OSbV^Lo;gS=*PAqZJPdP865I5SKF@4h%`B%k8AGQ-p^FDM&AfU#G_Da`NtL9*qt6es4>|NeIg)N{^rK1{ye5L30fdcu+ASV3C<9%gtz%>mI$6dseU5y zLfs{obY?`7yM*I9(_?kc6z`anYsN2+=Z;AOn1!wQ71J+^^$Q!59#H1x!sK$b_tatE z`&P?8W1oKH3@>i)I=IWOn{;r_*Hnma+|+0zcuPe17bFIelIjbFslWzLw{1s#Wh5X+ zJnoh|1snM+Upa;Ps?C`ztMn3z3cn%mqM zblrRGF3fBRb$+2Z{>iqk+f)I_E)0rh=>LvpSm6@N5AaI;`B|lQ5jtxb5do@~W@atU zd0mR%y2ve&p(wq2L$CfE>iB{Fx5F7csFhN7#mK!XVZcAK!=H-7T#T#ANZRUtz-JMw zP=XB86D8z)|2j1%^*|WC;^-2kw6(TIr`KxTlrPK1z!kC89ed7wN%D~ub84kWQCLrO zZcp0!V}ZSs7hiC|y9H;~HKbppTyR{h@mLw8U`*X#c3!TjChnh=4bl=CtQ02wcHH8L zJ0nr6gR)EvJ9QypOLV(8EfafkDjx(6*cV!b5g=0Mmoiduw}(6u5Hu9!VxlGEEIa2N z_sUm4e_~?hhC6Z7H$SictX=-ebbNP&Dxx8>xB)Xh6u02qKjR_!w0D~GIYcyi}9@Vr{G+H*S8-wKXuQ>{04mv?`5>t!6R7TT&x z9E!a%u``+d(@2nfZ{OXq!YJ>97NauO^tOU47W{>%JW=|6eNFm;_q|{U*zUdCmH9qS80OxEws-2a1KgPs#+hNk#jFfPOY4t= zhgVOQ1hl13%RR*O;1EpL_kUOYJanT;xwj*N*kHj-eiG1dC^w_fLA3|+#1pDO!VS|* zmjy|6)!jQj&#=#}hQ4CzE$nC}i8FFXtp1dd!8|EvlQ1>|{HE_=o{sy3Oa{%yu5Nd7i95!2?~q!6&Al|4z3? zQfp*UXByTBlR!a%u#pES$VnOnfz%NwZmWx-)@OhTWHWeX5$~`8{T!;k9PPx+*+&mK zx@OFRQ}tkEo!`;fV+Hy-Of(q*f3`DND%|_Wi*DtV3YT6Kd1}YhDOM811g8G%?dmhL z_rC5dyu`A4jYS3brLu=Mt+XEbTK6pjb5nb!;=$>e*3ctIX$-fv-|#twotUBHKN^lK zXgRW4Mn)dzGu@d0Sq#BULGtzImt{Ow@+U{$1E&4DR4Zi;ib7d3ZWJ1JGKi$1fux%Z z)_tlaB*b=7Dl)Y23lKpub2wjy%6|+*8_m%xfKeW^aa+gN;}wNb22|yjh%?00%|Dz` zSS6qzk&XCC!zFh-AZf`}s%ft^{YS1?nPLW|?Jsff>XpR#*ZzEoyJEOqG*uyQ-p;hH zFf@1fn{tU&n}QLZZ}v$Ah|8Z3{eZ*EGOz>RIHLi*MkU0{gXA80evSm~@Ao$>V6t^w zJlk;c?HY&TEV*n(g3kt6IPO|aF)^`NK;O^JK00zHh)?M%$VB$|KpDK``x+XRF9BDQ z?icJ)ZO8F}H#&rfs4W=TGx`>{E}YabE0$9!-RAU-02@9pyc)=&$d4k(Prp5WrlDZ( zRtp`nnLj~)3UkjMMx}OFlOx+kXXqX)vevRYAtG}Uk~>3MQ?^6YxCKOeW}x@Z;QL!N z-Wkj*6}-acwb#1UdOxV+yd@@{#WH1Gysnvx9H4lj0lDdw=hZwB9R2FaA@=(ze~QNf zG>Fx#tUtG^K(HfbxOM9j;G*4fHd9BdXcNlGZvKS)_du3NY|v_F@BSM4d9aL^;7;{n z#xQ5u+Uj89QpEdYWD|Z9MgoPn9<=U#f%fL z3nIw!@rvIfS>Y~cy&{hu$}s&H`n$;`x71$^Ts$FasWa`Txgo5#r=K0Dqa12*3jXh( z0u|dH!u(fXKNhgeIy@(?oppCCg3uM$mmUU}Q1F}|@;>eu6wtFU26`5A7w7k^JD_7V ziO-xeW*`lrNUw!x6k^3Wk{QTlB=f3th$STn@p!TwA`hGVVNzq$1T|SH>JU-R3OE0P zzf52Ff9_WEr|SwxHv509v>6c9OF92noJh&l@pn+Jc@*eS9k?!ztf~w?_#k(a5$xdp zdLYTHP%qY_(I{T(G<)H9PTVs4y@FDZm)YwQ;10(=7(wcqacE>|Jhztad^h!0>l6tD zGn~Xwoz1T6?fK7}W8SKIfr&a1b3q^d!UV!T|Iv7Ts}(q4+`v6>+a4 zbp9DMK|OITEe;Ua>qDESMl3Wewg8HNWy7Z&c$I*fqY}tqN7ZXV5{9lAHh9j0qS0lD zkFfFrpnR(wtTxkCClj7L0hp>YhF?nMPf?i&P&suv-=KTQTF7ety*kS&f(nllWo9ou z1~5lH1t_bVBOf>x5-!inqh~sB{}2negwW4}Eu6rry_hELhhx7BX+bu|#n|8~><}ib z%svA^doT*5AMK;Zk0p=AaAr@k{c4t^?ds~fa2x_MC3R=7v$VEd&&SJ4s3_mP0?f>U zZx=%*U;$=6v|SAMIpCum(#ue)%lW5u!>N}5!fs412Q5R8yb^?wHX7pc+jFpsp`1|T z?GtXpcQD2C#0TN83%Drd2)0S^kC*MGwt(XT2pDKbsOa5a3)dK5123J8#S3Sqp0n(F z6WywYMp4>$xNI1cq@>xO0sai2l6Z=eKHJx@$H&J@05s&z>>%b>dlWQ%xAtBZG1Q`+kVd{O;s5$9M_Gr@CMlZ{`~tUS6VSO^aSWI zfKDeb!;pdPTQO=8;w2fNS)dtFkx9n8bFikdem=#BI)w^15?EV@+3yP^<5iHvegB*L zmgkZ~-d$P(_(S|Bv6pc2>1HV>*-v*j{j1Do2%E9#|IFBkC^P~EFap(M1|1+!podmi zuwaK$f)%7Ox*(0`srCu7oxGeRS37hI0~r80$}({F*ER$}n*5vZq?|51Gv^ilC0j{I zfqV#B=~C$Vye&8UwG7*38|EmI>%GFmFM_Ae&tRGB69z!xp%%-=UTZ`Oei2MsC0`Ivf(R0PfpM zFXk8ziAE0I&aN~>cYG*4lne(HCKoUfb-XYjD)~bX(Ni`7QLCV}$cAv{nntkG8#-zb zqc*J|dJcGk|&LsdXai$WP}N#-s8ngGm> z2J1?UqNf9m1pV6x^iptujHERExqtl=4BYqX)B*LAr#|A73w%PrRG6;TWn75{G3Y-4 DJqNB_ literal 0 HcmV?d00001 diff --git a/rspec-distrib/docs/watchdog.png b/rspec-distrib/docs/watchdog.png new file mode 100644 index 0000000000000000000000000000000000000000..9679e988af499d4038c316427054567315d620fb GIT binary patch literal 16942 zcmZv@bySqy7dCv)Fu(xPjdUX&f`l}ZA|TS;NOwpK-65?=hbSN+(&_*bibzOz3nDEc z65ru@e$ThwwceLMSgbQQd-mC9pL6zg?dy)y(p1LBrNM z-D7zOL0Ei&x(41Z{>q+~?hg0eY;2t&=)T_Pv`Pt*i|6qhX1bV+H|!h*(qXyOUF*@1 zAfX*r=ta8uT=E7cjnG47ajtZJ#)c<)q8ka!)6%?eNy^L@&y=*u`t3-YTz> zq3Gj$^>>iCi0UuJL)NS_rrJB*=c#0I)C6A_4K5kJDjSP*o$F6+&5vfMI}J{KO^&l_ zPjfrgw^LftJ)WzfQayhW(rn%n)IP+2TgmBJAt8CJxvYwMPD*Vz^O||K9czaGA~VRI zv-|^b7utH^rA!6HHi?r<^_Z-uFKwVBHzg%(LI%@Rm>_o2R4ljd_T_)7{`^DXww zY5OwSE3J)P;C14E_qKkU7sk@T28)t}BcA>3Dy}IxhsAVjZKxyMc7`{vjYeoNOXu7E zT5JHl%?4wV-2Qsk0b^Vx%PsOfQ^=w|x+Q~ze$pXWMh%w=!ToQSreGhO?jVMn2b2{qkf2fjS@{|5-?oClj)2Qm?b|Xl^iKp0~v-4KrNPAweZ7BNFg_7D|zipT_ z-n2YQ^R9}B--FKq4Z3ftDMn)hgVuvXa4x()2tHroenfn;wTzDims!9e>V1T!ouuq( zaMbQsg}PvA0}gWbAD{SV`Nj-Ctv<3V7e0>l{WqeFCvpFM}G*@CxlS^BJ)0WocYz2 z(GzF;H&+2zIy%*)YijReJP14(R;p`E;`Opyq?NK;_nS#rR|Qz2eV31ywij;xEa> zDHvNBDLokLI?|jsvAY5?+a>i(#^lL(+NJ#E@(T_tZ}vamW}LP5C%M(L`evrnc(?IL z4?dEDr%soWxH9Zp-nS&WZgNRd;-9nDRNy-L1Y=z5ZM50U$hsJvNRcWXU!)e`(J(q-<bWekOX+S-KM^j6ru_J z^&8Q$?gPwn;h1nl_TtVCjmbg5%S*2xx)O83chBzoeLQ*bB1rh5>4Bcl+{t|Fqp{Db zktiY}xV$V98P>@(^va!AFCyx{5f}%&P>vDt?-PY6WGqBh`+dxY>3>6cJ;`*Easo)$ ze*@j}Xi}8f`#|!_vB%Z_S%tIgt)@SaAV$Fnq1&d&f1ZwyLm_1a|C&}){QJ$}ER1li zJy_v7-aocv1&N7ZoOT~8$FBW?F=@FY6d_z`Xf){GZ{lQ!Geh0K{N5S;YYsOVnCrks zJ?H<|*KS2wL}AXw8@>OT>$o;|3TYkwJF~)^>zTY=7<>Fbb9aC_Oa|!jhkt!0BZU#8 ztd84^c>gtrsSM05V#p3q{_Cw3Gw}9nh<@e&%q?D<3xX>C$QRBLogPh?&k6POJbXRA zFovG7WA4#SzGHrMmVNakn%d4iMf%KvYwp#{{GWWS8_ZXK21763&bWQN(7C;x z>-NFJ?fp^KgSN$Ryw+#-`Ql?cc$zMjur~s zJ8@hO2gFW78GI)#ce0FKxyn9XUH+Z>^>%PahLB1C{eG_#?{b&BrhSCB?N|M)R5qCd zrDT~)|E6vVGMGI>1&IzWe9Ib0Fa!7uvC{@574H>}JM zAbPc)>(&ZlIDVCGLmEbH|6+K*=E=#<8~ee0*EcQeU+ffgx3ZW}>LDvhV#jMsm%ZKe zrZkI9rhao{bb%X1^TwzNXmh5iM)G9V>tcaW=DhZL1!ix$wJ#NzKYmp<8_)e_qjA<# zU2LoBZEHXO$JsgeF8rOcY_o$+N@Hii2dv5B>r_|I0)G7d{=hHg?s3jv@W&j_+po)K zzc*Kor=4m>>J=+kYg*Q6{0=^I{5`jO?#Mn-V=m*j*MZ#zLRTM2hgdI7*Yq5exoXd~ ziszaQ+^YV;u9357dZn@fqml_}m)!qEdZE*c%@DMheV4+_-?C{?`fu|iJww~#OGub! z@kjoXZ|MbL7}%|@bFBexNO|eQufl_@!3G2okEeZmsO?YLQm5>42k_~oCh>2Ieodm2 z9G~;(rTZ|y+kTkrCU$ssvJkpUVIJi1U;_)EMnpUB*3mPzH(0${__K-Rqk0HbHj47c zhrc^sM3Aa9agN`7zy<4o4L31d6Pws>bQZt{X!|mzDZv0o^X|kH*tWalYS#or2Fzt}ZXssT8@xxSCeE`HR$)4hG?C%*c*vs>&+NJWo~^_8Znl7jhm32G9bryYb?tM zF|WWJ7Z%EB)YDtRpblb(gh$UBv&DBBn z#|#x1J%-7U$JT2yRdFwL0q0fyRnj>l8IO!b zm;&M}cd8=f&v*ZZY)d0=!;m~5D+Dt1L&}l`7qdxzQGVk?mS5(}cVd62e79eHO3|x_ z{ZNl~)FrF1>n_w@*xc*xry3uJ4muq8;kyv*=O(BUiDKG29F%EW%QF5J`pM^FB}JyS zX(43BZ8t*1C__eV@BDD^WRr&-^^pnTd{eQ77{lU?d;dpTm_-ZB6E!KMpfebWfQ0JN za6FCTihUu03CksB&J&QeRT;J$+bBp@bT8R(Mth!}=tTs9L|>?hQ}sJ3>iQri<*SR` zP<6)+97-u2ei-M3XnOz?j7rcd(%34FEvUsGN|yLpp_3<{Qg9SU+$0j*Lt;7p7UC~2bdFngXE(?=0Hg1nG97m{oKkmB^> z?pb3(AuSA9Jzk-cbgZRQDn%hokN5Bcf|$e$<=ng!U1mxUWarowe&oQgeft4?oG%}X zGz;duT_7uqM1Paq{T14Cz)m%YsP{gZ^An4NB8_QFTp|ji5R4k??kuoab`M1cxI4Ij zUj7trG%WAdMieIDK>VM;;f0aAUa@pup)d&cJ`$@AZiZ3O`r1JYy~L2_2jMM}HOqX=SPZr2K96T-VwIL& zvOx(v^|2{81)?-YrCmczRuP|`Dc+?GBJ_~;D2|*`vnM{Tdin4{UpyvB{XI>|&tfmk zzw^N&sY+dpM=^*)~4BTu6e=by=3AjN3cJSK!J9t^Iu`7XWJ3Y{o9dp1ZAx| zty@9uP=FCnS{{Zh{n8(}9tRm|e0HB-qya`BL^6-*zWP)-tC6xkc<9^v#nfW=&Gf*! z`` zVp}=3$bz~v@%M!jCFpz@cO1Bl);aMr;b!?d+StYX1OKI z0VByqNzRU+tQ;tbe*|aj&btY-Lp?#K#YVQFk%uzV@%G`qMul|rBvNc12(C9(tT%qT z)Z87^B$(b`%Qipr{gkI+aKGSDgmuYu>dPRBLDKwvv}Jt1}-zV9Zt$(O0Wn0Cn(bD5A4JeOcho$;Yc|mqr?%bVf!19)@*6<^CME zMu-y5hPcCkdZJtVyT$*d)5M$PJZ!|ll4~|Hj1yGfG^F>HK#~1w@0QnfO)fhCO6t)Y zdGs0)pt7gQ$hdc1=nn+~V@4?fMQUU)PF=ssI)Dyrkf2t-;iE@Uc?+J}ZT2t^_`(SX z7L_*E2%Jlhve@A@#sAdlwnRj*p*@FN;IsQt`p^D>29NK7FSXnFZ9xMpj}(@Q{DXtW z0U8+!{EGLe7%5xSYMew4muFM}4%t+^tv7f?jOsb25i9q&Y%%4UnVikc1)#xD4wX>Y zESK5F{g|8>W%TsB*DLUCDwq#{Di(xZ1}N~AsG1g%;DRYMBHagZDb>bqF+=Z|U;fZx zVuYvSmXF3spYJJ5R2w&qZf0r$D0a@g2MXGr`HGyzZf3~wsZQ&2V6XzG@l%Wk9 zA)u(xzuQf1#4Pukhf9>(K%Fx>ApYaU8~qwH*Jrj_9>%UMWyYZTeLlVkq86b?jTiRy z+T#{JZoQ~OMJWHtj`%+{=TboS}&82zg)c;-57y&A3yBtM1d8~dLM zhne9+4vLutM16c<6{fb^d;zbm-uroW@tcBKL9Vx`P#SY%UdQcz_-SQ zkSBg!_+lB`T`4k`lLI2(HSTZO6Qj!IDifn**FiY@OtQ%?%eq}|3CwMd%|SJEMSe?Z^osm?d`QG0vyP~b7R=z;R=w4uI}gC2 znkCoy_mOP75+zea%Vhu^k#-k+FFytN-J_o+w9elL8~BhTn{oknl^;~yi`=t7W}-&&@==#d2@fUfUl0Q}~RK_O8+%Em0I%5Z3rGZ=}! zz9T2j#pb-fm(xQw*HanqGR-7~^ImVgGWW{b$HaZyFh;7>h|mvbKP%%H4M90LNA^%d zn^)u2_XMakS`8;kb_amzisA&Z8E4_%Gu{J$KU?VU2C_;I(CjKvn(7Lk?635;ZI~buz7*ra;q1HwRpl5OK>*@(j$?a7>fHEh2Cy5)e z>AX{A7R#yAl;LQ)+2FI^`@PV2{i5s3!4)7FXRHkA2mj|HRuuP(PT7$9ZB;j&ahL_| zD>Z>)m_z#g)|st2i#qaNBjOfQ6 zs%nE5@E0OEtcD!WudKxd0Lo`L`!Z69IC(&FJn{~%Ml9BL5qO!l`R0|fCP&PH+Lhh& zPrc|@UxBJFt@*$$-X;JtHGi8L6nxi|92k%~{2EdqKD>5K38*<OgFiqS>pNi{`u+(3j~Djsu!5iiRdR=N_2*!%U9S;W8$g`j!?cuc zorSGLUY$*RY*iq<>z&5o!~|V~pC>zQ!n+BjwB6wBPZb?ea$wNx?aPX?32thnoP z$Mb_r;XgsYhn45?h)7T(N4Pk$qgx%AqIkRwTCX&|?St{;IqS z@ctRyjR6uVM~hB_Qj?y*k0BTA`!B|frSqTRkg|(JLP1~=cEG&7-wsN*5$-uhM&1!& z{&?@lunHX2dzd6Pzo%X#n$kViM4{1>NWCIZ!AMDE$D*A4yXPT5t~c>jXMS+NwUj8w z)FAqR)u)q4w{O8ISV2)vzf0~VdERtwE~;S?>+LZ5P6CZs3JOjV^Z3BO%EMF3&DM}m zIvei7cs+8UO+Eb`DvEgXkxHnm9es?y zc|e76Z)cN5MTS0!`%QCaI$hRx&Xb+CU0(pMo2<7SVig>!NA#@Hc?vVwl-^{9{Hp7f zx<>^qg#4|}_58whK}|}bSpbmS!@*2A`Du@-~vq^*3G zw(v@FHGJ-lg+lNvu?uyxpf@7YuY@lMz99{+VJy9MdtFcpcI@D2wxF295aR-ahc$^% zxL>C4avo|Zf}`|iL_({OtHaF|=Qy^kil3&*>=dvH_y;2#Ssmh+FFfNVS5p<#WQ5&$ z=uxWYfDAc8pfwY_@bR2(bNzrZ4uA_|YvPJ{cHqWq5kbiEy*%6&x>s+BxmmLo!Gr@tpfG)@%?QpVCe#rt#&9Zj6L6U1<&3-Ccga zlGpvXTU~D6;uGuvKszd2>Wv}tffd5G#RzCuTDrM?(P;pIN zMK8hPN*^$aC*mJOK|_>0H?=6f6nLscFh@bPv}!W@fA8IN5gD$3;O#v0v7M4r`3#q~ z_(F9mQD$Cpr0|9I0rJA3lp^R=XxYB1B+tG0lbj&pV`Cq&Y<2GxS!*@ z7!p56_>oIpvz~v$?U6lL{OxK|>Q_dsMn6o6QMXc^L|}LRtfq5#4tmnJV~!9b{X20u zn(_FW(c3LQ@;_wpz#==sGLg_~_#}ilVNX}1ffJ2$-zQ?EAU$&KIeynp0>zR#{nl{R z80Q?jdpy}?oRG=-;_J#tG(AM7wE;upzR83v-W5%@`t9y;{0LZo)@y96(Vp?y=5yr0 zr{wPFv7~P$`wn53BYS7trGsUk*mbW z+x|!>V=?RCk(NSoda_%xKlx%Ozm`|%e8&1(K&9*GE2onbg;c*H!5ctQ-D8HKxlz{mL98upF7Ic_?gqm?SobC$;l^GTzeyF1rP34ZLEy5+<9xvm zSNPLF-kl_0zOe7A*bljpvOvFu=joAnYBW*ZBNPRnZ<5@b3c^+`)+EB2$7oNpz}ra_ zOfNM+5s_Beld;TGASum=dn64avpi=>P)GbWk-n+)Li&zDR!_a+In;6dJvO|0{Mg`- zV{8{g&@vJVuy)@M*&_&Jn#X0K%v?__T~LBv%f8Wr3jkhH6@JQ@B}Yf_RUotqf%OKT zedLh&V2aFz4Nf$WB1d+8zo;od{!5m_MSjJ&2L0>Wjnv>)3^CuTLV0`LvD*SPU%Th}FO#p<(mB)u6n+R6OOG2Fh zYPywLnF*ERG+GxfuUE1*)4Js9i^GZL?xStO%SJ00WwAunF@NOw<=4~)@Z1{vIK6PF z0-+NHJ;2eFy@6A<#e4MmM?w}y!mp+z zq(~)6KNwnz3hH2Ikj8gqh^)fYGjJGeN$0dJW*oJfQ(0#)E$C662$Y0QqoKM%q(Pho zK1C|xXV-2JLlXiDkgJn-RT-3#q9dXUS)s0Sojo(~VS*LbAPp38p69S7+kV~UamvAb zTpD|xL+YQR0GFj-v{3g5678p~q+CP!JP7eO-@wkm^0BXf8Il^9p-sz9MAgG=dm!n; zW4APSRJ)yWWELrx+$l$+1ob6I-8UqzM7Qddu%#d%ywoyk_Q$z7wIa_( zaX4SEs)F>?>RThQeVq*nyb}CFR2lwAA)vhQ^ro#{c%>(xdJh=t9y`ylSYxhzTJ6cH z^La>y@KJ7ww#uR-<#Mqvw*RF7<)FV1Lk9{0i?0N>azwTG$#^JeNkuzX;}s*IflL>l z;z33(1Bb>@v>T^U20H@f_ieVFlLkJcvEfwpaKxykD&6|?Qlr=)Vl5KE_zA`zgAVaiNG;f`lig)ok?s|(1;+)>OJ%qb>QE=S7n+AL(;5t zo<-ZgVwlH3LWg*Yadk>pvUfU7g_+p>k$Z&(=h(JW2g-Ny`zEX$7^#}~R*=xR&-48N z(0Iy%3%n{JhI%?;iCKaej6Mz#5<#(=w3ig_k5|Imd9Y9|VIglnh%lQDzTgI-4=n0b zGVI+}=)sS?O?GR#BLbVl=%hK2%At`@*=MqgK{H(7{hp!>)}5XAwbH!|q3!lb1`Yy7 z?4qRX%(()utM#W{P|Z}&+G%(LSRqb&GDNy`eAwrdICzwUQ{kF>6&Z%s&Xf^S_I7Z5 zec6qw@(r#S)jzUOZG%=ud1Q&>=j^zeY1C%{1~MWwnv^J@KR_b)+zig^x2iT*+`#zo=;s&4AL zJ|YC9)3^bQE*mX{1j9mUXBmMbHI_H`cu*d*&FF*fyU9OT9B*WBbJN}=Qk0@tq=}LI zgA%;JQmt;-79BVh*(L6`@HbX)!}KW{uxyWf$3_sEAsvbLtINZ3a7gUy(NQ~IRyIHq zZ;hX)pG-dX0f?~_x3R=wwwx0q0kd}FYC|2xJw`BB7kepkBo5kXb4xJ{W{+*3Gs;#{ zI^~0Un!mcGV7ZY?59VSu0>A@J0(mD1uJJoG5wprCiTJl^4Z^SrUB8QY34}bHh@N-8 zm4>W_b>=B?lgV(hLoB2_%&ls_8)Hc8zZkXyI)EqkCofy@AX@_KRR@FBK9&@3gF|g` z(3YcHg)gCTO;S{k!W0+JSZkxJm5Ov9^5^64!ei${p{*VtyT8v2Wmu-0pQh=<-x9Cl zK+w&aYb^lk_9s0aB!yEZ;%;4I?!AABD}PZc{ZEpS9U=1N0Z0q7B3;8k6@~R&{d8|} z0NlG&Ur8Xtz;v>p$pb`()vgpG)#m}c)R{k6yjcm7a1eNHz?3QsSO(AWq5vmNdK?J5 zzxJsA+sMm7XZ#mOUgIZ<(1B>$7(LTa2%s<;%nK^+!QddRel0Jik`%|>ccz3SF}}zvHjTIwY`06IOiSKy&?^!5?Qg=Gt_hQ0BC_O-9`3)zX=ox@0hiY4?Xc@`YN! zCfVz@w-b}V$G|YOOtQ)PB1w4EBqn;^`&de!gczlU-1Gy~;0T}5@d>`4I0OF{bOBsYmb!?HnN2|5t_ zHpa{}L=2wuM*O@beEZr%+RLsinPj~+q{ADR^;Ld8oTZGKX3|A@G-x#wGO&@su2<}GL9)!0Q(z;KN#=mApYnH-?!giRlPjbDEX zx!2*m3E_LVysCnPe!TGt!+Xi=>6Hs45WUwT1EXs-1oyC>;iGG=#zHK-?uc+{gv8{E z2! zvQdEo{7hH?gk79N{;#aK9P$tpHF5o*$uN}o#Q~Pd-1fXGW z-#zZj4pxs}Hw>-t{c#3g~3FOqe|>^QWsd*j`AFL3Lnbh%1`c?hq~CE{N;yytJm=litg`@d<@!G< z1KVx^6G(?dXn>)6EU(XL;$`!Z5IP9-DtC$#_zv_gSyH5yWmpt1Pqt?!wgIgd@>2Gt zW*&(LIIuQ|9@@zl)&hUY8n@mK@3j-7IDZ2AEza*`@b1a1YdlxE${3d+J9JGf{RLFX z1w~)O_I=EF5?h)uhLF>al$hq~x@WEix%$#?5sMwz4Bmc#bA0UoH761l>2?yyjD?fwJP&cR$@unak5IkHqzB8u!7nk!#D^tdUc7=4**<4af`tyAwbU z-XA4k_ieuk7)PIL4a*b2WX^Ts@&5ucfg{~*mzRn)?FaEI494Q({Y8ccXbi$rHm08L z`hLEdDyyI0POfQ1v03+s6N_tw#qG5dK$W%t4NRRC{X)=T5)kP8L;A_!un&i{{2oyc zHr=bRmGkG%*#eEOw^@2(U`9Z+F_Zql8XY2-K zq5jJ_V*Yy;erKC6^{bvxAko(<4?r19eEcM5!BRgUI@1givJb@5%N39$?g_k&2HlN~ z#^Hk2J*StC8?8P7u~M)}AJAE?WC6K{Z$y!9!95}rL+JVN9MC$reiQ%U4Y*@f=dEk8 z%uf5Err6(KZ`%On=w0j$d|ypdKNm96L!v>8V#_+rcgi-qzCSdH9{G$7y4J$90r}I- zq?m*LJ8(c|+7Dp=YIwb=Z{iUp39jzB_b2mPnXCukdT#^bn{L53&yfA96iC$l5zf$9 z2}M+4FHl5%zI%0km=6_Bx_(N4mB9uc?Qt3()yEUQMuPA3g|4r>uoSZ5)9cj_w(M(#3<;JqfSrdA7x%0EK#`ojJaqf&|3|%{X zMS%UFP)Z8t?vrEhYu5KKP|3^znb0q-*Q-tlh!A%h3wtk&);*HjjCD_udgKjQv9WCN zkaPblg0FzorRUBvafq;_Pp+=b8;yC(H4Z+n&(W@lp%3h9irY_Ejt_*SFHfx(#vNbg zjTbFC@@GlrGbVS10QNYjc8k(HXi3j3z;^fcbC<@cAQVqWeWJ%sbz#Aud{Hy74j(kO zj7ON;g@;2r9?CxB;TpR9mM7VUFE%@r{n?iDspL0i?u$-+dIE&#uP}n6haM!R>#5q|KoM7wq?&tqK(54yF7^0C ztlW?xHxYgg4yqp4Nh5a_^3yB8k!WIh*j`pyT~U-!PUrAuma zoc}PSL~@ZM@knKC5AJ?Kblb+q?rM_(cEqPjzrKBGP z?u8ssMohhOK#7B@@)OmFvECd3Hk7<lMHEN2a86ESEWdNJ0N4n67;h^IzSlu_I|;e-|D`z-`F@iVNc z2Y!jE3L|$>K*Q5|(y#;eB-O~}o{dO%(bw<`3UeOKzkl*XK32=E@7^T;ZwBiK33(dM z0#Ttx^m9+@l-~imoAPGzJlUYl{lK7)Lqe0$H6vljEfx374)o2ABm+-1%!RWmFceAB zsMZNBOEvlkBD3h^tJA)cK+tk`G_^W7{slU~Hd%Ke#NwBv*Z^{@&2a;3e3_@hSuTE1 z!L##Py=7K)dg$li1r}hV59~*7!_jV4UMvkbmE~EeT%SXMIJrXjLG@`#r2RVqLzxlQd zQoO0;wb9U?7=;HFh6MuZ#t#shkPZ&_tO7}T&V>5SaN;g-%$S^!R9}w2sGRGVJ=bB0 zK-b#1NV~pR)P~6*AxS?*suz1WaaOzx%nIT}`bbDIRj~b=#Foa0z7f2_a(aDD95gP} zoDy%_6e)X#L!`SfY}!Th17FH;{U*Tn)sL;VqnH$;SHq!N1WHJ5xTr4)TD+w+ymZTU zL@`HkQz0YZWM}TKTqpdV_`!!5HTudM>Y1u}q34aDzxE`%p6g30eHS)E`R^w9Jy`4(hKs^*s%^L5U zV->?$Xu?elA<6R@c^rOfKG7~rjJpH3*cPCD%mh(j$ja<})8exFDu#{19z~I7VAjQDwF(79lGcwgsE5Vf)vRBHSrm6igz1zdY4kJ~)KMit0{hm)*7zl| z2D!F6c0zPOnWbNv3LTQY}R65-uY{{v!1}9Mcj8jvP005-eB46h+o4t*(pEO>KVn; z7g~*GRIAdWax;rgrZV7WD{N*+*3= z;y@I)`vs!$`2r4jn-H>B@?;y_?nN~}nV`hb42Nb{w|w|kHt3y|lOd=`pmt@zqhhFF zUc}v?5vhn&&46lk(?v8nv-j^83^RPPqcRkBgUqQ65=*`UBg zn56Thz+fX=w<0vbxK$zWy9!Bq2^7OWt3}2gJg9eofSC0DH&vI`(URdAYiOHr+CMzF8~B@Z46~mQk4WwnX^_fI2eDGPZHj%kj|=IcXHGA( z8pOzMFwk(b81A^_X6%P0T;4|#<6=kQ;H%Kv*g8tR zam3I%^$V`Q7>XKTRAQa(@Z7{Z<+Q!)z ztKN!lpOHkR-M;iQSQ)pgCYOPT_T``3Zf_~vyGHm-lA7E4advFH6!d3&P0E~=(vV?x z%&6R4e`FS^L2-X&?JqlLuXA9({L*%p2(8dD!nB%8 z6^{(UiUe?noI-Qh^wuXxQpxk}XITzNd76z&a3D{TkN(`NE zj?AYok<}{748aWx7z-L)xG$v{7)U%4L)1XH{MQ>HEJyP z^u#%)fkzt~9@W*s^~=^y@+Lcgx5!0%+V?TKvkU!X{Um%r(7B5Rtu$^4Nw=1%B3$R@C*h|e5iG3NRV3T^! zgjqq_kv|zH#&(2FYFIrC{Vc(yIQuHncwi%?6wjMecoSY+E3M~R_Ce!G2mT!n(?ZfL z^{Zj&#z)cF#F=e29Gh^C#$k_RSb&E#{6r)&@_B0nrk?al%Kqm`NWDi#1w#H>amH!P zO3#8!kqSD&U7||-yv*zDJ358#Cpxp3iZ84?W@YE7cAd%B=~$4>CZYPm7jckogkHi3 z5LzKO2yR8^LfWqZajy9atCF5i4aAiwtHQZaQhBRv-rLJ0)P3sH3D&>Xm4D`V;}EvVcB+RN(8sr(bLcvA zUWW%Q2EyZ>Xc&C!$aWgf^$Up3vPCb40>uW)H#6v=yGrXn0V22x8!Z~5Hc$PqXLDEO z7ARA`P_5lvfphf1S9n+NGsX) zwW;Eg1fUMCj^&-2t$Ntk=_Iqu2yzE21mg~zu2!=Zg(RT>FASWev7|BS{1*<4i?7XHH?8Se zt9-wC#kJXB55l)jJtz+sZ_847wLi4LVgFSxsHy;IeQln3>Ml+oB=^VT2!C*w5C>Y3 z!V#WlN^f3?Ny?~Gh`me+gqF{Y5ry;A_4xUeFoy_D2T$^x!rG7=PVue0&toRv2Y4jn zpqzf;KtaOzf}4Zx_CiY*?^%-AA&oPQDuYL#XeMpGP3;@VgEm}{m(ro%Eat5-5|k1D z)4EDi7a=}B4~>f0G`^({VIni!PkF>1;H=mu0A&9vlF}ROW=q;Iu3R-6zHLk~M-S9u z;c3g4+&kE0Os>vIxmh+zqVCd!fDNyFYw854j->ssw%TJv5Hix#KhnOVo%6+L+yvj7 z4@*im_n==R%vrGp$+2d*WYVO{(63g9b|NBgDZ(WZSy~wCg8pG^VbLmYWk&(&BFq( z)RMB(e(qUB+h!8wZQ!B}x4@-$$}5Yim|$)9j*XLa#3`Itw?X)gT7vg3EB#?JtxwP3@0a5t zjLarnQpL%R{t^Inx}Z#ml#KA`qhyzgSm-fs6d&R5rY@S|@bg+Felo=+%m2fm3^7I^ z2$?AqjpHCK^9%_=+*v(CRYkyFkHS7CK;F;>FQ@13`;+5z5AiTo_s<${OGiY}Ahfj` zwd}dUyKqt_`lo7?4U=Q7rEAgcfu)AoeV` zNDD||4$RG%gA#-YD1pR}f9u77lyK3aLYYEt0eDh^to;@);4kgZd}_lGCPjq$brNpGh?vFz#3RM;uAw4@;7BLED2I{Di@)~o&R+Zd zL)k|Fnlu`)s(3x&;Fqi(@xNN)bP}8}dB89ag}YL~9AJI7n8`sq;+2ICGn3YJuf=Ex zu-Ckhp;9TJn;FS9)L0R?Vb^9_fs7QBu`*T{S^F|4LY4Je;|#O8+eY466U(Hie}XmuX+U9AdDRFR5&Xe@xQM@YFw*r zi`&3B3cL>W&x@L=T)+_>Z_t6mP0djJ_vK8XDBwi=4hc*JmX`KU3m{Uq{8}A7Y|0FA z$?A|WFn>}uHUQrf4__oL37;@v3A$x z`)}Z%?<#Lq&jga~e;gX30uIqclz9H@GmPmU4~51|UtbgR*UyCYT*s7(HuIX2|7YNz zdA-`vvHeFj4u(Xo8RwqG3KQVwKhN0Uf&mSm>|5N|o`a!6L1YH-r2gUm?@6Sr9?_}H Y{pRB5%;FbMA@HNBq^bBu-Xi?}1H77)6aWAK literal 0 HcmV?d00001 diff --git a/rspec-distrib/docs/worker.png b/rspec-distrib/docs/worker.png new file mode 100644 index 0000000000000000000000000000000000000000..ce7a79511958ea525e5f4db15598122131047669 GIT binary patch literal 31627 zcmeFZWmJ@3+c!Kj)X+nR#Ly|AfFL;xEnT95gn$A{3eqqmqo9C_NJ@7|h=3v~AT6SF zBPAiyUC$o=*LAP^UhjII_rv?;eb;+^&^3$mJa-)Xh`o>B@7T`_^))HUSjiv|2&J}` znlS_d!$Ba>&v0V!muPWwY6t{g<8OMy%f(mI)7ITl&&}T90R*CVy{+VrT**liffsep z(}&EXV!sWk%=BZPKI|E>p7LBf!pM|~KAWRezjUHf{kOa5%26n4;?;G(*qo5wAx=US zWWp(VMhMLzL34;9@!1sWIGvXRc6nMn%;98S@x`n_O4e*1Yp}La^v+rD_OFuBD~F-* zeQH9jv!7Yn%R{z8XX&Chx=xZRr>U#H>_2Lp4yAs2N?>=Uaz>fkEnAfs|JAutNl@@0 zl$QFHaFEw3braUlLi#&pi>T96O`OQ@3<7!Bvtty7oh#UekeE=QGpqVAMB-)D2rI*x zr5p{h;%`C8L;6GKojXmO4GIBQ#0PrM%D?ndpQ@hG4&Lt$_UU_2`ufj^`hl`w{B@&C zn(Q_Or+pML~@86()B!_2O_bM%)LZz`nXH5<1*)^@C z^)`>6eK{HZNgg95StcvN6R^A(ko?g-D6wE9=%o%7?4Z3;nAo?;O89c;NUTgr*YjF} z`P-eiY*$5%PKuWqe4qH_OGtc?P!TJbss!3Eyrb*ROR09~CAL3@CLfmaibvhrgB;c6 z`vccwwCM5zSwvl;M0Kw!7nbkdye*4?QN_*aLD804Qn_~BWJk#-N^!h*D{iIzj%h6WYo27@`=y6`pP9bm6S?~} zjB?|KM=~>X_-;Tf?R=KNl#ixxmG_QO9dQPd-Bsn&Qcf4trvE-MH!R4U`IcsD2=jv-9NM_OPE8H6nLOy2k4u zQ0bc_S4R8col8KL|JQ48lOO7IHxt_@OZwbENTGG=S-5vL>KDFdTIH{6m{c!yT@+WedoX`FYEvcP#_g^B4iX9m0$ZqOVqb;rj}^MnT4MMd z*3T?0V!^^81@m(0anfdVL;0`bn{~>R9u+KgDBoqNb*02-{Uob0*{m8r++?>L-jQd( z=lP=aG&_GB5g0^KYYt@mX=3qX+TFcgm^_Ho|3!JieE$Qt)s@_)zbUAKMYtT)LXUyc zcjd{fUB(-6=}c*cVU1pzc9*9=?*~~pSP#zHOpBzHn13_OB^t0yxtU@0Gc|B^@*~v) zh7l$o3GBvI*M_Da&r7kTxbFd??`le8uF0xWm&Zu3dy>{(*P`0_yKcMbmDK-9dpy5L z?p_#QO)~{gNTXz}WF^CIzJC78?h!~4_aHf|XXY!hlpm(P#@#9HoW89#|FBiQ=2IIh zlIPFcr1ng{`wgED%<4tPWE5Wi`Bf$5EZ=EnaWPH7CdfV&PRHcQUSM}OIo9~)w?!z2 zeePi+Z|SX*4l}1X@~2r4QtGV^xN;BG>SK}Ft18@S`=p$CoLx;(-M3}56AbqJKG_jy zX}y>hJo+3|YZtZQ;Sf%7G?|gte#qPZV~E!)=|;5SuT*#X?1-Br*D75%b~4}lyzz`; zY0I!@b#NR?y7rI@j`;FU{(9kZ9fZU>T??KjZ%%k5n3Mj7@CPdKy`EYbXSel;Bem3g z)Q{;&Gl`2tE9D;BElNv59M7wJPWWmIj+1IKNqYiXhZAfjKmMhekNdJD(nPUi)n^gr z!8GzW(4$P##H`hxQnZmoM_b<#j5-4p#TY8PY%5F6@ zo*ov?6LW~Kz0JBf>XOj;VCAp7o`7JFH?4q_h|!xcd!aj=Twj;}BwK!#{r&RE)|1?u zVhU@UTHmTa*Jzfuj=U~BMD1D4+`iwicI6<5!y#Duh_-JkR!g_6g5$fv{E6j}-Ro;r zJmmfyB?CgD|h^D%FretyPnW^?{Z<(R;|E$T>J?m!XSgv< zZ3LU@>-XFe(O%b$F0yv?kaeq09@?d!s7Mv(79=OlUWVp8R?yBb))p$My{R2%{qC5z zaIA@oO?Q-}TWRR&X_Y{z+W1glPOQas-z5eqBW8EaD-(U+n0+p+D`nQr6_?p`T%I?Y z4X-Gc|K9tlr7)^zD9MhP$VKLTzpF;jpyQV>F6QtH0u|`!Tg*zNRGG2$or)Z`lC%%T zQsZfi#=Y;0(ZP3f=Cdeh+EYxmw)GIQGu@_k%XJLDFS#WaF8&pm8lgEU6x52RUvx0w z4z=_VXbTlnsLY5J4k+r)%$DV~=pHOBWOA2aWcU>z@als$@%uG%Wtq-zY#gD?-q(iG zOCTPPQP@|a@|4nDJIAJ`5OLc1SF z5R0VTyTHr&pxsTFjac8@J3OxW9hI8|WApgt+y&^4wfjw$6SQ6Mz3o)QAjWG;FX_-< zpXUXFR0&tNSuHTxA0SX96-!PT^tB(BaxXm3TBAYh)hpR+Y;sPn_Z6ktf3$GZRK_`G z=x<2+9_Svfjjc}^X&TT6pneB4z9H#b3{+O}`ov6hc)6D%)8OJkL&;{y#RZf8kQ$#i zXW|E5X>M!`eITRsMN z-`bKn%=Yq0(#PwBO|`TeFyZr_reAwFk3z5#(j^P8J+9sU=4=854QDd_OLFdl}d`d~M5{bdzEJR?oR?pBi>4k;Y9{m~SlK1)h?qux9 z?L_{VBIMEPDBsvexA@M^&QaxkmeZG0MbR|jOH&qh65VF2z8FE~*(q9nM z{Z^**3U!KsxA%^#a^K#<&)Sgf?+~xV2DhpDy1=8=kCljQ0rQiOK21BP6v|URQ$Y?p%LC3$19UZ~ zA5+~=g7ZSEhO7dF0;B{Xgi?b0C&eLYe(e12s01dodk9zKOdZqSk}_*xt#a31O?SY0 zNJBRE;+9Xa^7lu@_2vXwdu|PBzYZ;x_a5gl_17U*iRO{7ilHOa+Z&klkdm{gl{)1i zYL>~B-pL^_>bB;HVCEmx7ag|}mHQf+zso=RBPe;V$Ml3~$p47)&pIrVi5dEOKV+!n zb}W0Dz!o)!@Amg~xAEDNn=$-){bw6KAMSqZZ%Djmf-$>i#0JCDQQyPGw7$OOw{e#YZk zy|-7K(aqVjDJ|=6Cz5a0_T1QwplX+lfca%XFummARyHl6kx%s|+#V`c94HP~J{U1p zjI>oiRhaArHF6}9=+rN_gxaJ8tQ(^dUp;?jh+2EPd{&c9r>G6THjq7Bs->OY9iOX@+nxq|ak81aD z3Zh7{8%aCo!d;6Y<28oX$w6edm86zQG>_TNT0_Sk6OlV}%I<{b@3}>au+oxHvE_3Z zTYPlASbVF}IAle6Upa1S!-uw&17^!z^AvBEL$#I9Y2lX|XGqxta@P08s~sdSD2#yTEY=`V7-P-xa^Mfh^*@YhGemz~LVK zl$x{TH0lzy9cR^>X}7fo=-VoOSVnQG&^Ixdt<&Mv0G|ZZ{#PdJ_At(1LC%ol_oH(S zjz74UMn2?iFaA(HiHh1zk@qQi(pVu@BRl5nC=2|Dx(LR>r7_)#;mZ_6^mz3|=DI4v zW8AIbq@SbHb>eeD$V$W3s?WvIMu-e8oFIH|nvW1ol5)3c&KSGC`f)0bT)fusbiYlx z6ZHE|*ME29-u&T8skuRiKw4mV^2461p8LPTME316VdoBJnhbduK0yB`tn~`ZiQCEE zQeIlgt~C|21ZsCGB&725^K##fJ48>z*dHaxNyt1HmXOwet51mT zdp;V?iS4CQR&#wqiac)SJVkz-3a*hO%BRyGp7foK5;ksE`^P>15W<*@K$<&Z1iEmf zbp2^!w?DW4(Bv%U7!9LhcWz;ZWzet`ci~WuDLR>#+6fU&#&WrFRM=mtSS1P>U{Zm>p>jX)2YA#<&L(MGoZPRV(& z=_4C=Z>C(X*b3U4Fh3G%O?N;5M>ap`O^E(@?9hEH1{E&&%jM$)f5YicX|+#pLRbRL zdF-$xf;{mJ`o``vZe*xJ*z|bLbKwpwL$c)Z{d{d?i;PZb{h@R9-ekalk7go>P+C8(KuzmXA^Y`!GG&D^`8%YzjtA6)P4W+~p{Z<}(JO&%3{y@H!urr!yOK9PT@1#~ zv`4}9hhsm*w?>?1_6`n}cyRotRpQjJj3A|FEt#@d#FSj{O7X|J%m^{z!V(Bct$n56 z)`BOnk>B03z(7p9ZV4p2qX$`cJ}651CHOO6LKN-Cxt)z&N=DQhhUZlVy!|bWT=xdn1LAc8Uwr zI=rJjFxT(?PWAN$NU4nSAU_0rbxWI<@hcH-t#<2$si|rHiO|=_LptJhydvf$rgFCK zJq$kx5Hl93%)K9SLQYn!muu2Ky+G}A#%}77N*8j_pp8))A}ub6NV)G+1f6 zpsIkqzn=X@ngyq!Bzngj)Nhaee0X__0i}Gl*Kk&B)fjmS;<@~ed-zdW$f@5BBK%Pa z_2o*}iCa4YDICH3whr=#_P={l2J)tQ&pcjHy^{Bw({dXB^#1YnhSQR*kuqCV7r$^h zEB~dO`n|^BNR#2WQ5&|67-%`rz%IL&iu_@0p5fbnCoH1qID_{y{Fw*ss5bi_-5FoM zKu+#-8^17R2Gy4`x4$%)KVAP?AS z7-OE~a8e|MYfKlggr4T=z3u14e3`7-;GhZjm~HK7#1iBL9qn#*ij)c>*%W-s9?MX4 zLbNb^KW)R+>U0l z4m85D1;03cpZ%1STt_*fU83xs2vdy6G&3r4b30g7S1{NilKE2R^lprsv7l_Tf(!z# zW72;G*KFkBkx<5hCPIM#Hd+_FCxX5yt2XWe4 zRb>)nPr@bD5kg0>0v!64DZO28qJ+F8diW)f$bap6TNCNkW!?I&k^v4i=*!hJ*}=zk zdW;_(gp`h)KaxS@dVc)uo%I&f^F^N2Ip`ydn5f<_|C1gmU2f(^Nt1=w8yjt$rTi4S z!h*Bo55JqedQumjAd1jy#Q2VTv+{%An>>NbD%3(Dizf8yJJmIGC?VMI{(yj5mB_fN!ja&EHW7 zggUNEJW;~pl%Cb+#t@?^9Tx9n(A&{521+;8JJ%7pJTs1xi=n;}YN`GqQ};{jay!5H zVn_rgPH8T->|^W6QS$R|OtQfqUHBH0eFyW8tL$}ZZ#e8=o8JegB(a9^VT%P*ir|!C zf`mi<4!_TW-E@qmhS%w|=6{^JaYJjRb7+qg_auL*r;-ru6X?)yY=K1tJc8(I`EF|P z>Iq=jA=CythXcGfV6EQ1tZj3cmsLFIa4nX&LWsV9qPckU0s)%68h;Ql^Md`5R@*8| zo;#uK$_V55Y*ArEz}5M*IKj~i{f1WJxi(y<&i7W^FIG``u3H$-ZTqL~Q2sp2`8#)- z=bk%;$t-E9-DM#}XHB$qc0V9Ms|`av6$5rn{kdpg21hu?O%g=zu0&ZbDdHyi>_lm@ z3}){Qi)A)sW~5-k*}0v<*OwGApGYiJ%2o+Z$cu=y7Ta$89YFbZ@WtmZ`Wqs*IfVUY zo(c35u%m`b8v;1*dN=gXLx;;JLgC1&=4p<_SQf71?LLD6)ekc0gzB(fH&J_BZ>Q`1 zSR+Do(Kt=Yr7D=M7*Uxeq0P2FBSchf4DuqpoQo_B%3;ud<1*SFCh>HhQI|Npkdjj= z@Z%i>SIacz^cIKk4^()Lg&O=)HJATB*bx;fCj{26E%u?{sFRrpanm{4XqDk`E!e3I zDcLT8eOmCLlk`nn1BST5msb~z{eNViMN*h3J<1xwt7bk_!4sEZ*ac1NX82L2_9V};vto#nsM z8VB>@A=PcGU|x0U3BA8ehN<1D4}&#Ol>EkSXr+Y4hbeN@x5Q0KMbPZ`dXfv4N%Fs_ zkPbgPafZaDckm)6L(Wb%Kct=LvfJ-(rJdE1FcU-3`*bw^Cwx3=yHJ_x^dV|^2BlQf zE?lkthq!mDZ@S#?f&~ zG^4^`cWdDWbH!A8WMEL5POC~lj>QlSd}1^H(*0RNv|a3a2VB7SeRS-oXqW!7u%?L94d>_`Zwo{AQ-1^tP_yyNnS1 zYo-$QH#}hJhcnZJakfxfS>Ino%}7*!5TP>irEz9j;wkZ~ z^s_|=LkJi5Q;dKpL%_%T=^?Y}#2oFl%HO-`s7GpNtLyL7AFPh^4IsCO0g8NR<7iceL$bZ^QeK?QE@JH9M?^ihY z#c-G{x|NVz3xY(JKe_Ks2JDdSb<&|W^cm*1ZbdjE(~=UWV&#WB z($j!TH#UnO8paVfH=L}xEjNbhO;?Om9p0*Rc&RG4I<4@8Sl$lNaCcA!d(Ki=ttWICfZ$l?!x#3v9`cck)~69woNF4 z8~P4%G0}RwkqbJEHGgfB_Kpcovq84mnU&9t8~*ql&UNfahhXeu^fP!ww7Cu>Vcvv;O4yVP>LHvUXvOF1I!!(+k`Ki92ZjS$fS~gn<*;Qua zYt`plM|$tox_FdhsT;a767?ysHXO~x`lQb424k})2Nkcc0nfrivfUi6pJweM9Gmkj zLu)|1Pez|mo?zsYFYi81?-8Jbd!QL?p3;r_XDRcyq{n1(B9_IgDK1>mO__eGDC_;) z!S>GuQF{2px)OLPJKBpzXFNP`CRV_Axc%75RSel8Gz&(4nG#2>u#U0xneZlSsYB)m zfs>^NEn}&1#|~=}(uJaHbAQyJYTlD=ym9Cn+0~n_%Y=SaAEAT=@q4Vv^%~ZVZ%Hz> z&Ua#D;%e5!SPB1c#e&gbMi;haFz-#np$2Q#1c>Fm*!hQ6RZQaxHQ!sOe!rzBt7I9q z*J!KtvxjBuiAWyGiJsksnO%`T#vq2c%fl9#PHn6?X_0g^MGsIQJCF!Go0WNXgGZ5d z*kkhxwr-g+P`HODJxzNzpIt@GKl2#=@JuP4=W!zI7%@)PFZIC1!{tM8ybV`crx8NG3-ZU4-s^BE%(BY-E~2SoI^gox>cI8(xmsS>LLn{k4qRcWn0bMpov37D^cQPBQblUwwGIi!nJ&@v6%uKQq*i3jpe-c zf{$a$Z!q?W#|=2l$knPb8maeF8cATDO?8;7$+~aS3nT0b9ucKF=K~Z2S$K{|-+4u%sD3?^fxLs=iGflPnq%W7G z)#MAlM}f}>jjmfp%s)(~*W`rt&O!-j(A%n_-)0G~#_jE>aP>p;+J4-VxH2taRIlW; zOn9|(7iL@Kp*=)8w6aGvGsRqrsML78dZ?o_qF*u#C9)&g{vtCL;nX+!8!Rag?m$CQ zCd2^g)#t`%u}X&Eb`H(gK07@bz&%%CKt9!OUbEZXr+qx7Ir}~vmm~8>wYdcwxISSR zI1eEWy%-naLB^mHbU6Rfzk@lNm7@2?_0~P}_kp3x`*p#;B>CA?{{U{yGmD)IX*Ft#kEU)6BZ0jYp_DfaMDzsNt znhk+~2$Z1TzyZz7|IOxlbmd7K9ZVZBBwmea~ z`?5V1$i{=3pE$^qemYDrbWgoG>7AO#8OUYlYp zJmulgTOn7d{O-V7a5;4R%DTA2IU#;5ume&kA{@teQI~DfP4TU(`vE^u-#iWe?52)X zvmhbl{`G|#iF4#ck_LD{pN7-lNfY4sxjvnHM7JDq37Y2Lud@l!Todw|Te*Fi>ZgGB zSyLz{_oMLZ+_a=)>y2R>e2b)>>|8BAT_>%4{p3ZQzP&93oXZg;X8m-^4S&kJ-drhz z25>!$jVO12CA4@y(X+A&A8K#hvaZ~5#ph>9N5tYmkjf4xhRjG@ohD_bfdOlp6&hR1 z+6WiCgXl|xHDki&PUN(ekb!^RxAm8o>~V_F(;Jq;@X)} z6xKzkH&k?(skT3tZ1;ntICLs~J6p|#$_PVGu7sETdXukzfLX5`vN8#ki%TtQc{0x~ zx^?HOg}~b0%lF3-+N@o-WWPN($xU>CZP%j0OC8un=ua9t`HM^^i6dp?CU=#iC`Lv*gQ<%_!7V-e*Ry%*kOTpeeEo!U(wCBb4RW z8W^ru(`|EbO{BT?p&I3}(%sV1HXJs)fhIyLrH4dWv*O~Gqe=rRR?L+8pO zpPyp1SoNuSY?V2Jt=cn4Q36#qHV#}`X%s7j)l;7c8`V>`nOw>a)2Sme3~bms6fQkj zp7XC_j4xe#*zu{N-g#w$5N)AISALrxW7t4U^rI@D&WblO2$#6_G-C7xZ+w_Gv(EZt z_}zq^uWI3n>61g7xSLk5H;Ee?t#fLfvk{8AwU2UNZTh-YfS8swFc$X{j;r$Jz9X*- z;nF3hpV3xY#?4#kVr1vvCgzG4i}VS1W{0#nFAW8o_viBEDyT_kJyr>SDo7Pr7>}Ui zu_t8s;>pd?oq0$B63(!^)*FOa35^Xq(Igae*1iX$3mJU&ooZD%=TZ8#*5Dg_OdLj< zVywn(8gx!{4DPp}$%{mRPY*bB^N2jhebrWIyj#hz)-$i)&r=Qrt5 z^h`#)b=Fsw(BEzuS>L@V9zW^gb8YR)kJ50pV(}{1=~o6-;gVPh=a*_G99Rj~lwy%5 zctpYx0TSZhFk(3CIBSZ_ZIzQ8(h3nAw4vdlFO8j3u0+2JJ5=MxI+4j4Lz1W1~ zJn~&t5(&}d$tXI2D7d4cFfMHGv!}N*!==-N{>=r5CTyZ5^4fU>%P30CngAGwS}_Fs z3dQXGHX=&_X3JJ+VaSi|g$eW$0>muY5QEY}dS!yC^8SBgs{fhZe^&4RwRZU5m6Lqv zZ;noUXN6Or{onEGa@%I&ov;o)q9hqNvHiZf0H39)hK7p!Lp|S~->{Je2I{CHNijhP zvt6Vey9vSe>Nm95nt6Lyc7r@Mzit8&g3h%U?tO2k$)Xha31A5Fi3}9)uj8FV__wa4 zcci$%Pu#au;kX~mF87_rs$60KMuk=d7&18{OQ;Bw?;H4!^&0}saPs4;jC z06>%KznHa<6q@Goo)Z;LF$(LA1m(^H!tvd?#l-ZDn;U!edx}ng^MM_Iokj#knQ(kv z42{$cjSpJZLh2UGHWcmlq{z?LAN`F1kb9m*IQ>w)NuIvve60DkiO=39@6ct1bing{ zx%GU^93+u;t$z&C!6V|tXk_X^^6IkR=A2splUPA3Y}O08Yx)WsEi;l@9Vz&Eu_l2H z1*H|h7jvw3OLw0$h|Hx1{LSDuMJ@Li@BnZ(s^Y=OYv;Lh7pAL9!iC8F(nZ+x#($(? zbhb6hq%_Vp6J}_$KK;dW)G5!10%kjX1@n;xz;KZ)a$buUv2HU@27x<2mq9zd-#D_1J1(b^Ci9 zipPMNd4b!n%hfrJc#sc}8$4{XHneL1Aa!MZ$EoHOhyc2(Zw3A>zC={Q^)@>AH~&=d zKx!=WPj?i`G0z(P@wrZGhRy>AXe8p+6634}4gA}+Ye1EfT#OzqXdo;4#4{|^krcP@ zh>#=y^TvPV@gM#8mo>PvzhkLGsX!#S`&t`;8y?j;Wn0(Co8x#F#Jkss74uJoTtG|=8P8HoI6_y5zEsC#1 zFp0;#V3#jE3)%jm>DfY~nI8hMpQTNJ2@P72G#&tN#4B?B-LUGA{93I^N%aD20U-RX zu8Gxc8~e>gYX%1O=IUKwk#c->{O6u>%hDB^O?OPYf~oNVtxA0)@d7@Q z;anN85?;@WMUleD+Le-;xzYQjW~})9;AWokiT26S!SA$qqb!X9XM#n7W3FFGX=lg3 zm#6l|AHF;W{HW_^rxRmB|MZiGmAp3A{d#Nu6a0{2GLXO6!<^+@vv|X$w14=&4kom2DA=!%i02q!=0E4Et zGoS1Z5Qs%tFOL2P0NTXxsh!0BcL^}9NbM&Dmgn63fd!~8dysdrEGEg_KF#mtEQN~?ABX^Nb;v9k3?*AxXnPPpv! zlXoXXjXYl5y5w|~1cy;XrRj$wXXH|9QNDm1)dk{iO_)ztXXjjDQHe24_nkk&xz*=s z$RkC}A zYYGp7*enXgY%27zga^<_#q9u%G)jBPRKp1NE97J-q+`Nos(4#PYVFhP3px~6`dWFH zSWFm)zt%}fdAaf3Z%KS7M|dp!LB3q z4p`%fsRVNPhY>Sf0v@0ttbiThOxTout=gL%+j}vD;Ctu^;mK@6{d&;eq2gaCIHU!}_tcPe$Zh<-O|h1l5FT=NG}R>@a?quSk+6tq zuP92$1xzaqX_&3y0G{FPxgrP4usFSZ_ae48;f=lnJs$#{K*OUUn1F6NZ5VTFxUKP$ zqga0nR#x^V(xiPjE$rl}K#JGv=hIxpfHS))q9%tYp>ED~)ZfDQ6%Te#)+c{Y_NL-_ zUMgJfb+T$)2E;@A=|=4Gq%Kk|5l@;qWG_j%`Y`v_i(P(kM5e18Y`Kp3V1?j{3QD@K zCq&mz#jxEwa&4hQo~l#!?#k$B}Qr1>8VL|Dcpp_EQRXJm^l z%^G_=0+$u{58x|24 zH$S4GK$?=BUZXmbZwJE6SY7jm%SDZyszNac$RJl!wDf(|4=J{_JJLOI=Ac2Rc38en@*B_iNs>Cfkc@U#B&%#%c2o2Fs zd|#_6aB*6TI6#N_;K~iMQ{SotihNmAN)@GLZ)GqYGgnq)5z zI(o#^ofAbPh`9&_r-9Hnu^w{m{4)5=iAFDjo%mxO)E}|%HVDjC;q#c`GALsbu?{}S zq0H?iCZ}fxC!A!!Fk`%Nr<4?b)~YM+OLibcKfEOuONZ?h@?nvFG*D>f3HD(+An*1g zFflQJ4Ae$wmelRu8Uc*SaY-d~1St;iY2I2qP^W?0#VAcaCdF+mbNVca*KNNJ-0TqW zc>873?)r3o){ED41`3z%iu@7BXHhB|V&h!7uw`D9DcxAqV)j@h%WXix2d5lm2e9pG z-L%(m73$5=VX8jf?8*VPBd#nQpw?1)7Q!KeIxd%Ut)~PJr6fE(Rl%n~S-5ly;`_{8V78x+ z*&ST)sv$ULVd+2vpEXum@MPhN9D#~XegX*2?CZ{=;jdKCnR$hF& z$%h!7i-NmwAv6T9L_Ajm66C#{SB^MOB~qs0Q;FwBZrmnsx{5F80qJ7`3k9FQmN!4S z+fJ{L*SP~~`%h3C*7y?6pvW_UyLi~=P)>`8Ze4m{dq6}ioXQ2G@?8Q zgIP~G@!(gJ)HVr7_||sAZLih+c~nV?b3GE*28m))oVN4#k>&G@GtjM_^05Ja#OKup z-kSy>^T=PINC(oT-(brieG6-u$$1ZIFeB9;F)nBQvraF*!2h?cf^6%Tw?zHXG)@2g z%TBoMav{N84K-kcno-wu^GVItT62pU&*76QbMLZ@fDITg%wlAR{3#ylYNzF$Q?#PQv#JO#n-w@0KW;+E}85jsU+CmN7PU3IeBngq+JidGp*$FG0m`68a(+g1-x7Y zph?U>@FU%7`>p~&c>@G_kC?eAv~S$i0kGxFxzf@Nc zAET@?^h_IX6UpX&T9LoSVF_gIUhJC#US59$UXCG?&4jekYqF_J>KKEv&DVjoBN{}q zzGI6FqiRIsW*EEYHT7BZU% zhv_YmMhQk7k?ZG#^rS9B&62tMwSCI&enkf7tQdypqskf9iO^z{ykkoR1?=%Y#Iz{kq>xn~zE+n*(l2oU&z!KF&o0Dz=vJBf8oU3VgA>d^)V-I?@OD%Q*Xhw8|`e zktQKNn`$U%r9s37!uewUtm72`=$az0fd-8Vzm?~sw!!SOhLZV=p?Hb{D4||V?sAl1 z5Yz$|i;s2SulXoBV28PVYwx`Mfse{<8+s>%ZzbIkw4$!lcA)@62U{0a_U*lMFd;1qMFmZ+!L+#?X%Up5H?8HyWUbHic2hZV?*sbeKLs2)#yw zq*Cz@5K$loIC|X`GDV7kemsP}d!FAhKm{DTSNlz=STcIhN*VlzFZubDT)+_e7CZ)J z{{15!--;m)w8E`6d(pI!3$#LYH6r(XrZd3nFvo9-#*?Xm_yu?=H_7m=G&+Ho^lM(W znt;Vd{Er(C+OMF9UITiE#K@_`>iIPY8el%G(b0*9(EC6>w^mqLHSo^5OcM&rP^?eq zGeuH>)@<`T70w4v3WS>=5mqi|ic|&8`7I&7Lk!s*{sM;cOBa%_1L;>BIVzmbhZTJL zW3k&fA7u$fGHaYVe?D+FF!04!9~^}l1kpgaG?^QW=fWMuugtgEcW(dwA&ze)9S&Mi z4{x=|$Ebo<*gYBba?Y112TVk4{-zj1IM}P$O38ho$G3v%123gK;*83J`GF}d#D+7S zFHt^NE*B6~@XJ|#(lvTsJRAv>PKujHG6OaUR- zrjYr06|c%;Ar+})b6~;R zcUACnO^0E@T!>#kV={sY0Qt%-U10^g0tiF66%g!p(g!tRL@dzP{5N;60?jalw&7p; zi%K#e^2A&C*15tMVo_jL1RX6A`Is2+E5++V)Okg=S}72v|F@gm4B^*6d#2Yuu;B-a znFTMg`yjvN!7MM|z3{I#j6u_Jg~+u$*e?E}jk@{_hx0CrzzA>6Q?aUo3pl{~3JXY{ zI}%=V(4uEhSCSD>-7)-Bz}tj)BhuoxQX@(Reg+)19Hq`x*B`IC_j_myP^O^kEaRkk zpcg12HsDbgKM^=7{ZIRcfu*D1r4!GD>4P4@E>J4z-2Ur;Bo?LbJu^iLgZ7MszIo12e-PV04HwjcfpE0UUYM4*q=j4hRUBlh8M>Ws|*$kH;6m34?0P3Mtod zS;G-p|+ofcB>wKl0(3MxR( z?=_iz2h+g1H{nQo$W)pOFso#M*DvZtr+m1`;|J&(Xz?(j;k?H7Xr6g+@m!A+GdTJ* z{NNV0xdxp@`To=vNGy*!05V%ofq{!gKI15Hpe`~7RGAuWy%9FNew{zw((@U(D}kp7 zmIL7;AQp1D@$L~XSwaFQP{B{npBQ3)(!(#nL4~adxZFc#k34RH|Myz!=7#i;B@ZZV z-YM28K(%AATY5N#GDE1z76G;w{re}c@T&q$0acpbO*LW~F8fRvzJ@m0qZMQ{uMi1O zhr@CgNMIQ}RC$R@*Bed4_DnqHJ7b0Az=Z@AFaS-|xxE3$p(+|-o3wx4464sjH?^TA z()IRLcA5gE(O)?Eh0_^iPqzC8M4T?u0jGi^8_vTN9AHt7X>Lw54B_9d&gg zvpiTO8NPs5?DHZw&^bZ|cNF#pW_?e5?e#)br~QpIxu1EVS~s5>%Br{OHB(h^BrPVx za^-5Rzk&*I{T(7&d|!@zxlzXek>o;@Ya2pI=q03;>FK$S2V!dGiqr?Lr3NKEW1Gm+ zeew6Urhi4TiYhFF;9?lR99|4-<_7>-YzF`nZ0TpYgU*)>0&oLM<8R`rfVH|tjoFHg zD49vX=Xu90o(C&wN62*(hB^6dJLxn^C@DWy>fi9Pb~UPz4{CtA8M{MaaH zdR>4*Ge`{lI+?3p%6$tf0>H9N_2F_6f%SLtRYM5PG(cwRkan(hJJ&%3ehwlq;w2Dg zSS@PNyv|v|b|h7&_0#VD$x25^l%t>?YYPvBP7X}l6f6k1T;RR;T&amED&f8bjrpt1 zqi5!Z*gN=1+Y_NhDGg3s{cc>?->q=InaYQj^>^39t!G9@A{b{4!G_0E$fmFvobg6_ zGGR@YW6$BZr3V4;y!Xixk~qs@8P*K7DRthG(HV7}_Ks_EKWh~C5Kq@U<6hmxuP@A2 z&LY|gx$VlgDV%iQ;s?ztQ`Z1bfKFbW)vU``(}a zK$z6L)q*O~BlGj&=YDWVByI{6_Ovikda@rNY@-i!Q&k0(@I4S_d;S+g{1Pi*%VbY( zH+0b>4qf;*ont_MwyTU?X7_eqixsbY5))X>gI&ij2;P-l2d?ZGFK|%#^uRV3Ba!A` zUBz(8=9PisX(}20KiYRA`Ls5IOsxB@+AL-`ox>F8V{^nINvEOhTN0=wA;3?S*i7bBPe^XvLU+XUaGkFvU>-& z?FrY$z$JY5&)kIT86wHx4-0gVy7$KHikr_ci!<-NAdr^?sd&@GcpR7vS&iJYs=0idy`W3c5y}a zjm)x@Rk%XKmQ9&uB`UHvnfaZo@B01m`{(!g{rT&^K9B2j_W7K1-p|+TJPEG?v0J&L zzmmA!dHT)Ovs`5OO_yXo8}xUhxHqg1UtHP$^Rk#_;SwkBY%+e(-VLw1MC7c66}+md zQKK!b@*+06Qz}g7Fv+CXgfZND~p6OjKI?$USFX9Z>s9nEXIUM_9ljJX%)t0@eaaKJpD=)^~R+<>cy>qvCXXqoYDtmTsD?8V`VP#SEpChc^o&bO+3b(N^@`tPxZMu%#|PlX6D65jf1;$P^@>K`T; z-k;w5c9~z7g9a)y*+^XZw!GKJnV`6MX1?LP@D0z01GKjV^o;~AAMYlQFV3Rg|llFHM?DF@aRMyU*GX}&g&b7k8D}JN-d7RUZlv#6VZ6KK;EIcN4V@%(7lrLorXSx4w)^o|T z6D||7YRszsv=%=L2^$T(sKebNP=rT*XBChQ`jbMb7LPbwCa(n!s!0u6MFC#e`tQ^e zO-h8vkQx@26G0a}D?v@iX!kp~l7&25quMNDCT0nBW8?lOV*^Y*$w8!1ORHk*qmy7- zpg;iWztW3u0db?&HMfs)_hhjG*~(?jD~mB}1Pe|B)Q9`oS&nFAW#SOA4kT^mOYG?3 z{X*e_IQ6~Dy}PlM4#z!Q>SuR7ut(#Veq}iSzYm1@;xvht1{Co_bgPI@?N;R{+2YgI zt{DDj$GP2kWK5%Unb!>*<~_0R&C+81M$EMlxblZenK#2;j1KL<*rOaBhpTo%ac&(@ zvngL8#_d19{f{Ife0$kr4e(PL1p=I>8!gD3CZ~oSgnP>`a+iL^)pi#|v0uJ9$_@P0 zy1xjpvxXsYzS`QqKOZh3$dcV_OMtq512sv-?> zpLO2yvX$;M*O6kk9jDT-o*u5#mCdWB^{QzVSwfD#oZ@Bvef&l~#_H(F= zdE&abDHEl%j>C{0R)yTHh2rPwTy$K+5w82@4drD=$p;yXY!zis_O$PGCx%BljJ)MC zv>G$|P&9;w6)7|m;P!je=Cjv>dTjs5T@AALu#2VMy+2&^ShU5eA4>BH#&6YNx(_fQ z=*IMp=P$S%cW2@g@|&rxZBgW7S6Gcjz!V00UEgFZfv2@G)c7^M*MO!VAjvZdQy=_hRiuyd((*RUdSwzCNQ1>!7)Lch)&oBVk zb`^s?g(ra8)FubTJ!bf6xO7slCQ2xpzj49VtOCCxeSG|fnAtly_;5abxZFgjHeT-T z7yfquNZOe^1?*EdINOiQ@mlE5yt?!Ib2=cPbvU)Nt{sFWd+snfEk@wYeyYj>fVV;) zFk<0jmi9Gda@52_*3Yg7A3RXHw3?Gpdf3W@RNmXXhEA+;pTIG3pDb%GHJ;7s&%AXV z*xstU$NuWhWByl&Oz~PG%IbRI8~KG_U-4y6zDc255%2vX=~ZO4voKs@9acL1-b{ta z@44@zh*n^f$ioo-vyTIBpoHIDiB&8NWS5&L+%%-~x!%LKB#9kYuSdMXI|~gHOX&cQnnbB(?xo4v@(Rlj z##W(+j*<%*_71(NJZ>%RDO`VNR|r8o&478T)1j>Wcx(;8gJc%-V9M$lGqGm=7md9I zQQX}Cv=C{50G*!&S6A)Y)r9EhND+_M-8e_FJ28Q~w3)I7zjk!J)JW+r828kjzSrzn zSi`OL360+2UiMoN)F!%u1hG)#OS`Uo%wc=ZbQAc9rpBaF%vSLJ>I4~y@VYgi5;^8g zw94BW$n>NEv@+(OAz*7kh}d5mQ6r*a;WvHPx`)PjG#zpZWw4b*Zn=SYt(zP#X95Ti z4iMvae@{^{NJ?)gLJm~+aJ~{TymErtTj?@>Z&l2}VsU}XU?b zG;N-C7%wS|C5Ig%0&?>L%Hv%E@V<$rkPpv*%MaPn<}bpQHl!GcRKZiOswL9OL_zK+ z%K!EEk+<1OR-(MkO!gkSIoMNEjCVL%D_k8ej)Q7VM~0JAu$*;%#-0rqYdV59H<|r2 zZlA;>eci^UrPYADg)Wi=Mf<14_k>dSh3ppLMv8EE(3Fzeq8ym#gA#;FsU3({mP_pm}2w z4Lbd+%A?vo5#RdB)~?d@oFgLIxgoo5DwO?me7WR<0;*Vtzzf9Wm&H>ZWsiE7fqfvZ zSHpF%wp>)wp{gQu$1(GRM~XSaM?$OKbj_;m$Z8_0k@@*~w+kvcc%+nS)-+8;lwxxS zoJI8XXWkgAHD&KG9mA_512KmLt+11X@3zmn#wG4eii+jTvB-4W$9L=_Ptlb-T@nd* zH~&<3v1ISf(c5!(>Rb3N=0}%F5>p*PkV1$fIH#VxjCWz9<%3qZ+XTad=yznHN>H1h zbC*e*+r_HLr;2Urk7(!ASW6{|59b0~xI53Zv)$y>$b*O|m{Mz5ors8qseXhu&%#9% zK2nJnYW2fNp4m{sKH5_%9|2lzEI2OE6^R+BA|<8j8)Ru^Agr_DO6&JAM0jy5++!ZU zE*S%MgB37EYwl;;QM>E1>b)yB5oSar{4{2b7KB32UXOBNn^Wj@Ln)T~I#snDI%36e zA>X82-&o(A;26Vs?|7C<$^Pxmt7n6>L0nxn8Lk1IQF%;wXf|+nkxl8*g^R26s5cjw zAy|P6I=`R$<@7G#HMp~wR%LmDAHQAV(aJ44gTUEF z(~vTRr*{kq#IESCaY9X|W;@_as4SdGH&~q0fH?={Qs~!S&%=XSq>3We_}Qv9))2!) zO!dc&2j^SdO`qb`kP(=^x-5@!3fV6OTJ|gKwd!>zYl-w(U<-_T6GUJt& zGR96l&@&=LOm8g%E4o-zyDEe$kJ2kI#|x;U@>rCGzXC9FUl2w6Mn#nF>`@Yy{&hz) z9ukM2mhi1Y^|Exp4w5f?ke;mhuSVb>ChmEkUKQiYvH*dBA5rXgjnBBil3+%`^_zK` z$24#GB_XV;<$DdI0A{l~$Ea1Q!_o2d$399Z(F@P%buAYO86*a0FS;sW5EPUYT57v^ zSGd~x2SAWicEf@5o6AtZ5O%P2usS&cnm9yAz!{lV-l%H-c6!vR|z^kXzay(nf8 z8lsym3~AP0QA0O2b|xPmK_&J^jFLmH1tK64O6|ZGDenZ3F|dx1^vXY6VnB#DW~dHRmO3d_(us`lA@l<-$ZTfa#^IXp|L8_Un(6?G0b@}Tr%EPA$Cl< zO)(-1WIN8PBttxVO-jl~db+a6g+N|lY$Q8tX9mv#=66urgxF~%mq7nIp=Pi#7NYf) zS7!PnspqhNQ_LEtnEK#L|2J#BSCK>^N4=wd-_y5M6F}odI-Y*D?w?KYbvKdw6N6svi@{jHZ<-g7IP@u@F_|uOe))mZ)las)d;oZO|Q6 zQ(gIx6kc?7Qnpn2L~S%W5()~u)Hc*!R~CfP@m8K48+u*ts1gE%X8j>$y#o>POz81i zMJsv#-Haeep?b`-AI_$pf1vsF`Za58${`mU1~qqP8h75Z-fkDB>?(qioaR-{r}hj7 zHG_5iHzIZ~=ds|)`(67?42ByCZv1>V-Jkf3`4qobc*kEkgEV^i>^|LvWgU!ai~@Ad zrzJ7bc!|Sa9;w{Ty7@xsufI9;gbf$v2p4t3bXA)uva%~kqH`^VgiZ}(xO9}^KFXAu zO))w77goz5n*?Q>gfTCX%F~ehz;r5E@qjsqdH?&p`#8>nE+ukr%HYGGIc+e?)RK7>5ZzvO7_@MpiBh zp%kjJjc@0LQ?eil+z=8iN>ZIl@y>yXgLhmI@03`Ml4DtR5EhVB3>*{K{Lb12iYhTF z5k)M&)0!eSg){t|B}t3QC#GzG+;4{{_D5$(pGMGU5mRxnvg#dlMLrkZX}y zvlRIhj|c^u05?bS+qMwt85h?FUH1w%Qy|Q+kWNUx17~~kNl9;em2nK-`h6K$?oO*48YKGWIpmhi?WNnBxvzXFZ5G||N$!q#$NpW5W}-o*(Mpx0 zA-BqrJX6jmO?2rdN}NAOU`JY%3{K>8XDe|hFjpe@5foh3_(59mEBwU8T7NRDD2@2QS$nS(nBX8Ux!#P~=AGnm!GDoUB z(E#>>e_`I&xsdux6ihaBWOZ|XXeUTY4!IuHl{Z^WzMYrZlutK_^L~G$$z)_3or*dE zt>%U3fKMx`T`5!8B;d1w1RlVWiNlbG1Z?cJ8GIK{KQ_dKUCj%B$7EjEG+-S~h_@J)QG> z4k~dW4yTV)a8*7uODFPTm1#VCqSd)Kclk`Kq+o+Bd{x3}Y=;6Nc1dgn7^$VmsK7ln zd89r6$4CKV@2Z;=KVRsKM~K>eB>8>Ta;@Hbo=s@dLG-}BV|z4nR_Q*j8|m`SfUjeQ(|x0k`iNmp8q}k97eX?r)3q?(A*A5bQwqrJNYS_7D!f0jzu!x z`^?XpB9WFBx?ht!wodiw&Ky)wT?(N{w!!BLiHfe@;Ah@vMv81DWT6T6GrzvN8dtv3 zx1A8@I3zcq7}W}$u9+$}#SzXDU=U_X;9L$tykBSAK|DLJ4H12o2uCgiEr}w5m#eH- zrtiY!@3ltpJ2ueGM*?awrtD>pT{zJcw4npUD2?SGb==BczcXb^N)PJthCa#2a(3y0DU0FuG8ZqGWE?mv zWhcnq0|<8>NX<*5J}Is#IuKmU3MR!8j#SEPz!2*MQT3Kw(e-Qm%wBW!{hh?070 zcQviN8k#N?c*mU)th%{?4|5d5>czjEIq-___Gbh{qi^NZOe&NDSn0Y!;Ei%a3SpPx zH|P;!PxDp|{+NSX8G%JyCu|8wp(m+6`T31JZs73fLrpiark80Juo**qXh3B!(sg(~ z{d2>b8^$H3T`Wpq%SroVs_WHnErY(!^1$5`!R99EH?om0PC9~1P;B@>;WM~mOca9O z>Scla@5sF%6kB4Fy*)P#oq8-jG`oU|F-Q&Nx;FHCj|=I;+!%DSnMHZp!~Kj&`Iitk z0vN7cVR$)%Mf4~Yb&_KgR)Ig?I2jD`5hj>NQ(4|!q04hm@*Zk2O7WTS81c{pJxKJA zHVKq|Y?!|g9k&XI6-JJrOk>dv>11z<{*~EjXM*+bCh=j1Q0VRByYAFqk4d-6*q_}e zMkuOX3w{UbGn1N+Mf}Lh{k!|p?F>o1?=`olt+0;>T-%1uJ@=a_EihC^(=`-c$o`3i z;3Q7$M2atxX5 zx%tf`c!IccXVAL0RnGpzI?_ZA+22nUMAL|3`1l%$ikY`-xHE9Mk-wt&z*-yFr?UgF z#FzIxUOKsCZLR|MH2~pQKW381D>E(mh|_PQ76T+)XPZ48TIhZm!zTgej2k1X-#*k= zD{)Bd#TU>q_YRcJ>V_APL6)sNXk*lib=Xe5zAXckq=rr7D=~gySz3gODQ+S{yf*3Q zWzV+^EfzCG`k+Ys+o|t6wTTEd_0Pg9K_f?-6^1!K_enb{BWZerR=TS0AB~(*DWREB zpFMpmLzz2EXEehxcXr%-X7Eyv;5LIxV{UC*T(Qvm#R8z9jImU(mX6#Q`|+WU6G#;P zqY$7tf?B(jF(OJZfU~O#x%PT#>Ar{7+!BQb2=Z*eZfAK$f zBFTSmnCLG-!^ko~6jG7+8FR#A!Zw7mW13+LH5`0M2o*K2Xxu143H}ErFvS=y{5;h8 z)kyw%56MW}Z!?#AeI9uCZeXKz1-|U+bN-hB?eULGA#fg1{+IN~L>gcccBSt=r#ydg zoA(>~FT)T2g}Z-Jr2c4PErp9vH*tEG9ijRFH)O#9S5KrPCuEyR65U*F!Nee zt+&Pbdl&tcxQ79ofNk3n=bc6W>!GkuFCPDkw!NqkxCqQe{Juyr<0nUudl#XK;-CMj zs_U{Mb^-Xa&?eU3{zs+&OzVNTcF{TEApHE^W1#wJuKBX`FGpHxLAC>JfXQu)b3xL@Sb`*01@j0DkyhZ1Kp3R51SBDbPK<^v_rQbu$4Ord!k^$F zH~#!RMxeX&yGT(4tp=Hn{{|QRFG#hJ`#Fjss+T{K>@6NYymvMf$g76~4%TwlURo`q2NCN8p=@l_jaldGG5U!O#wcQCq#j0N0yG;EPU$z2W{W=X-4i`)dZ$n z;LaDQWnbLt?K}W{T;exHjOT$sQ(q-e>bHE|Q3UwiG#+e?tTU`xb?D7ugDZqR(id;utEYXFFy@c}Mky`yOAd)~DglFDx?j5hT=h4!j zyFfz&ssqsv;cJCkKp}Qp>@Q~>@+KlV=k8E_=(vwDpgqg(Mxk0cl-+OIRjG4|!06@N*AmQ#ufc0!)A)7UD z#+!xWLt=#Om^!Ledinr>j{ zK!;`z?fjJkg`gTVUVzZ?aC2A~kbb8FMDc^kMRP3xWpjh-SHeEc1avK;9{Ry|4R3^j zaZ>F{WS$2O6=VWoPT*%?H0pz)lIm8&4~oJL7-WXbQT`ZE8bW}SYN2V&;NbZ_;5WOC z&mmP3e6M|L-0%cIn9+D-@)v)W21xF|$LasZHLI6_5)=id?G=FD0;(Mt8!iJgJw2ey z0r&L$=rBZ+y)6JEi%xe4;GN*wC@_K}>0Mv|6zv2Zohdz)5fmWV26PfqU%_%!_%Imi z!UHqT-#~qG<25?@v0H)QFY)$yi41GN#diVouQfkF`u~#?AL6TW8elry?h}I1TUc5O&~R#z1c&6)ivxEJ5F#-gyr;$z zP$cRym&=~ih6h6LX0fw&;enw?qEyt>U-60kN{@qg1QGTsx6Y;+QVOemi literal 0 HcmV?d00001 diff --git a/rspec-distrib/exe/rspec-distrib b/rspec-distrib/exe/rspec-distrib new file mode 100755 index 0000000..cbe9476 --- /dev/null +++ b/rspec-distrib/exe/rspec-distrib @@ -0,0 +1,40 @@ +#!/usr/bin/env ruby + +usage = < { puts 'on_finish called' } + + config.add_leader_formatter('progress') + config.add_leader_formatter(FeaturesFormatter) + config.add_worker_formatter(FeaturesFormatter) + + config.error_handler.retryable_exceptions = %w[RetryThisError] + config.error_handler.retry_attempts = 2 + config.error_handler.fatal_worker_failures = [] + config.error_handler.failed_workers_threshold = 0 + + config.before_test_report = lambda { |_file_path, example_groups, _exception| + example_groups.each { |eg| eg.metadata[:custom_metadata_field] = 'present' } + } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_after_all/fail_spec.rb b/rspec-distrib/features/fixtures/specs/failing_after_all/fail_spec.rb new file mode 100644 index 0000000..0684ee4 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_after_all/fail_spec.rb @@ -0,0 +1,7 @@ +RSpec.describe 'Fail spec' do + after(:all) do + raise StandardError + end + + it { expect(3).to eq 3 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_after_all/pass_spec.rb b/rspec-distrib/features/fixtures/specs/failing_after_all/pass_spec.rb new file mode 100644 index 0000000..436c3a8 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_after_all/pass_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe 'Pass spec' do + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_before_all/bar_spec.rb b/rspec-distrib/features/fixtures/specs/failing_before_all/bar_spec.rb new file mode 100644 index 0000000..0c37a63 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_before_all/bar_spec.rb @@ -0,0 +1,7 @@ +RSpec.describe 'Bar' do + before(:all) do + raise StandardError + end + + it { expect(3).to eq 3 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_before_all/foo_spec.rb b/rspec-distrib/features/fixtures/specs/failing_before_all/foo_spec.rb new file mode 100644 index 0000000..3774eeb --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_before_all/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe 'Foo' do + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_inside_examples/bar_spec.rb b/rspec-distrib/features/fixtures/specs/failing_inside_examples/bar_spec.rb new file mode 100644 index 0000000..8e3b88b --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_inside_examples/bar_spec.rb @@ -0,0 +1,4 @@ +RSpec.describe 'Bar' do + it { expect(2).to eq 3 } + it { expect(3).to eq 3 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_inside_examples/foo_spec.rb b/rspec-distrib/features/fixtures/specs/failing_inside_examples/foo_spec.rb new file mode 100644 index 0000000..4f59dfa --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_inside_examples/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe 'Foo' do + it { expect(1).to eq 'potato' } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/1_pass_spec.rb b/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/1_pass_spec.rb new file mode 100644 index 0000000..d3a3517 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/1_pass_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe '1 Pass spec' do + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/2_fail_spec.rb b/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/2_fail_spec.rb new file mode 100644 index 0000000..d043406 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/2_fail_spec.rb @@ -0,0 +1,7 @@ +RSpec.describe '2 Fail spec' do + after do + fail StandardError, '2' + end + + it { fail StandardError, '1' } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/3_pass_spec.rb b/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/3_pass_spec.rb new file mode 100644 index 0000000..c5d0dbc --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_multiple_exceptions/3_pass_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe '3 Pass spec' do + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_outside_examples/bar_spec.rb b/rspec-distrib/features/fixtures/specs/failing_outside_examples/bar_spec.rb new file mode 100644 index 0000000..8c17467 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_outside_examples/bar_spec.rb @@ -0,0 +1,7 @@ +RSpec.describe 'Bar' do + before do + raise StandardError + end + + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_outside_examples/baz_spec.rb b/rspec-distrib/features/fixtures/specs/failing_outside_examples/baz_spec.rb new file mode 100644 index 0000000..9bc3bd0 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_outside_examples/baz_spec.rb @@ -0,0 +1,11 @@ +RSpec.describe 'Baz' do + it { expect(3).to eq 3 } + + context do + after do + raise StandardError + end + + it { expect(1).to eq 1 } + end +end diff --git a/rspec-distrib/features/fixtures/specs/failing_outside_examples/foo_spec.rb b/rspec-distrib/features/fixtures/specs/failing_outside_examples/foo_spec.rb new file mode 100644 index 0000000..3774eeb --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_outside_examples/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe 'Foo' do + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/fixtures/specs/failing_outside_examples/fur_spec.rb b/rspec-distrib/features/fixtures/specs/failing_outside_examples/fur_spec.rb new file mode 100644 index 0000000..760d616 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/failing_outside_examples/fur_spec.rb @@ -0,0 +1,5 @@ +RSpec.describe 'Fur' do + let!(:fail) { raise StandardError } + + it { expect(3).to eq 3 } +end diff --git a/rspec-distrib/features/fixtures/specs/features_formatter.rb b/rspec-distrib/features/fixtures/specs/features_formatter.rb new file mode 100644 index 0000000..3904345 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/features_formatter.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class FeaturesFormatter + start_events = %i[seed start] + example_events = %i[example_group_started example_started example_passed + example_failed example_pending example_finished] + finish_events = %i[stop start_dump dump_pending dump_failures deprecation_summary + dump_profile dump_summary seed close] + other_events = %i[deprecation message] + custom_events = %i[example_will_be_retried] + events = start_events + example_events + finish_events + other_events + custom_events + + RSpec::Core::Formatters.register self, *events + + def initialize(output) + @output = output + end + + events.each do |event| + define_method event do |notification| + output.puts "FORMATTER: #{event} #{notification.inspect}" + end + end + + private + + attr_reader :output +end diff --git a/rspec-distrib/features/fixtures/specs/flaky_retries/foo_spec.rb b/rspec-distrib/features/fixtures/specs/flaky_retries/foo_spec.rb new file mode 100644 index 0000000..1e59975 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/flaky_retries/foo_spec.rb @@ -0,0 +1,24 @@ +class RetryThisError < StandardError +end + +RSpec.describe 'Foo' do + it do + case (Object.const_get(:FLAKY_RAISED) if Object.constants.include?(:FLAKY_RAISED)) + when nil + puts 'Failing on first time' + Object.const_set(:FLAKY_RAISED, 1) + raise RetryThisError + when 1 + puts 'Wrapping and failing on second time' + Object.const_set(:FLAKY_RAISED, 2) + begin + raise RetryThisError + rescue + raise RuntimeError + end + when 2 + puts 'Pass on third time' + expect(1).to eq 1 + end + end +end diff --git a/rspec-distrib/features/fixtures/specs/flaky_retries/zap_spec.rb b/rspec-distrib/features/fixtures/specs/flaky_retries/zap_spec.rb new file mode 100644 index 0000000..3597ac0 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/flaky_retries/zap_spec.rb @@ -0,0 +1,5 @@ +RSpec.describe 'Zap' do + it do + expect(1).to eq 1 + end +end diff --git a/rspec-distrib/features/fixtures/specs/passing/bar_spec.rb b/rspec-distrib/features/fixtures/specs/passing/bar_spec.rb new file mode 100644 index 0000000..78acee6 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/passing/bar_spec.rb @@ -0,0 +1,4 @@ +RSpec.describe 'Bar' do + it { expect(2).to eq 2 } + it { expect(3).to eq 3 } +end diff --git a/rspec-distrib/features/fixtures/specs/passing/foo_spec.rb b/rspec-distrib/features/fixtures/specs/passing/foo_spec.rb new file mode 100644 index 0000000..3774eeb --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/passing/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe 'Foo' do + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/fixtures/specs/prevent_eval/foo_spec.rb b/rspec-distrib/features/fixtures/specs/prevent_eval/foo_spec.rb new file mode 100644 index 0000000..6f32604 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/prevent_eval/foo_spec.rb @@ -0,0 +1,11 @@ +runner = ObjectSpace.each_object.with_object([]) do |object, collector| + if object.is_a? RSpec::Distrib::Worker::RSpecRunner + collector << object + break collector + end +end.first + +leader = runner.instance_variable_get :@leader + +leader.instance_eval 'undef :instance_eval' +leader.instance_eval 'puts "HACKED!"' diff --git a/rspec-distrib/features/fixtures/specs/signals_handling/foo_spec.rb b/rspec-distrib/features/fixtures/specs/signals_handling/foo_spec.rb new file mode 100644 index 0000000..5b28d28 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/signals_handling/foo_spec.rb @@ -0,0 +1,6 @@ +RSpec.describe 'Foo' do + specify do + sleep(2) + expect(1).to eq(1) + end +end diff --git a/rspec-distrib/features/fixtures/specs/spec_helper.rb b/rspec-distrib/features/fixtures/specs/spec_helper.rb new file mode 100644 index 0000000..a1fddd6 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/spec_helper.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +abort('Abort worker in root') if ENV['RSPEC_DISTRIB_ABORT_WORKER'] == 'true' + +require 'rspec' +require 'stringio' + +RSpec.configure do |config| + if ENV['RSPEC_DISTRIB_MULTIPLE_WORKERS'] == 'true' + config.before(:all) do + sleep 1 # let other workers pick the rest of specs + end + end + + if ENV['RSPEC_DISTRIB_FAIL_BEFORE_SUITE'] == 'true' + config.before(:suite) do + raise StandardError, 'Fail before suite' + end + end + + if ENV['RSPEC_DISTRIB_FAIL_AFTER_SUITE'] == 'true' + config.after(:suite) do + raise StandardError, 'Fail after suite' + end + end + + config.deprecation_stream = StringIO.new +end + +if ENV['RSPEC_DISTRIB_FAIL_CONFIGURATION'] == 'true' + raise StandardError, 'Fail configuration' +end diff --git a/rspec-distrib/features/fixtures/specs/syntax_error/bar_spec.rb b/rspec-distrib/features/fixtures/specs/syntax_error/bar_spec.rb new file mode 100644 index 0000000..6d8805c --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/syntax_error/bar_spec.rb @@ -0,0 +1,5 @@ +RSpec.describe 'Bar' do + it { expect(2).to eq 2 } + it { expect(3).to eq 3 } + a = # syntax error +end diff --git a/rspec-distrib/features/fixtures/specs/syntax_error/foo_spec.rb b/rspec-distrib/features/fixtures/specs/syntax_error/foo_spec.rb new file mode 100644 index 0000000..3774eeb --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/syntax_error/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe 'Foo' do + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/fixtures/specs/timeout_of_spec/bar_spec.rb b/rspec-distrib/features/fixtures/specs/timeout_of_spec/bar_spec.rb new file mode 100644 index 0000000..dd396c7 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/timeout_of_spec/bar_spec.rb @@ -0,0 +1,5 @@ +RSpec.describe 'Bar' do + it do + 3.times { sleep(1) } + end +end diff --git a/rspec-distrib/features/fixtures/specs/timeout_of_spec/foo_spec.rb b/rspec-distrib/features/fixtures/specs/timeout_of_spec/foo_spec.rb new file mode 100644 index 0000000..dc50b52 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/timeout_of_spec/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe 'Foo' do + it { expect(2).to eq 2 } +end diff --git a/rspec-distrib/features/fixtures/specs/timeout_processing_stopped/bar_spec.rb b/rspec-distrib/features/fixtures/specs/timeout_processing_stopped/bar_spec.rb new file mode 100644 index 0000000..3f3bb57 --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/timeout_processing_stopped/bar_spec.rb @@ -0,0 +1,2 @@ +puts 'Wait till timeout' +sleep 3 diff --git a/rspec-distrib/features/fixtures/specs/timeout_processing_stopped/foo_spec.rb b/rspec-distrib/features/fixtures/specs/timeout_processing_stopped/foo_spec.rb new file mode 100644 index 0000000..3774eeb --- /dev/null +++ b/rspec-distrib/features/fixtures/specs/timeout_processing_stopped/foo_spec.rb @@ -0,0 +1,3 @@ +RSpec.describe 'Foo' do + it { expect(1).to eq 1 } +end diff --git a/rspec-distrib/features/flaky_retries_spec.rb b/rspec-distrib/features/flaky_retries_spec.rb new file mode 100644 index 0000000..836289d --- /dev/null +++ b/rspec-distrib/features/flaky_retries_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require_relative 'feature_helper' + +RSpec.describe 'Flaky retries' do + include_context 'base pipeline' + + specify do + run_distrib(:flaky_retries) + + expect(leader.output).to include '2 files have been enqueued' + expect(leader.output).to match(/2 examples, 0 failures$/) + expect(leader.status).to eq(0) + + expect(worker.output).to include 'Failing on first time' + expect(worker.output).to include 'Wrapping and failing on second time' + expect(worker.output).to include 'Pass on third time' + expect(worker.output).to match(/Foo.*Zap.*Foo.*Foo/m) + + common_checks + end +end diff --git a/rspec-distrib/features/passing_spec.rb b/rspec-distrib/features/passing_spec.rb new file mode 100644 index 0000000..6f71249 --- /dev/null +++ b/rspec-distrib/features/passing_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require_relative 'feature_helper' + +RSpec.describe 'Passing specs' do + include_context 'base pipeline' + + specify do + run_distrib(:passing, workers_count: 2) + + expect(leader.output).to include '2 files have been enqueued' + expect(leader.output).to match(/custom_metadata_field=>"present"/) + expect(leader.output).to match(/0 failures$/) + + expect(worker_outputs).to include match(/1 example/) + expect(worker_outputs).to include match(/2 examples/) + expect(worker_outputs).to include match(/0 failures/) + + common_checks + end +end diff --git a/rspec-distrib/features/prevent_eval_spec.rb b/rspec-distrib/features/prevent_eval_spec.rb new file mode 100644 index 0000000..32518b0 --- /dev/null +++ b/rspec-distrib/features/prevent_eval_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require_relative 'feature_helper' + +RSpec.describe 'Prevent eval' do + include_context 'base pipeline' + + specify do + run_distrib(:prevent_eval) + + expect(leader.output).not_to include 'HACKED!' + end +end diff --git a/rspec-distrib/features/signals_handling_spec.rb b/rspec-distrib/features/signals_handling_spec.rb new file mode 100644 index 0000000..16acd3c --- /dev/null +++ b/rspec-distrib/features/signals_handling_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require_relative 'feature_helper' + +RSpec.describe 'Signals handling' do + include_context 'base pipeline' + + def with_delay + sleep(1.5) # let a process to load + yield + sleep(0.1) # let a process to react + end + + it 'finishes current file and exits' do + run_distrib(:signals_handling) do |(_, worker)| + with_delay { Process.kill('INT', worker.pid) } + end + + expect(worker.output).to include('Received INT') + expect(worker.status).to eq(2) + + expect(leader.output).to include('1 example, 0 failures') + expect(leader.output).to include('Build succeeded. Files processed: 1') + expect(leader.status).to eq(0) + + common_checks + end + + it 'exits a worker immediately' do + run_distrib(:signals_handling) do |(_, worker)| + with_delay do + Process.kill('INT', worker.pid) + sleep(0.1) + Process.kill('INT', worker.pid) + end + end + + expect(worker.output).to include('Received INT') + expect(worker.output).to include('Received second SIGINT') + expect(worker.status).to eq(2) + + expect(leader.output).to include('0 examples, 0 failures') + expect(leader.output).to include('Build failed') + expect(leader.output).to include('Files left: 1') + expect(leader.status).to eq(1) + + common_checks + end + + it 'does not send results to the leader' do + run_distrib(:signals_handling) do |(_, worker)| + with_delay { Process.kill('TERM', worker.pid) } + end + + expect(worker.output).to include('Received TERM') + expect(worker.status).to eq(15) + + expect(leader.output).to include('Build failed') + expect(leader.output).to include('Files left: 1') + expect(leader.output).to include(' 1 tests not executed') + expect(leader.status).to eq(1) + + common_checks + end + + it 'exits leader properly' do + run_distrib(:signals_handling, workers_count: 0) do |(leader, _)| + with_delay do + Process.kill('TERM', leader.pid) + end + end + + expect(leader.output).to include('Build failed') + expect(leader.output).to include('Files left: 1') + expect(leader.status).to eq(15) + + common_leader_checks + end +end diff --git a/rspec-distrib/features/support/shared_contexts/base_pipeline.rb b/rspec-distrib/features/support/shared_contexts/base_pipeline.rb new file mode 100644 index 0000000..0012c73 --- /dev/null +++ b/rspec-distrib/features/support/shared_contexts/base_pipeline.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require 'tempfile' + +# Class representing the result of child process execution. +# It is used to inspect output as well as exit status of +# rspec-distrib leader or worker processes +class Result + attr_accessor :output, :pid, :status, :type, :file + + def initialize(file:, pid:, type:, output: nil, status: nil) + @file = file + @output = output + @pid = pid + @status = status + @type = type + end +end + +shared_context 'base pipeline' do + let(:features) { Pathname(__dir__).join('..', '..') } + let(:fixtures) { features.join('fixtures') } + let(:specs) { fixtures.join('specs') } + let!(:leader_output) { Tempfile.new('leader') } + let(:additional_worker_env) { {} } + let(:common_formatter_events) do + %i[seed start stop start_dump dump_pending + dump_failures deprecation_summary dump_summary close] + end + + before do + @leader = nil + @workers = [] + end + + attr_reader :leader, :workers + + def worker_outputs + @workers.map(&:output) + end + + def worker + raise 'You are asking for one worker when there are many' if workers.count > 1 + + workers.first + end + + def run_distrib(folder_name, workers_count: 1) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + results = {} + + folder_path = specs.join(folder_name.to_s) + raise(ArgumentError, "Folder #{folder_path} doesn't exist") unless File.directory?(folder_path) + + worker_env = additional_worker_env.each_with_object({}) { |(k, v), acc| acc[k.to_s] = v.to_s } + worker_env['RSPEC_DISTRIB_MULTIPLE_WORKERS'] = 'true' if workers_count > 1 + + workers_count.times do + spawn_process(:worker, worker_env).tap { |res| results[res.pid] = res } + end + + sleep(1) if workers_count > 1 # allow workers to start + + spawn_process(:leader, 'RSPEC_DISTRIB_FOLDER' => folder_name.to_s).tap { |res| results[res.pid] = res } + + yield results.values.sort_by(&:type) if block_given? + + statuses = Process.waitall + + statuses.each do |_, status| # rubocop:disable Style/HashEachMethods + results[status.pid].status = status.exited? ? status.exitstatus : status.to_i + results[status.pid].output = results[status.pid].file.read + end + + results.values.sort_by(&:type).tap do |res| # leader first, than workers + @leader, *@workers = res + end + ensure + results.each do |_, result| # rubocop:disable Style/HashEachMethods + result.file.close + result.file.unlink + end + end + + def spawn_process(type, env) + file = Tempfile.new(type.to_s) + distrib_command = 'bundle exec rspec-distrib' + cmd = type == :leader ? 'start' : 'join 127.0.0.1' + pid = spawn(env, [distrib_command, cmd].join(' '), %i[out err] => file.path.to_s, chdir: specs) + Result.new(pid:, file:, type:) + end + + def common_checks + common_leader_checks + common_worker_checks + end + + def common_leader_checks # rubocop:disable Metrics/AbcSize + expect(leader.output).to include 'Using seed' + expect(leader.output).to include 'Finished in' + # expect(leader.output).not_to include 'Unable to read failed line' + expect(leader.output).to include 'on_finish called' + + check_formatter( + leader.output, + leader.output.match(/Using seed (\d+)/)[1], + common_formatter_events + ) + end + + def common_worker_checks # rubocop:disable Metrics/AbcSize + leader_seed = leader.output.match(/Using seed (\d+)/)[1] + worker_outputs.each do |output| + expect(output).to include "Randomized with seed #{leader_seed}" + expect(output).to include 'Finished in' + expect(output).not_to include 'has already been initialized with' + end + + check_formatter(worker_outputs, leader_seed, common_formatter_events) + end + + def check_formatter(results, seed, *events) + results = [results] unless results.is_a?(Array) + events = events.flatten + + results.each do |result| + expect(result).to match(/FORMATTER: seed.*seed=#{seed}/) + events.each do |event| + expect(result).to include "FORMATTER: #{event}" + end + end + end +end diff --git a/rspec-distrib/features/syntax_error_spec.rb b/rspec-distrib/features/syntax_error_spec.rb new file mode 100644 index 0000000..6f951b9 --- /dev/null +++ b/rspec-distrib/features/syntax_error_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require_relative 'feature_helper' + +RSpec.describe 'Syntax error in spec file' do + include_context 'base pipeline' + + specify do + run_distrib(:syntax_error, workers_count: 2) + + expect(leader.output).to include '2 files have been enqueued' + expect(leader.output).to include '0 failures, 1 error occurred outside of examples' + expect(leader.status).to eq(1) + + expect(worker_outputs).to include match(/1 example/) + expect(worker_outputs).to include match(/1 error/) + + common_checks + end +end diff --git a/rspec-distrib/features/timeout_no_spec_picked_spec.rb b/rspec-distrib/features/timeout_no_spec_picked_spec.rb new file mode 100644 index 0000000..111beb4 --- /dev/null +++ b/rspec-distrib/features/timeout_no_spec_picked_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require_relative 'feature_helper' + +RSpec.describe 'Timeout when no spec picked at all' do + include_context 'base pipeline' + + specify do + started = Time.now + run_distrib(:passing, workers_count: 0) + finished = Time.now + + expect(leader.output).to include '2 files have been enqueued' + expect(leader.output).to include 'Leader has reached the time limit' + expect(leader.output).to include '0 examples, 0 failures' + expect(leader.status).to eq(1) + + time_match = leader.output.match(/Finished in (\d+(?:\.\d*)?) seconds/) + time = time_match[1].to_f + expect(time).to be_between(2, 3) + + expect(finished - started).to be < 3 + + common_leader_checks + end +end diff --git a/rspec-distrib/features/timeout_of_spec.rb b/rspec-distrib/features/timeout_of_spec.rb new file mode 100644 index 0000000..09dcfa9 --- /dev/null +++ b/rspec-distrib/features/timeout_of_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative 'feature_helper' + +RSpec.describe 'Timeout of spec' do + include_context 'base pipeline' + + specify do + run_distrib(:timeout_of_spec, workers_count: 2) + + expect(leader.output).to include '2 files have been enqueued' + expect(leader.output).to match(/1 example, 0 failures$/) + expect(leader.output).to match(/Files completed: 1/) + expect(leader.output).to match(/Files left: 1/) + expect(leader.status).to eq(1) + + common_checks + end +end diff --git a/rspec-distrib/features/timeout_processing_stopped_spec.rb b/rspec-distrib/features/timeout_processing_stopped_spec.rb new file mode 100644 index 0000000..b59a370 --- /dev/null +++ b/rspec-distrib/features/timeout_processing_stopped_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require_relative 'feature_helper' + +RSpec.describe 'Timeout when one spec picked but second did not (worker died)' do + include_context 'base pipeline' + + specify do + started = Time.now + run_distrib(:timeout_processing_stopped) + finished = Time.now + + expect(leader.output).to include '2 files have been enqueued' + expect(leader.output).to include 'Workers did not pick tests for too long!' + expect(leader.output).to include 'Aborting...' + expect(leader.output).to match(/1 example, 0 failures$/) + expect(leader.status).to eq(1) + + expect(finished - started).to be < 5 + + expect(worker.output).to include 'Foo' + expect(worker.output).to include 'Wait till timeout' + + common_checks + end +end diff --git a/rspec-distrib/lib/rspec/distrib.rb b/rspec-distrib/lib/rspec/distrib.rb new file mode 100644 index 0000000..1b8ece4 --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib.rb @@ -0,0 +1,20 @@ +require 'distrib_core' +require 'rspec/distrib/configuration' + +# @see https://github.com/rspec/rspec +module RSpec + # Core module to store configuration and some other global vars. + module Distrib + extend ::DistribCore::Distrib + + class << self + # @return [RSpec::Distrib::Configuration] + def configuration + @configuration ||= ::RSpec::Distrib::Configuration.new + end + end + end +end + +# init default configuration +RSpec::Distrib.configuration diff --git a/rspec-distrib/lib/rspec/distrib/configuration.rb b/rspec-distrib/lib/rspec/distrib/configuration.rb new file mode 100644 index 0000000..b8e0606 --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/configuration.rb @@ -0,0 +1,124 @@ +require 'distrib_core/configuration' +require 'rspec/distrib/leader/tests_provider' +require 'rspec/distrib/leader/rspec_helper' + +module RSpec + module Distrib + # Configuration holder + # + # Override default list of the test files: + # + # RSpec::Distrib.configure do |config| + # config.tests_provider = -> { + # Dir.glob(['spec/**/*_spec.rb', 'engines/**/*_spec.rb']) + # } + # end + # + # Specify which errors should fail leader. Other errors will be retried, here you can specify how many times. + # + # RSpec::Distrib.configure do |config| + # config.error_handler.retryable_exceptions = ['Elasticsearch::ServiceUnavailable'] + # config.error_handler.retry_attempts = 2 + # config.error_handler.fatal_worker_failures = ['NameError'] + # config.error_handler.failed_workers_threshold = 2 + # end + # + # Or set your own logic for retries. + # It should respond to `#retry_test?`, `#ignore_worker_failure?` methods + # + # RSpec::Distrib.configure do |config| + # config.error_handler = MyErrorHandler + # end + # + # Set equal timeout for all spec files to 30 seconds: + # + # RSpec::Distrib.configure do |config| + # config.test_timeout = 30 # seconds + # end + # + # Or you can specify timeout per spec file. An object that responds to `call` and receives + # the spec file path as an argument. The proc returns the timeout in seconds. + # + # RSpec::Distrib.configure do |config| + # config.test_timeout = ->(spec_file) do + # 10 + 2 * average_execution_in_seconds(spec_file) + # end + # end + # + # Set how long leader will wait before first spec processed by workers. Leader will exit if + # no specs picked in this period + # + # RSpec::Distrib.configure do |config| + # config.first_test_picked_timeout = 10*60 # 10 minutes + # end + # + # Set how long leader will wait if workers stopped processing the queue. Leader will exit if + # no specs picked in this period + # + # RSpec::Distrib.configure do |config| + # config.tests_processing_stopped_timeout = 5*60 # 5 minutes + # end + # + # Specify which formatters you want to use using `add_leader_formatter` or `add_worker_formatter` methods. + # See `RSpec.configuration.add_formatter` for more info + # + # RSpec::Distrib.configure do |config| + # config.add_leader_formatter('html', 'summary.html') # add HTML formatter which writes to 'summary.html' file + # config.add_worker_formatter('progress') # add progress formatter (prints dots to the console) + # end + # + # Specify custom options for DRb service. Defaults are `{ safe_level: 1 }` + # See `DRb::DRbServer.new` for complete list + # + # RSpec::Distrib.configure do |config| + # config.drb = {safe_level: 1} + # end + # + # Specify custom block to pre-process examples before reporting them to the leader. + # Useful to add additional information about workers. + # + # RSpec::Distrib.configure do |config| + # config.before_test_report = -> (file_name, example_groups) do + # example_groups.each { |eg| eg.metadata[:custom] = 'foo' } + # end + # end + # + class Configuration + include ::DistribCore::Configuration + # Sets RSpec's `--color` option for workers with "force" mode, rewriting existing one + # Possible values: :on, :off; by default it's :automatic + # See https://github.com/rspec/rspec-core/blob/7510b747cdb028dea4feb56cef8062cea14640a5/lib/rspec/core/configuration.rb#L937 + attr_accessor :worker_color_mode + + def initialize + super + @tests_provider = ::RSpec::Distrib::Leader::TestsProvider + @error_handler = ::DistribCore::Leader::ErrorHandler.new( + ::RSpec::Distrib::Leader::RSpecHelper + ) + end + + # @return [Array] + def leader_formatters + @leader_formatters ||= [] + end + + # @param formatter [Object] + # @param output [IO, String] + def add_leader_formatter(formatter, output = nil) + leader_formatters << (output ? [formatter, output] : [formatter]) + end + + # @return [Array] + def worker_formatters + @worker_formatters ||= [] + end + + # @param formatter [Object] + # @param output [IO, String] + def add_worker_formatter(formatter, output = nil) + worker_formatters << (output ? [formatter, output] : [formatter]) + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/example_group.rb b/rspec-distrib/lib/rspec/distrib/example_group.rb new file mode 100644 index 0000000..797ce97 --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/example_group.rb @@ -0,0 +1,224 @@ +RSpec::Support.require_rspec_core 'formatters/exception_presenter' + +module RSpec + module Distrib + # Helper to proxy getter methods to metadata attributes. + module DelegateToMetadata + # Defines methods that fetch attributes from metadata hash + # @param keys [Array] + def delegate_to_metadata(*keys) + keys.each { |key| define_method(key) { @metadata[key] } } + end + end + + # Objects that mimic an RSpec::Core::ExampleGroup on the Reporters. + # + # This is necessary because the RSpec::Core::ExampleGroup is quite large and not + # Marshalable. + # + # So we send this object to the Leader instead of the real ExampleGroup. + # @api private + class ExampleGroup + extend DelegateToMetadata + + attr_reader :class_name, :examples, :metadata, :children, :parent_example_group, :description + + delegate_to_metadata :described_class, :file_path, :location + + # @param [RSpec::Core::ExampleGroup] example_group + # @param [RSpec::Distrib::ExampleGroup] parent_example_group + def initialize(example_group, parent_example_group = nil) + initialize_metadata(example_group.metadata) + @class_name = example_group.name + @parent_example_group = parent_example_group + @children = example_group.children.map { |eg| ExampleGroup.new(eg, self) } + @examples = example_group.filtered_examples.map { |e| ExampleResult.new(e, self) } + @description = example_group.description + end + + def superclass_metadata + parent_example_group&.metadata + end + + def descendant_filtered_examples + @descendant_filtered_examples ||= [examples, children.map(&:descendant_filtered_examples)].flatten + end + + def top_level? + !parent_example_group + end + + def top_level_description + parent_groups.last.description + end + + def parent_groups + groups = [self] + current_group = self + + while (current_group = current_group.parent_example_group) + groups << current_group + end + + groups + end + + private + + def initialize_metadata(metadata) + @metadata = metadata.slice(:description, :full_description, + :file_path, :line_number, :location, :absolute_file_path, + :rerun_file_path, :scoped_id) + @metadata[:described_class] = metadata[:described_class]&.to_s + @metadata[:description_args] = metadata[:description_args]&.map(&:to_s) + end + end + + # Objects that mimic an RSpec::Core::Example on the Reporters. + # + # This is necessary because the RSpec::Core::Example is quite large and not + # Marshalable. + # + # So we send this object to the Leader instead of the real Example. + # @api private + class ExampleResult + extend DelegateToMetadata + + attr_reader :example_group, :description, :location_rerun_argument, + :metadata, :example, :formatted_backtrace + + delegate_to_metadata :execution_result, :file_path, :full_description, + :location, :pending, :skip + + def initialize(example, example_group) + initialize_metadata(example.metadata) + @description = example.description + @location_rerun_argument = example.location_rerun_argument + @example_group = example_group + exception_presenter = RSpec::Core::Formatters::ExceptionPresenter::Factory.new(example).build + + if example.exception # rubocop:disable Style/GuardClause + colorizer = ::RSpec::Core::Notifications::NullColorizer + # FIXME: figure out how to pass proper failure_number + @fully_formatted_lines = exception_presenter.fully_formatted_lines(1, colorizer) + @formatted_backtrace = exception_presenter.formatted_backtrace + end + end + + def exception + execution_result.exception + end + + def id + "#{metadata[:rerun_file_path]}[#{metadata[:scoped_id]}]" + end + + def fully_formatted_lines(_failure_number = nil, _colorizer = nil) + @fully_formatted_lines + end + + private + + def initialize_metadata(metadata) + @metadata = metadata.slice(:extra_failure_lines, + :rerun_file_path, :file_path, :full_description, + :location, :pending, :skip, :scoped_id) + @metadata[:execution_result] = ExecutionResults.new(metadata[:execution_result]) + @metadata[:shared_group_inclusion_backtrace] = metadata[:shared_group_inclusion_backtrace].map do |frame| + SharedExampleGroupInclusionStackFrame.new(frame) + end + end + end + + # Objects that mimic an RSpec::Core::Example::ExecutionResult on the Reporters. + # + # This is necessary because the RSpec::Core::Example is quite large and not + # Marshalable. + # @api private + class ExecutionResults + attr_reader :status, :pending_exception, :pending_message, :exception, + :run_time, :pending_fixed, :example_skipped + + def initialize(execution_results) + @status = execution_results.status + @pending_exception = Exception.new(execution_results.pending_exception) if execution_results.pending_exception + @pending_message = execution_results.pending_message + @exception = Exception.new(execution_results.exception) if execution_results.exception + @example_skipped = execution_results.example_skipped? + @pending_fixed = execution_results.pending_fixed? + @run_time = execution_results.run_time + end + + alias example_skipped? example_skipped + alias pending_fixed? pending_fixed + + # Objects that mimic an Exception on the Reporters. + # + # This is necessary because some exceptions are quite large and not Marshalable. + class Exception + attr_reader :backtrace, :cause, :message, :original_class + + def initialize(exception) # rubocop:disable Metrics/MethodLength + @original_class = if exception.is_a?(Class) + exception.to_s + else + exception.class.to_s + end + + if multiple_exceptions?(exception) + initialize_as_multiple_exceptions(exception) + return + end + + @backtrace = exception.backtrace + @cause = Exception.new(exception.cause) if exception.cause + @message = exception.message + end + + def set_backtrace(backtrace) # rubocop:disable Naming/AccessorMethodName as on original interface + @backtrace = backtrace + end + + private + + def multiple_exceptions?(exception) + defined?(RSpec::Core::MultipleExceptionError) && + exception.is_a?(RSpec::Core::MultipleExceptionError) + end + + def initialize_as_multiple_exceptions(exception) + @backtrace = backtrace_for_multiple_exceptions(exception) + @message = message_for_multiple_exceptions(exception) + cause = exception.all_exceptions.first.cause + @cause = Exception.new(cause) if cause + end + + def backtrace_for_multiple_exceptions(exception) + exceptions = exception.all_exceptions + exceptions.map(&:backtrace).zip(Array.new(exceptions.count - 1, 'AND')).flatten.compact + end + + def message_for_multiple_exceptions(exception) + exceptions = exception.all_exceptions + messages = exceptions.map { |e| "#{e.class.name}: #{e.message}" }.join("\n\nAND\n\n") + "#{exception.summary}:\n#{messages}" + end + end + end + + # Objects that mimic an RSpec::Core::SharedExampleGroupInclusionStackFrame on the Reporters. + # + # This is necessary because the original object refers to objects which can't be accessed on the leader. + # @api private + class SharedExampleGroupInclusionStackFrame + attr_reader :shared_group_name, :inclusion_location, :formatted_inclusion_location, :description + + def initialize(frame) + @shared_group_name = frame.shared_group_name.to_s + @inclusion_location = frame.inclusion_location + @formatted_inclusion_location = frame.formatted_inclusion_location + @description = frame.description + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/leader.rb b/rspec-distrib/lib/rspec/distrib/leader.rb new file mode 100644 index 0000000..5ae856c --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/leader.rb @@ -0,0 +1,199 @@ +require 'drb/drb' + +require 'rspec/distrib' +require 'rspec/distrib/leader/reporter' +require 'rspec/distrib/leader/tests_provider' + +module RSpec + module Distrib + # Interface exposed over the network that Workers connect to in order to + # receive spec file names and report back the results to. + # + # Transport used is [DRb](https://rubydoc.info/stdlib/drb/DRb) + class Leader # rubocop:disable Metrics/ClassLength + include ::DistribCore::Leader + # Used to interpolate with leader ip in order to generate the actual DRb server URL + DRB_SERVER_URL = 'druby://%s:8787'.freeze + # We can't calculate total amount of examples. But we need to provide a big number to prevent warnings + FAKE_TOTAL_EXAMPLES_COUNT = 1_000_000_000 + + class << self + # Starts the DRb server and Watchdog thread + # + # @param seed [Integer] a seed for workers to randomize order of examples + # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + def start_service(seed = nil) + files = ::DistribCore::Leader::QueueBuilder.tests + queue = ::DistribCore::Leader::QueueWithLease.new(files) + + logger.info "#{files.count} files have been enqueued" + + seed ||= rand(0xFFFF) # Mimic how RSpec::Core::Ordering::ConfigurationManager randomizes it + RSpec.configuration.seed = seed # it is going to be used by reporter + + reporter = Leader::Reporter.new + + leader = new(queue, reporter, seed) + + watchdog = ::DistribCore::Leader::Watchdog.new(queue) + watchdog.start + + DRb.start_service(DRB_SERVER_URL % '0.0.0.0', leader, RSpec::Distrib.configuration.drb) + logger.info 'Leader ready' + ::DistribCore::Metrics.queue_exposed + DRb.thread.join + + reporter.finish + RSpec::Distrib.configuration.on_finish&.call + + failed = reporter.failures? || watchdog.failed? || leader.non_example_exception + count_mismatch = (queue.size + queue.completed_size != files.count) + + if failed || ::DistribCore::ReceivedSignals.any? || count_mismatch + print_failure_status(reporter, watchdog, leader, queue, count_mismatch) + Kernel.exit(::DistribCore::ReceivedSignals.any? ? ::DistribCore::ReceivedSignals.exit_code : 1) + else + logger.info "Build succeeded. Files processed: #{queue.completed_size}" + end + end + # rubocop:enable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + + private + + # rubocop:disable Metrics/AbcSize + def print_failure_status(reporter, watchdog, leader, queue, count_mismatch) + logger.info 'Build failed' + logger.debug(::DistribCore::ReceivedSignals.message) if ::DistribCore::ReceivedSignals.any? + logger.debug 'Reporter failed' if reporter.failures? + logger.debug 'Watchdog failed' if watchdog.failed? + logger.debug 'Non example exception' if leader.non_example_exception + logger.debug "Files completed: #{queue.completed_size}" + logger.debug "Files left: #{queue.size}" if queue.size + logger.warn("Amount of processed files doesn't match amount of enqueued files") if count_mismatch + end + # rubocop:enable Metrics/AbcSize + end + + # Object shared through DRb is open for any calls. Including eval calls + # A simple way to prevent it - undef + undef :instance_eval + undef :instance_exec + + attr_reader :seed, :non_example_exception + + def initialize(queue, reporter, seed) + @queue = queue + @reporter = reporter + @seed = seed + logger.info "Using seed #{@seed}" + end + + # Get the next spec from the queue + # @return [String] spec file name + # @example + # leader.next_file_to_run # => 'spec/services/user_service_spec.rb' + drb_callable def next_file_to_run + ::DistribCore::Metrics.test_taken + + queue.lease.tap do |file| + logger.debug "Serving #{file}" + end + end + + # Report example group results for a spec file + # + # @param file_path [String] ex: './spec/services/user_service_spec.rb' + # @param example_groups [Array] + # @param exception [RSpec::Distrib:: + # @see RSpec::Distrib::ExampleGroup + # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength + drb_callable def report_file(file_path, example_groups, exception = nil) + message = "Reported #{file_path} with #{example_groups.count} example groups" + message += " and exception #{exception.original_class}" if exception + logger.debug message + + return if queue.completed?(file_path) + + if RSpec::Distrib.configuration.error_handler.retry_test?(file_path, example_groups, exception) + logger.debug("Retrying #{file_path}") + will_be_retried = true + queue.repush(file_path) + example_groups.each { |example_group| reporter.report(example_group, will_be_retried: true) } + return + end + + RSpec.configuration.loaded_spec_files << File.expand_path(file_path) + + example_groups.each { |example_group| reporter.report(example_group) } + + handle_failed_worker(exception, file_path) if exception + nil + ensure + queue.release(file_path) unless will_be_retried + log_completed_percent + end + # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength + + # Notifies non example error to RSpec reporter and stops the service. + # If the error occurs in one worker, it's likely something that will break + # all workers, like a SyntaxError while loading some file, so it does + # not make sense to continue executing. + drb_callable def notify_non_example_exception(exception, context_description) + logger.info("Worker failed with non_example_exception #{exception.original_class}") + + return if RSpec::Distrib.configuration.error_handler.ignore_worker_failure?(exception) + + logger.info("Leader will stop since worker failed with non_example_exception #{exception.original_class}") + + reporter.notify_non_example_exception(exception, context_description) + + handle_non_example_exception + + nil + end + + drb_callable def report_worker_configuration_error(exception) + logger.info "Worker failed during startup with #{exception.original_class}" + + return if RSpec::Distrib.configuration.error_handler.ignore_worker_failure?(exception) + + handle_failed_worker(exception) + nil + end + + private + + attr_reader :queue, :reporter + + def handle_failed_worker(exception, file = nil) + message = "Leader will stop since worker failed with #{exception.original_class}" + message += " on file #{file}:" if file + message += "\n#{exception.message}" + logger.error message + logger.debug exception.backtrace&.join("\n") + logger.debug exception.cause.inspect if exception.cause + + handle_non_example_exception + end + + def handle_non_example_exception + @non_example_exception = true + DRb.current_server.stop_service + end + + def log_completed_percent # rubocop:disable Metrics/AbcSize + @logged_percents ||= [] + log_every = 10 + + completed_percent = (queue.completed_size.to_f / (queue.size + queue.completed_size) * 100).to_i + bucket = completed_percent / log_every * log_every # convert 35 to 30 + + return if @logged_percents.include?(bucket) + + @logged_percents << bucket + + logger.debug "Completed: #{completed_percent}%" + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/leader/reporter.rb b/rspec-distrib/lib/rspec/distrib/leader/reporter.rb new file mode 100644 index 0000000..b03d8ba --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/leader/reporter.rb @@ -0,0 +1,91 @@ +require 'rspec/core' +require 'rspec/core/profiler' + +require 'rspec/distrib/example_group' + +module RSpec + module Distrib + class Leader + # RSpec reporter local to the leader, but reachable by Workers. + # Used to accumulate the example execution results from worker machines. + class Reporter + # Failed statuses for an example + FAILED_STATUSES = %w[failed].freeze + # Possible example statuses + REPORTABLE_EXAMPLE_STATUSES = (%w[passed pending] + FAILED_STATUSES).freeze + + def initialize + @reporter = init_reporter + @reporter.start(Leader::FAKE_TOTAL_EXAMPLES_COUNT) + end + + # To report the file and all its specs results. + # + # We're doing this by file, so that the results keeps readable in any + # format, even if multiple workers are sending results at the same time. + # + # @param example_group [RSpec::Distrib::ExampleGroup] + # @see RSpec::Distrib::ExampleGroup + # + # @note This is only supporting RSpec "Progress" formatter for now. + def report(example_group, will_be_retried: false) + reporter.example_group_started(example_group) + + example_group.examples.each { |example| report_example(example, will_be_retried:) } + example_group.children.each do |inner_example_group| + report(inner_example_group, will_be_retried:) + end + + reporter.example_group_finished(example_group) + end + + # Print the final results of the test suite + def finish + reporter.finish + end + + # Notifies RSpec about exceptions unrelated to an example in order to halt execution + # + # @param exception [Exception] + def notify_non_example_exception(exception, context_description) + reporter.notify_non_example_exception(exception, context_description) + end + + def failures? + reporter.failed_examples.any? + end + + private + + attr_reader :reporter + + def init_reporter + RSpec::Distrib.configuration.leader_formatters.each do |(formatter, *args)| + RSpec.configuration.add_formatter(formatter, *args) + end + + RSpec.configuration.reporter + end + + # Adds example to report. Notifies formatters + def report_example(example_result, will_be_retried:) + status = example_result.execution_result.status.to_s + + raise "Example status not valid: '#{status}'" unless REPORTABLE_EXAMPLE_STATUSES.include?(status) + + if will_be_retried + # We retry the whole file, but we only want to + # report the specs that actually failed and cause the retry + return unless FAILED_STATUSES.include?(status) + + reporter.publish(:example_will_be_retried, example: example_result) + else + reporter.example_started(example_result) + reporter.example_finished(example_result) + reporter.public_send("example_#{status}", example_result) + end + end + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/leader/rspec_helper.rb b/rspec-distrib/lib/rspec/distrib/leader/rspec_helper.rb new file mode 100644 index 0000000..61697d6 --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/leader/rspec_helper.rb @@ -0,0 +1,35 @@ +module RSpec + module Distrib + class Leader + # Helper that handles some cases with RSpec. + module RSpecHelper + class << self + # Goes through all example_group tree and returns all failures as array. + def failures_of(example_groups) + fetch_all_failures_recursively(example_groups).flatten.compact.uniq + end + + # Extract all causes for exceptions to a list. + def unpack_causes(exceptions) + exceptions.map do |exception| + causes = [] + while exception + causes << exception + exception = exception.cause + end + causes + end + end + + private + + def fetch_all_failures_recursively(example_groups) + example_groups.map do |eg| + eg.examples.map(&:exception) + fetch_all_failures_recursively(eg.children) + end + end + end + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/leader/tests_provider.rb b/rspec-distrib/lib/rspec/distrib/leader/tests_provider.rb new file mode 100644 index 0000000..b2574dd --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/leader/tests_provider.rb @@ -0,0 +1,20 @@ +module RSpec + module Distrib + class Leader + # Default strategy to get a list of spec files to serve from + # the queue. Gets spec files from spec directory. + class TestsProvider + class << self + # @return [Array] list of spec files to enqueue + # @example ['spec/user_spec.rb', 'spec/users_controller_spec.rb'] + # + # An application with a very long test suite might have better + # results by ordering the specs by average execution time descending. + def call + Dir.glob('spec/**/*_spec.rb') + end + end + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/worker.rb b/rspec-distrib/lib/rspec/distrib/worker.rb new file mode 100644 index 0000000..f7c0c86 --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/worker.rb @@ -0,0 +1,19 @@ +require 'rspec/distrib' +require 'rspec/distrib/worker/rspec_runner' + +module RSpec + module Distrib + # Wrapper around {RSpec::Distrib::RSpecRunner} + module Worker + # Start a worker instance with a given leader ip. + # + # @param leader_ip [String] the ip address of the DRb server of the leader + def self.join(leader_ip) + raise 'Leader IP should be specified' unless leader_ip && !leader_ip.empty? + + status = RSpec::Distrib::Worker::RSpecRunner.run_from_leader(leader_ip) + exit(status) if status != 0 + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/worker/configuration.rb b/rspec-distrib/lib/rspec/distrib/worker/configuration.rb new file mode 100644 index 0000000..07c7918 --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/worker/configuration.rb @@ -0,0 +1,28 @@ +module RSpec + module Distrib + module Worker + # @api private + # Custom configuration for RSpec which we use on workers to replace + # regular reporter with {LeaderReporter}. + class Configuration < RSpec::Core::Configuration + # @return [DRbObject(RSpec::Distrib::Leader)] + attr_accessor :leader + + # Overridden method which wraps original reporter with {LeaderReporter} + # @return RSpec::Core::Formatters::Loader + def formatter_loader + @formatter_loader ||= begin + original_reporter = RSpec::Core::Reporter.new(self) + wrapped_reporter = LeaderReporter.new(leader, original_reporter) + RSpec::Core::Formatters::Loader.new(wrapped_reporter) + end + end + + # Always true because seed comes from leader + def seed_used? + true + end + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/worker/leader_reporter.rb b/rspec-distrib/lib/rspec/distrib/worker/leader_reporter.rb new file mode 100644 index 0000000..3f35717 --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/worker/leader_reporter.rb @@ -0,0 +1,35 @@ +require 'delegate' + +module RSpec + module Distrib + module Worker + # @api private + # Custom reporter to notify leader about non_example_exception + class LeaderReporter < SimpleDelegator + # @param leader [DRbObject(RSpec::Distrib::Leader)] + def initialize(leader, *) + super(*) + @leader = leader + end + + # Calls original behaviour and notifies leader + # @param exception [Exception] + def notify_non_example_exception(exception, context_description) + super + + return if force_exit_signal? + + converted_exception = RSpec::Distrib::ExecutionResults::Exception.new(exception) + @leader.notify_non_example_exception(converted_exception, context_description) + end + + private + + def force_exit_signal? + DistribCore::ReceivedSignals.received?('TERM') || + DistribCore::ReceivedSignals.force_int? + end + end + end + end +end diff --git a/rspec-distrib/lib/rspec/distrib/worker/rspec_runner.rb b/rspec-distrib/lib/rspec/distrib/worker/rspec_runner.rb new file mode 100644 index 0000000..1a188af --- /dev/null +++ b/rspec-distrib/lib/rspec/distrib/worker/rspec_runner.rb @@ -0,0 +1,205 @@ +require 'drb/drb' +require 'rspec/core' +require 'rspec/core/configuration_options' + +require 'distrib_core/core_ext/drb_tcp_socket' +require 'distrib_core/worker' +require 'distrib_core/drb_helper' + +require 'rspec/distrib/example_group' +require 'rspec/distrib/leader' +require 'rspec/distrib/worker/leader_reporter' +require 'rspec/distrib/worker/configuration' + +module RSpec + module Distrib + module Worker + # Modified RSpec runner to consume files from the leader + # + # @see https://github.com/rspec/rspec-core/blob/master/lib/rspec/core/runner.rb + class RSpecRunner < RSpec::Core::Runner # rubocop:disable Metrics/ClassLength + include ::DistribCore::Worker + + # Public method that invokes the runner. + # + # @param leader_ip [String] + def self.run_from_leader(leader_ip) + leader = DRbObject.new_with_uri(Leader::DRB_SERVER_URL % leader_ip) + new(leader).run + end + + # @see RSpec::Core::Runner#initialize + # @param leader [DRbObject] + def initialize(leader) # rubocop:disable Metrics/MethodLength + @success = true + @leader = leader + @seed = connect_to_leader_with_timeout { leader.seed } + + handle_configuration_failure do + @options = RSpec::Core::ConfigurationOptions.new(["--seed=#{@seed}"]) + # as long as there is this assignment to global variable + # the test have to restore RSpec.configuration after the example + # see `around` block in spec/rspec/distrib/worker/rspec_runner_spec.rb + @configuration = RSpec.configuration = RSpec::Distrib::Worker::Configuration.new + @configuration.leader = leader + init_formatters + @world = RSpec.world + setup($stdout, $stderr) + end + end + + # @see RSpec::Core::Runner#run + # @see RSpec::Core::Runner#run_specs + # + # @note + # Originally it makes setup and runs specs. + # We patch this method to consume from the leader, instead of the given + # example_groups param. + def run(*) # rubocop:disable Metrics/MethodLength + handle_configuration_failure do + @configuration.reporter.report(Leader::FAKE_TOTAL_EXAMPLES_COUNT) do |reporter| + @configuration.with_suite_hooks do + # Disable handler since consume_queue has it's own handler + @handle_configuration_failure = false + consume_queue(reporter) + end + end + end + persist_example_statuses + + return ::DistribCore::ReceivedSignals.exit_code if received_any_signal? + + success && !world.non_example_failure ? 0 : @configuration.failure_exit_code + end + + private + + attr_reader :leader, :world, :success + + def init_formatters + RSpec::Distrib.configuration.worker_formatters.each do |(formatter, *args)| + RSpec.configuration.add_formatter(formatter, *args) + end + end + + # Runs specs from the leader until it is empty. + # + # @param reporter [RSpec::Core::Reporter] + # + # @return [Boolean] true in case there were no failures. + # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + def consume_queue(reporter) + @reported_examples = [] + return if received_any_signal? + + while (file_path = leader.next_file_to_run) + logger.debug "Running #{file_path}" + reset_reporter(reporter) + load_spec_file(file_path) + + @success &= world.ordered_example_groups.map { |example_group| example_group.run(reporter) }.all? + + # Should not send a possible broken report for the leader + # A report can be broken because other services (e.g. Redis, Elasticsearch) could have already terminated + break if received_term? || received_force_int? + + report_file_to_leader(file_path, world.ordered_example_groups) + + break if received_int? || world.non_example_failure + end + rescue DRb::DRbConnError + # It raises when Leader is disconnected = a.k.a. queue is empty + logger.info 'Disconnected from leader, finishing' + rescue Exception => e # rubocop:disable Lint/RescueException + # TODO: I'm unsure about this rescue, but we need to report all cases to leader + silently do + report_file_to_leader(file_path, world.ordered_example_groups, e) + end + raise + ensure + # Putting examples back to the local reporter, so that results are consistent. + reporter.examples.concat(@reported_examples) + false + end + # rubocop:enable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + + # Send the just processed example groups to leader, and concat these with an + # array with the previous specs. + # + # We're doing that to keep consistency between Leader and Worker reports. + # rubocop:disable Metrics/MethodLength, Metrics/AbcSize + def report_file_to_leader(file_path, example_groups, exception = nil) + message = "Reporting #{file_path} with #{example_groups.count} example groups" + message += " and exception #{exception.class}" if exception + logger.debug message + + converted_example_groups = example_groups.map { |example_group| ExampleGroup.new(example_group) } + converted_exception = RSpec::Distrib::ExecutionResults::Exception.new(exception) if exception + + if RSpec::Distrib.configuration.before_test_report + instance_exec( + file_path, + converted_example_groups, + converted_exception, + &RSpec::Distrib.configuration.before_test_report + ) + end + + leader.report_file(file_path, converted_example_groups, converted_exception) + rescue DRb::DRbConnError => e + dump_failed = ::DistribCore::DRbHelper.dump_failed?(e, converted_example_groups + [exception]) + world.non_example_failure = true if dump_failed + raise + end + # rubocop:enable Metrics/MethodLength, Metrics/AbcSize + + def handle_configuration_failure + @handle_configuration_failure = true + yield + rescue Exception => e # rubocop:disable Lint/RescueException + # TODO: I'm unsure about this rescue, but we need to report all cases to leader + if @handle_configuration_failure + silently do + converted_exception = RSpec::Distrib::ExecutionResults::Exception.new(e) + leader.report_worker_configuration_error(converted_exception) + end + end + raise + end + + def silently + # we don't care if it fails + yield unless received_term? || received_force_int? + rescue StandardError + nil + end + + # This is a hack to be make world recognize the next spec file to run from + # the leader. + # + # @param path [String] ex: 'spec/apq/actions/ba/talent/walkthrough/update_profile_spec.rb' + # + # @see RSpec::Core::Runner#configure + # @see RSpec::Core::Runner#setup + def load_spec_file(path) + world.example_groups.clear + @options = RSpec::Core::ConfigurationOptions.new(["--seed=#{@seed}", path]) + @options.configure(@configuration) + if RSpec::Distrib.configuration.worker_color_mode + @configuration.force(color_mode: RSpec::Distrib.configuration.worker_color_mode) + end + @configuration.load_spec_files + end + + def reset_reporter(reporter) + @reported_examples.concat(reporter.examples) + reporter.examples.clear + end + + def logger + RSpec::Distrib.configuration.logger + end + end + end + end +end diff --git a/rspec-distrib/rspec-distrib.gemspec b/rspec-distrib/rspec-distrib.gemspec new file mode 100644 index 0000000..9b737de --- /dev/null +++ b/rspec-distrib/rspec-distrib.gemspec @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +Gem::Specification.new do |s| + s.name = 'rspec-distrib' + s.version = '0.0.1' + s.authors = ['Toptal, LLC'] + s.email = ['open-source@toptal.com'] + s.license = 'MIT' + + s.summary = 'RSpec extension for distributed running of specs from a queue.' + s.description = '' + s.homepage = 'https://github.com/toptal/test-distrib' + s.required_ruby_version = '>= 3.2.4' + + s.files = Dir['lib/**/*.rb'] + Dir['exe/*'] + s.bindir = 'exe' + s.executables = ['rspec-distrib'] + + s.add_dependency 'rspec-core', '~> 3.12' + + s.metadata['rubygems_mfa_required'] = 'true' +end diff --git a/rspec-distrib/spec/rspec/distrib/configuration_spec.rb b/rspec-distrib/spec/rspec/distrib/configuration_spec.rb new file mode 100644 index 0000000..438c7fc --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/configuration_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'distrib_core/spec/configuration' + +RSpec.describe RSpec::Distrib::Configuration do + subject(:configuration) { RSpec::Distrib.configuration } + + around do |example| + config = RSpec::Distrib.instance_variable_get(:@configuration) + RSpec::Distrib.instance_variable_set(:@configuration, nil) + example.run + RSpec::Distrib.instance_variable_set(:@configuration, config) + end + + it '#tests_provider' do + expect(configuration.tests_provider) + .to eq(RSpec::Distrib::Leader::TestsProvider) + end + + it '#error_handler' do + expect(configuration.error_handler) + .to be_a(RSpec::Distrib::Leader::ErrorHandler) + end + + it 'has no leader_formatters' do + expect(configuration.leader_formatters).to be_empty + end + + it 'has no worker_formatters' do + expect(configuration.worker_formatters).to be_empty + end + + it 'can add leader_formatters' do + RSpec::Distrib.configure do |config| + config.add_leader_formatter('html', 'output.html') + end + + expect(configuration.leader_formatters).to include(%w[html output.html]) + end + + it 'can add worker_formatters' do + RSpec::Distrib.configure do |config| + config.add_worker_formatter('html', 'output.html') + end + + expect(configuration.worker_formatters).to include(%w[html output.html]) + end + + include_examples 'DistribCore configuration' +end diff --git a/rspec-distrib/spec/rspec/distrib/example_group_spec.rb b/rspec-distrib/spec/rspec/distrib/example_group_spec.rb new file mode 100644 index 0000000..0feb925 --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/example_group_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +RSpec.describe RSpec::Distrib::ExampleGroup do + subject(:example_group) { described_class.new(rspec_example_group, parent_example_group) } + + let(:rspec_example_group) do + class_double(RSpec::Core::ExampleGroup, + metadata:, + children: [], + filtered_examples: []) + .as_null_object + end + + let(:parent_example_group) do + class_double(described_class) + .as_null_object + end + + let(:metadata) { {} } + + describe 'metadata delegation' do + let(:metadata) do + { + described_class: 'Foo', + file_path: 'foo.rb', + location: 'foo.rb:42' + } + end + + it 'delegates to metadata' do + expect(example_group.described_class).to eq('Foo') + expect(example_group.file_path).to eq('foo.rb') + expect(example_group.location).to eq('foo.rb:42') + end + end + + describe '#top_level?' do + context 'when parent_group exists' do + it 'returns false' do + expect(example_group.top_level?).to be(false) + end + end + + context 'when parent_group does not exist' do + let(:parent_example_group) { nil } + + it 'returns true' do + expect(example_group.top_level?).to be(true) + end + end + end + + describe 'parent_groups' do + let(:parent_example_group) do + described_class.new(rspec_example_group, parent_2) + end + let(:parent_2) do + described_class.new(rspec_example_group) + end + + it 'returns all parent_groups' do + expect(example_group.parent_groups).to eq([example_group, parent_example_group, parent_2]) + end + end +end diff --git a/rspec-distrib/spec/rspec/distrib/execution_results/exception_spec.rb b/rspec-distrib/spec/rspec/distrib/execution_results/exception_spec.rb new file mode 100644 index 0000000..5ec55dd --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/execution_results/exception_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +RSpec.describe RSpec::Distrib::ExecutionResults::Exception do + subject(:exception) { described_class.new(original_exception) } + + before do + stub_const('DummyError', Class.new(StandardError)) + end + + it do + original_exception_cause = StandardError.new('foo') + original_exception_cause.set_backtrace %w[4 5 6] + original_exception = DummyError.new('bar') + original_exception.set_backtrace %w[1 2 3] + allow(original_exception).to receive(:cause).and_return(original_exception_cause) + + exception = described_class.new(original_exception) + + expect(exception.message).to eq original_exception.message + expect(exception.backtrace).to eq original_exception.backtrace + expect(exception.original_class).to eq 'DummyError' + expect(exception.cause.message).to eq original_exception_cause.message + expect(exception.cause.backtrace).to eq original_exception_cause.backtrace + expect(exception.cause.original_class).to eq 'StandardError' + expect(exception.cause.cause).to be_nil + end + + it do + exception1 = StandardError.new('Foo') + exception1.set_backtrace %w[1 2 3] + exception2 = DummyError.new('Bar') + exception2.set_backtrace %w[4 5 6] + + multiple_exceptions_error = RSpec::Core::MultipleExceptionError.new(exception1, exception2) + exception = described_class.new(multiple_exceptions_error) + + expect(exception.message).to eq "Got 0 failures and 2 other errors:\nStandardError: Foo\n\nAND\n\nDummyError: Bar" + expect(exception.backtrace).to eq exception1.backtrace + ['AND'] + exception2.backtrace + end +end diff --git a/rspec-distrib/spec/rspec/distrib/leader/reporter_spec.rb b/rspec-distrib/spec/rspec/distrib/leader/reporter_spec.rb new file mode 100644 index 0000000..b6b3a2a --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/leader/reporter_spec.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +# rubocop:disable RSpec/MessageChain +RSpec.describe RSpec::Distrib::Leader::Reporter do + subject(:leader_reporter) { described_class.new } + + let(:reporter) { instance_double(RSpec::Core::Reporter) } + + before do + allow(RSpec.configuration).to receive(:reporter).and_return(reporter) + allow(RSpec.configuration).to receive(:add_formatter) + allow(reporter).to receive(:start).with(RSpec::Distrib::Leader::FAKE_TOTAL_EXAMPLES_COUNT) + allow(reporter).to receive(:example_group_started) + allow(reporter).to receive(:example_group_finished) + allow(reporter).to receive(:example_started) + allow(reporter).to receive(:example_finished) + allow(reporter).to receive(:example_passed) + allow(reporter).to receive(:example_failed) + allow(reporter).to receive(:example_pending) + end + + describe '#report' do + let(:execution_result) { instance_double(RSpec::Distrib::ExecutionResults, status: :passed) } + let(:example_result) { instance_double(RSpec::Distrib::ExampleResult, execution_result:) } + let(:example_group) { instance_double(RSpec::Distrib::ExampleGroup, children:, examples:) } + let(:children) { [] } + let(:examples) { [example_result] } + let(:progress_formatter) { instance_double(RSpec::Core::Formatters::ProgressFormatter) } + let(:html_formatter) { instance_double(RSpec::Core::Formatters::HtmlFormatter) } + + before do + allow(RSpec.configuration).to receive(:formatters).and_return([progress_formatter, html_formatter]) + allow(progress_formatter) + .to receive_messages(example_passed: true, example_failed: true, example_pending: true) + allow(html_formatter).to receive_messages(example_failed: true, example_pending: true) + allow(progress_formatter).to receive(:is_a?) { |klass| klass == RSpec::Core::Formatters::ProgressFormatter } + allow(html_formatter).to receive(:is_a?) { |klass| klass == RSpec::Core::Formatters::HtmlFormatter } + allow(reporter).to receive_message_chain(:examples, :<<) + end + + it 'only starts the reporter once' do + another_results = [instance_double(RSpec::Distrib::ExampleResult, execution_result:)] + another_group = instance_double(RSpec::Distrib::ExampleGroup, children: [], examples: another_results) + expect(reporter).to receive(:start).once + leader_reporter.report(example_group) + leader_reporter.report(another_group) + end + + context 'when example group has nested groups' do + let(:all_examples) do + Array.new(3) do + instance_double(RSpec::Distrib::ExampleResult, execution_result:) + end + end + let(:examples) { all_examples[2..] } + let(:children) do + [instance_double(RSpec::Distrib::ExampleGroup, children: [], examples: all_examples[0..2])] + end + + it 'reports all the examples' do + all_examples.each do |example_result| + expect(reporter).to receive(:example_started).with(example_result) + end + leader_reporter.report(example_group) + end + end + + context 'when example status is :passed' do + it 'adds to examples' do + result = instance_double(RSpec::Distrib::ExampleResult, execution_result:) + example_group = instance_double(RSpec::Distrib::ExampleGroup, children: [], examples: [result]) + expect(reporter).to receive(:example_passed).with(result) + leader_reporter.report(example_group) + end + end + + context 'when example status is :failed' do + let(:execution_result) { instance_double(RSpec::Distrib::ExecutionResults, status: :failed) } + + it 'adds to examples and failed_examples' do + expect(reporter).to receive(:example_started).with(example_result) + expect(reporter).to receive(:example_failed).with(example_result) + leader_reporter.report(example_group) + end + end + + context 'when example status is :pending' do + let(:execution_result) { instance_double(RSpec::Distrib::ExecutionResults, status: :pending) } + + it 'adds to examples and pending_examples' do + expect(reporter).to receive(:example_started).with(example_result) + expect(reporter).to receive(:example_pending).with(example_result) + leader_reporter.report(example_group) + end + end + + context 'when example status is not acceptable' do + it 'adds to examples and pending_examples' do + unacceptable_result = instance_double(RSpec::Distrib::ExecutionResults, status: :unacceptable) + result = instance_double(RSpec::Distrib::ExampleResult, execution_result: unacceptable_result) + example_group = instance_double(RSpec::Distrib::ExampleGroup, children: [], examples: [result]) + expect { leader_reporter.report(example_group) } + .to raise_error(/Example status not valid: 'unacceptable'/) + end + end + + context 'when it will be retried' do + context 'when the example is failed' do + let(:execution_result) { instance_double(RSpec::Distrib::ExecutionResults, status: :failed) } + + it 'reports it only as a retry' do + expect(reporter).not_to receive(:example_started) + expect(reporter).not_to receive(:example_finished) + + expect(reporter).to receive(:publish).with(:example_will_be_retried, example: example_result) + + leader_reporter.report(example_group, will_be_retried: true) + end + end + + context 'when the example is passed' do + let(:execution_result) { instance_double(RSpec::Distrib::ExecutionResults, status: :passed) } + + it 'does not report as retry' do + expect(reporter).not_to receive(:publish) + + leader_reporter.report(example_group, will_be_retried: true) + end + end + end + end + + describe '#finish' do + it 'finishes the report' do + expect(reporter).to receive(:finish) + leader_reporter.finish + end + end + + describe '#failures?' do + it 'returns true if there are failures' do + allow(reporter).to receive(:failed_examples).and_return(%i[failed.rb]) + expect(leader_reporter.failures?).to be(true) + end + end + + describe '#notify_non_example_exception' do + it 'notifies rspec reporter' do + expect(reporter).to receive(:notify_non_example_exception).with(:exception, :context) + leader_reporter.notify_non_example_exception(:exception, :context) + end + end +end +# rubocop:enable RSpec/MessageChain diff --git a/rspec-distrib/spec/rspec/distrib/leader/rspec_helper_spec.rb b/rspec-distrib/spec/rspec/distrib/leader/rspec_helper_spec.rb new file mode 100644 index 0000000..7e64674 --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/leader/rspec_helper_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +RSpec.describe RSpec::Distrib::Leader::RSpecHelper do # rubocop:disable RSpec/FilePath, RSpec/SpecFilePathFormat + describe '.failures_of' do + it do + errors = [ + RSpec::Distrib::ExecutionResults::Exception.new(StandardError.new('QWE')), + RSpec::Distrib::ExecutionResults::Exception.new(ArgumentError.new('ASD')), + RSpec::Distrib::ExecutionResults::Exception.new(RuntimeError.new('ZXC')) + ] + + example_groups = [] + errors.each do |error| + example_groups = [ + instance_double( + RSpec::Distrib::ExampleGroup, + examples: [instance_double(RSpec::Distrib::ExampleResult, exception: error)], + children: example_groups + ) + ] + end + + expect(described_class.failures_of(example_groups)).to eq errors.reverse + end + end + + describe '.unpack_causes' do + it do + errors = [StandardError.new('ASD')] + errors_tree = [errors[0]] + + begin + begin + raise 'QWE' + rescue StandardError => e + errors << e + raise ArgumentError, 'ZXC' + end + rescue StandardError => e + errors << e + errors_tree << e + end + + expect(described_class.unpack_causes(errors_tree)).to contain_exactly([errors[0]], [errors[2], errors[1]]) + end + end +end diff --git a/rspec-distrib/spec/rspec/distrib/leader/tests_provider_spec.rb b/rspec-distrib/spec/rspec/distrib/leader/tests_provider_spec.rb new file mode 100644 index 0000000..e58ea15 --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/leader/tests_provider_spec.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +RSpec.describe RSpec::Distrib::Leader::TestsProvider do + describe '.call' do + it 'calls Dir.glob' do + expect(Dir).to receive(:glob).with('spec/**/*_spec.rb') + described_class.call + end + end +end diff --git a/rspec-distrib/spec/rspec/distrib/leader_spec.rb b/rspec-distrib/spec/rspec/distrib/leader_spec.rb new file mode 100644 index 0000000..7fdeae6 --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/leader_spec.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +RSpec.describe RSpec::Distrib::Leader do + subject(:leader) { described_class.new(queue, reporter, 123) } + + let(:reporter) { instance_double(RSpec::Distrib::Leader::Reporter, 'reporter') } + + before do + skip "Leader specs can't run in distrib mode" if already_running_in_distrib? + end + + # Some specs try to start a thread with a leader, and loop forever on the main + # thread waiting to become available. If you run those tests inside rspec-distrib itself + # they get hung up forever because the port for the leader is taken. + # + # Other tests stub stuff on DRb, which causes issues for the operation of the running leader + def already_running_in_distrib? + RSpec::Distrib.kind # This is only set when running in distrib. Otherwise it raises + rescue StandardError + false + end + + describe '.start_service' do + it do + seed = 123 + file_paths = instance_double(Array, 'file_paths', length: 10_000, count: 10_000) + queue = instance_double(DistribCore::Leader::QueueWithLease, 'queue_with_lease') + allow(queue).to receive_messages(size: 0, completed_size: file_paths.length) + + expect(DistribCore::Leader::QueueBuilder).to receive(:tests).and_return(file_paths) + expect(DistribCore::Leader::QueueWithLease) + .to receive(:new).with(file_paths).and_return(queue) + + expect(RSpec::Distrib::Leader::Reporter).to receive(:new).and_return(reporter) + + leader = instance_double(described_class, 'leader') + expect(described_class).to receive(:new).with(queue, reporter, seed).and_return(leader) + + watchdog = instance_double(DistribCore::Leader::Watchdog, 'watchdog') + expect(leader).to receive(:non_example_exception) + expect(DistribCore::Leader::Watchdog).to receive(:new).with(queue).and_return(watchdog) + expect(watchdog).to receive(:start) + expect(watchdog).to receive(:failed?).and_return(false) + + expect(DRb).to receive(:start_service).with('druby://0.0.0.0:8787', leader, instance_of(Hash)) + expect(DRb).to receive_message_chain(:thread, :join) # rubocop:disable RSpec/MessageChain + + expect(reporter).to receive(:finish) + expect(reporter).to receive(:failures?) + + allow(RSpec::Distrib.configuration.broadcaster).to receive(:info) + expect(RSpec::Distrib.configuration.broadcaster).to receive(:info).with('10000 files have been enqueued') + described_class.start_service(seed) + end + + describe 'exit codes' do + include RSpec::Support::InSubProcess + + it 'exits with 1 on non_example_exception' do + in_sub_process_if_possible(false) do + allow(STDOUT).to receive(:puts) + + leader_thread = Thread.new do + expect { described_class.start_service }.to raise_error(SystemExit) do |error| + expect(error.status).to eq(1) + end + end + + loop do + sleep 0.1 + break if DRb.current_server + rescue DRb::DRbServerNotFound + retry + end + + remote_leader = DRbObject.new_with_uri(RSpec::Distrib::Leader::DRB_SERVER_URL % 'localhost') + error = instance_double(RSpec::Distrib::ExecutionResults::Exception, + original_class: 'StandardError', + cause: nil, + message: 'foo', + backtrace: []) + remote_leader.notify_non_example_exception(error, '') + leader_thread.join + end + end + end + end + + describe '#next_file_to_run' do + let(:queue) { DistribCore::Leader::QueueWithLease.new(file_paths) } + + context 'with a spec file in the queue' do + let(:file_paths) { ['file_path'] } + + it 'returns a spec from the top of the queue' do + expect(leader.next_file_to_run).to eq('file_path') + end + end + + context 'with a couple of spec files in the queue' do + let(:file_paths) { %w[file_path another_file_path] } + + it 'does not return the same spec when called twice' do + expect(leader.next_file_to_run).to eq('another_file_path') + expect(leader.next_file_to_run).to eq('file_path') + end + end + end + + describe '#report_file' do + let(:queue) { instance_double(DistribCore::Leader::QueueWithLease, size: 1, completed_size: 1) } + let(:file_path) { 'file_path' } + let(:example_groups) { [instance_double(RSpec::Distrib::ExampleGroup)] } + + before { allow(RSpec::Distrib::Leader::Reporter).to receive(:new).and_return(reporter) } + + context 'when spec is not released (not reported by other worker)' do + it 'delegates to the reporter' do + allow(queue).to receive(:completed?).with(file_path).and_return(false) + allow(queue).to receive(:release).with(file_path).and_return(true) + expect(reporter).to receive(:report).with(example_groups.first) + leader.report_file(file_path, example_groups) + end + end + + context 'when spec is released (already reported by other worker)' do + it 'does not delegate to the reporter' do + allow(queue).to receive(:completed?).with(file_path).and_return(true) + allow(queue).to receive(:release).with(file_path).and_return(false) + expect(reporter).not_to receive(:report) + leader.report_file(file_path, example_groups) + end + end + + context 'when spec failed but should be retried' do + it 'placed back to queue and notify reporter' do + spec = './some_spec.rb' + allow(RSpec::Distrib.configuration.error_handler) + .to receive(:retry_test?).with(spec, example_groups, nil).and_return(true) + allow(queue).to receive(:completed?).with(spec).and_return(false) + expect(queue).to receive(:repush).with(spec) + expect(reporter).to receive(:report).with(example_groups.first, will_be_retried: true) + + leader.report_file(spec, example_groups) + end + end + end + + describe '#seed' do + let(:queue) { DistribCore::Leader::QueueWithLease.new([]) } + + it 'returns constant numerical seed' do + expect(leader.seed) + .to eq(leader.seed) + .and be_between(0, 65_535) + end + end + + describe '#notify_non_example_exception' do + let(:exception) do + instance_double(RSpec::Distrib::ExecutionResults::Exception, + original_class: 'FooError', + cause: nil, + message: 'foo', + backtrace: []) + end + let(:context_description) { '' } + let(:queue) { DistribCore::Leader::QueueWithLease.new(%i[a b c]) } + + it 'notifies reporter and stops the service' do + expect(reporter).to receive(:notify_non_example_exception) + .with(exception, context_description) + server = instance_double(DRb::DRbServer) + expect(DRb).to receive(:current_server).and_return(server) + expect(server).to receive(:stop_service) + + leader.notify_non_example_exception(exception, context_description) + end + + context 'when failure should be ignored' do + it 'ignores the failure' do + allow(RSpec::Distrib.configuration.error_handler) + .to receive(:ignore_worker_failure?).with(exception).and_return(true) + + expect(reporter).not_to receive(:notify_non_example_exception) + expect(DRb).not_to receive(:current_server) + + leader.notify_non_example_exception(exception, context_description) + end + end + end + + describe '#report_worker_configuration_error' do + let(:exception) do + instance_double(RSpec::Distrib::ExecutionResults::Exception, + original_class: 'FooError', + cause: nil, + message: 'foo', + backtrace: []) + end + let(:queue) { DistribCore::Leader::QueueWithLease.new(%i[a b c]) } + + it 'stops the service' do + server = instance_double(DRb::DRbServer) + expect(DRb).to receive(:current_server).and_return(server) + expect(server).to receive(:stop_service) + + leader.report_worker_configuration_error(exception) + end + + context 'when failure should be ignored' do + it 'ignores the failure' do + allow(RSpec::Distrib.configuration.error_handler) + .to receive(:ignore_worker_failure?).with(exception).and_return(true) + + expect(DRb).not_to receive(:current_server) + + leader.report_worker_configuration_error(exception) + end + end + end +end diff --git a/rspec-distrib/spec/rspec/distrib/worker/configuration_spec.rb b/rspec-distrib/spec/rspec/distrib/worker/configuration_spec.rb new file mode 100644 index 0000000..46f109f --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/worker/configuration_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +RSpec.describe RSpec::Distrib::Worker::Configuration do + subject(:configuration) { described_class.new.tap { |o| o.leader = leader } } + + let(:leader) { instance_double(RSpec::Distrib::Leader) } + + it do + expect(configuration.formatter_loader.reporter).to be_an_instance_of(RSpec::Distrib::Worker::LeaderReporter) + end + + it { expect(configuration.seed_used?).to be true } +end diff --git a/rspec-distrib/spec/rspec/distrib/worker/leader_reporter_spec.rb b/rspec-distrib/spec/rspec/distrib/worker/leader_reporter_spec.rb new file mode 100644 index 0000000..52db980 --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/worker/leader_reporter_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +RSpec.describe RSpec::Distrib::Worker::LeaderReporter do + describe '#notify_non_example_exception' do + subject(:reporter) { described_class.new(leader, rspec_reporter) } + + let(:leader) { instance_double(RSpec::Distrib::Leader) } + let(:rspec_reporter) { instance_double(RSpec::Core::Reporter).as_null_object } + + it 'notifies the leader' do + expect(leader).to receive(:notify_non_example_exception) + .with(an_instance_of(RSpec::Distrib::ExecutionResults::Exception), :context_description) + reporter.notify_non_example_exception(Exception.new, :context_description) + end + end +end diff --git a/rspec-distrib/spec/rspec/distrib/worker/rspec_runner_spec.rb b/rspec-distrib/spec/rspec/distrib/worker/rspec_runner_spec.rb new file mode 100644 index 0000000..83a5084 --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/worker/rspec_runner_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'tempfile' + +RSpec.describe RSpec::Distrib::Worker::RSpecRunner do # rubocop:disable RSpec/FilePath, RSpec/SpecFilePathFormat + include RSpec::Support::InSubProcess + + # restore the original value to prevent leaking leader double to other examples + # see described_class initializer setting it to global RSpec.configuration + around do |example| + old_configuration = RSpec.configuration + example.run + ensure + RSpec.configuration = old_configuration + end + + let(:leader) { instance_double(RSpec::Distrib::Leader, seed: 1234) } + + it 'gets seed from the leader' do + # Isolate seed setting to configuration of this example only. + in_sub_process_if_possible do + runner = described_class.new(leader) + expect(runner.configuration.seed).to eq(1234) + end + end + + context 'when running' do + let(:temp_file) { Tempfile.new } + + before do + allow(leader).to receive(:report_file) + allow(leader).to receive(:next_file_to_run).and_return(temp_file.path, nil) + end + + after { temp_file.unlink } + + def mock_reporter + report = double.as_null_object + allow(report).to receive(:examples).and_return([]) + + reporter = double.as_null_object + allow(reporter).to receive(:report).and_yield(report) + + reporter + end + + it "consumes leader's queue" do + in_sub_process_if_possible do + RSpec.configuration.reset_reporter + allow(RSpec.configuration).to receive(:reporter).and_return(mock_reporter) + + expect(leader).to receive(:next_file_to_run).and_return(temp_file.path).once + + described_class.new(leader).run($stdout, $stdout) + end + end + end + + describe '.run_from_leader' do + let(:leader) { instance_double(DRbObject) } + let(:runner) { instance_double(described_class) } + + it 'initializes a runner and calls #run' do + allow(DRbObject).to receive(:new_with_uri) + .with(RSpec::Distrib::Leader::DRB_SERVER_URL % 'leader_ip') + .and_return(leader) + expect(described_class).to receive(:new).with(leader).and_return(runner) + expect(runner).to receive(:run) + described_class.run_from_leader('leader_ip') + end + end + + describe '#load_spec_file' do + let(:leader) { instance_double(RSpec::Distrib::Leader, seed: 42) } + let(:runner) { described_class.new(leader) } + + it 'sets color_mode' do + allow(runner.configuration).to receive(:load_spec_files) + expect(runner.configuration.color_mode).to eq(:automatic) + + RSpec::Distrib.configuration.worker_color_mode = :off + runner.__send__ :load_spec_file, 'path/to/spec.rb' + expect(runner.configuration.color_mode).to eq(:off) + + RSpec::Distrib.configuration.worker_color_mode = :on + runner.__send__ :load_spec_file, 'path/to/spec.rb' + expect(runner.configuration.color_mode).to eq(:on) + + RSpec::Distrib.configuration.worker_color_mode = nil + runner.__send__ :load_spec_file, 'path/to/spec.rb' + expect(runner.configuration.color_mode).to eq(:automatic) + end + end +end diff --git a/rspec-distrib/spec/rspec/distrib/worker_spec.rb b/rspec-distrib/spec/rspec/distrib/worker_spec.rb new file mode 100644 index 0000000..2853c43 --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib/worker_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +RSpec.describe RSpec::Distrib::Worker do + describe '.join' do + subject(:join) { described_class.join(leader_ip) } + + context 'when leader ip is specified' do + let(:leader_ip) { '127.0.0.1' } + + it 'joins the leader' do + expect(RSpec::Distrib::Worker::RSpecRunner) + .to receive(:run_from_leader) + .with(leader_ip) + .and_return(0) + + join + end + end + + context 'when leader ip is nil' do + let(:leader_ip) { nil } + + it 'raises an error' do + expect { join }.to raise_error('Leader IP should be specified') + end + end + + context 'when leader ip is empty' do + let(:leader_ip) { '' } + + it 'raises an error' do + expect { join }.to raise_error('Leader IP should be specified') + end + end + end +end diff --git a/rspec-distrib/spec/rspec/distrib_spec.rb b/rspec-distrib/spec/rspec/distrib_spec.rb new file mode 100644 index 0000000..d6c0c0c --- /dev/null +++ b/rspec-distrib/spec/rspec/distrib_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'distrib_core/spec/distrib' + +RSpec.describe RSpec::Distrib do + subject(:root) do + described_class + end + + around do |example| + config = described_class.instance_variable_get(:@configuration) + described_class.instance_variable_set(:@configuration, nil) + example.run + described_class.instance_variable_set(:@configuration, config) + end + + include_examples 'DistribCore root module' +end diff --git a/rspec-distrib/spec/spec_helper.rb b/rspec-distrib/spec/spec_helper.rb new file mode 100644 index 0000000..4953f12 --- /dev/null +++ b/rspec-distrib/spec/spec_helper.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'bundler/setup' + +if ENV['DISABLE_SIMPLECOV'] != '1' + require 'simplecov' + + SimpleCov.start do + add_filter '_spec.rb' + command_name "rspec-distrib-#{Process.pid}" + end + + SimpleCov.at_exit { SimpleCov.instance_variable_set('@result', nil) } +end + +require 'English' +require 'rspec/distrib/leader' +require 'rspec/distrib/worker' + +require 'rspec/support/spec/in_sub_process' +require 'rspec/support/spec/stderr_splitter' + +RSpec.configure do |config| + if ENV['DISABLE_SIMPLECOV'] != '1' + config.after(:suite) do + SimpleCov.result.format! + end + end + + config.order = :random + Kernel.srand config.seed +end