diff --git a/.github/workflows/batch_rector.yaml b/.github/workflows/batch_rector.yaml new file mode 100644 index 00000000000..a48f54543ec --- /dev/null +++ b/.github/workflows/batch_rector.yaml @@ -0,0 +1,44 @@ +# Test workflow to show how this feature works. DO NOT MERGE, REMOVE IN FINAL VERSION OF PR +name: Rector + +on: + pull_request: null + +jobs: + no_batch_rector: + + runs-on: ubuntu-latest + + steps: + - + uses: actions/checkout@v4 + + - + uses: shivammathur/setup-php@v2 + with: + php-version: 8.2 + coverage: none + + - run: composer install --no-progress --ansi + + - run: bin/rector process --ansi + batch_rector: + strategy: + matrix: + batches: [0, 1, 2, 3] + + runs-on: ubuntu-latest + + steps: + - + uses: actions/checkout@v4 + + - + uses: shivammathur/setup-php@v2 + with: + php-version: 8.2 + coverage: none + + - run: composer install --no-progress --ansi + + - run: bin/rector process --batch-index=${{ strategy.job-index }} --batch-total=${{ strategy.job-total }} --ansi diff --git a/.github/workflows/e2e_with_batches.yaml b/.github/workflows/e2e_with_batches.yaml new file mode 100644 index 00000000000..adf7a99b04d --- /dev/null +++ b/.github/workflows/e2e_with_batches.yaml @@ -0,0 +1,42 @@ +# This workflow runs system tests: Use the Rector application from the source +# checkout to process "fixture" projects in e2e/batch-run directory +# to see if those can be processed successfully in batches +name: End to End tests with batches + +on: + pull_request: null + +env: + # see https://github.com/composer/composer/issues/9368#issuecomment-718112361 + COMPOSER_ROOT_VERSION: "dev-main" + +jobs: + end_to_end: + runs-on: ubuntu-latest + timeout-minutes: 3 + strategy: + fail-fast: false + matrix: + batches: [0, 1] + + name: End to end test with batches - batch ${{ strategy.job-index }} + + steps: + - uses: actions/checkout@v4 + + - uses: shivammathur/setup-php@v2 + with: + php-version: 8.2 + coverage: none + + # run in root rector-src + - run: composer install --ansi + + # run in e2e subdir + - + run: composer install --ansi + working-directory: e2e/batch-run + + # run e2e test + - run: php ../e2eTestRunnerWithBatches.php ${{ strategy.job-index }} ${{ strategy.job-total }} + working-directory: e2e/batch-run diff --git a/e2e/batch-run/.gitignore b/e2e/batch-run/.gitignore new file mode 100644 index 00000000000..61ead86667c --- /dev/null +++ b/e2e/batch-run/.gitignore @@ -0,0 +1 @@ +/vendor diff --git a/e2e/batch-run/composer.json b/e2e/batch-run/composer.json new file mode 100644 index 00000000000..5468cd74606 --- /dev/null +++ b/e2e/batch-run/composer.json @@ -0,0 +1,7 @@ +{ + "require": { + "php": "^8.1" + }, + "minimum-stability": "dev", + "prefer-stable": true +} diff --git a/e2e/batch-run/expected-output-0.diff b/e2e/batch-run/expected-output-0.diff new file mode 100644 index 00000000000..c7308a5391b --- /dev/null +++ b/e2e/batch-run/expected-output-0.diff @@ -0,0 +1,22 @@ +1 file with changes +=================== + +1) src/RenameDocblock.php:0 + + ---------- begin diff ---------- +@@ @@ + paths([ + __DIR__ . '/src', + ]); + + $rectorConfig->ruleWithConfiguration(RenameClassRector::class, [ + 'DateTime' => 'DateTimeInterface' + ]); + $rectorConfig->ruleWithConfiguration(DowngradeAttributeToAnnotationRector::class, [ + new DowngradeAttributeToAnnotation('Symfony\Component\Routing\Annotation\Route') + ]); + + $rectorConfig->rule(RemoveUselessVarTagRector::class); +}; diff --git a/e2e/batch-run/src/AlreadyChangedDocblock.php b/e2e/batch-run/src/AlreadyChangedDocblock.php new file mode 100644 index 00000000000..3c839fbfa12 --- /dev/null +++ b/e2e/batch-run/src/AlreadyChangedDocblock.php @@ -0,0 +1,8 @@ +create(); + +$matchedExpectedOutput = false; +$expectedOutput = trim(file_get_contents($expectedDiff)); +if ($output === $expectedOutput) { + $symfonyStyle->success('End-to-end test successfully completed'); + exit(Command::SUCCESS); +} + +// print color diff, to make easy find the differences +$consoleDiffer = new ConsoleDiffer(new ColorConsoleDiffFormatter()); +$diff = $consoleDiffer->diff($output, $expectedOutput); +$symfonyStyle->writeln($diff); + +exit(Command::FAILURE); diff --git a/rector.php b/rector.php index 4a6d3db3656..6d0456dc79a 100644 --- a/rector.php +++ b/rector.php @@ -7,6 +7,7 @@ use Rector\DeadCode\Rector\ConstFetch\RemovePhpVersionIdCheckRector; use Rector\DeadCode\Rector\Property\RemoveUnusedPrivatePropertyRector; use Rector\Php55\Rector\String_\StringClassNameToClassConstantRector; +use Rector\TypeDeclaration\Rector\Expression\InlineVarDocTagToAssertRector; return RectorConfig::configure() ->withPreparedSets( @@ -57,4 +58,7 @@ ], RemoveUnusedPrivatePropertyRector::class => [__DIR__ . '/src/Configuration/RectorConfigBuilder.php'], - ]); + ]) + // This is just to show how it works in github actions. DO NOT MERGE, REMOVE IN FINAL VERSION OF PR + ->withRules([InlineVarDocTagToAssertRector::class]) +; diff --git a/src/Application/ApplicationFileProcessor.php b/src/Application/ApplicationFileProcessor.php index 852a6e32de0..091d0a85eb6 100644 --- a/src/Application/ApplicationFileProcessor.php +++ b/src/Application/ApplicationFileProcessor.php @@ -16,6 +16,7 @@ use Rector\Reporting\MissConfigurationReporter; use Rector\Testing\PHPUnit\StaticPHPUnitEnvironment; use Rector\Util\ArrayParametersMerger; +use Rector\Util\BatchSplitter; use Rector\ValueObject\Application\File; use Rector\ValueObject\Configuration; use Rector\ValueObject\Error\SystemError; @@ -61,6 +62,13 @@ public function run(Configuration $configuration, InputInterface $input): Proces $this->missConfigurationReporter->reportVendorInPaths($filePaths); $this->missConfigurationReporter->reportStartWithShortOpenTag(); + $batchSplitter = new BatchSplitter(); + $filePaths = $batchSplitter->getItemsInBatch( + $filePaths, + $configuration->getBatchIndex(), + $configuration->getBatchTotal() + ); + // no files found if ($filePaths === []) { return new ProcessResult([], []); diff --git a/src/Configuration/ConfigurationFactory.php b/src/Configuration/ConfigurationFactory.php index 9fb6eb0672f..952f64d2d79 100644 --- a/src/Configuration/ConfigurationFactory.php +++ b/src/Configuration/ConfigurationFactory.php @@ -41,7 +41,9 @@ public function createForTests(array $paths): Configuration false, null, false, - false + false, + 0, + 0, ); } @@ -65,11 +67,14 @@ public function createFromInput(InputInterface $input): Configuration $isParallel = SimpleParameterProvider::provideBoolParameter(Option::PARALLEL); $parallelPort = (string) $input->getOption(Option::PARALLEL_PORT); $parallelIdentifier = (string) $input->getOption(Option::PARALLEL_IDENTIFIER); + $batchIndex = (int) $input->getOption(Option::BATCH_INDEX); + $batchTotal = (int) $input->getOption(Option::BATCH_TOTAL); $isDebug = (bool) $input->getOption(Option::DEBUG); - // using debug disables parallel, so emitting exception is straightforward and easier to debug + // using debug disables parallel and batch running, so emitting exception is straightforward and easier to debug if ($isDebug) { $isParallel = false; + $batchTotal = 0; } $memoryLimit = $this->resolveMemoryLimit($input); @@ -90,6 +95,8 @@ public function createFromInput(InputInterface $input): Configuration $memoryLimit, $isDebug, $isReportingWithRealPath, + $batchIndex, + $batchTotal, ); } diff --git a/src/Configuration/Option.php b/src/Configuration/Option.php index 9a000853944..0145af0db32 100644 --- a/src/Configuration/Option.php +++ b/src/Configuration/Option.php @@ -184,6 +184,16 @@ final class Option */ public const PARALLEL_PORT = 'port'; + /** + * @var string + */ + public const BATCH_INDEX = 'batch-index'; + + /** + * @var string + */ + public const BATCH_TOTAL = 'batch-total'; + /** * @internal Use @see \Rector\Config\RectorConfig::parallel() instead with pass int $jobSize parameter * @var string diff --git a/src/Console/Command/ProcessCommand.php b/src/Console/Command/ProcessCommand.php index acfa8e8c31c..3097e34c8cb 100644 --- a/src/Console/Command/ProcessCommand.php +++ b/src/Console/Command/ProcessCommand.php @@ -83,6 +83,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int } $configuration = $this->configurationFactory->createFromInput($input); + + if ($configuration->getBatchTotal() !== 0 && $configuration->getBatchIndex() >= $configuration->getBatchTotal()) { + $this->symfonyStyle->error('The job index needs to be less than the job total'); + return ExitCode::FAILURE; + } + $this->memoryLimiter->adjust($configuration); // disable console output in case of json output formatter diff --git a/src/Console/ConsoleApplication.php b/src/Console/ConsoleApplication.php index ce3940f44d1..0c1f70a4da8 100644 --- a/src/Console/ConsoleApplication.php +++ b/src/Console/ConsoleApplication.php @@ -129,6 +129,20 @@ private function addCustomOptions(InputDefinition $inputDefinition): void InputOption::VALUE_NONE, 'Clear cache' )); + + $inputDefinition->addOption(new InputOption( + Option::BATCH_INDEX, + null, + InputOption::VALUE_REQUIRED, + 'Index of the current job when running in batch mode' + )); + + $inputDefinition->addOption(new InputOption( + Option::BATCH_TOTAL, + null, + InputOption::VALUE_REQUIRED, + 'Total number of jobs when running in batch mode' + )); } private function getDefaultConfigPath(): string diff --git a/src/Util/BatchSplitter.php b/src/Util/BatchSplitter.php new file mode 100644 index 00000000000..4d3fbf0e131 --- /dev/null +++ b/src/Util/BatchSplitter.php @@ -0,0 +1,27 @@ + 0) { + $chunks = array_chunk($items, $chunkSize); + $items = $chunks[$batchIndex] ?? []; + } + } + + return $items; + } +} diff --git a/src/ValueObject/Configuration.php b/src/ValueObject/Configuration.php index b99186a5894..9545638f893 100644 --- a/src/ValueObject/Configuration.php +++ b/src/ValueObject/Configuration.php @@ -26,7 +26,9 @@ public function __construct( private bool $isParallel = false, private string|null $memoryLimit = null, private bool $isDebug = false, - private bool $reportingWithRealPath = false + private bool $reportingWithRealPath = false, + private int $batchIndex = 0, + private int $batchTotal = 0, ) { } @@ -101,4 +103,14 @@ public function isReportingWithRealPath(): bool { return $this->reportingWithRealPath; } + + public function getBatchIndex(): int + { + return $this->batchIndex; + } + + public function getBatchTotal(): int + { + return $this->batchTotal; + } } diff --git a/tests/Util/BatchSplitterTest.php b/tests/Util/BatchSplitterTest.php new file mode 100644 index 00000000000..212b40cfc6b --- /dev/null +++ b/tests/Util/BatchSplitterTest.php @@ -0,0 +1,51 @@ +batchSplitter = $this->make(BatchSplitter::class); + } + + /** + * @param int[] $items + * @param int[] $expectedItems + */ + #[DataProvider('provideData')] + public function testGetItemsInBatch( + array $items, + int $batchIndex, + int $batchTotal, + array $expectedItems + ): void { + $batchItems = $this->batchSplitter->getItemsInBatch($items, $batchIndex, $batchTotal); + $this->assertSame($expectedItems, $batchItems); + } + + public static function provideData(): Iterator + { + yield [[], 1, 4, []]; + yield [[1, 2, 3, 4], 0, 0, [1, 2, 3, 4]]; + yield [[1, 2, 3, 4], 3, 2, [1, 2, 3, 4]]; + yield [[1, 2, 3, 4], 0, 2, [1, 2]]; + yield [[1, 2, 3, 4], 1, 2, [3, 4]]; + yield [[1, 2, 3], 0, 2, [1, 2]]; + yield [[1, 2, 3], 1, 2, [3]]; + yield [[1, 2, 3, 4], 0, 3, [1, 2]]; + yield [[1, 2, 3, 4], 1, 3, [3, 4]]; + yield [[1, 2, 3, 4], 2, 3, []]; + yield [[1, 2, 3], 0, 4, [1]]; + yield [[1, 2, 3], 3, 4, []]; + } +}