From 8b2cd89890ffefc883e900c8cc62ced8a4c34646 Mon Sep 17 00:00:00 2001 From: przemyslaw-zan <69513154+przemyslaw-zan@users.noreply.github.com> Date: Wed, 18 Sep 2024 17:48:29 +0200 Subject: [PATCH] Separated feature tests on ci into batches. --- .circleci/template.yml | 12 +- scripts/ci/generate-circleci-configuration.js | 112 ++++++++++++++++-- 2 files changed, 105 insertions(+), 19 deletions(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index 19198f60fa6..546ead18ec1 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -263,7 +263,7 @@ jobs: workflows: version: 2 - # `cke5_tests_framework` and `cke5_tests_features` jobs are generated by the `scripts/ci/generate-config-tests.js` script. + # `cke5_tests_framework` and `cke5_tests_features_batch_n` jobs are generated by the `scripts/ci/generate-config-tests.js` script. main: when: and: @@ -275,7 +275,7 @@ workflows: branches: ignore: - stable - - cke5_tests_features: + - cke5_tests_features_batch_n: filters: branches: ignore: @@ -283,7 +283,7 @@ workflows: - cke5_coverage: requires: - cke5_tests_framework - - cke5_tests_features + - cke5_tests_features_batch_n filters: branches: only: @@ -304,7 +304,7 @@ workflows: - cke5_trigger_uber_ci: requires: - cke5_tests_framework - - cke5_tests_features + - cke5_tests_features_batch_n - cke5_coverage - cke5_validators - cke5_manual @@ -316,7 +316,7 @@ workflows: - cke5_trigger_release_process: requires: - cke5_tests_framework - - cke5_tests_features + - cke5_tests_features_batch_n - cke5_validators - cke5_manual - release_prepare @@ -336,7 +336,7 @@ workflows: when: << pipeline.parameters.isNightly >> jobs: - cke5_tests_framework - - cke5_tests_features + - cke5_tests_features_batch_n - cke5_validators - cke5_manual - release_prepare diff --git a/scripts/ci/generate-circleci-configuration.js b/scripts/ci/generate-circleci-configuration.js index 398bd10c536..7c3a9b298da 100755 --- a/scripts/ci/generate-circleci-configuration.js +++ b/scripts/ci/generate-circleci-configuration.js @@ -24,6 +24,18 @@ const IS_COMMUNITY_PR = require( './is-community-pr' ); const CKEDITOR5_ROOT_DIRECTORY = upath.join( __dirname, '..', '..' ); const CIRCLECI_CONFIGURATION_DIRECTORY = upath.join( CKEDITOR5_ROOT_DIRECTORY, '.circleci' ); +const FEATURE_BATCH_NAME_PLACEHOLDER = 'cke5_tests_features_batch_n'; + +/** + * This variable determines amount and size of feature test batches. + * + * If there are more feature packages than the sum of all batches defined here, + * one batch will be automatically added to cover remaining tests. + */ +const FEATURE_BATCH_SIZES = [ + 30 +]; + const NON_FULL_COVERAGE_PACKAGES = [ 'ckeditor5-minimap' ]; @@ -67,6 +79,29 @@ const persistToWorkspace = fileName => ( { await fs.readFile( upath.join( CIRCLECI_CONFIGURATION_DIRECTORY, 'template.yml' ) ) ); + const featureTestBatches = featurePackages.reduce( ( output, packageName, packageIndex ) => { + let currentBatch = FEATURE_BATCH_SIZES.findIndex( ( batchSize, batchIndex, allBatches ) => { + return packageIndex < allBatches.slice( 0, batchIndex + 1 ).reduce( ( a, b ) => a + b ); + } ); + + // Additional batch for the remaining tests not included in defined batch sizes. + if ( currentBatch === -1 ) { + currentBatch = FEATURE_BATCH_SIZES.length; + } + + if ( !output[ currentBatch ] ) { + output[ currentBatch ] = []; + } + + output[ currentBatch ].push( packageName ); + + return output; + }, [] ); + + const featureTestBatchNames = featureTestBatches.map( ( batch, batchIndex ) => { + return FEATURE_BATCH_NAME_PLACEHOLDER.replace( /(?<=_)n$/, batchIndex + 1 ); + } ); + config.jobs.cke5_tests_framework = { machine: true, steps: [ @@ -81,19 +116,60 @@ const persistToWorkspace = fileName => ( { ] }; - config.jobs.cke5_tests_features = { - machine: true, - steps: [ - ...bootstrapCommands(), - prepareCodeCoverageDirectories(), - ...generateTestSteps( featurePackages, { - checkCoverage: true, - coverageFile: '.out/combined_features.info' - } ), - 'community_verification_command', - persistToWorkspace( 'combined_features.info' ) - ] - }; + // Adding batches to the root `jobs`. + featureTestBatches.forEach( ( batch, batchIndex ) => { + config.jobs[ featureTestBatchNames[ batchIndex ] ] = { + machine: true, + steps: [ + ...bootstrapCommands(), + prepareCodeCoverageDirectories(), + ...generateTestSteps( batch, { + checkCoverage: true, + coverageFile: '.out/combined_features.info' + } ), + 'community_verification_command', + persistToWorkspace( 'combined_features.info' ) + ] + }; + } ); + + Object.values( config.workflows ).forEach( workflow => { + if ( !( workflow instanceof Object ) ) { + return; + } + + if ( !workflow.jobs ) { + return; + } + + // Replacing the placeholder batch names in `requires` arrays in `workflows`. + workflow.jobs.forEach( job => { + const { requires } = Object.values( job )[ 0 ]; + + if ( requires ) { + replacePlaceholderBatchNameInArray( requires, featureTestBatchNames ); + } + } ); + + // Replacing the placeholder batch names in `jobs` arrays in `workflows`. + replacePlaceholderBatchNameInArray( workflow.jobs, featureTestBatchNames ); + + // Replacing the placeholder batch objects in `jobs` arrays in `workflows`. + const placeholderJobIndex = workflow.jobs.findIndex( job => job[ FEATURE_BATCH_NAME_PLACEHOLDER ] ); + + if ( placeholderJobIndex === -1 ) { + return; + } + + const placeholderJobContent = workflow.jobs[ placeholderJobIndex ][ FEATURE_BATCH_NAME_PLACEHOLDER ]; + const newBatchJobs = featureTestBatchNames.map( featureTestBatchName => { + return { + [ featureTestBatchName ]: placeholderJobContent + }; + } ); + + workflow.jobs.splice( placeholderJobIndex, 1, ...newBatchJobs ); + } ); if ( IS_COMMUNITY_PR ) { // CircleCI does not understand custom cloning when a PR comes from the community. @@ -158,6 +234,16 @@ function replaceShortCheckout( config, jobName ) { } ); } +function replacePlaceholderBatchNameInArray( array, featureTestBatchNames ) { + const placeholderIndex = array.findIndex( item => item === FEATURE_BATCH_NAME_PLACEHOLDER ); + + if ( placeholderIndex === -1 ) { + return; + } + + array.splice( placeholderIndex, 1, ...featureTestBatchNames ); +} + /** * This type partially covers supported options on CircleCI. * To see the complete guide, follow: https://circleci.com/docs/configuration-reference.