From 8b2cd89890ffefc883e900c8cc62ced8a4c34646 Mon Sep 17 00:00:00 2001 From: przemyslaw-zan <69513154+przemyslaw-zan@users.noreply.github.com> Date: Wed, 18 Sep 2024 17:48:29 +0200 Subject: [PATCH 1/4] Separated feature tests on ci into batches. --- .circleci/template.yml | 12 +- scripts/ci/generate-circleci-configuration.js | 112 ++++++++++++++++-- 2 files changed, 105 insertions(+), 19 deletions(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index 19198f60fa6..546ead18ec1 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -263,7 +263,7 @@ jobs: workflows: version: 2 - # `cke5_tests_framework` and `cke5_tests_features` jobs are generated by the `scripts/ci/generate-config-tests.js` script. + # `cke5_tests_framework` and `cke5_tests_features_batch_n` jobs are generated by the `scripts/ci/generate-config-tests.js` script. main: when: and: @@ -275,7 +275,7 @@ workflows: branches: ignore: - stable - - cke5_tests_features: + - cke5_tests_features_batch_n: filters: branches: ignore: @@ -283,7 +283,7 @@ workflows: - cke5_coverage: requires: - cke5_tests_framework - - cke5_tests_features + - cke5_tests_features_batch_n filters: branches: only: @@ -304,7 +304,7 @@ workflows: - cke5_trigger_uber_ci: requires: - cke5_tests_framework - - cke5_tests_features + - cke5_tests_features_batch_n - cke5_coverage - cke5_validators - cke5_manual @@ -316,7 +316,7 @@ workflows: - cke5_trigger_release_process: requires: - cke5_tests_framework - - cke5_tests_features + - cke5_tests_features_batch_n - cke5_validators - cke5_manual - release_prepare @@ -336,7 +336,7 @@ workflows: when: << pipeline.parameters.isNightly >> jobs: - cke5_tests_framework - - cke5_tests_features + - cke5_tests_features_batch_n - cke5_validators - cke5_manual - release_prepare diff --git a/scripts/ci/generate-circleci-configuration.js b/scripts/ci/generate-circleci-configuration.js index 398bd10c536..7c3a9b298da 100755 --- a/scripts/ci/generate-circleci-configuration.js +++ b/scripts/ci/generate-circleci-configuration.js @@ -24,6 +24,18 @@ const IS_COMMUNITY_PR = require( './is-community-pr' ); const CKEDITOR5_ROOT_DIRECTORY = upath.join( __dirname, '..', '..' ); const CIRCLECI_CONFIGURATION_DIRECTORY = upath.join( CKEDITOR5_ROOT_DIRECTORY, '.circleci' ); +const FEATURE_BATCH_NAME_PLACEHOLDER = 'cke5_tests_features_batch_n'; + +/** + * This variable determines amount and size of feature test batches. + * + * If there are more feature packages than the sum of all batches defined here, + * one batch will be automatically added to cover remaining tests. + */ +const FEATURE_BATCH_SIZES = [ + 30 +]; + const NON_FULL_COVERAGE_PACKAGES = [ 'ckeditor5-minimap' ]; @@ -67,6 +79,29 @@ const persistToWorkspace = fileName => ( { await fs.readFile( upath.join( CIRCLECI_CONFIGURATION_DIRECTORY, 'template.yml' ) ) ); + const featureTestBatches = featurePackages.reduce( ( output, packageName, packageIndex ) => { + let currentBatch = FEATURE_BATCH_SIZES.findIndex( ( batchSize, batchIndex, allBatches ) => { + return packageIndex < allBatches.slice( 0, batchIndex + 1 ).reduce( ( a, b ) => a + b ); + } ); + + // Additional batch for the remaining tests not included in defined batch sizes. + if ( currentBatch === -1 ) { + currentBatch = FEATURE_BATCH_SIZES.length; + } + + if ( !output[ currentBatch ] ) { + output[ currentBatch ] = []; + } + + output[ currentBatch ].push( packageName ); + + return output; + }, [] ); + + const featureTestBatchNames = featureTestBatches.map( ( batch, batchIndex ) => { + return FEATURE_BATCH_NAME_PLACEHOLDER.replace( /(?<=_)n$/, batchIndex + 1 ); + } ); + config.jobs.cke5_tests_framework = { machine: true, steps: [ @@ -81,19 +116,60 @@ const persistToWorkspace = fileName => ( { ] }; - config.jobs.cke5_tests_features = { - machine: true, - steps: [ - ...bootstrapCommands(), - prepareCodeCoverageDirectories(), - ...generateTestSteps( featurePackages, { - checkCoverage: true, - coverageFile: '.out/combined_features.info' - } ), - 'community_verification_command', - persistToWorkspace( 'combined_features.info' ) - ] - }; + // Adding batches to the root `jobs`. + featureTestBatches.forEach( ( batch, batchIndex ) => { + config.jobs[ featureTestBatchNames[ batchIndex ] ] = { + machine: true, + steps: [ + ...bootstrapCommands(), + prepareCodeCoverageDirectories(), + ...generateTestSteps( batch, { + checkCoverage: true, + coverageFile: '.out/combined_features.info' + } ), + 'community_verification_command', + persistToWorkspace( 'combined_features.info' ) + ] + }; + } ); + + Object.values( config.workflows ).forEach( workflow => { + if ( !( workflow instanceof Object ) ) { + return; + } + + if ( !workflow.jobs ) { + return; + } + + // Replacing the placeholder batch names in `requires` arrays in `workflows`. + workflow.jobs.forEach( job => { + const { requires } = Object.values( job )[ 0 ]; + + if ( requires ) { + replacePlaceholderBatchNameInArray( requires, featureTestBatchNames ); + } + } ); + + // Replacing the placeholder batch names in `jobs` arrays in `workflows`. + replacePlaceholderBatchNameInArray( workflow.jobs, featureTestBatchNames ); + + // Replacing the placeholder batch objects in `jobs` arrays in `workflows`. + const placeholderJobIndex = workflow.jobs.findIndex( job => job[ FEATURE_BATCH_NAME_PLACEHOLDER ] ); + + if ( placeholderJobIndex === -1 ) { + return; + } + + const placeholderJobContent = workflow.jobs[ placeholderJobIndex ][ FEATURE_BATCH_NAME_PLACEHOLDER ]; + const newBatchJobs = featureTestBatchNames.map( featureTestBatchName => { + return { + [ featureTestBatchName ]: placeholderJobContent + }; + } ); + + workflow.jobs.splice( placeholderJobIndex, 1, ...newBatchJobs ); + } ); if ( IS_COMMUNITY_PR ) { // CircleCI does not understand custom cloning when a PR comes from the community. @@ -158,6 +234,16 @@ function replaceShortCheckout( config, jobName ) { } ); } +function replacePlaceholderBatchNameInArray( array, featureTestBatchNames ) { + const placeholderIndex = array.findIndex( item => item === FEATURE_BATCH_NAME_PLACEHOLDER ); + + if ( placeholderIndex === -1 ) { + return; + } + + array.splice( placeholderIndex, 1, ...featureTestBatchNames ); +} + /** * This type partially covers supported options on CircleCI. * To see the complete guide, follow: https://circleci.com/docs/configuration-reference. From 528a6412604956fdeb052f6b6c610513ebe8334c Mon Sep 17 00:00:00 2001 From: przemyslaw-zan <69513154+przemyslaw-zan@users.noreply.github.com> Date: Wed, 18 Sep 2024 18:35:18 +0200 Subject: [PATCH 2/4] Enabled coverage, added info step. --- .circleci/template.yml | 2 +- scripts/ci/generate-circleci-configuration.js | 48 ++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index 546ead18ec1..ae55fe71692 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -161,7 +161,7 @@ jobs: at: .out - run: name: Merge the code coverage of the framework and features - command: cat .out/combined_framework.info .out/combined_features.info > .out/combined_lcov.info + command: cat .out/combined_framework.info .out/combined_features_batch_n.info > .out/combined_lcov.info - run: name: Preparing the environment variables command: | diff --git a/scripts/ci/generate-circleci-configuration.js b/scripts/ci/generate-circleci-configuration.js index 7c3a9b298da..6065942a9d9 100755 --- a/scripts/ci/generate-circleci-configuration.js +++ b/scripts/ci/generate-circleci-configuration.js @@ -24,7 +24,8 @@ const IS_COMMUNITY_PR = require( './is-community-pr' ); const CKEDITOR5_ROOT_DIRECTORY = upath.join( __dirname, '..', '..' ); const CIRCLECI_CONFIGURATION_DIRECTORY = upath.join( CKEDITOR5_ROOT_DIRECTORY, '.circleci' ); -const FEATURE_BATCH_NAME_PLACEHOLDER = 'cke5_tests_features_batch_n'; +const FEATURE_TEST_BATCH_NAME_PLACEHOLDER = 'cke5_tests_features_batch_n'; +const FEATURE_COVERAGE_BATCH_FILENAME_PLACEHOLDER = '.out/combined_features_batch_n.info'; /** * This variable determines amount and size of feature test batches. @@ -55,6 +56,21 @@ const prepareCodeCoverageDirectories = () => ( { } } ); +const listBatchPackages = packageNames => { + const text = [ + `A total of ${ packageNames.length } packages will be ran in this test batch:`, + packageNames.map( packageName => ` - ${ packageName }` ).join( '\\n' ) + ].join( '\\n\\n' ); + + return { + run: { + when: 'always', + name: 'List of packages running in this test batch', + command: `printf "${ text }"` + } + }; +}; + const persistToWorkspace = fileName => ( { persist_to_workspace: { root: '.out', @@ -99,7 +115,10 @@ const persistToWorkspace = fileName => ( { }, [] ); const featureTestBatchNames = featureTestBatches.map( ( batch, batchIndex ) => { - return FEATURE_BATCH_NAME_PLACEHOLDER.replace( /(?<=_)n$/, batchIndex + 1 ); + return FEATURE_TEST_BATCH_NAME_PLACEHOLDER.replace( /(?<=_)n$/, batchIndex + 1 ); + } ); + const featureCoverageBatchFilenames = featureTestBatches.map( ( batch, batchIndex ) => { + return FEATURE_COVERAGE_BATCH_FILENAME_PLACEHOLDER.replace( /(?<=_)n(?=\.info$)/, batchIndex + 1 ); } ); config.jobs.cke5_tests_framework = { @@ -123,9 +142,10 @@ const persistToWorkspace = fileName => ( { steps: [ ...bootstrapCommands(), prepareCodeCoverageDirectories(), + listBatchPackages( batch ), ...generateTestSteps( batch, { checkCoverage: true, - coverageFile: '.out/combined_features.info' + coverageFile: featureCoverageBatchFilenames[ batchIndex ] } ), 'community_verification_command', persistToWorkspace( 'combined_features.info' ) @@ -155,13 +175,13 @@ const persistToWorkspace = fileName => ( { replacePlaceholderBatchNameInArray( workflow.jobs, featureTestBatchNames ); // Replacing the placeholder batch objects in `jobs` arrays in `workflows`. - const placeholderJobIndex = workflow.jobs.findIndex( job => job[ FEATURE_BATCH_NAME_PLACEHOLDER ] ); + const placeholderJobIndex = workflow.jobs.findIndex( job => job[ FEATURE_TEST_BATCH_NAME_PLACEHOLDER ] ); if ( placeholderJobIndex === -1 ) { return; } - const placeholderJobContent = workflow.jobs[ placeholderJobIndex ][ FEATURE_BATCH_NAME_PLACEHOLDER ]; + const placeholderJobContent = workflow.jobs[ placeholderJobIndex ][ FEATURE_TEST_BATCH_NAME_PLACEHOLDER ]; const newBatchJobs = featureTestBatchNames.map( featureTestBatchName => { return { [ featureTestBatchName ]: placeholderJobContent @@ -171,6 +191,22 @@ const persistToWorkspace = fileName => ( { workflow.jobs.splice( placeholderJobIndex, 1, ...newBatchJobs ); } ); + // Replacing the coverage filename placeholder in coverage steps. + Object.values( config.jobs.cke5_coverage.steps ).forEach( step => { + if ( !( step instanceof Object ) ) { + return; + } + + if ( !step.run || !step.run.command ) { + return; + } + + step.run.command = step.run.command.replace( + FEATURE_COVERAGE_BATCH_FILENAME_PLACEHOLDER, + featureCoverageBatchFilenames.join( ' ' ) + ); + } ); + if ( IS_COMMUNITY_PR ) { // CircleCI does not understand custom cloning when a PR comes from the community. // In such a case, the goal to use the built-in command. @@ -235,7 +271,7 @@ function replaceShortCheckout( config, jobName ) { } function replacePlaceholderBatchNameInArray( array, featureTestBatchNames ) { - const placeholderIndex = array.findIndex( item => item === FEATURE_BATCH_NAME_PLACEHOLDER ); + const placeholderIndex = array.findIndex( item => item === FEATURE_TEST_BATCH_NAME_PLACEHOLDER ); if ( placeholderIndex === -1 ) { return; From 0d10672d8f6262a55b93cf5d4f3682361ae34650 Mon Sep 17 00:00:00 2001 From: przemyslaw-zan <69513154+przemyslaw-zan@users.noreply.github.com> Date: Wed, 18 Sep 2024 18:42:55 +0200 Subject: [PATCH 3/4] Updated message. --- scripts/ci/generate-circleci-configuration.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/ci/generate-circleci-configuration.js b/scripts/ci/generate-circleci-configuration.js index 6065942a9d9..88fdd723e00 100755 --- a/scripts/ci/generate-circleci-configuration.js +++ b/scripts/ci/generate-circleci-configuration.js @@ -58,14 +58,15 @@ const prepareCodeCoverageDirectories = () => ( { const listBatchPackages = packageNames => { const text = [ - `A total of ${ packageNames.length } packages will be ran in this test batch:`, - packageNames.map( packageName => ` - ${ packageName }` ).join( '\\n' ) + `A total of ${ packageNames.length } packages will be tested in this batch:`, + packageNames.map( packageName => ` - ${ packageName }` ).join( '\\n' ), + '' ].join( '\\n\\n' ); return { run: { when: 'always', - name: 'List of packages running in this test batch', + name: 'List packages tested in this batch', command: `printf "${ text }"` } }; From 62b0b563a0ba4a2329103d6b49b7df1edef70af8 Mon Sep 17 00:00:00 2001 From: przemyslaw-zan <69513154+przemyslaw-zan@users.noreply.github.com> Date: Thu, 19 Sep 2024 11:37:31 +0200 Subject: [PATCH 4/4] Fixed coverage. --- scripts/ci/generate-circleci-configuration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/generate-circleci-configuration.js b/scripts/ci/generate-circleci-configuration.js index 88fdd723e00..b614163966b 100755 --- a/scripts/ci/generate-circleci-configuration.js +++ b/scripts/ci/generate-circleci-configuration.js @@ -149,7 +149,7 @@ const persistToWorkspace = fileName => ( { coverageFile: featureCoverageBatchFilenames[ batchIndex ] } ), 'community_verification_command', - persistToWorkspace( 'combined_features.info' ) + persistToWorkspace( featureCoverageBatchFilenames[ batchIndex ].replace( /^\.out\//, '' ) ) ] }; } );