diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml index ab67e915de8..d1db47b30b2 100644 --- a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -48,6 +48,12 @@ parameters: - name: PRMatrixSetting type: string default: 'ArtifactPackageNames' +- name: PRJobBatchSize + type: number + default: 10 +- name: PRMatrixIndirectFilters + type: object + default: [] # Mappings to OS name required at template compile time by 1es pipeline templates - name: Pools type: object @@ -126,7 +132,6 @@ jobs: - ${{ else }}: - ${{ each pool in parameters.Pools }}: - pwsh: | - # dump the conglomerated CI matrix '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` @@ -135,7 +140,9 @@ jobs: -PRMatrixSetting ${{ parameters.PRMatrixSetting }} ` -DisplayNameFilter '$(displayNameFilter)' ` -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' ` - -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + -IndirectFilters '${{ join(''',''', parameters.PRMatrixIndirectFilters) }}' ` + -Replace '${{ join(''',''', parameters.MatrixReplace) }}' ` + -PackagesPerPRJob ${{ parameters.PRJobBatchSize }} displayName: Create ${{ pool.name }} PR Matrix name: vm_job_matrix_pr_${{ pool.name }} diff --git a/eng/common/pipelines/templates/steps/verify-readmes.yml b/eng/common/pipelines/templates/steps/verify-readmes.yml index 09628b37389..40994767e8c 100644 --- a/eng/common/pipelines/templates/steps/verify-readmes.yml +++ b/eng/common/pipelines/templates/steps/verify-readmes.yml @@ -13,13 +13,20 @@ parameters: - name: Condition type: string default: succeeded() +- name: IncludeIndirect + type: boolean + default: true steps: - pwsh: | + $includeIndirect = $${{ parameters.IncludeIndirect }} $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackagePropertiesFolder }}" *.json - $paths = @() + if (-not $includeIndirect) { + $packageProperties = $packageProperties | Where-Object { (Get-Content -Raw $_ | ConvertFrom-Json).IncludedForValidation -eq $false } + } + foreach($propertiesFile in $packageProperties) { $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 index 1d86cfcd79d..8e18e2479ca 100644 --- a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 @@ -23,6 +23,12 @@ It generates the matrix by the following algorithm: - add the combined property name to the parameters of the matrix item - add the matrix item to the overall result +.PARAMETER IndirectFilters +Any array of strings representing filters that will only be applied to the matrix generation for indirect packages. This is useful for +filtering out OTHER parameter settings othan than PRMatrixSetting that are only relevant to direct packages. + +For .NET, this value will be AdditionalTestArguments=/p:UseProjectReferenceToAzureClients=true + .EXAMPLE ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 ` -PackagePropertiesFolder "path/to/populated/PackageInfo" ` @@ -36,13 +42,158 @@ param ( [Parameter(Mandatory = $true)][string] $PRMatrixSetting, [Parameter(Mandatory = $False)][string] $DisplayNameFilter, [Parameter(Mandatory = $False)][array] $Filters, + [Parameter(Mandatory = $False)][array] $IndirectFilters, [Parameter(Mandatory = $False)][array] $Replace, + [Parameter(Mandatory = $False)][int] $PackagesPerPRJob = 10, [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID) ) +Set-StrictMode -Version 4 . $PSScriptRoot/job-matrix-functions.ps1 . $PSScriptRoot/../Helpers/Package-Helpers.ps1 -$BATCHSIZE = 10 +. $PSScriptRoot/../Package-Properties.ps1 +$BATCHSIZE = $PackagesPerPRJob + +# this function takes an array of objects, takes a copy of the first item, and moves that item to the back of the array +function QueuePop([ref]$queue) { + + if ($queue.Value.Length -eq 1) { + return ($queue.Value[0] | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable) + } + + # otherwise we can rotate stuff + $first = $queue.Value[0] + $rest = $queue.Value[1..($queue.Value.Length - 1)] + + $queue.Value = $rest + $first + + return ($first | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable) +} + +function GeneratePRMatrixForBatch { + param ( + [Parameter(Mandatory = $true)][array] $Packages + ) + + $OverallResult = @() + if (!$Packages) { + Write-Host "Unable to generate matrix for empty package list" + return ,$OverallResult + } + + # this check assumes that we have properly separated the direct and indirect package lists + $directBatch = $Packages[0].IncludedForValidation -eq $false + Write-Host "Generating matrix for $($directBatch ? 'direct' : 'indirect') packages" + + # The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig + # to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config. + $matrixBatchesByConfig = Group-ByObjectKey $Packages "CIMatrixConfigs" + + foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { + $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] + $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs + + $matrixResults = @() + foreach ($matrixConfig in $matrixConfigs) { + Write-Host "Generating config for $($matrixConfig.Path)" + + $matrixResults = @() + if ($directBatch) { + $matrixResults = GenerateMatrixForConfig ` + -ConfigPath $matrixConfig.Path ` + -Selection $matrixConfig.Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace + + if ($matrixResults) { + Write-Host "We have the following direct matrix results: " + Write-Host ($matrixResults | Out-String) + } + } + else { + $matrixResults = GenerateMatrixForConfig ` + -ConfigPath $matrixConfig.Path ` + -Selection $matrixConfig.Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters ($Filters + $IndirectFilters) ` + -Replace $Replace + + if ($matrixResults) { + Write-Host "We have the following indirect matrix results: " + Write-Host ($matrixResults | Out-String) + } + else { + Write-Host "No indirect matrix results found for $($matrixConfig.Path)" + continue + } + } + + $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE + + # we only need to modify the generated job name if there is more than one matrix config + batch + $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1 + + # if we are doing direct packages, we need to walk the batches and duplicate the matrix config for each batch, fully assigning + # the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages + if ($directBatch) { + $batchSuffixNecessary = $packageBatches.Length -gt 1 + $batchCounter = 1 + + foreach ($batch in $packageBatches) { + $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," + + foreach ($matrixOutputItem in $matrixResults) { + # we need to clone this, as each item is an object with possible children + $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable + # we just need to iterate across them, grab the parameters hashtable, and add the new key + # if there is more than one batch, we will need to add a suffix including the batch name to the job name + $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch + + if ($matrixSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name + } + + if ($batchSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_b$batchCounter" + } + + $OverallResult += $outputItem + } + $batchCounter += 1 + } + } + # in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely, + # we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages + # as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are + else { + $batchSuffixNecessary = $packageBatches.Length -gt 0 + $batchCounter = 1 + foreach ($batch in $packageBatches) { + $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," + $outputItem = QueuePop -queue ([ref]$matrixResults) + + $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch + + if ($matrixSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name + } + + if ($batchSuffixNecessary) { + $outputItem["name"] = $outputItem["name"] + "_ib$batchCounter" + } + # now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results + # we will add the cloned version to OverallResult + $OverallResult += $outputItem + $batchCounter += 1 + } + } + } + } + + + return ,$OverallResult +} if (!(Test-Path $PackagePropertiesFolder)) { Write-Error "Package Properties folder doesn't exist" @@ -58,71 +209,35 @@ Write-Host "Generating PR job matrix for $PackagePropertiesFolder" $configs = Get-Content -Raw $PRMatrixFile | ConvertFrom-Json -# calculate general targeting information and create our batches prior to generating any matrix +# get all the package property objects loaded $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json ` | ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json } -# set default matrix config for each package if there isn't an override +# enhance the package props with a default matrix config if one isn't present $packageProperties | ForEach-Object { if (-not $_.CIMatrixConfigs) { $_.CIMatrixConfigs = $configs } } -# The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig -# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config. -$matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs" +$directPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $false } +$indirectPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $true } $OverallResult = @() -foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) { - $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey] - $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs - - $matrixResults = @() - foreach ($matrixConfig in $matrixConfigs) { - Write-Host "Generating config for $($matrixConfig.Path)" - $matrixResults = GenerateMatrixForConfig ` - -ConfigPath $matrixConfig.Path ` - -Selection $matrixConfig.Selection ` - -DisplayNameFilter $DisplayNameFilter ` - -Filters $Filters ` - -Replace $Replace - - $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE - - # we only need to modify the generated job name if there is more than one matrix config + batch - $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1 - $batchSuffixNecessary = $packageBatches.Length -gt 1 - $batchCounter = 1 - - foreach ($batch in $packageBatches) { - $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join "," - # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys: - # [ - # { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} }, - # ] - foreach ($matrixOutputItem in $matrixResults) { - # we need to clone this, as each item is an object with possible children - $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable - # we just need to iterate across them, grab the parameters hashtable, and add the new key - # if there is more than one batch, we will need to add a suffix including the batch name to the job name - $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch - - if ($matrixSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name - } - - if ($batchSuffixNecessary) { - $outputItem["name"] = $outputItem["name"] + "_b$batchCounter" - } - - $OverallResult += $outputItem - } - $batchCounter += 1 - } +if ($directPackages) { + Write-Host "Discovered $($directPackages.Length) direct packages" + foreach($artifact in $directPackages) { + Write-Host "-> $($artifact.ArtifactName)" } + $OverallResult += GeneratePRMatrixForBatch -Packages $directPackages +} +if ($indirectPackages) { + Write-Host "Discovered $($indirectPackages.Length) indirect packages" + foreach($artifact in $indirectPackages) { + Write-Host "-> $($artifact.ArtifactName)" + } + $OverallResult += GeneratePRMatrixForBatch -Packages $indirectPackages } - $serialized = SerializePipelineMatrix $OverallResult Write-Output $serialized.pretty