Sync eng/common directory with azure-sdk-tools for PR 9601 (#8413)

* updates to dynamic matrix generation. moves indirectly included packages to sparse usage. adds additional parameters to GeneratePRMatrix that enable further filtering of the matrix generated for the indirect packages --------- Co-authored-by: Scott Beddall <[email protected]> Co-authored-by: Wes Haggard <[email protected]>
Azure · Jan 17, 2025 · cf92da3 · cf92da3
1 parent e80bf56
commit cf92da3
Show file tree

Hide file tree

Showing 3 changed files with 185 additions and 56 deletions.
diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml
@@ -48,6 +48,12 @@ parameters:
 - name: PRMatrixSetting
   type: string
   default: 'ArtifactPackageNames'
+- name: PRJobBatchSize
+  type: number
+  default: 10
+- name: PRMatrixIndirectFilters
+  type: object
+  default: []
 # Mappings to OS name required at template compile time by 1es pipeline templates
 - name: Pools
   type: object
@@ -126,7 +132,6 @@ jobs:
     - ${{ else }}:
       - ${{ each pool in parameters.Pools }}:
         - pwsh: |
-            # dump the conglomerated CI matrix
             '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json
 
             ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
@@ -135,7 +140,9 @@ jobs:
               -PRMatrixSetting ${{ parameters.PRMatrixSetting }} `
               -DisplayNameFilter '$(displayNameFilter)' `
               -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' `
-              -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+              -IndirectFilters '${{ join(''',''', parameters.PRMatrixIndirectFilters) }}' `
+              -Replace '${{ join(''',''', parameters.MatrixReplace) }}' `
+              -PackagesPerPRJob ${{ parameters.PRJobBatchSize }}
           displayName: Create ${{ pool.name }} PR Matrix
           name: vm_job_matrix_pr_${{ pool.name }}
 

diff --git a/eng/common/pipelines/templates/steps/verify-readmes.yml b/eng/common/pipelines/templates/steps/verify-readmes.yml
@@ -13,13 +13,20 @@ parameters:
 - name: Condition
   type: string
   default: succeeded()
+- name: IncludeIndirect
+  type: boolean
+  default: true
 
 steps:
 - pwsh: |
+    $includeIndirect = $${{ parameters.IncludeIndirect }}
     $packageProperties = Get-ChildItem -Recurse "${{ parameters.PackagePropertiesFolder }}" *.json
-
     $paths = @()
 
+    if (-not $includeIndirect) {
+      $packageProperties = $packageProperties | Where-Object { (Get-Content -Raw $_ | ConvertFrom-Json).IncludedForValidation -eq $false }
+    }
+
     foreach($propertiesFile in $packageProperties) {
       $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json
 

diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1
@@ -23,6 +23,12 @@ It generates the matrix by the following algorithm:
             - add the combined property name to the parameters of the matrix item
             - add the matrix item to the overall result
 
+.PARAMETER IndirectFilters
+Any array of strings representing filters that will only be applied to the matrix generation for indirect packages. This is useful for
+filtering out OTHER parameter settings othan than PRMatrixSetting that are only relevant to direct packages.
+
+For .NET, this value will be AdditionalTestArguments=/p:UseProjectReferenceToAzureClients=true
+
 .EXAMPLE
 ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
   -PackagePropertiesFolder "path/to/populated/PackageInfo" `
@@ -36,13 +42,158 @@ param (
   [Parameter(Mandatory = $true)][string] $PRMatrixSetting,
   [Parameter(Mandatory = $False)][string] $DisplayNameFilter,
   [Parameter(Mandatory = $False)][array] $Filters,
+  [Parameter(Mandatory = $False)][array] $IndirectFilters,
   [Parameter(Mandatory = $False)][array] $Replace,
+  [Parameter(Mandatory = $False)][int] $PackagesPerPRJob = 10,
   [Parameter()][switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
 )
 
+Set-StrictMode -Version 4
 . $PSScriptRoot/job-matrix-functions.ps1
 . $PSScriptRoot/../Helpers/Package-Helpers.ps1
-$BATCHSIZE = 10
+. $PSScriptRoot/../Package-Properties.ps1
+$BATCHSIZE = $PackagesPerPRJob
+
+# this function takes an array of objects, takes a copy of the first item, and moves that item to the back of the array
+function QueuePop([ref]$queue) {
+
+  if ($queue.Value.Length -eq 1) {
+    return ($queue.Value[0] | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable)
+  }
+
+  # otherwise we can rotate stuff
+  $first = $queue.Value[0]
+  $rest = $queue.Value[1..($queue.Value.Length - 1)]
+
+  $queue.Value = $rest + $first
+
+  return ($first | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable)
+}
+
+function GeneratePRMatrixForBatch {
+  param (
+    [Parameter(Mandatory = $true)][array] $Packages
+  )
+
+  $OverallResult = @()
+  if (!$Packages) {
+    Write-Host "Unable to generate matrix for empty package list"
+    return ,$OverallResult
+  }
+
+  # this check assumes that we have properly separated the direct and indirect package lists
+  $directBatch = $Packages[0].IncludedForValidation -eq $false
+  Write-Host "Generating matrix for $($directBatch ? 'direct' : 'indirect') packages"
+
+  # The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig
+  # to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config.
+  $matrixBatchesByConfig = Group-ByObjectKey $Packages "CIMatrixConfigs"
+
+  foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) {
+    $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey]
+    $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs
+
+    $matrixResults = @()
+    foreach ($matrixConfig in $matrixConfigs) {
+      Write-Host "Generating config for $($matrixConfig.Path)"
+
+      $matrixResults = @()
+      if ($directBatch) {
+        $matrixResults = GenerateMatrixForConfig `
+          -ConfigPath $matrixConfig.Path `
+          -Selection $matrixConfig.Selection `
+          -DisplayNameFilter $DisplayNameFilter `
+          -Filters $Filters `
+          -Replace $Replace
+
+        if ($matrixResults) {
+          Write-Host "We have the following direct matrix results: "
+          Write-Host ($matrixResults | Out-String)
+        }
+      }
+      else {
+        $matrixResults = GenerateMatrixForConfig `
+          -ConfigPath $matrixConfig.Path `
+          -Selection $matrixConfig.Selection `
+          -DisplayNameFilter $DisplayNameFilter `
+          -Filters ($Filters + $IndirectFilters) `
+          -Replace $Replace
+
+        if ($matrixResults) {
+          Write-Host "We have the following indirect matrix results: "
+          Write-Host ($matrixResults | Out-String)
+        }
+        else {
+          Write-Host "No indirect matrix results found for $($matrixConfig.Path)"
+          continue
+        }
+      }
+
+      $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE
+
+      # we only need to modify the generated job name if there is more than one matrix config + batch
+      $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1
+
+      # if we are doing direct packages, we need to walk the batches and duplicate the matrix config for each batch, fully assigning
+      # the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages
+      if ($directBatch) {
+        $batchSuffixNecessary = $packageBatches.Length -gt 1
+        $batchCounter = 1
+
+        foreach ($batch in $packageBatches) {
+          $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
+
+          foreach ($matrixOutputItem in $matrixResults) {
+            # we need to clone this, as each item is an object with possible children
+            $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable
+            # we just need to iterate across them, grab the parameters hashtable, and add the new key
+            # if there is more than one batch, we will need to add a suffix including the batch name to the job name
+            $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
+
+            if ($matrixSuffixNecessary) {
+              $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
+            }
+
+            if ($batchSuffixNecessary) {
+              $outputItem["name"] = $outputItem["name"] + "_b$batchCounter"
+            }
+
+            $OverallResult += $outputItem
+          }
+          $batchCounter += 1
+        }
+      }
+      # in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely,
+      # we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages
+      # as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are
+      else {
+        $batchSuffixNecessary = $packageBatches.Length -gt 0
+        $batchCounter = 1
+        foreach ($batch in $packageBatches) {
+          $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
+          $outputItem = QueuePop -queue ([ref]$matrixResults)
+
+          $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
+
+          if ($matrixSuffixNecessary) {
+            $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
+          }
+
+          if ($batchSuffixNecessary) {
+            $outputItem["name"] = $outputItem["name"] + "_ib$batchCounter"
+          }
+          # now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results
+          # we will add the cloned version to OverallResult
+          $OverallResult += $outputItem
+          $batchCounter += 1
+        }
+      }
+    }
+  }
+
+
+  return ,$OverallResult
+}
 
 if (!(Test-Path $PackagePropertiesFolder)) {
   Write-Error "Package Properties folder doesn't exist"
@@ -58,71 +209,35 @@ Write-Host "Generating PR job matrix for $PackagePropertiesFolder"
 
 $configs = Get-Content -Raw $PRMatrixFile | ConvertFrom-Json
 
-# calculate general targeting information and create our batches prior to generating any matrix
+# get all the package property objects loaded
 $packageProperties = Get-ChildItem -Recurse "$PackagePropertiesFolder" *.json `
 | ForEach-Object { Get-Content -Path $_.FullName | ConvertFrom-Json }
 
-# set default matrix config for each package if there isn't an override
+# enhance the package props with a default matrix config if one isn't present
 $packageProperties | ForEach-Object {
   if (-not $_.CIMatrixConfigs) {
     $_.CIMatrixConfigs = $configs
   }
 }
 
-# The key here is that after we group the packages by the matrix config objects, we can use the first item's MatrixConfig
-# to generate the matrix for the group, no reason to have to parse the key value backwards to get the matrix config.
-$matrixBatchesByConfig = Group-ByObjectKey $packageProperties "CIMatrixConfigs"
+$directPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $false }
+$indirectPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $true }
 
 $OverallResult = @()
-foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) {
-  $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey]
-  $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs
-
-  $matrixResults = @()
-  foreach ($matrixConfig in $matrixConfigs) {
-    Write-Host "Generating config for $($matrixConfig.Path)"
-    $matrixResults = GenerateMatrixForConfig `
-      -ConfigPath $matrixConfig.Path `
-      -Selection $matrixConfig.Selection `
-      -DisplayNameFilter $DisplayNameFilter `
-      -Filters $Filters `
-      -Replace $Replace
-
-    $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE
-
-    # we only need to modify the generated job name if there is more than one matrix config + batch
-    $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1
-    $batchSuffixNecessary = $packageBatches.Length -gt 1
-    $batchCounter = 1
-
-    foreach ($batch in $packageBatches) {
-      $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
-      # to understand this iteration, one must understand that the matrix is a list of hashtables, each with a couple keys:
-      # [
-      #  { "name": "jobname", "parameters": { matrixSetting1: matrixValue1, ...} },
-      # ]
-      foreach ($matrixOutputItem in $matrixResults) {
-        # we need to clone this, as each item is an object with possible children
-        $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable
-        # we just need to iterate across them, grab the parameters hashtable, and add the new key
-        # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-        $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
-
-        if ($matrixSuffixNecessary) {
-          $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
-        }
-
-        if ($batchSuffixNecessary) {
-          $outputItem["name"] = $outputItem["name"] + "_b$batchCounter"
-        }
-
-        $OverallResult += $outputItem
-      }
-      $batchCounter += 1
-    }
+if ($directPackages) {
+  Write-Host "Discovered $($directPackages.Length) direct packages"
+  foreach($artifact in $directPackages) {
+    Write-Host "-> $($artifact.ArtifactName)"
   }
+  $OverallResult += GeneratePRMatrixForBatch -Packages $directPackages
+}
+if ($indirectPackages) {
+  Write-Host "Discovered $($indirectPackages.Length) indirect packages"
+  foreach($artifact in $indirectPackages) {
+    Write-Host "-> $($artifact.ArtifactName)"
+  }
+  $OverallResult += GeneratePRMatrixForBatch -Packages $indirectPackages
 }
-
 $serialized = SerializePipelineMatrix $OverallResult
 
 Write-Output $serialized.pretty