Adjustments to PR Matrix generation (#9836)

* adjustments to iterating across matrix configs for packages that belong to multiple matrices
Azure · Feb 14, 2025 · b92990a · b92990a
1 parent 45d12c1
commit b92990a
Showing 1 changed file with 87 additions and 81 deletions.
diff --git a/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1
@@ -94,110 +94,116 @@ function GeneratePRMatrixForBatch {
   $matrixBatchesByConfig = Group-ByObjectKey $Packages "CIMatrixConfigs"
 
   foreach ($matrixBatchKey in $matrixBatchesByConfig.Keys) {
+    # recall that while we have grouped the package info by the matrix config object, each package still has knowledge about
+    # every other matrix that it belongs to.
+    # so we actually need to get a valid matrix config object from the first package in the batch, but we need to be certain
+    # that the matrix config object we get is the SAME ONE that we are iterating through
     $matrixBatch = $matrixBatchesByConfig[$matrixBatchKey]
-    $matrixConfigs = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs
-
+    $allPossibleMatrixConfigsForFirstPackage = $matrixBatch | Select-Object -First 1 -ExpandProperty CIMatrixConfigs
+    $matrixConfig = $allPossibleMatrixConfigsForFirstPackage | Where-Object { (Get-ObjectKey $_) -eq $matrixBatchKey }
     $matrixResults = @()
-    foreach ($matrixConfig in $matrixConfigs) {
-      Write-Host "Generating config for $($matrixConfig.Path)"
-      $nonSparse = $matrixConfig.PSObject.Properties['NonSparseParameters'] ? $matrixConfig.NonSparseParameters : @()
-
-      $matrixResults = @()
-      if ($directBatch) {
-        $matrixResults = GenerateMatrixForConfig `
-          -ConfigPath $matrixConfig.Path `
-          -Selection $matrixConfig.Selection `
-          -DisplayNameFilter $DisplayNameFilter `
-          -Filters $Filters `
-          -Replace $Replace `
-          -NonSparseParameters $nonSparse
-
-        if ($matrixResults) {
-          Write-Host "We have the following direct matrix results: "
-          Write-Host ($matrixResults | Out-String)
-        }
+
+    if (!$matrixConfig) {
+      Write-Error "Unable to find matrix config for $matrixBatchKey. Check the package properties for the package $($matrixBatch[0].ArtifactName)."
+      exit 1
+    }
+
+    Write-Host "Generating config for $($matrixConfig.Path)"
+    $nonSparse = $matrixConfig.PSObject.Properties['NonSparseParameters'] ? $matrixConfig.NonSparseParameters : @()
+
+    if ($directBatch) {
+      $matrixResults = GenerateMatrixForConfig `
+        -ConfigPath $matrixConfig.Path `
+        -Selection $matrixConfig.Selection `
+        -DisplayNameFilter $DisplayNameFilter `
+        -Filters $Filters `
+        -Replace $Replace `
+        -NonSparseParameters $nonSparse
+
+      if ($matrixResults) {
+        Write-Host "We have the following direct matrix results: "
+        Write-Host ($matrixResults | Out-String)
+      }
+    }
+    else {
+      $matrixResults = GenerateMatrixForConfig `
+        -ConfigPath $matrixConfig.Path `
+        -Selection $matrixConfig.Selection `
+        -DisplayNameFilter $DisplayNameFilter `
+        -Filters ($Filters + $IndirectFilters) `
+        -Replace $Replace `
+        -NonSparseParameters $nonSparse
+
+      if ($matrixResults) {
+        Write-Host "We have the following indirect matrix results: "
+        Write-Host ($matrixResults | Out-String)
       }
       else {
-        $matrixResults = GenerateMatrixForConfig `
-          -ConfigPath $matrixConfig.Path `
-          -Selection $matrixConfig.Selection `
-          -DisplayNameFilter $DisplayNameFilter `
-          -Filters ($Filters + $IndirectFilters) `
-          -Replace $Replace `
-          -NonSparseParameters $nonSparse
-
-        if ($matrixResults) {
-          Write-Host "We have the following indirect matrix results: "
-          Write-Host ($matrixResults | Out-String)
-        }
-        else {
-          Write-Host "No indirect matrix results found for $($matrixConfig.Path)"
-          continue
-        }
+        Write-Host "No indirect matrix results found for $($matrixConfig.Path)"
+        continue
       }
+    }
 
-      $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE
-
-      # we only need to modify the generated job name if there is more than one matrix config + batch
-      $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1
-
-      # if we are doing direct packages (or a full indirect matrix), we need to walk the batches and duplicate the matrix config for each batch, fully assigning
-      # the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages
-      if ($directBatch -or $FullSparseMatrix) {
-        $batchSuffixNecessary = $packageBatches.Length -gt $($directBatch ? 1 : 0)
-        $batchCounter = 1
-
-        foreach ($batch in $packageBatches) {
-          $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
-
-          foreach ($matrixOutputItem in $matrixResults) {
-            # we need to clone this, as each item is an object with possible children
-            $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable
-            # we just need to iterate across them, grab the parameters hashtable, and add the new key
-            # if there is more than one batch, we will need to add a suffix including the batch name to the job name
-            $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
+    $packageBatches = Split-ArrayIntoBatches -InputArray $matrixBatch -BatchSize $BATCHSIZE
 
-            if ($matrixSuffixNecessary) {
-              $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
-            }
+    # we only need to modify the generated job name if there is more than one matrix config + batch
+    $matrixSuffixNecessary = $matrixBatchesByConfig.Keys.Count -gt 1
 
-            if ($batchSuffixNecessary) {
-              $outputItem["name"] = $outputItem["name"] + "$batchNamePrefix$batchCounter"
-            }
+    # if we are doing direct packages (or a full indirect matrix), we need to walk the batches and duplicate the matrix config for each batch, fully assigning
+    # the each batch's packages to the matrix config. This will generate a _non-sparse_ matrix for the incoming packages
+    if ($directBatch -or $FullSparseMatrix) {
+      $batchSuffixNecessary = $packageBatches.Length -gt $($directBatch ? 1 : 0)
+      $batchCounter = 1
 
-            $OverallResult += $outputItem
-          }
-          $batchCounter += 1
-        }
-      }
-      # in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely,
-      # we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages
-      # as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are
-      else {
-        $batchSuffixNecessary = $packageBatches.Length -gt 0
-        $batchCounter = 1
-        foreach ($batch in $packageBatches) {
-          $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
-          $outputItem = QueuePop -queue ([ref]$matrixResults)
+      foreach ($batch in $packageBatches) {
+        $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
 
+        foreach ($matrixOutputItem in $matrixResults) {
+          # we need to clone this, as each item is an object with possible children
+          $outputItem = $matrixOutputItem | ConvertTo-Json -Depth 100 | ConvertFrom-Json -AsHashtable
+          # we just need to iterate across them, grab the parameters hashtable, and add the new key
+          # if there is more than one batch, we will need to add a suffix including the batch name to the job name
           $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
 
           if ($matrixSuffixNecessary) {
             $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
           }
 
           if ($batchSuffixNecessary) {
-            $outputItem["name"] = $outputItem["name"]  + "_$batchNamePrefix$batchCounter"
+            $outputItem["name"] = $outputItem["name"] + "$batchNamePrefix$batchCounter"
           }
-          # now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results
-          # we will add the cloned version to OverallResult
+
           $OverallResult += $outputItem
-          $batchCounter += 1
         }
+        $batchCounter += 1
       }
     }
-  }
+    # in the case of indirect packages, instead of walking the batches and duplicating their matrix config entirely,
+    # we instead will walk each each matrix, create a parameter named for the PRMatrixSetting, and add the targeted packages
+    # as an array. This will generate a _sparse_ matrix for for whatever the incoming packages are
+    else {
+      $batchSuffixNecessary = $packageBatches.Length -gt 0
+      $batchCounter = 1
+      foreach ($batch in $packageBatches) {
+        $namesForBatch = ($batch | ForEach-Object { $_.ArtifactName }) -join ","
+        $outputItem = QueuePop -queue ([ref]$matrixResults)
+
+        $outputItem["parameters"]["$PRMatrixSetting"] = $namesForBatch
+
+        if ($matrixSuffixNecessary) {
+          $outputItem["name"] = $outputItem["name"] + "_" + $matrixConfig.Name
+        }
 
+        if ($batchSuffixNecessary) {
+          $outputItem["name"] = $outputItem["name"]  + "_$batchNamePrefix$batchCounter"
+        }
+        # now we need to take an item from the front of the matrix results, clone it, and add it to the back of the matrix results
+        # we will add the cloned version to OverallResult
+        $OverallResult += $outputItem
+        $batchCounter += 1
+      }
+    }
+  }
 
   return ,$OverallResult
 }