Enable testing of internal servicing builds

eng/Get-DropVersions.ps1

@@ -29,13 +29,13 @@ param(
     [switch]
     $UpdateDependencies,
 
-    # SAS query string used to access the internal blob storage location of the build
+    # PAT used to access the versions repo in AzDO
     [string]
-    $BlobStorageSasQueryString,
+    $AzdoVersionsRepoInfoAccessToken,
 
-    # PAT used to access the versions repo in AzDO
+    # PAT used to access internal AzDO build artifacts
     [string]
-    $AzdoVersionsRepoInfoAccessToken
+    $InternalAccessToken
 )
 
 Import-Module -force $PSScriptRoot/DependencyManagement.psm1
@@ -94,7 +94,7 @@ function GetSdkVersionInfo([string]$sdkUrl) {
     }
 }
 
-function ResolveSdkUrl([string]$sdkVersion, [string]$queryString, [bool]$useStableBranding) {
+function ResolveSdkUrl([string]$sdkVersion, [bool]$useStableBranding) {
     if ($useStableBranding) {
         $sdkStableVersion = ($sdkVersion -split "-")[0]
     }
@@ -107,7 +107,7 @@ function ResolveSdkUrl([string]$sdkVersion, [string]$queryString, [bool]$useStab
     $containerVersion = $sdkVersion.Replace(".", "-")
 
     if ($UseInternalBuild) {
-        $sdkUrl = "https://dotnetstage.blob.core.windows.net/$containerVersion-internal/Sdk/$sdkVersion/$zipFile$queryString"
+        $sdkUrl = "https://dotnetstage.blob.core.windows.net/$containerVersion-internal/Sdk/$sdkVersion/$zipFile"
     }
     else {
         $sdkUrl = "https://dotnetbuilds.blob.core.windows.net/public/Sdk/$sdkVersion/$zipFile"
@@ -150,11 +150,21 @@ function GetDependencyVersion([string]$dependencyName, [xml]$versionDetails) {
 }
 
 function GetVersionInfoFromBuildId([string]$buildId) {
-    $configPath = Join-Path $tempDir "config.json"
+    $configFilename = "config.json"
+    $configPath = Join-Path $tempDir $configFilename
 
     try {
-        write-host here
-        az pipelines runs artifact download --organization https://dev.azure.com/dnceng/ --project internal --run-id $buildId --path $tempDir --artifact-name drop
+        New-Item -Path $tempDir -ItemType Directory -Force | Out-Null
+
+        $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalAccessToken"))
+        $headers = @{
+            "Authorization" = "Basic $base64AccessToken"
+        }
+
+        $url = GetArtifactUrl 'drop'
+        $url = $url.Replace("content?format=zip", "content?format=file&subPath=%2F$configFilename")
+
+        Invoke-WebRequest -OutFile $configPath $url -Headers $headers
 
         $config = $(Get-Content -Path $configPath | Out-String) | ConvertFrom-Json
 
@@ -178,29 +188,69 @@ function GetVersionInfoFromBuildId([string]$buildId) {
     }
 }
 
+function GetArtifactUrl([string]$artifactName) {
+    $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalAccessToken"))
+    $headers = @{
+        "Authorization" = "Basic $base64AccessToken"
+    }
+
+    $artifactsUrl = "https://dev.azure.com/dnceng/internal/_apis/build/builds/$BuildId/artifacts?api-version=6.0"
+    $response = Invoke-RestMethod -Uri $artifactsUrl -Method Get -Headers $headers
+
+    $url = $null
+    foreach ($artifact in $response.value) {
+        if ($artifact.name -eq $artifactName) {
+            $url = $artifact.resource.downloadUrl
+            break
+        }
+    }
+
+    if ($url -eq $null) {
+        Write-Error "Artifact '$artifactName' was not found in build# $BuildId"
+        exit 1
+    }
+
+    return $url
+}
+
+function GetInternalBaseUrl() {
+    $shippingUrl = GetArtifactUrl 'shipping'
+
+    # Format artifact URL into base-url
+    return $shippingUrl.Replace("content?format=zip", "content?format=file&subPath=%2Fassets")
+}
+
 $ErrorActionPreference = 'Stop'
 $ProgressPreference = 'SilentlyContinue'
 Set-StrictMode -Version 2.0
 
-$tempDir = Join-Path ([System.IO.Path]::GetTempPath()) "dotnet-docker-get-dropversions" ([System.Guid]::NewGuid())
+$tempDir = Join-Path ([System.IO.Path]::GetTempPath()) -ChildPath "dotnet-docker-get-dropversions" | Join-Path -ChildPath ([System.Guid]::NewGuid())
 
 if ($UseInternalBuild) {
     if ($Channel)
     {
         $Channel = "internal/$Channel"
     }
 
-    $queryString = "$BlobStorageSasQueryString"
-}
-else {
-    $queryString = ""
+    if ($BuildId) {
+        if (!$InternalAccessToken) {
+            $InternalAccessToken = az account get-access-token --query accessToken --output tsv
+            if ($LASTEXITCODE -ne 0) {
+                Write-Error "Failed to obtain access token using Azure CLI"
+                Write-Error "Please provide 'InternalAccessToken' parameter when using 'BuildId' option"
+                exit 1
+            }
+        }
+
+        $internalBaseUrl = GetInternalBaseUrl
+    }
 }
 
 $sdkVersionInfos = @()
 
 if ($Channel) {
     $sdkFile = "dotnet-sdk-win-x64.zip"
-    $akaMsUrl = "https://aka.ms/dotnet/$Channel/$sdkFile$queryString"
+    $akaMsUrl = "https://aka.ms/dotnet/$Channel/$sdkFile"
 
     $sdkUrl = Resolve-DotnetProductUrl $akaMsUrl
     $sdkVersionInfos += GetSdkVersionInfo $sdkUrl
@@ -209,7 +259,7 @@ if ($Channel) {
 foreach ($sdkVersion in $SdkVersions)
 {
     $useStableBranding = Get-IsStableBranding -Version $sdkVersion
-    $sdkUrl = ResolveSdkUrl $sdkVersion $queryString $useStableBranding
+    $sdkUrl = ResolveSdkUrl $sdkVersion $useStableBranding
     $sdkVersionInfo = GetSdkVersionInfo $sdkUrl
     $sdkVersionInfos += $sdkVersionInfo
 }
@@ -256,6 +306,13 @@ foreach ($sdkVersionInfo in $SdkVersionInfos) {
 
 if ($UpdateDependencies)
 {
+    $additionalArgs = @{}
+
+    if ($internalBaseUrl -and $InternalAccessToken) {
+        $additionalArgs += @{ InternalBaseUrl = "$internalBaseUrl" }
+        $additionalArgs += @{ InternalAccessToken = "$InternalAccessToken" }
+    }
+
     foreach ($versionInfo in $versionInfos) {
         Write-Host "Dockerfile version: $($versionInfo.DockerfileVersion)"
         Write-Host "SDK version: $($versionInfo.SdkVersion)"
@@ -269,6 +326,7 @@ if ($UpdateDependencies)
             -RuntimeVersion $versionInfo.RuntimeVersion `
             -AspnetVersion $versionInfo.AspnetVersion `
             -SdkVersion $versionInfo.SdkVersion `
+            @additionalArgs
 
         Write-Host "`r`nDone: Updates for .NET $($versionInfo.RuntimeVersion)/$($versionInfo.SdkVersion)`r`n"
     }

eng/Set-DotnetVersions.ps1

@@ -48,22 +48,22 @@ param(
     [string]
     $AzdoVariableName,
 
-    # SAS query string used to access files in the binary blob container
-    [string]
-    $BinarySasQueryString,
-
-    # SAS query string used to access files in the checksum blob container
-    [string]
-    $ChecksumSasQueryString,
-
     # File containing checksums for each product asset; used to override the behavior of locating the checksums from blob storage accounts.
     [string]
     $ChecksumsFile,
 
     # The release state of the product assets
     [ValidateSet("Prerelease", "Release")]
     [string]
-    $ReleaseState
+    $ReleaseState,
+
+    # PAT used to access internal AzDO build artifacts
+    [string]
+    $InternalAccessToken,
+
+    # Base Url for internal AzDO build artifacts
+    [string]
+    $InternalBaseUrl
 )
 
 Import-Module -force $PSScriptRoot/DependencyManagement.psm1
@@ -102,14 +102,6 @@ if ($ComputeShas) {
     $updateDepsArgs += "--compute-shas"
 }
 
-if ($BinarySasQueryString) {
-    $updateDepsArgs += "--binary-sas=$BinarySasQueryString"
-}
-
-if ($ChecksumSasQueryString) {
-    $updateDepsArgs += "--checksum-sas=$ChecksumSasQueryString"
-}
-
 if ($ChecksumsFile) {
     $updateDepsArgs += "--checksums-file=$ChecksumsFile"
 }
@@ -122,6 +114,14 @@ if ($ReleaseState) {
     $updateDepsArgs += "--release-state=$ReleaseState"
 }
 
+if ($InternalAccessToken) {
+    $updateDepsArgs += "--internal-access-token=$InternalAccessToken"
+}
+
+if ($InternalBaseUrl) {
+    $updateDepsArgs += "--internal-base-url=$InternalBaseUrl"
+}
+
 $versionSourceName = switch ($PSCmdlet.ParameterSetName) {
     "DotnetSdk" { "dotnet/sdk" }
     "DotnetMonitor" { "dotnet/dotnet-monitor/$ProductVersion" }

eng/common/templates/jobs/build-images.yml

@@ -10,12 +10,14 @@ parameters:
   publicProjectName: null
   internalVersionsRepoRef: null
   publicVersionsRepoRef: null
+  isInternalServicingValidation: false
 
 jobs:
 - job: ${{ parameters.name }}
-  condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
+  condition: and(${{ parameters.matrix }}, not(canceled()), or(in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), eq(${{ parameters.isInternalServicingValidation }}, 'true')))
   dependsOn:
-  - PreBuildValidation
+  - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}:
+    - PreBuildValidation
   - CopyBaseImages
   - GenerateBuildMatrix
   pool: ${{ parameters.pool }}
@@ -97,7 +99,7 @@ jobs:
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
       $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json"
-      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
+      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") {
         $imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr-staging.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push"
       }
 
@@ -141,7 +143,7 @@ jobs:
       displayName: Publish Image Info File Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
     # The following task depends on the SBOM Manifest Generator task installed on the agent.
     # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves.
     - powershell: |
@@ -193,11 +195,11 @@ jobs:
         }
       displayName: Generate SBOMs
       condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+  - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}:
     - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
       parameters:
         condition: ne(variables.testScriptPath, '')
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
     - template: /eng/common/templates/steps/publish-artifact.yml@self
       parameters:
         path: $(sbomDirectory)