From ecdce4c870c9ad4f54c7cd568ad73b014b0d912e Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Fri, 27 Sep 2024 15:18:13 -0700 Subject: [PATCH 01/15] Enable dependency and dockerfile updates for internal builds --- eng/Get-DropVersions.ps1 | 74 +++++++++++++++++-- eng/Set-DotnetVersions.ps1 | 18 ++++- eng/update-dependencies/BaseUrlUpdater.cs | 8 +- .../DockerfileShaUpdater.cs | 11 ++- eng/update-dependencies/NuGetConfigUpdater.cs | 2 +- eng/update-dependencies/Options.cs | 12 ++- 6 files changed, 109 insertions(+), 16 deletions(-) diff --git a/eng/Get-DropVersions.ps1 b/eng/Get-DropVersions.ps1 index f71544d344..c515b097ba 100644 --- a/eng/Get-DropVersions.ps1 +++ b/eng/Get-DropVersions.ps1 @@ -35,7 +35,11 @@ param( # PAT used to access the versions repo in AzDO [string] - $AzdoVersionsRepoInfoAccessToken + $AzdoVersionsRepoInfoAccessToken, + + # PAT used to access internal AzDO build artifacts + [string] + $InternalArtifactsAccessToken ) Import-Module -force $PSScriptRoot/DependencyManagement.psm1 @@ -150,11 +154,21 @@ function GetDependencyVersion([string]$dependencyName, [xml]$versionDetails) { } function GetVersionInfoFromBuildId([string]$buildId) { - $configPath = Join-Path $tempDir "config.json" + $configFilename = "config.json" + $configPath = Join-Path $tempDir $configFilename try { - write-host here - az pipelines runs artifact download --organization https://dev.azure.com/dnceng/ --project internal --run-id $buildId --path $tempDir --artifact-name drop + New-Item -Path $tempDir -ItemType Directory -Force | Out-Null + + $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalArtifactsAccessToken")) + $headers = @{ + "Authorization" = "Basic $base64AccessToken" + } + + $url = GetArtifactUrl 'drop' + $url = $url.Replace("content?format=zip", "content?format=file&subPath=%2F$configFilename") + + Invoke-WebRequest -OutFile $configPath $url -Headers $headers $config = $(Get-Content -Path $configPath | Out-String) | ConvertFrom-Json @@ -178,11 +192,43 @@ function GetVersionInfoFromBuildId([string]$buildId) { } } +function GetArtifactUrl([string]$artifactName) { + $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalArtifactsAccessToken")) + $headers = @{ + "Authorization" = "Basic $base64AccessToken" + } + + $artifactsUrl = "https://dev.azure.com/dnceng/internal/_apis/build/builds/$BuildId/artifacts?api-version=6.0" + $response = Invoke-RestMethod -Uri $artifactsUrl -Method Get -Headers $headers + + $url = $null + foreach ($artifact in $response.value) { + if ($artifact.name -eq $artifactName) { + $url = $artifact.resource.downloadUrl + break + } + } + + if ($url -eq $null) { + Write-Error "Artifact '$artifactName' was not found in build# $BuildId" + exit 1 + } + + return $url +} + +function GetInternalBaseUrl() { + $shippingUrl = GetArtifactUrl 'shipping' + + # Format artifact URL into base-url + return $shippingUrl.Replace("content?format=zip", "content?format=file&subPath=%2Fassets") +} + $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' Set-StrictMode -Version 2.0 -$tempDir = Join-Path ([System.IO.Path]::GetTempPath()) "dotnet-docker-get-dropversions" ([System.Guid]::NewGuid()) +$tempDir = Join-Path ([System.IO.Path]::GetTempPath()) -ChildPath "dotnet-docker-get-dropversions" | Join-Path -ChildPath ([System.Guid]::NewGuid()) if ($UseInternalBuild) { if ($Channel) @@ -190,6 +236,16 @@ if ($UseInternalBuild) { $Channel = "internal/$Channel" } + if ($BuildId) { + if ($InternalArtifactsAccessToken) { + $internalBaseUrl = GetInternalBaseUrl + } + else { + Write-Error "'InternalArtifactsAccessToken' parameter is required for obtaining internal base-url, when specifying 'BuildId' option" + exit 1 + } + } + $queryString = "$BlobStorageSasQueryString" } else { @@ -256,6 +312,13 @@ foreach ($sdkVersionInfo in $SdkVersionInfos) { if ($UpdateDependencies) { + $additionalArgs = @{} + + if ($internalBaseUrl -and $InternalArtifactsAccessToken) { + $additionalArgs += @{ InternalBaseUrl = "$internalBaseUrl" } + $additionalArgs += @{ InternalPat = "$InternalArtifactsAccessToken" } + } + foreach ($versionInfo in $versionInfos) { Write-Host "Dockerfile version: $($versionInfo.DockerfileVersion)" Write-Host "SDK version: $($versionInfo.SdkVersion)" @@ -269,6 +332,7 @@ if ($UpdateDependencies) -RuntimeVersion $versionInfo.RuntimeVersion ` -AspnetVersion $versionInfo.AspnetVersion ` -SdkVersion $versionInfo.SdkVersion ` + @additionalArgs Write-Host "`r`nDone: Updates for .NET $($versionInfo.RuntimeVersion)/$($versionInfo.SdkVersion)`r`n" } diff --git a/eng/Set-DotnetVersions.ps1 b/eng/Set-DotnetVersions.ps1 index 091dfcd86a..b0a3837b75 100644 --- a/eng/Set-DotnetVersions.ps1 +++ b/eng/Set-DotnetVersions.ps1 @@ -63,7 +63,15 @@ param( # The release state of the product assets [ValidateSet("Prerelease", "Release")] [string] - $ReleaseState + $ReleaseState, + + # PAT used to access internal AzDO build artifacts + [string] + $InternalPat, + + # Base Url for internal AzDO build artifacts + [string] + $InternalBaseUrl ) Import-Module -force $PSScriptRoot/DependencyManagement.psm1 @@ -122,6 +130,14 @@ if ($ReleaseState) { $updateDepsArgs += "--release-state=$ReleaseState" } +if ($InternalArtifactsAccessToken) { + $updateDepsArgs += "--internal-pat=$InternalPat" +} + +if ($InternalBaseUrl) { + $updateDepsArgs += "--internal-base-url=$InternalBaseUrl" +} + $versionSourceName = switch ($PSCmdlet.ParameterSetName) { "DotnetSdk" { "dotnet/sdk" } "DotnetMonitor" { "dotnet/dotnet-monitor/$ProductVersion" } diff --git a/eng/update-dependencies/BaseUrlUpdater.cs b/eng/update-dependencies/BaseUrlUpdater.cs index ffff926400..7798624255 100644 --- a/eng/update-dependencies/BaseUrlUpdater.cs +++ b/eng/update-dependencies/BaseUrlUpdater.cs @@ -42,14 +42,12 @@ protected override string TryGetDesiredValue(IEnumerable depend if (_options.IsInternal) { - if (!_options.ProductVersions.TryGetValue("sdk", out string? sdkVersion) || string.IsNullOrEmpty(sdkVersion)) + if (string.IsNullOrEmpty(_options.InternalBaseUrl)) { - throw new InvalidOperationException("The sdk version must be set in order to derive the build's blob storage location."); + throw new InvalidOperationException("InternalBaseUrl must be set in order to update base url for internal builds"); } - sdkVersion = sdkVersion.Replace(".", "-"); - - unresolvedBaseUrl = $"https://dotnetstage.blob.core.windows.net/{sdkVersion}-internal"; + unresolvedBaseUrl = _options.InternalBaseUrl; } else if (_options.ReleaseState.HasValue) { diff --git a/eng/update-dependencies/DockerfileShaUpdater.cs b/eng/update-dependencies/DockerfileShaUpdater.cs index 8e575993fa..517dcc5f9f 100644 --- a/eng/update-dependencies/DockerfileShaUpdater.cs +++ b/eng/update-dependencies/DockerfileShaUpdater.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Net.Http.Headers; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; @@ -114,6 +115,14 @@ public DockerfileShaUpdater( } return (JObject)variables; }); + + if (!string.IsNullOrEmpty(_options.InternalPat)) + { + s_httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", + _options.InternalPat)))); + } } private string GetRpmArchFormat() => _arch == "arm64" ? "aarch64" : "$ARCH"; @@ -326,7 +335,7 @@ private static string ApplySasQueryStringIfNecessary(string url, string sasQuery private async Task GetDotNetBinaryStorageChecksumsShaAsync(string productDownloadUrl) { string? sha = null; - string shaExt = _productName.Contains("sdk", StringComparison.OrdinalIgnoreCase) ? ".sha" : ".sha512"; + string shaExt = _options.IsInternal || !_productName.Contains("sdk", StringComparison.OrdinalIgnoreCase) ? ".sha512" : ".sha"; string shaUrl = productDownloadUrl .Replace("/dotnetcli", "/dotnetclichecksums") diff --git a/eng/update-dependencies/NuGetConfigUpdater.cs b/eng/update-dependencies/NuGetConfigUpdater.cs index 7955911b6e..9f3f9c5910 100644 --- a/eng/update-dependencies/NuGetConfigUpdater.cs +++ b/eng/update-dependencies/NuGetConfigUpdater.cs @@ -28,7 +28,7 @@ public NuGetConfigUpdater(string repoRoot, Options options) _repoRoot = repoRoot; _options = options; - string configSuffix = (_options.SourceBranch == "nightly" ? ".nightly" : string.Empty); + string configSuffix = (_options.SourceBranch == "nightly" ? ".nightly" : _options.IsInternal ? ".internal" : string.Empty); _configPath = Path.Combine(_repoRoot, $"tests/Microsoft.DotNet.Docker.Tests/TestAppArtifacts/NuGet.config{configSuffix}"); } diff --git a/eng/update-dependencies/Options.cs b/eng/update-dependencies/Options.cs index 75a05cf0f4..f4e178c447 100644 --- a/eng/update-dependencies/Options.cs +++ b/eng/update-dependencies/Options.cs @@ -11,6 +11,8 @@ namespace Dotnet.Docker { public class Options { + public string InternalBaseUrl { get; } + public string InternalPat { get; } public string BinarySasQueryString { get; } public string ChecksumSasQueryString { get; } public bool ComputeChecksums { get; } @@ -29,13 +31,13 @@ public class Options public string VersionSourceName { get; } public bool UseStableBranding { get; } public bool UpdateOnly => Email == null || Password == null || User == null || TargetBranch == null; - public bool IsInternal => !string.IsNullOrEmpty(BinarySasQueryString) || !string.IsNullOrEmpty(ChecksumSasQueryString); + public bool IsInternal => !string.IsNullOrEmpty(InternalBaseUrl) || !string.IsNullOrEmpty(BinarySasQueryString) || !string.IsNullOrEmpty(ChecksumSasQueryString); public string ChecksumsFile { get; } public ReleaseState? ReleaseState { get; } public Options(string dockerfileVersion, string[] productVersion, string versionSourceName, string email, string password, string user, bool computeShas, bool stableBranding, string binarySas, string checksumSas, string sourceBranch, string targetBranch, string org, - string project, string repo, string checksumsFile, ReleaseState? releaseState) + string project, string repo, string checksumsFile, ReleaseState? releaseState, string internalBaseUrl, string internalPat) { DockerfileVersion = dockerfileVersion; ProductVersions = productVersion @@ -51,6 +53,8 @@ public Options(string dockerfileVersion, string[] productVersion, string version BinarySasQueryString = binarySas; ChecksumSasQueryString = checksumSas; SourceBranch = sourceBranch; + InternalBaseUrl = internalBaseUrl; + InternalPat = internalPat; // Default TargetBranch to SourceBranch if it's not explicitly provided TargetBranch = string.IsNullOrEmpty(targetBranch) ? sourceBranch : targetBranch; @@ -91,7 +95,9 @@ public static IEnumerable GetCliSymbols() => new Option("--project", "Name of the AzDO project"), new Option("--repo", "Name of the AzDO repo"), new Option("--checksums-file", "File containing a list of checksums for each product asset"), - new Option("--release-state", "The release state of the product assets") + new Option("--release-state", "The release state of the product assets"), + new Option("--internal-base-url", "Base Url for internal build artifacts"), + new Option("--internal-pat", "PAT for accessing internal build artifacts") }; } From 9cd85476709ede8522c9a43c7d18b6edb9dce8b5 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Fri, 27 Sep 2024 15:30:39 -0700 Subject: [PATCH 02/15] Dockerfile support for internal builds --- .../Dockerfile.linux.download-file | 5 ++++- .../Dockerfile.windows.download-file | 7 ++++++- eng/dockerfile-templates/aspnet/Dockerfile.linux | 6 ++---- .../aspnet/Dockerfile.linux-composite | 5 ++--- eng/dockerfile-templates/aspnet/Dockerfile.windows | 5 ++--- eng/dockerfile-templates/runtime-deps/Dockerfile | 5 ++--- eng/dockerfile-templates/runtime/Dockerfile.linux | 6 ++---- eng/dockerfile-templates/runtime/Dockerfile.windows | 5 ++--- eng/dockerfile-templates/sdk/Dockerfile.linux | 11 ++++------- eng/dockerfile-templates/sdk/Dockerfile.windows | 5 ++--- .../sdk/Dockerfile.windows.install-components | 5 +++-- 11 files changed, 31 insertions(+), 34 deletions(-) diff --git a/eng/dockerfile-templates/Dockerfile.linux.download-file b/eng/dockerfile-templates/Dockerfile.linux.download-file index f7faf1e044..662f9ab695 100644 --- a/eng/dockerfile-templates/Dockerfile.linux.download-file +++ b/eng/dockerfile-templates/Dockerfile.linux.download-file @@ -7,7 +7,10 @@ sha: Expected checksum of the downloaded file sha-var-name: Name of variable that stores the checksum ^ + set isInternal to find(ARGS["url"], "content?format=file&subPath") >= 0 ^ + set additionalWgetArgs to when(isInternal, '--header="Authorization: Basic `echo $ACCESSTOKEN:$ACCESSTOKEN | base64 -w 0`" ', '') ^ + set additionalCurlArgs to when(isInternal, '-u :$ACCESSTOKEN --basic ', '') ^ set isAlpine to find(OS_VERSION, "alpine") >= 0 -}}{{if isAlpine:wget -O^else:curl -fSL --output}} {{ARGS["out-file"]}} {{ARGS["url"]}}{{if ARGS["sha"]: \ +}}{{if isAlpine:wget {{additionalWgetArgs}}-O^else:curl {{additionalCurlArgs}}-fSL --output}} {{ARGS["out-file"]}} {{if isInternal:"{{ARGS["url"]}}"^else:{{ARGS["url"]}}}}{{if ARGS["sha"]: \ && {{ARGS["sha-var-name"]}}='{{ARGS["sha"]}}' \ && echo "${{ARGS["sha-var-name"]}} {{ARGS["out-file"]}}" | sha512sum -c -}} diff --git a/eng/dockerfile-templates/Dockerfile.windows.download-file b/eng/dockerfile-templates/Dockerfile.windows.download-file index be8b9d58aa..765827168e 100644 --- a/eng/dockerfile-templates/Dockerfile.windows.download-file +++ b/eng/dockerfile-templates/Dockerfile.windows.download-file @@ -8,8 +8,13 @@ sha-var-name: Name of variable that stores the checksum hash-algorithm: Algorithm type to use to get the checksum. Defaults to sha512 ^ + set isInternal to find(ARGS["url"], "content?format=file&subPath") >= 0 ^ set hashAlgorithm to when(ARGS["hash-algorithm"], ARGS["hash-algorithm"], "sha512") -}}Invoke-WebRequest -OutFile {{ARGS["out-file"]}} {{ARGS["url"]}}; ` +}}{{if isInternal:` +$Base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(\":$($Env:ACCESSTOKEN)\")); ` +$Headers = @@{Authorization = \"Basic $Base64AuthInfo\"}; ` +Invoke-WebRequest -OutFile {{ARGS["out-file"]}} \"{{ARGS["url"]}}\" -Headers $Headers; ` +`^else:Invoke-WebRequest -OutFile {{ARGS["out-file"]}} {{ARGS["url"]}}; `}} ${{ARGS["sha-var-name"]}} = '{{ARGS["sha"]}}'; ` if ((Get-FileHash {{ARGS["out-file"]}} -Algorithm {{hashAlgorithm}}).Hash -ne ${{ARGS["sha-var-name"]}}) { ` Write-Host 'CHECKSUM VERIFICATION FAILED!'; ` diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.linux b/eng/dockerfile-templates/aspnet/Dockerfile.linux index 53f2ae661a..97807f953c 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.linux +++ b/eng/dockerfile-templates/aspnet/Dockerfile.linux @@ -10,7 +10,7 @@ set isFullAzureLinux to isAzureLinux && !isDistroless ^ set isDistrolessAzureLinux to isAzureLinux && isDistroless ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "msrc") >= 0 || find(baseUrl, "internal") >= 0 ^ + set isInternal to find(baseUrl, "content?format=file&subPath") ^ set isRpmInstall to isFullAzureLinux && dotnetVersion = "6.0" ^ set isSingleStage to (isAlpine || isRpmInstall) && !isInternal ^ set runtimeDepsVariant to when(ARGS["is-extra"], "-extra", "") ^ @@ -59,7 +59,7 @@ # Installer image FROM {{installerImageTag}} AS installer {{if isInternal: -ARG SAS_QUERY_STRING +ARG ACCESSTOKEN }}{{if isDistrolessAzureLinux: {{InsertTemplate("../Dockerfile.linux.distroless-azurelinux-installer-prereqs")}} ^elif isFullAzureLinux && !isRpmInstall: @@ -74,7 +74,6 @@ RUN {{InsertTemplate("../Dockerfile.linux.install-pkgs", "install-method": when(isInternal && isRpmInstall, "download", "download-and-install"), "use-local-version-var": "true", "is-internal": isInternal, - "url-suffix": when(isInternal, "$SAS_QUERY_STRING", ""), "is-rpm-install": isRpmInstall ])}} @@ -88,7 +87,6 @@ FROM {{runtimeBaseTag}} [ "install-method": "copy-and-install", "is-internal": isInternal, - "url-suffix": when(isInternal, "$SAS_QUERY_STRING", ""), "installer-stage": "installer", "is-rpm-install": isRpmInstall ])}}^else: diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.linux-composite b/eng/dockerfile-templates/aspnet/Dockerfile.linux-composite index 9cc898f3ca..2dba2867ec 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.linux-composite +++ b/eng/dockerfile-templates/aspnet/Dockerfile.linux-composite @@ -6,7 +6,7 @@ set isFullAzureLinux to isAzureLinux && !isDistroless ^ set isDistrolessAzureLinux to isAzureLinux && isDistroless ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "msrc") >= 0 || find(baseUrl, "internal") >= 0 ^ + set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ set isSingleStage to isAlpine && !isInternal ^ set runtimeDepsVariant to when(ARGS["is-extra"], "-extra", "") ^ set tagVersion to when(dotnetVersion = "6.0" || dotnetVersion = "8.0", @@ -55,7 +55,7 @@ else:{{ # Installer image FROM {{installerImageTag}} AS installer {{ if isInternal: -ARG SAS_QUERY_STRING +ARG ACCESSTOKEN }}{{ if isDistrolessAzureLinux: {{InsertTemplate("../Dockerfile.linux.distroless-azurelinux-installer-prereqs")}} ^elif isFullAzureLinux: @@ -71,7 +71,6 @@ RUN {{InsertTemplate("../Dockerfile.linux.install-pkgs", "install-method": "download-and-install", "use-local-version-var": "true", "is-internal": isInternal, - "url-suffix": when(isInternal, "$SAS_QUERY_STRING", ""), "is-rpm-install": false "is-composite-runtime": "true", ])}} diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.windows b/eng/dockerfile-templates/aspnet/Dockerfile.windows index 50c566315b..4bf16f04eb 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.windows +++ b/eng/dockerfile-templates/aspnet/Dockerfile.windows @@ -1,7 +1,7 @@ {{ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "msrc") >= 0 || find(baseUrl, "internal") >= 0 ^ + set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ set isSingleStage to (find(OS_VERSION, "windowsservercore") >= 0 && !isInternal) ^ set tagVersion to when(dotnetVersion = "6.0" || dotnetVersion = "8.0", VARIABLES[cat("dotnet|", dotnetVersion, "|product-version")] @@ -26,12 +26,11 @@ ARG REPO=mcr.microsoft.com/dotnet/runtime }}# Installer image FROM mcr.microsoft.com/windows/servercore:{{OS_VERSION_NUMBER}}-amd64 AS installer {{if isInternal: -ARG SAS_QUERY_STRING +ARG ACCESSTOKEN }} {{InsertTemplate("Dockerfile.windows.install-aspnet", [ "use-local-version-var": "true", - "url-suffix": when(isInternal, "$Env:SAS_QUERY_STRING", ""), "is-internal": isInternal ])}} diff --git a/eng/dockerfile-templates/runtime-deps/Dockerfile b/eng/dockerfile-templates/runtime-deps/Dockerfile index 18409f47a5..89570ebed4 100644 --- a/eng/dockerfile-templates/runtime-deps/Dockerfile +++ b/eng/dockerfile-templates/runtime-deps/Dockerfile @@ -12,7 +12,7 @@ set isAzureLinux to isCblMariner || defined(match(OS_VERSION, "^azurelinux\d+\.\d+$")) ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "msrc") >= 0 || find(baseUrl, "internal") >= 0 ^ + set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ set baseImageRepo to when(isAlpine, cat(ARCH_VERSIONED, "/alpine"), when(isDebian, @@ -32,7 +32,6 @@ set secondStageName to when(isMultiStage && nonRootUserSupported, "installer") ^ set stagingDir to "/staging" ^ set createUserHome to is ^ - set urlSuffix to when(isInternal, "$SAS_QUERY_STRING", "") ^ set rpmFilename to "dotnet-runtime-deps.rpm" ^ set utilPkgs to when(isAzureLinux && nonRootUserSupported, ["shadow-utils"], []) ^ set useGlobalizationInvariantMode to !ARGS["is-extra"] && (isDistroless || isAlpine) ^ @@ -42,7 +41,7 @@ set nonRootUserComment to "# Create a non-root user and group" }}FROM {{baseImageRepo}}:{{baseImageTag}}{{if isMultiStage: AS {{firstStageName}}}}{{if isRpmInstall && isInternal: -ARG SAS_QUERY_STRING +ARG ACCESSTOKEN RUN {{InsertTemplate("Dockerfile.download-runtime-deps-pkg", [ diff --git a/eng/dockerfile-templates/runtime/Dockerfile.linux b/eng/dockerfile-templates/runtime/Dockerfile.linux index d3caeea325..646db98d33 100644 --- a/eng/dockerfile-templates/runtime/Dockerfile.linux +++ b/eng/dockerfile-templates/runtime/Dockerfile.linux @@ -10,7 +10,7 @@ set isFullAzureLinux to isAzureLinux && !isDistroless ^ set isDistrolessAzureLinux to isAzureLinux && isDistroless ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "msrc") >= 0 || find(baseUrl, "internal") >= 0 ^ + set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ set isRpmInstall to isFullAzureLinux && dotnetVersion = "6.0" ^ set isSingleStage to (isAlpine || isRpmInstall) && !isInternal ^ set runtimeDepsVariant to when(ARGS["is-extra"], "-extra", "") ^ @@ -60,7 +60,7 @@ _ MULTI STAGE # Installer image FROM {{installerImageTag}} AS installer {{if isInternal: -ARG SAS_QUERY_STRING +ARG ACCESSTOKEN }}{{ if isDistrolessAzureLinux: {{InsertTemplate("../Dockerfile.linux.distroless-azurelinux-installer-prereqs")}} ^elif isFullAzureLinux && !isRpmInstall: @@ -76,7 +76,6 @@ RUN {{InsertTemplate("../Dockerfile.linux.install-pkgs", "dest-dir": when(isDistroless, "/usr/share/dotnet", "/dotnet"), "use-local-version-var": "true", "is-internal": isInternal, - "url-suffix": when(isInternal, "$SAS_QUERY_STRING", ""), "is-rpm-install": isRpmInstall ])}}{{ if isDistroless: @@ -94,7 +93,6 @@ FROM {{runtimeDepsBaseTag}} "install-method": "copy-and-install", "dest-dir": when(isDistroless, "/usr/share/dotnet", "/dotnet"), "is-internal": isInternal, - "url-suffix": when(isInternal, "$SAS_QUERY_STRING", ""), "installer-stage": "installer", "is-rpm-install": isRpmInstall ])}}}}{{ if isDistroless: diff --git a/eng/dockerfile-templates/runtime/Dockerfile.windows b/eng/dockerfile-templates/runtime/Dockerfile.windows index 38ed875e6c..b82e1b5a4c 100644 --- a/eng/dockerfile-templates/runtime/Dockerfile.windows +++ b/eng/dockerfile-templates/runtime/Dockerfile.windows @@ -2,7 +2,7 @@ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ set isServerCore to find(OS_VERSION, "windowsservercore") >= 0 ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "msrc") >= 0 || find(baseUrl, "internal") >= 0 ^ + set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ set isSingleStage to (find(OS_VERSION, "windowsservercore") >= 0 && !isInternal) ^ set serverCoreBaseTag to cat("mcr.microsoft.com/windows/servercore:", OS_VERSION_NUMBER, "-amd64") ^ set finalStageBaseRepo to when(isInternal && isServerCore, "servercore", "nanoserver") @@ -28,13 +28,12 @@ else:{{ }}# Installer image FROM {{serverCoreBaseTag}} AS installer {{if isInternal: -ARG SAS_QUERY_STRING +ARG ACCESSTOKEN }} # Retrieve .NET Runtime {{InsertTemplate("Dockerfile.windows.install-runtime", [ "use-local-version-var": "true", - "url-suffix": when(isInternal, "$Env:SAS_QUERY_STRING", ""), "is-internal": isInternal ])}} diff --git a/eng/dockerfile-templates/sdk/Dockerfile.linux b/eng/dockerfile-templates/sdk/Dockerfile.linux index 85334d13f7..4d32d94be6 100644 --- a/eng/dockerfile-templates/sdk/Dockerfile.linux +++ b/eng/dockerfile-templates/sdk/Dockerfile.linux @@ -3,10 +3,9 @@ set isAlpine to find(OS_VERSION, "alpine") >= 0 ^ set isMariner to find(OS_VERSION, "cbl-mariner") >= 0 ^ set isAzureLinux to isMariner || find(OS_VERSION, "azurelinux") >= 0 ^ - set isFullAzureLinux to defined(match(OS_VERSION, "^cbl-mariner\d+\.\d+$")) ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "msrc") >= 0 || find(baseUrl, "internal") >= 0 ^ - set isRpmInstall to isFullAzureLinux && dotnetVersion = "6.0" ^ + set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ + set isRpmInstall to isAzureLinux && dotnetVersion = "6.0" ^ set tagVersion to when(dotnetVersion = "6.0" || dotnetVersion = "8.0", VARIABLES[cat("dotnet|", dotnetVersion, "|product-version")] VARIABLES[cat("dotnet|", dotnetVersion, "|fixed-tag")]) ^ @@ -78,8 +77,8 @@ {{if isInternal || useNobleArm32Workaround:# Installer image FROM {{internalInstallerBase}} AS installer{{if isInternal: -ARG SAS_QUERY_STRING}} -{{if isFullAzureLinux && !isRpmInstall: +ARG ACCESSTOKEN}} +{{if isAzureLinux && !isRpmInstall: RUN {{InsertTemplate("../Dockerfile.linux.install-pkgs", [ "pkgs": ["tar"] @@ -89,7 +88,6 @@ RUN {{InsertTemplate("../Dockerfile.linux.install-pkgs", [ "install-method": when(isRpmInstall, "download", "download-and-install"), "is-internal": isInternal, - "url-suffix": when(isInternal, "$SAS_QUERY_STRING", ""), "is-rpm-install": isRpmInstall, "disable-first-run-experience": useNobleArm32Workaround, "no-version-env-var": useNobleArm32Workaround @@ -111,7 +109,6 @@ if isRpmInstall:{{InsertTemplate("Dockerfile.linux.install-sdk", [ "install-method": "copy-and-install", "is-internal": isInternal, - "url-suffix": "$SAS_QUERY_STRING", "installer-stage": "installer", "is-rpm-install": isRpmInstall ])}}^ diff --git a/eng/dockerfile-templates/sdk/Dockerfile.windows b/eng/dockerfile-templates/sdk/Dockerfile.windows index 2884fe4fca..06098b9d05 100644 --- a/eng/dockerfile-templates/sdk/Dockerfile.windows +++ b/eng/dockerfile-templates/sdk/Dockerfile.windows @@ -1,7 +1,7 @@ {{ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "msrc") >= 0 || find(baseUrl, "internal") >= 0 ^ + set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ set isSingleStage to (find(OS_VERSION, "windowsservercore") >= 0 && !isInternal) ^ set tagVersion to when(dotnetVersion = "6.0" || dotnetVersion = "8.0", VARIABLES[cat("dotnet|", dotnetVersion, "|product-version")] @@ -39,13 +39,12 @@ ARG REPO=mcr.microsoft.com/dotnet/aspnet }}# Installer image FROM mcr.microsoft.com/windows/servercore:{{OS_VERSION_NUMBER}}-amd64 AS installer {{if isInternal: -ARG SAS_QUERY_STRING +ARG ACCESSTOKEN }} {{InsertTemplate("Dockerfile.windows.install-components", [ "use-local-version-var": "true", "dotnet-is-internal": isInternal, - "dotnet-url-suffix": when(isInternal, "$Env:SAS_QUERY_STRING", "") ])}} # SDK image diff --git a/eng/dockerfile-templates/sdk/Dockerfile.windows.install-components b/eng/dockerfile-templates/sdk/Dockerfile.windows.install-components index 747a2f6856..2aab6793a1 100644 --- a/eng/dockerfile-templates/sdk/Dockerfile.windows.install-components +++ b/eng/dockerfile-templates/sdk/Dockerfile.windows.install-components @@ -52,7 +52,8 @@ RUN powershell -Command " ` "dest-dir": mingitDir ], " ")}}" -}}RUN powershell -Command " ` +}}{{if ARGS["dotnet-is-internal"]:SHELL ["powershell", "-command"] +RUN `^else:RUN powershell -Command " `}} $ErrorActionPreference = 'Stop'; ` $ProgressPreference = 'SilentlyContinue'; ` ` @@ -74,4 +75,4 @@ RUN powershell -Command " ` # Delete everything in the dotnet folder that's not needed in the SDK layer but will instead be derived from base layers{{ for i, group in sdkExtractGroups: Get-ChildItem -Exclude {{join(map(group.paths, getFormattedPath), ",")}} -Path {{group.dir}} ` - | Remove-Item -Force -Recurse{{if i < len(group.paths) - 1:; `}}}}}}" + | Remove-Item -Force -Recurse{{if i < len(group.paths) - 1:; `}}}}}}{{if !ARGS["dotnet-is-internal"]:"}} From 0eda96e8886b88906af1b800d5aa294e2c504157 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Fri, 27 Sep 2024 15:31:51 -0700 Subject: [PATCH 03/15] Pipeline changes for internal testing --- eng/common/templates/jobs/build-images.yml | 14 ++- .../jobs/copy-base-images-staging.yml | 4 + .../templates/jobs/copy-base-images.yml | 4 + .../jobs/test-images-linux-client.yml | 2 + .../jobs/test-images-windows-client.yml | 2 + .../stages/build-test-publish-repo.yml | 107 ++++++++++-------- .../stages/dotnet/build-test-publish-repo.yml | 2 + .../templates/steps/copy-base-images.yml | 5 +- .../steps/test-images-linux-client.yml | 11 +- .../steps/test-images-windows-client.yml | 9 +- .../dotnet-core-internal-testing.yml | 38 +++++++ .../stages/build-test-publish-repo.yml | 10 +- .../steps/set-custom-test-variables.yml | 6 +- 13 files changed, 145 insertions(+), 69 deletions(-) create mode 100644 eng/pipelines/dotnet-core-internal-testing.yml diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index 2b6ba00735..b567c4523f 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -10,12 +10,14 @@ parameters: publicProjectName: null internalVersionsRepoRef: null publicVersionsRepoRef: null + isInternalServicingValidation: false jobs: - job: ${{ parameters.name }} - condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped')) + condition: and(${{ parameters.matrix }}, not(canceled()), or(in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), eq(${{ parameters.isInternalServicingValidation }}, 'true'))) dependsOn: - - PreBuildValidation + - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: + - PreBuildValidation - CopyBaseImages - GenerateBuildMatrix pool: ${{ parameters.pool }} @@ -97,7 +99,7 @@ jobs: # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates # the environment variable for us. $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json" - if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { + if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") { $imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr-staging.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push" } @@ -141,7 +143,7 @@ jobs: displayName: Publish Image Info File Artifact internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: # The following task depends on the SBOM Manifest Generator task installed on the agent. # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves. - powershell: | @@ -193,11 +195,11 @@ jobs: } displayName: Generate SBOMs condition: and(succeeded(), ne(variables['BuildImages.builtImages'], '')) - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}: - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self parameters: condition: ne(variables.testScriptPath, '') - - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - template: /eng/common/templates/steps/publish-artifact.yml@self parameters: path: $(sbomDirectory) diff --git a/eng/common/templates/jobs/copy-base-images-staging.yml b/eng/common/templates/jobs/copy-base-images-staging.yml index 169eeb1441..b1c24a6e49 100644 --- a/eng/common/templates/jobs/copy-base-images-staging.yml +++ b/eng/common/templates/jobs/copy-base-images-staging.yml @@ -14,6 +14,9 @@ parameters: - name: continueOnError type: string default: false +- name: isInternalServicingValidation + type: string + default: false jobs: - template: /eng/common/templates/jobs/copy-base-images.yml@self @@ -28,3 +31,4 @@ jobs: subscription: $(acr-staging.subscription) resourceGroup: $(acr-staging.resourceGroup) repoPrefix: $(mirrorRepoPrefix) + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/jobs/copy-base-images.yml b/eng/common/templates/jobs/copy-base-images.yml index c75974216d..6b8725bf52 100644 --- a/eng/common/templates/jobs/copy-base-images.yml +++ b/eng/common/templates/jobs/copy-base-images.yml @@ -23,6 +23,9 @@ parameters: - name: forceDryRun type: boolean default: false +- name: isInternalServicingValidation + type: string + default: false jobs: - job: ${{ parameters.name }} @@ -37,3 +40,4 @@ jobs: additionalOptions: ${{ parameters.additionalOptions }} continueOnError: ${{ parameters.continueOnError }} forceDryRun: ${{ parameters.forceDryRun }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/jobs/test-images-linux-client.yml b/eng/common/templates/jobs/test-images-linux-client.yml index 4e1c0fd440..248d54ab38 100644 --- a/eng/common/templates/jobs/test-images-linux-client.yml +++ b/eng/common/templates/jobs/test-images-linux-client.yml @@ -6,6 +6,7 @@ parameters: preBuildValidation: false internalProjectName: null customInitSteps: [] + isInternalServicingValidation: false jobs: - job: ${{ parameters.name }} @@ -24,3 +25,4 @@ jobs: preBuildValidation: ${{ parameters.preBuildValidation }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/jobs/test-images-windows-client.yml b/eng/common/templates/jobs/test-images-windows-client.yml index 3a0b2fd917..dbe1c0806e 100644 --- a/eng/common/templates/jobs/test-images-windows-client.yml +++ b/eng/common/templates/jobs/test-images-windows-client.yml @@ -5,6 +5,7 @@ parameters: testJobTimeout: 60 internalProjectName: null customInitSteps: [] + isInternalServicingValidation: false jobs: - job: ${{ parameters.name }} @@ -19,3 +20,4 @@ jobs: parameters: internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml index 29a72a48fc..c70d2564b3 100644 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ b/eng/common/templates/stages/build-test-publish-repo.yml @@ -25,6 +25,8 @@ parameters: internalVersionsRepoRef: null publicVersionsRepoRef: null + isInternalServicingValidation: false + linuxAmd64Pool: vmImage: $(defaultLinuxAmd64PoolImage) linuxArm32Pool: @@ -46,30 +48,32 @@ stages: - stage: Build condition: and(succeeded(), contains(variables['stages'], 'build')) jobs: - - template: /eng/common/templates/jobs/test-images-linux-client.yml@self - parameters: - name: PreBuildValidation - pool: ${{ parameters.linuxAmd64Pool }} - testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} - preBuildValidation: true - internalProjectName: ${{ parameters.internalProjectName }} - customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables + - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: + - template: /eng/common/templates/jobs/test-images-linux-client.yml@self + parameters: + name: PreBuildValidation + pool: ${{ parameters.linuxAmd64Pool }} + testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} + preBuildValidation: true + internalProjectName: ${{ parameters.internalProjectName }} + customInitSteps: + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self parameters: name: CopyBaseImages pool: ${{ parameters.linuxAmd64Pool }} additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/generate-matrix.yml@self parameters: matrixType: ${{ parameters.buildMatrixType }} @@ -92,6 +96,7 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Linux_arm64 @@ -105,6 +110,7 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Linux_arm32 @@ -118,6 +124,7 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Windows1809_amd64 @@ -131,6 +138,7 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: Windows2022_amd64 @@ -144,6 +152,7 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/build-images.yml@self parameters: name: WindowsLtsc2016_amd64 @@ -157,6 +166,7 @@ stages: publicProjectName: ${{ parameters.publicProjectName }} internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }} publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} ################################################################################ # Post-Build @@ -174,7 +184,7 @@ stages: ################################################################################ # Test Images ################################################################################ -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - stage: Test dependsOn: Post_Build condition: " @@ -205,6 +215,7 @@ stages: testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: Linux_arm64 @@ -213,6 +224,7 @@ stages: testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: Linux_arm32 @@ -221,6 +233,7 @@ stages: testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self parameters: name: Windows1809_amd64 @@ -229,6 +242,7 @@ stages: testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self parameters: name: Windows2022_amd64 @@ -237,6 +251,7 @@ stages: testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self parameters: name: WindowsLtsc2016_amd64 @@ -245,44 +260,46 @@ stages: testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} ################################################################################ # Publish Images ################################################################################ -- stage: Publish - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - dependsOn: Test - ${{ else }}: - dependsOn: Post_Build - condition: " - and( - not(canceled()), +- ${{ if eq(parameters.isInternalServicingValidation, 'false') }}: + - stage: Publish + ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + dependsOn: Test + ${{ else }}: + dependsOn: Post_Build + condition: " and( - contains(variables['stages'], 'publish'), - or( + not(canceled()), + and( + contains(variables['stages'], 'publish'), or( - and( - and( - contains(variables['stages'], 'build'), - succeeded('Post_Build')), - and( - contains(variables['stages'], 'test'), - in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), or( and( - not(contains(variables['stages'], 'build')), + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build')), and( contains(variables['stages'], 'test'), in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), - and( - not(contains(variables['stages'], 'test')), + or( and( - contains(variables['stages'], 'build'), - succeeded('Post_Build'))))), - not( - or( - contains(variables['stages'], 'build'), - contains(variables['stages'], 'test'))))))" + not(contains(variables['stages'], 'build')), + and( + contains(variables['stages'], 'test'), + in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))), + and( + not(contains(variables['stages'], 'test')), + and( + contains(variables['stages'], 'build'), + succeeded('Post_Build'))))), + not( + or( + contains(variables['stages'], 'build'), + contains(variables['stages'], 'test'))))))" jobs: - template: /eng/common/templates/jobs/publish.yml@self parameters: diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml index 53f9e8a637..683a8c21b7 100644 --- a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml @@ -18,6 +18,7 @@ parameters: linuxAmd64Pool: "" buildMatrixType: platformDependencyGraph testMatrixType: platformVersionedOs + isInternalServicingValidation: false stages: - template: /eng/common/templates/stages/build-test-publish-repo.yml@self @@ -25,6 +26,7 @@ stages: noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} testMatrixCustomBuildLegGroupArgs: ${{ parameters.testMatrixCustomBuildLegGroupArgs }} customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps}} diff --git a/eng/common/templates/steps/copy-base-images.yml b/eng/common/templates/steps/copy-base-images.yml index e24fc0306e..a032f15614 100644 --- a/eng/common/templates/steps/copy-base-images.yml +++ b/eng/common/templates/steps/copy-base-images.yml @@ -18,9 +18,12 @@ parameters: - name: forceDryRun type: boolean default: false +- name: isInternalServicingValidation + type: string + default: false steps: -- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}: - script: echo "##vso[task.setvariable variable=dryRunArg]--dry-run" - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: diff --git a/eng/common/templates/steps/test-images-linux-client.yml b/eng/common/templates/steps/test-images-linux-client.yml index 6a747016c8..644cd79e73 100644 --- a/eng/common/templates/steps/test-images-linux-client.yml +++ b/eng/common/templates/steps/test-images-linux-client.yml @@ -3,15 +3,16 @@ parameters: internalProjectName: null condition: true customInitSteps: [] + isInternalServicingValidation: false steps: - template: /eng/common/templates/steps/init-docker-linux.yml@self parameters: setupImageBuilder: false setupTestRunner: true - # Clean only up when we're running an internal build, not a PR, and not doing pre-build validation. + # Clean only up when we're running an internal build, not a PR, not doing internal servicing validation, and not doing pre-build validation. # i.e. when we're building something important. - cleanupDocker: ${{ and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.preBuildValidation, 'false')) }} + cleanupDocker: ${{ and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.preBuildValidation, 'false'), eq(parameters.isInternalServicingValidation, 'false')) }} condition: ${{ parameters.condition }} - ${{ parameters.customInitSteps }} - script: | @@ -21,7 +22,7 @@ steps: if [ "${{ parameters.preBuildValidation }}" == "true" ]; then additionalTestArgs="$additionalTestArgs -TestCategories pre-build" else - if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then + if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ] && [ "${{ parameters.isInternalServicingValidation }}" != "true" ]; then additionalTestArgs="$additionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json" if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" @@ -41,7 +42,7 @@ steps: $(imageNames.testRunner.withrepo) displayName: Start Test Runner Container condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self parameters: displayName: Docker login @@ -72,7 +73,7 @@ steps: $(additionalTestArgs)" displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - script: docker exec $(testRunner.container) docker logout $(acr-staging.server) displayName: Docker logout condition: and(always(), ${{ parameters.condition }}) diff --git a/eng/common/templates/steps/test-images-windows-client.yml b/eng/common/templates/steps/test-images-windows-client.yml index 04b099d523..597493629c 100644 --- a/eng/common/templates/steps/test-images-windows-client.yml +++ b/eng/common/templates/steps/test-images-windows-client.yml @@ -2,9 +2,10 @@ parameters: internalProjectName: null condition: true customInitSteps: [] + isInternalServicingValidation: false steps: -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - template: /eng/common/templates/steps/init-docker-windows.yml@self parameters: cleanupDocker: true @@ -22,7 +23,7 @@ steps: docker login $(acr-staging.server) -u 00000000-0000-0000-0000-000000000000 -p $accessToken - ${{ parameters.customInitSteps }} - powershell: | - if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { + if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") { $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json" } echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" @@ -31,7 +32,7 @@ steps: - powershell: Get-ChildItem -Path tests -r | Where {$_.Extension -match "trx"} | Remove-Item displayName: Cleanup Old Test Results condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - template: /eng/common/templates/steps/download-build-artifact.yml@self parameters: targetPath: $(Build.ArtifactStagingDirectory) @@ -46,7 +47,7 @@ steps: $(additionalTestArgs) displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: - script: docker logout $(acr-staging.server) displayName: Docker logout condition: and(always(), ${{ parameters.condition }}) diff --git a/eng/pipelines/dotnet-core-internal-testing.yml b/eng/pipelines/dotnet-core-internal-testing.yml new file mode 100644 index 0000000000..3392007699 --- /dev/null +++ b/eng/pipelines/dotnet-core-internal-testing.yml @@ -0,0 +1,38 @@ +trigger: + batch: true + branches: + include: + - nightly + paths: + include: + - manifest.json + - manifest.versions.json + - src/* +pr: none + +resources: + repositories: + - repository: InternalVersionsRepo + type: github + endpoint: dotnet + name: dotnet/versions + +variables: +- template: /eng/pipelines/variables/internal-core.yml@self +- name: IsInternalServicingValidation + value: true +- name: imageBuilder.pathArgs + value: "--path '*9.0*'" +- name: buildId + value: 2525144 + +extends: + template: /eng/common/templates/1es-official.yml@self + parameters: + stages: + - template: stages/build-test-publish-repo.yml + parameters: + internalProjectName: ${{ variables.internalProjectName }} + publicProjectName: ${{ variables.publicProjectName }} + isInternalServicingValidation: ${{ variables.IsInternalServicingValidation }} + noCache: true diff --git a/eng/pipelines/stages/build-test-publish-repo.yml b/eng/pipelines/stages/build-test-publish-repo.yml index a5b2055afd..3d9a8439a3 100644 --- a/eng/pipelines/stages/build-test-publish-repo.yml +++ b/eng/pipelines/stages/build-test-publish-repo.yml @@ -6,6 +6,7 @@ parameters: internalProjectName: null publicProjectName: null linuxAmd64Pool: "" + isInternalServicingValidation: false stages: - template: /eng/common/templates/stages/dotnet/build-test-publish-repo.yml@self @@ -13,8 +14,9 @@ stages: noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} linuxAmd64Pool: ${{ parameters.linuxAmd64Pool }} - ${{ if or(eq(variables['System.TeamProject'], parameters.publicProjectName), and(eq(variables['System.TeamProject'], parameters.internalProjectName), eq(variables['Build.Reason'], 'PullRequest'))) }}: + ${{ if or(eq(variables['System.TeamProject'], parameters.publicProjectName), and(eq(variables['System.TeamProject'], parameters.internalProjectName), or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')))) }}: buildMatrixType: platformVersionedOs buildMatrixCustomBuildLegGroupArgs: --custom-build-leg-group pr-build --custom-build-leg-group test-dependencies ${{ elseif eq(variables['System.TeamProject'], parameters.internalProjectName) }}: @@ -26,9 +28,9 @@ stages: - template: /eng/pipelines/steps/set-public-source-branch-var.yml@self - powershell: | $imageBuilderBuildArgs = "$IMAGEBUILDERBUILDARGS" - if ("$(publishRepoPrefix)".Contains("internal/")) { - $sasQueryString = "$(dotnetstage-account-sas-read-token)" - $imageBuilderBuildArgs += " --build-arg SAS_QUERY_STRING='$sasQueryString'" + if ("${{ parameters.IsInternalServicingValidation }}" -eq "true") { + $accessToken = "$(System.AccessToken)" + $imageBuilderBuildArgs += " --build-arg ACCESSTOKEN='$accessToken'" } echo "##vso[task.setvariable variable=imageBuilderBuildArgs]$imageBuilderBuildArgs" displayName: Set Custom Build Variables diff --git a/eng/pipelines/steps/set-custom-test-variables.yml b/eng/pipelines/steps/set-custom-test-variables.yml index 1857664e4d..167ad2dcf9 100644 --- a/eng/pipelines/steps/set-custom-test-variables.yml +++ b/eng/pipelines/steps/set-custom-test-variables.yml @@ -8,17 +8,15 @@ steps: $testInit="" if ("$(publishRepoPrefix)".Contains("internal/")) { - $sasQueryString = "$(dotnetstage-account-sas-read-token)" - if ($Env:AGENT_OS -eq 'Linux') { - $testRunnerOptions="$testRunnerOptions -e SAS_QUERY_STRING='$sasQueryString' -e NUGET_FEED_PASSWORD='$(dn-bot-dnceng-artifact-feeds-rw)'" + $testRunnerOptions="$testRunnerOptions -e INTERNAL_TESTING='1' -e NUGET_FEED_PASSWORD='$(System.AccessToken)'" } if ($Env:AGENT_OS -eq 'Windows_NT') { # Be sure to use a verbatim string when referencing the environment variables. We don't want the # variables to be resolved in this script. We're generating the script here to be executed by the # test step. - $testInit='$Env:SAS_QUERY_STRING=' + "'$sasQueryString'" + '; $Env:NUGET_FEED_PASSWORD=''$(dn-bot-dnceng-artifact-feeds-rw)''' + $testInit='$Env:INTERNAL_TESTING=''1'' ; $Env:NUGET_FEED_PASSWORD=''$(System.AccessToken)''' } } From 3dfae62dffa14c156646936475aff34d631dbd88 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Fri, 27 Sep 2024 15:32:51 -0700 Subject: [PATCH 04/15] Test infra changes for internal testing --- tests/Microsoft.DotNet.Docker.Tests/Config.cs | 7 ++----- .../Microsoft.DotNet.Docker.Tests/SdkImageTests.cs | 13 ++++++++----- .../TestAppArtifacts/NuGet.config.internal | 7 +++++++ .../TestScenarios/TestDockerfile.cs | 3 ++- tests/Microsoft.DotNet.Docker.Tests/TestSolution.cs | 4 ++++ 5 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 tests/Microsoft.DotNet.Docker.Tests/TestAppArtifacts/NuGet.config.internal diff --git a/tests/Microsoft.DotNet.Docker.Tests/Config.cs b/tests/Microsoft.DotNet.Docker.Tests/Config.cs index 0346bf880b..7d2fa5762b 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/Config.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/Config.cs @@ -40,11 +40,8 @@ public static class Config Environment.GetEnvironmentVariable("DOCKERFILE_PATHS")? .Split(',', StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty(); - public static bool IsInternal(string dotnetVersion) - { - string versionBaseUrl = GetBaseUrl(dotnetVersion); - return versionBaseUrl.Contains("msrc") || versionBaseUrl.Contains("internal"); - } + public static bool IsInternal { get; } = + Environment.GetEnvironmentVariable("INTERNAL_TESTING") != null; private static bool GetIsNightlyRepo() { diff --git a/tests/Microsoft.DotNet.Docker.Tests/SdkImageTests.cs b/tests/Microsoft.DotNet.Docker.Tests/SdkImageTests.cs index 8b92011e47..e67ab12dc4 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/SdkImageTests.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/SdkImageTests.cs @@ -8,7 +8,9 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Net.Http.Headers; using System.Security.Cryptography; +using System.Text; using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Readers; @@ -322,6 +324,11 @@ private async Task> GetExpectedSdkContentsAsync( string sdkFile = Path.GetTempFileName(); using HttpClient httpClient = new(); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", + Config.NuGetFeedPassword)))); + await httpClient.DownloadFileAsync(new Uri(sdkUrl), sdkFile); files = EnumerateArchiveContents(sdkFile) @@ -336,7 +343,7 @@ private async Task> GetExpectedSdkContentsAsync( private string GetSdkUrl(ProductImageData imageData) { - bool isInternal = Config.IsInternal(imageData.VersionString); + bool isInternal = Config.IsInternal; string sdkBuildVersion = Config.GetBuildVersion(ImageRepo, imageData.VersionString); string sdkFileVersionLabel = isInternal ? imageData.GetProductVersion(ImageRepo, ImageRepo, DockerHelper) @@ -359,10 +366,6 @@ private string GetSdkUrl(ProductImageData imageData) string fileType = DockerHelper.IsLinuxContainerModeEnabled ? "tar.gz" : "zip"; string baseUrl = Config.GetBaseUrl(imageData.VersionString); string url = $"{baseUrl}/Sdk/{sdkBuildVersion}/dotnet-sdk-{sdkFileVersionLabel}-{osType}-{architecture}.{fileType}"; - if (isInternal) - { - url += Config.SasQueryString; - } return url; } diff --git a/tests/Microsoft.DotNet.Docker.Tests/TestAppArtifacts/NuGet.config.internal b/tests/Microsoft.DotNet.Docker.Tests/TestAppArtifacts/NuGet.config.internal new file mode 100644 index 0000000000..bff97919cb --- /dev/null +++ b/tests/Microsoft.DotNet.Docker.Tests/TestAppArtifacts/NuGet.config.internal @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/TestDockerfile.cs b/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/TestDockerfile.cs index a681a4f6a9..af5269a0d8 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/TestDockerfile.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/TestDockerfile.cs @@ -47,7 +47,7 @@ COPY NuGet.config . ? DockerOS.Linux : DockerOS.Windows; - private static bool s_useNuGetConfig = Config.IsNightlyRepo; + private static bool s_useNuGetConfig = Config.IsNightlyRepo || Config.IsInternal; private static string[] s_commonArgs = [ "sdk_image", @@ -140,6 +140,7 @@ public static TestDockerfile GetBlazorWasmDockerfile(bool useWasmTools) StringBuilder buildStageBuilder = new( $""" FROM $sdk_image AS {TestDockerfile.BuildStageName} + ARG NuGetFeedPassword ARG port EXPOSE $port """); diff --git a/tests/Microsoft.DotNet.Docker.Tests/TestSolution.cs b/tests/Microsoft.DotNet.Docker.Tests/TestSolution.cs index fb88629caa..109d826401 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/TestSolution.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/TestSolution.cs @@ -61,6 +61,10 @@ private string CreateTestSolutionWithSdkImage(string solutionDir, string appType { nuGetConfigFileName += ".nightly"; } + else if(Config.IsInternal) + { + nuGetConfigFileName += ".internal"; + } File.Copy(Path.Combine(DockerHelper.TestArtifactsDir, nuGetConfigFileName), Path.Combine(solutionDir, "NuGet.config")); File.Copy(Path.Combine(DockerHelper.TestArtifactsDir, ".dockerignore"), Path.Combine(solutionDir, ".dockerignore")); From 042d5f1173dcfd803d8947f33e436418dc6807ed Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Mon, 30 Sep 2024 08:42:23 -0700 Subject: [PATCH 05/15] Remove unnecessary variables --- eng/pipelines/dotnet-core-internal-testing.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/pipelines/dotnet-core-internal-testing.yml b/eng/pipelines/dotnet-core-internal-testing.yml index 3392007699..e85193e3d6 100644 --- a/eng/pipelines/dotnet-core-internal-testing.yml +++ b/eng/pipelines/dotnet-core-internal-testing.yml @@ -21,10 +21,6 @@ variables: - template: /eng/pipelines/variables/internal-core.yml@self - name: IsInternalServicingValidation value: true -- name: imageBuilder.pathArgs - value: "--path '*9.0*'" -- name: buildId - value: 2525144 extends: template: /eng/common/templates/1es-official.yml@self From 11e015167b5a9a29d48a424c168d023c1cb75d6e Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Tue, 1 Oct 2024 09:34:58 -0700 Subject: [PATCH 06/15] Remove unused url-suffix references --- .../aspnet/Dockerfile.linux.install-aspnet | 3 +-- .../aspnet/Dockerfile.windows.install-aspnet | 3 +-- eng/dockerfile-templates/runtime-deps/Dockerfile | 2 -- .../Dockerfile.download-runtime-deps-pkg | 3 +-- .../Dockerfile.install-runtime-deps-pkg | 2 -- .../runtime/Dockerfile.linux.install-runtime | 12 +++++------- .../runtime/Dockerfile.windows.install-runtime | 3 +-- .../sdk/Dockerfile.linux.install-sdk | 13 +++++-------- .../sdk/Dockerfile.windows.install-components | 6 ++---- .../sdk/Dockerfile.windows.install-sdk | 5 ++--- 10 files changed, 18 insertions(+), 34 deletions(-) diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.linux.install-aspnet b/eng/dockerfile-templates/aspnet/Dockerfile.linux.install-aspnet index 2ab72fa934..59e724b9d0 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.linux.install-aspnet +++ b/eng/dockerfile-templates/aspnet/Dockerfile.linux.install-aspnet @@ -4,7 +4,6 @@ - use-local-version-var (optional): Whether to define a local variable for the ASP.NET Core runtime version instead of referencing the environment variable. - is-internal (optional): Whether the Dockerfile is targeting an internal build of the product. - - url-suffix (optional): Suffix string to append the end of the URL. - installer-stage (optional): Name of the Dockerfile stage responsible for installation - is-rpm-install (optional): Whether to install RPM versus tarball ^ @@ -33,7 +32,7 @@ set url to cat( VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])], "/aspnetcore/Runtime/", aspnetVersionDir, "/aspnetcore-runtime-", aspnetVersionFile, - filePlatform, "-", fileArch, ".", fileExt, ARGS["url-suffix"]) ^ + filePlatform, "-", fileArch, ".", fileExt) ^ set files to [ [ "filename": cat("aspnetcore.", fileExt), diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.windows.install-aspnet b/eng/dockerfile-templates/aspnet/Dockerfile.windows.install-aspnet index 4c563aeebd..ee10650de9 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.windows.install-aspnet +++ b/eng/dockerfile-templates/aspnet/Dockerfile.windows.install-aspnet @@ -2,7 +2,6 @@ _ ARGS: - use-local-version-var (optional): Whether to define a local variable for the ASP.NET Core runtime version instead of referencing the environment variable. - - url-suffix (optional): Suffix string to append the end of the URL. - is-internal (optional): Whether the Dockerfile is targeting an internal build of the product. ^ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ @@ -20,7 +19,7 @@ aspnetVersion) ^ set url to cat( VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])], - "/aspnetcore/Runtime/", aspnetVersion, "/aspnetcore-runtime-", aspnetVersionFile, "-win-x64.zip", ARGS["url-suffix"]) + "/aspnetcore/Runtime/", aspnetVersion, "/aspnetcore-runtime-", aspnetVersionFile, "-win-x64.zip") }}# Install ASP.NET Core Runtime RUN powershell -Command ` $ErrorActionPreference = 'Stop'; ` diff --git a/eng/dockerfile-templates/runtime-deps/Dockerfile b/eng/dockerfile-templates/runtime-deps/Dockerfile index 89570ebed4..1bad2daf80 100644 --- a/eng/dockerfile-templates/runtime-deps/Dockerfile +++ b/eng/dockerfile-templates/runtime-deps/Dockerfile @@ -45,7 +45,6 @@ ARG ACCESSTOKEN RUN {{InsertTemplate("Dockerfile.download-runtime-deps-pkg", [ - "url-suffix": urlSuffix, "filename": rpmFilename, "is-internal": isInternal ], @@ -80,7 +79,6 @@ RUN {{InsertTemplate("../Dockerfile.linux.install-deps", {{InsertTemplate("Dockerfile.install-runtime-deps-pkg", [ "skip-download": isInternal - "url-suffix": urlSuffix, "filename": rpmFilename ] )}}}}{{if isMultiStage && nonRootUserSupported: diff --git a/eng/dockerfile-templates/runtime-deps/Dockerfile.download-runtime-deps-pkg b/eng/dockerfile-templates/runtime-deps/Dockerfile.download-runtime-deps-pkg index 446335a0dc..1064c859c0 100644 --- a/eng/dockerfile-templates/runtime-deps/Dockerfile.download-runtime-deps-pkg +++ b/eng/dockerfile-templates/runtime-deps/Dockerfile.download-runtime-deps-pkg @@ -1,6 +1,5 @@ {{ _ ARGS: - url-suffix (optional): Suffix string to append the end of the URL. filename: Name of the file to download is-internal (optional): Whether the Dockerfile is targeting an internal build of the product. ^ @@ -18,7 +17,7 @@ && {{InsertTemplate("../Dockerfile.linux.download-file", [ "out-file": ARGS["filename"], - "url": cat(VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])], "/Runtime/$dotnet_version/dotnet-runtime-deps-", runtimeVersionFile, "-cm.", marinerMajorVersion, "-", rpmFileArch, ".rpm", ARGS["url-suffix"]), + "url": cat(VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])], "/Runtime/$dotnet_version/dotnet-runtime-deps-", runtimeVersionFile, "-cm.", marinerMajorVersion, "-", rpmFileArch, ".rpm"), "sha": VARIABLES[cat("runtime-deps-cm.", marinerMajorVersion, "|", dotnetVersion, "|linux-rpm|", ARCH_SHORT, "|sha")], "sha-var-name": "dotnet_sha512" ])}} diff --git a/eng/dockerfile-templates/runtime-deps/Dockerfile.install-runtime-deps-pkg b/eng/dockerfile-templates/runtime-deps/Dockerfile.install-runtime-deps-pkg index 90fc624576..be071683c7 100644 --- a/eng/dockerfile-templates/runtime-deps/Dockerfile.install-runtime-deps-pkg +++ b/eng/dockerfile-templates/runtime-deps/Dockerfile.install-runtime-deps-pkg @@ -1,12 +1,10 @@ {{ _ ARGS: skip-download (optional): Skip downloading the RPM - url-suffix (optional): Suffix string to append the end of the URL. filename: Name of the file to install }}# Install dotnet-runtime-deps package RUN {{if !ARGS["skip-download"]:{{InsertTemplate("Dockerfile.download-runtime-deps-pkg", [ - "url-suffix": ARGS["url-suffix"], "filename": ARGS["filename"] ], " ")}} \ && }}{{InsertTemplate("../Dockerfile.linux.install-rpms", diff --git a/eng/dockerfile-templates/runtime/Dockerfile.linux.install-runtime b/eng/dockerfile-templates/runtime/Dockerfile.linux.install-runtime index fa4a6aba57..d81cb9df86 100644 --- a/eng/dockerfile-templates/runtime/Dockerfile.linux.install-runtime +++ b/eng/dockerfile-templates/runtime/Dockerfile.linux.install-runtime @@ -6,7 +6,6 @@ use-local-version-var (optional): Whether to define a local variable for the .NET runtime version instead of referencing the environment variable. is-internal (optional): Whether the Dockerfile is targeting an internal build of the product. - url-suffix (optional): Suffix string to append the end of the URL. installer-stage (optional): Name of the Dockerfile stage responsible for installation is-rpm-install (optional): Whether to install RPM versus tarball is-composite-runtime (optional): Whether to install aspnetcore composite version ^ @@ -39,7 +38,7 @@ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ set runtimeBaseUrl to cat(baseUrl, "/Runtime/", runtimeVersionDir, "/") ^ set aspnetCompositeUrl to cat(baseUrl, "/aspnetcore/Runtime/", aspnetCompositeVersionDir, "/aspnetcore-runtime-composite-", aspnetCompositeVersionFile, - filePlatform, "-", fileArch, ".", fileExt, ARGS["url-suffix"]) ^ + filePlatform, "-", fileArch, ".", fileExt) ^ set localVersionVar to when(ARGS["is-composite-runtime"], VARIABLES[cat("runtime|", dotnetVersion, "|build-version")], VARIABLES[cat("aspnet|", dotnetVersion, "|build-version")]) ^ @@ -47,19 +46,19 @@ set rpms to [ [ "filename": "dotnet-host.rpm", - "url": cat(runtimeBaseUrl, "dotnet-host-", runtimeVersionFile, "-", rpmFileArch, ".rpm", ARGS["url-suffix"]), + "url": cat(runtimeBaseUrl, "dotnet-host-", runtimeVersionFile, "-", rpmFileArch, ".rpm"), "sha": VARIABLES[join(["runtime-host", dotnetVersion, "linux-rpm", ARCH_SHORT, "sha"], "|")], "sha-var-name": "dotnet_sha512" ], [ "filename": "dotnet-hostfxr.rpm", - "url": cat(runtimeBaseUrl, "dotnet-hostfxr-", runtimeVersionFile, "-", rpmFileArch, ".rpm", ARGS["url-suffix"]), + "url": cat(runtimeBaseUrl, "dotnet-hostfxr-", runtimeVersionFile, "-", rpmFileArch, ".rpm"), "sha": VARIABLES[join(["runtime-hostfxr", dotnetVersion, "linux-rpm", ARCH_SHORT, "sha"], "|")], "sha-var-name": "dotnet_sha512" ], [ "filename": "dotnet-runtime.rpm", - "url": cat(runtimeBaseUrl, "dotnet-runtime-", runtimeVersionFile, "-", rpmFileArch, ".rpm", ARGS["url-suffix"]), + "url": cat(runtimeBaseUrl, "dotnet-runtime-", runtimeVersionFile, "-", rpmFileArch, ".rpm"), "sha": VARIABLES[join(["runtime", dotnetVersion, "linux-rpm", ARCH_SHORT, "sha"], "|")], "sha-var-name": "dotnet_sha512" ] @@ -76,8 +75,7 @@ [ [ "filename": "dotnet.tar.gz", - "url": cat(runtimeBaseUrl, "dotnet-runtime-", runtimeVersionFile, "-", varPlatform, "-", ARCH_SHORT, ".tar.gz", - ARGS["url-suffix"]), + "url": cat(runtimeBaseUrl, "dotnet-runtime-", runtimeVersionFile, "-", varPlatform, "-", ARCH_SHORT, ".tar.gz"), "sha": VARIABLES[join(["runtime", dotnetVersion, varPlatform, ARCH_SHORT, "sha"], "|")], "sha-var-name": "dotnet_sha512" ] diff --git a/eng/dockerfile-templates/runtime/Dockerfile.windows.install-runtime b/eng/dockerfile-templates/runtime/Dockerfile.windows.install-runtime index c6ec1a151a..bc107d3d30 100644 --- a/eng/dockerfile-templates/runtime/Dockerfile.windows.install-runtime +++ b/eng/dockerfile-templates/runtime/Dockerfile.windows.install-runtime @@ -2,7 +2,6 @@ _ ARGS: - use-local-version-var (optional): Whether to define a local variable for the ASP.NET Core runtime version instead of referencing the environment variable. - - url-suffix (optional): Suffix string to append the end of the URL. - is-internal (optional): Whether the Dockerfile is targeting an internal build of the product. ^ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ @@ -22,7 +21,7 @@ runtimeVersionDir) ^ set url to cat( VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])], - "/Runtime/", runtimeVersionDir, "/dotnet-runtime-", runtimeVersionFile, "-win-x64.zip", ARGS["url-suffix"]) + "/Runtime/", runtimeVersionDir, "/dotnet-runtime-", runtimeVersionFile, "-win-x64.zip") }}RUN powershell -Command ` $ErrorActionPreference = 'Stop'; ` $ProgressPreference = 'SilentlyContinue'; ` diff --git a/eng/dockerfile-templates/sdk/Dockerfile.linux.install-sdk b/eng/dockerfile-templates/sdk/Dockerfile.linux.install-sdk index e493222b8d..624b2203f6 100644 --- a/eng/dockerfile-templates/sdk/Dockerfile.linux.install-sdk +++ b/eng/dockerfile-templates/sdk/Dockerfile.linux.install-sdk @@ -4,7 +4,6 @@ use-local-version-var: Whether to define a local variable for the .NET SDK version instead of referencing the environment variable. is-internal (optional): Whether the Dockerfile is targeting an internal build of the product. - url-suffix (optional): Suffix string to append the end of the URL. installer-stage (optional): Name of the Dockerfile stage responsible for installation is-rpm-install (optional): Whether to install RPM versus tarball @@ -47,15 +46,14 @@ set rpms to [ [ "filename": "dotnet.rpm", - "url": cat(baseUrl, "/Sdk/", sdkVersionDir, "/dotnet-sdk-", sdkVersionFile, "-", rpmFileArch, ".rpm", - ARGS["url-suffix"]), + "url": cat(baseUrl, "/Sdk/", sdkVersionDir, "/dotnet-sdk-", sdkVersionFile, "-", rpmFileArch, ".rpm"), "sha": VARIABLES[join(["sdk", dotnetVersion, "linux-rpm", ARCH_SHORT, "sha"], "|")], "sha-var-name": commonShaVarName ], [ "filename": "apphost.rpm", "url": cat(baseUrl, "/Runtime/", runtimeVersionDir, "/dotnet-apphost-pack-", runtimeVersionFile, "-", - rpmFileArch, ".rpm", ARGS["url-suffix"]), + rpmFileArch, ".rpm"), "sha": VARIABLES[join(["runtime-apphost-pack", dotnetVersion, "linux-rpm", ARCH_SHORT, "sha"], "|")], "sha-var-name": commonShaVarName ], @@ -63,14 +61,14 @@ "filename": "targeting-pack.rpm", "url": cat(baseUrl, "/Runtime/", VARIABLES[cat("runtime|", dotnetVersion, "|targeting-pack-version")], "/dotnet-targeting-pack-", runtimeTargetingPackVersionFile, - "-", rpmFileArch, ".rpm", ARGS["url-suffix"]), + "-", rpmFileArch, ".rpm"), "sha": VARIABLES[join(["runtime-targeting-pack", dotnetVersion, "linux-rpm", ARCH_SHORT, "sha"], "|")], "sha-var-name": commonShaVarName ], [ "filename": "aspnetcore-targeting-pack.rpm", "url": cat(baseUrl, "/aspnetcore/Runtime/", VARIABLES[cat("aspnet|", dotnetVersion, "|targeting-pack-version")], - "/aspnetcore-targeting-pack-", aspnetTargetingPackVersionFile, "-", rpmFileArch, ".rpm", ARGS["url-suffix"]), + "/aspnetcore-targeting-pack-", aspnetTargetingPackVersionFile, "-", rpmFileArch, ".rpm"), "sha": VARIABLES[join(["aspnet-runtime-targeting-pack", dotnetVersion, "linux-rpm", ARCH_SHORT, "sha"], "|")], "sha-var-name": commonShaVarName ] @@ -86,8 +84,7 @@ set tarballs to [ [ "filename": "dotnet.tar.gz", - "url": cat(baseUrl, "/Sdk/", sdkVersionDir, "/dotnet-sdk-", sdkVersionFile, "-", platform, "-", ARCH_SHORT, ".tar.gz", - ARGS["url-suffix"]), + "url": cat(baseUrl, "/Sdk/", sdkVersionDir, "/dotnet-sdk-", sdkVersionFile, "-", platform, "-", ARCH_SHORT, ".tar.gz"), "sha": VARIABLES[join(["sdk", dotnetVersion, platform, ARCH_SHORT, "sha"], "|")], "sha-var-name": commonShaVarName, "extract-paths": [ diff --git a/eng/dockerfile-templates/sdk/Dockerfile.windows.install-components b/eng/dockerfile-templates/sdk/Dockerfile.windows.install-components index 2aab6793a1..f5b86d9b12 100644 --- a/eng/dockerfile-templates/sdk/Dockerfile.windows.install-components +++ b/eng/dockerfile-templates/sdk/Dockerfile.windows.install-components @@ -2,8 +2,7 @@ _ ARGS: use-local-version-var (optional): Whether to define a local variable for the SDK version instead of referencing the environment variable. - dotnet-is-internal (optional): Whether the Dockerfile is targeting an internal build of the .NET product. - dotnet-url-suffix (optional): Suffix string to append the end of .NET URLs. ^ + dotnet-is-internal (optional): Whether the Dockerfile is targeting an internal build of the .NET product. ^ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ set isServerCore to find(OS_VERSION, "windowsservercore") >= 0 ^ @@ -62,8 +61,7 @@ RUN `^else:RUN powershell -Command " `}} [ "use-local-version-var": ARGS["use-local-version-var"], "sdk-extract-groups": when(isSingleStage, sdkExtractGroups, []), - "is-internal": ARGS["dotnet-is-internal"], - "url-suffix": ARGS["dotnet-url-suffix"] + "is-internal": ARGS["dotnet-is-internal"] ], " ")}} ` ` {{InsertTemplate("Dockerfile.windows.install-powershell", diff --git a/eng/dockerfile-templates/sdk/Dockerfile.windows.install-sdk b/eng/dockerfile-templates/sdk/Dockerfile.windows.install-sdk index 16f9b94a2f..b837c3d1cb 100644 --- a/eng/dockerfile-templates/sdk/Dockerfile.windows.install-sdk +++ b/eng/dockerfile-templates/sdk/Dockerfile.windows.install-sdk @@ -3,8 +3,7 @@ use-local-version-var (optional): Whether to define a local variable for the SDK version instead of referencing the environment variable. sdk-extract-groups: Metadata groups describing the paths to be extracted from the SDK zip. - is-internal (optional): Whether the Dockerfile is targeting an internal build of the product. - url-suffix (optional): Suffix string to append the end of the URL. ^ + is-internal (optional): Whether the Dockerfile is targeting an internal build of the product. ^ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ set isServerCore to find(OS_VERSION, "windowsservercore") >= 0 ^ @@ -24,7 +23,7 @@ sdkVersionDir) ^ set url to cat( VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])], - "/Sdk/", sdkVersionDir, "/dotnet-sdk-", sdkVersionFile, "-win-x64.zip", ARGS["url-suffix"]) ^ + "/Sdk/", sdkVersionDir, "/dotnet-sdk-", sdkVersionFile, "-win-x64.zip") ^ _ The sdk-extract-groups arg is an array of maps. We want to reduce that to just a single array of formatted paths. The "for" keyword doesn't work here because we don't want to echo any output. Instead, we'll call From a44b4c0ea12609125ec32f65388ee8a99d36b8c6 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Tue, 1 Oct 2024 09:56:36 -0700 Subject: [PATCH 07/15] Revert unwanted dry-run --- eng/common/templates/steps/copy-base-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/templates/steps/copy-base-images.yml b/eng/common/templates/steps/copy-base-images.yml index a032f15614..13221ee424 100644 --- a/eng/common/templates/steps/copy-base-images.yml +++ b/eng/common/templates/steps/copy-base-images.yml @@ -23,7 +23,7 @@ parameters: default: false steps: -- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}: +- ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: - script: echo "##vso[task.setvariable variable=dryRunArg]--dry-run" - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: From a930c7ba9e34675b8ea63d4a09a1326d0d149526 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Tue, 1 Oct 2024 12:34:49 -0700 Subject: [PATCH 08/15] Remove isInternalServicingValidation parameter from templates not used in internal testing --- eng/common/templates/jobs/test-images-linux-client.yml | 2 -- eng/common/templates/jobs/test-images-windows-client.yml | 2 -- eng/common/templates/stages/build-test-publish-repo.yml | 6 ------ eng/common/templates/steps/test-images-linux-client.yml | 9 ++++----- .../templates/steps/test-images-windows-client.yml | 9 ++++----- 5 files changed, 8 insertions(+), 20 deletions(-) diff --git a/eng/common/templates/jobs/test-images-linux-client.yml b/eng/common/templates/jobs/test-images-linux-client.yml index 248d54ab38..4e1c0fd440 100644 --- a/eng/common/templates/jobs/test-images-linux-client.yml +++ b/eng/common/templates/jobs/test-images-linux-client.yml @@ -6,7 +6,6 @@ parameters: preBuildValidation: false internalProjectName: null customInitSteps: [] - isInternalServicingValidation: false jobs: - job: ${{ parameters.name }} @@ -25,4 +24,3 @@ jobs: preBuildValidation: ${{ parameters.preBuildValidation }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/jobs/test-images-windows-client.yml b/eng/common/templates/jobs/test-images-windows-client.yml index dbe1c0806e..3a0b2fd917 100644 --- a/eng/common/templates/jobs/test-images-windows-client.yml +++ b/eng/common/templates/jobs/test-images-windows-client.yml @@ -5,7 +5,6 @@ parameters: testJobTimeout: 60 internalProjectName: null customInitSteps: [] - isInternalServicingValidation: false jobs: - job: ${{ parameters.name }} @@ -20,4 +19,3 @@ jobs: parameters: internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml index c70d2564b3..6ba96f904c 100644 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ b/eng/common/templates/stages/build-test-publish-repo.yml @@ -215,7 +215,6 @@ stages: testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: Linux_arm64 @@ -224,7 +223,6 @@ stages: testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: Linux_arm32 @@ -233,7 +231,6 @@ stages: testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self parameters: name: Windows1809_amd64 @@ -242,7 +239,6 @@ stages: testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self parameters: name: Windows2022_amd64 @@ -251,7 +247,6 @@ stages: testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self parameters: name: WindowsLtsc2016_amd64 @@ -260,7 +255,6 @@ stages: testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} customInitSteps: ${{ parameters.customTestInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} ################################################################################ # Publish Images diff --git a/eng/common/templates/steps/test-images-linux-client.yml b/eng/common/templates/steps/test-images-linux-client.yml index 644cd79e73..a7aade7e01 100644 --- a/eng/common/templates/steps/test-images-linux-client.yml +++ b/eng/common/templates/steps/test-images-linux-client.yml @@ -3,7 +3,6 @@ parameters: internalProjectName: null condition: true customInitSteps: [] - isInternalServicingValidation: false steps: - template: /eng/common/templates/steps/init-docker-linux.yml@self @@ -12,7 +11,7 @@ steps: setupTestRunner: true # Clean only up when we're running an internal build, not a PR, not doing internal servicing validation, and not doing pre-build validation. # i.e. when we're building something important. - cleanupDocker: ${{ and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.preBuildValidation, 'false'), eq(parameters.isInternalServicingValidation, 'false')) }} + cleanupDocker: ${{ and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.preBuildValidation, 'false')) }} condition: ${{ parameters.condition }} - ${{ parameters.customInitSteps }} - script: | @@ -22,7 +21,7 @@ steps: if [ "${{ parameters.preBuildValidation }}" == "true" ]; then additionalTestArgs="$additionalTestArgs -TestCategories pre-build" else - if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ] && [ "${{ parameters.isInternalServicingValidation }}" != "true" ]; then + if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then additionalTestArgs="$additionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json" if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" @@ -42,7 +41,7 @@ steps: $(imageNames.testRunner.withrepo) displayName: Start Test Runner Container condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self parameters: displayName: Docker login @@ -73,7 +72,7 @@ steps: $(additionalTestArgs)" displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - script: docker exec $(testRunner.container) docker logout $(acr-staging.server) displayName: Docker logout condition: and(always(), ${{ parameters.condition }}) diff --git a/eng/common/templates/steps/test-images-windows-client.yml b/eng/common/templates/steps/test-images-windows-client.yml index 597493629c..04b099d523 100644 --- a/eng/common/templates/steps/test-images-windows-client.yml +++ b/eng/common/templates/steps/test-images-windows-client.yml @@ -2,10 +2,9 @@ parameters: internalProjectName: null condition: true customInitSteps: [] - isInternalServicingValidation: false steps: -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/steps/init-docker-windows.yml@self parameters: cleanupDocker: true @@ -23,7 +22,7 @@ steps: docker login $(acr-staging.server) -u 00000000-0000-0000-0000-000000000000 -p $accessToken - ${{ parameters.customInitSteps }} - powershell: | - if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") { + if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json" } echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" @@ -32,7 +31,7 @@ steps: - powershell: Get-ChildItem -Path tests -r | Where {$_.Extension -match "trx"} | Remove-Item displayName: Cleanup Old Test Results condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/steps/download-build-artifact.yml@self parameters: targetPath: $(Build.ArtifactStagingDirectory) @@ -47,7 +46,7 @@ steps: $(additionalTestArgs) displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}: +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - script: docker logout $(acr-staging.server) displayName: Docker logout condition: and(always(), ${{ parameters.condition }}) From 21ede77fceaa0a185a700b943fa1a9578b12952d Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Tue, 1 Oct 2024 13:37:03 -0700 Subject: [PATCH 09/15] Obtain token if not provided --- eng/Get-DropVersions.ps1 | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/eng/Get-DropVersions.ps1 b/eng/Get-DropVersions.ps1 index c515b097ba..24bd2d7727 100644 --- a/eng/Get-DropVersions.ps1 +++ b/eng/Get-DropVersions.ps1 @@ -237,13 +237,16 @@ if ($UseInternalBuild) { } if ($BuildId) { - if ($InternalArtifactsAccessToken) { - $internalBaseUrl = GetInternalBaseUrl - } - else { - Write-Error "'InternalArtifactsAccessToken' parameter is required for obtaining internal base-url, when specifying 'BuildId' option" - exit 1 + if (!$InternalArtifactsAccessToken) { + $InternalArtifactsAccessToken = az account get-access-token --query accessToken --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to obtain access token using Azure CLI" + Write-Error "Please provide 'InternalArtifactsAccessToken' parameter when using 'BuildId' option" + exit 1 + } } + + $internalBaseUrl = GetInternalBaseUrl } $queryString = "$BlobStorageSasQueryString" From b87ee96f0da4d950e6cee55cf309ce7a081b0a8b Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Tue, 1 Oct 2024 13:46:00 -0700 Subject: [PATCH 10/15] Fix indentation --- .../templates/stages/build-test-publish-repo.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml index 6ba96f904c..4053cb6eb3 100644 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ b/eng/common/templates/stages/build-test-publish-repo.yml @@ -294,10 +294,10 @@ stages: or( contains(variables['stages'], 'build'), contains(variables['stages'], 'test'))))))" - jobs: - - template: /eng/common/templates/jobs/publish.yml@self - parameters: - pool: ${{ parameters.linuxAmd64Pool }} - internalProjectName: ${{ parameters.internalProjectName }} - customPublishVariables: ${{ parameters.customPublishVariables }} - customInitSteps: ${{ parameters.customPublishInitSteps }} + jobs: + - template: /eng/common/templates/jobs/publish.yml@self + parameters: + pool: ${{ parameters.linuxAmd64Pool }} + internalProjectName: ${{ parameters.internalProjectName }} + customPublishVariables: ${{ parameters.customPublishVariables }} + customInitSteps: ${{ parameters.customPublishInitSteps }} From 1f57830d557d74a3390a210b0db1b72d6788642a Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Tue, 1 Oct 2024 16:33:33 -0700 Subject: [PATCH 11/15] Update internal url check --- eng/dockerfile-templates/Dockerfile.linux.download-file | 2 +- eng/dockerfile-templates/Dockerfile.windows.download-file | 2 +- eng/dockerfile-templates/aspnet/Dockerfile.linux | 2 +- eng/dockerfile-templates/aspnet/Dockerfile.linux-composite | 2 +- eng/dockerfile-templates/aspnet/Dockerfile.windows | 2 +- eng/dockerfile-templates/runtime-deps/Dockerfile | 2 +- eng/dockerfile-templates/runtime/Dockerfile.linux | 2 +- eng/dockerfile-templates/runtime/Dockerfile.windows | 2 +- eng/dockerfile-templates/sdk/Dockerfile.linux | 2 +- eng/dockerfile-templates/sdk/Dockerfile.windows | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/dockerfile-templates/Dockerfile.linux.download-file b/eng/dockerfile-templates/Dockerfile.linux.download-file index 662f9ab695..b72066439e 100644 --- a/eng/dockerfile-templates/Dockerfile.linux.download-file +++ b/eng/dockerfile-templates/Dockerfile.linux.download-file @@ -7,7 +7,7 @@ sha: Expected checksum of the downloaded file sha-var-name: Name of variable that stores the checksum ^ - set isInternal to find(ARGS["url"], "content?format=file&subPath") >= 0 ^ + set isInternal to find(ARGS["url"], "artifacts.visualstudio.com") >= 0 ^ set additionalWgetArgs to when(isInternal, '--header="Authorization: Basic `echo $ACCESSTOKEN:$ACCESSTOKEN | base64 -w 0`" ', '') ^ set additionalCurlArgs to when(isInternal, '-u :$ACCESSTOKEN --basic ', '') ^ set isAlpine to find(OS_VERSION, "alpine") >= 0 diff --git a/eng/dockerfile-templates/Dockerfile.windows.download-file b/eng/dockerfile-templates/Dockerfile.windows.download-file index 765827168e..4f9af660a1 100644 --- a/eng/dockerfile-templates/Dockerfile.windows.download-file +++ b/eng/dockerfile-templates/Dockerfile.windows.download-file @@ -8,7 +8,7 @@ sha-var-name: Name of variable that stores the checksum hash-algorithm: Algorithm type to use to get the checksum. Defaults to sha512 ^ - set isInternal to find(ARGS["url"], "content?format=file&subPath") >= 0 ^ + set isInternal to find(ARGS["url"], "artifacts.visualstudio.com") >= 0 ^ set hashAlgorithm to when(ARGS["hash-algorithm"], ARGS["hash-algorithm"], "sha512") }}{{if isInternal:` $Base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(\":$($Env:ACCESSTOKEN)\")); ` diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.linux b/eng/dockerfile-templates/aspnet/Dockerfile.linux index 97807f953c..18da9e6912 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.linux +++ b/eng/dockerfile-templates/aspnet/Dockerfile.linux @@ -10,7 +10,7 @@ set isFullAzureLinux to isAzureLinux && !isDistroless ^ set isDistrolessAzureLinux to isAzureLinux && isDistroless ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "content?format=file&subPath") ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") ^ set isRpmInstall to isFullAzureLinux && dotnetVersion = "6.0" ^ set isSingleStage to (isAlpine || isRpmInstall) && !isInternal ^ set runtimeDepsVariant to when(ARGS["is-extra"], "-extra", "") ^ diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.linux-composite b/eng/dockerfile-templates/aspnet/Dockerfile.linux-composite index 2dba2867ec..2bc0af62b5 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.linux-composite +++ b/eng/dockerfile-templates/aspnet/Dockerfile.linux-composite @@ -6,7 +6,7 @@ set isFullAzureLinux to isAzureLinux && !isDistroless ^ set isDistrolessAzureLinux to isAzureLinux && isDistroless ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") >= 0 ^ set isSingleStage to isAlpine && !isInternal ^ set runtimeDepsVariant to when(ARGS["is-extra"], "-extra", "") ^ set tagVersion to when(dotnetVersion = "6.0" || dotnetVersion = "8.0", diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.windows b/eng/dockerfile-templates/aspnet/Dockerfile.windows index 4bf16f04eb..4f1a665199 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.windows +++ b/eng/dockerfile-templates/aspnet/Dockerfile.windows @@ -1,7 +1,7 @@ {{ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") >= 0 ^ set isSingleStage to (find(OS_VERSION, "windowsservercore") >= 0 && !isInternal) ^ set tagVersion to when(dotnetVersion = "6.0" || dotnetVersion = "8.0", VARIABLES[cat("dotnet|", dotnetVersion, "|product-version")] diff --git a/eng/dockerfile-templates/runtime-deps/Dockerfile b/eng/dockerfile-templates/runtime-deps/Dockerfile index 1bad2daf80..44d60b90c0 100644 --- a/eng/dockerfile-templates/runtime-deps/Dockerfile +++ b/eng/dockerfile-templates/runtime-deps/Dockerfile @@ -12,7 +12,7 @@ set isAzureLinux to isCblMariner || defined(match(OS_VERSION, "^azurelinux\d+\.\d+$")) ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") >= 0 ^ set baseImageRepo to when(isAlpine, cat(ARCH_VERSIONED, "/alpine"), when(isDebian, diff --git a/eng/dockerfile-templates/runtime/Dockerfile.linux b/eng/dockerfile-templates/runtime/Dockerfile.linux index 646db98d33..eb1e1f9077 100644 --- a/eng/dockerfile-templates/runtime/Dockerfile.linux +++ b/eng/dockerfile-templates/runtime/Dockerfile.linux @@ -10,7 +10,7 @@ set isFullAzureLinux to isAzureLinux && !isDistroless ^ set isDistrolessAzureLinux to isAzureLinux && isDistroless ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") >= 0 ^ set isRpmInstall to isFullAzureLinux && dotnetVersion = "6.0" ^ set isSingleStage to (isAlpine || isRpmInstall) && !isInternal ^ set runtimeDepsVariant to when(ARGS["is-extra"], "-extra", "") ^ diff --git a/eng/dockerfile-templates/runtime/Dockerfile.windows b/eng/dockerfile-templates/runtime/Dockerfile.windows index b82e1b5a4c..e3c5ef5735 100644 --- a/eng/dockerfile-templates/runtime/Dockerfile.windows +++ b/eng/dockerfile-templates/runtime/Dockerfile.windows @@ -2,7 +2,7 @@ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ set isServerCore to find(OS_VERSION, "windowsservercore") >= 0 ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") >= 0 ^ set isSingleStage to (find(OS_VERSION, "windowsservercore") >= 0 && !isInternal) ^ set serverCoreBaseTag to cat("mcr.microsoft.com/windows/servercore:", OS_VERSION_NUMBER, "-amd64") ^ set finalStageBaseRepo to when(isInternal && isServerCore, "servercore", "nanoserver") diff --git a/eng/dockerfile-templates/sdk/Dockerfile.linux b/eng/dockerfile-templates/sdk/Dockerfile.linux index 4d32d94be6..016e29edea 100644 --- a/eng/dockerfile-templates/sdk/Dockerfile.linux +++ b/eng/dockerfile-templates/sdk/Dockerfile.linux @@ -4,7 +4,7 @@ set isMariner to find(OS_VERSION, "cbl-mariner") >= 0 ^ set isAzureLinux to isMariner || find(OS_VERSION, "azurelinux") >= 0 ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") >= 0 ^ set isRpmInstall to isAzureLinux && dotnetVersion = "6.0" ^ set tagVersion to when(dotnetVersion = "6.0" || dotnetVersion = "8.0", VARIABLES[cat("dotnet|", dotnetVersion, "|product-version")] diff --git a/eng/dockerfile-templates/sdk/Dockerfile.windows b/eng/dockerfile-templates/sdk/Dockerfile.windows index 06098b9d05..29abdc0986 100644 --- a/eng/dockerfile-templates/sdk/Dockerfile.windows +++ b/eng/dockerfile-templates/sdk/Dockerfile.windows @@ -1,7 +1,7 @@ {{ set dotnetVersion to join(slice(split(PRODUCT_VERSION, "."), 0, 2), ".") ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "content?format=file&subPath") >= 0 ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") >= 0 ^ set isSingleStage to (find(OS_VERSION, "windowsservercore") >= 0 && !isInternal) ^ set tagVersion to when(dotnetVersion = "6.0" || dotnetVersion = "8.0", VARIABLES[cat("dotnet|", dotnetVersion, "|product-version")] From 887e9fdc5d36034a08200686b7d8837480eaa4a9 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Tue, 1 Oct 2024 16:33:56 -0700 Subject: [PATCH 12/15] Remove unused parameters --- eng/common/templates/jobs/copy-base-images-staging.yml | 4 ---- eng/common/templates/jobs/copy-base-images.yml | 4 ---- eng/common/templates/steps/copy-base-images.yml | 3 --- 3 files changed, 11 deletions(-) diff --git a/eng/common/templates/jobs/copy-base-images-staging.yml b/eng/common/templates/jobs/copy-base-images-staging.yml index b1c24a6e49..169eeb1441 100644 --- a/eng/common/templates/jobs/copy-base-images-staging.yml +++ b/eng/common/templates/jobs/copy-base-images-staging.yml @@ -14,9 +14,6 @@ parameters: - name: continueOnError type: string default: false -- name: isInternalServicingValidation - type: string - default: false jobs: - template: /eng/common/templates/jobs/copy-base-images.yml@self @@ -31,4 +28,3 @@ jobs: subscription: $(acr-staging.subscription) resourceGroup: $(acr-staging.resourceGroup) repoPrefix: $(mirrorRepoPrefix) - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/jobs/copy-base-images.yml b/eng/common/templates/jobs/copy-base-images.yml index 6b8725bf52..c75974216d 100644 --- a/eng/common/templates/jobs/copy-base-images.yml +++ b/eng/common/templates/jobs/copy-base-images.yml @@ -23,9 +23,6 @@ parameters: - name: forceDryRun type: boolean default: false -- name: isInternalServicingValidation - type: string - default: false jobs: - job: ${{ parameters.name }} @@ -40,4 +37,3 @@ jobs: additionalOptions: ${{ parameters.additionalOptions }} continueOnError: ${{ parameters.continueOnError }} forceDryRun: ${{ parameters.forceDryRun }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/steps/copy-base-images.yml b/eng/common/templates/steps/copy-base-images.yml index 13221ee424..e24fc0306e 100644 --- a/eng/common/templates/steps/copy-base-images.yml +++ b/eng/common/templates/steps/copy-base-images.yml @@ -18,9 +18,6 @@ parameters: - name: forceDryRun type: boolean default: false -- name: isInternalServicingValidation - type: string - default: false steps: - ${{ if or(eq(parameters.forceDryRun, true), eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: From f6cf4a1bad95ae3fe45127fe19d4f3b2d17ce9f0 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Wed, 2 Oct 2024 11:38:34 -0700 Subject: [PATCH 13/15] Follow up on removing unneeded parameters --- eng/common/templates/stages/build-test-publish-repo.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/common/templates/stages/build-test-publish-repo.yml b/eng/common/templates/stages/build-test-publish-repo.yml index 4053cb6eb3..e0c8096432 100644 --- a/eng/common/templates/stages/build-test-publish-repo.yml +++ b/eng/common/templates/stages/build-test-publish-repo.yml @@ -73,7 +73,6 @@ stages: pool: ${{ parameters.linuxAmd64Pool }} additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} - isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} - template: /eng/common/templates/jobs/generate-matrix.yml@self parameters: matrixType: ${{ parameters.buildMatrixType }} From f9700ce5d5ad34b5b17e3e2cb43895171a16f44a Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Wed, 2 Oct 2024 18:31:12 -0700 Subject: [PATCH 14/15] Cleanup naming --- eng/Get-DropVersions.ps1 | 33 +++++++------------ eng/Set-DotnetVersions.ps1 | 22 ++----------- .../steps/set-custom-test-variables.yml | 4 +-- .../steps/update-dotnet-dependencies.yml | 5 ++- .../DockerfileShaUpdater.cs | 25 +++----------- eng/update-dependencies/NuGetConfigUpdater.cs | 2 +- eng/update-dependencies/Options.cs | 14 +++----- tests/Microsoft.DotNet.Docker.Tests/Config.cs | 6 ++-- .../EnvironmentVariableInfo.cs | 2 +- .../SdkImageTests.cs | 2 +- .../ProjectTemplateTestScenario.cs | 12 +++---- .../TestScenarios/TestDockerfile.cs | 6 ++-- tests/run-tests.ps1 | 13 +++----- 13 files changed, 47 insertions(+), 99 deletions(-) diff --git a/eng/Get-DropVersions.ps1 b/eng/Get-DropVersions.ps1 index 24bd2d7727..d8eacb8321 100644 --- a/eng/Get-DropVersions.ps1 +++ b/eng/Get-DropVersions.ps1 @@ -29,17 +29,13 @@ param( [switch] $UpdateDependencies, - # SAS query string used to access the internal blob storage location of the build - [string] - $BlobStorageSasQueryString, - # PAT used to access the versions repo in AzDO [string] $AzdoVersionsRepoInfoAccessToken, # PAT used to access internal AzDO build artifacts [string] - $InternalArtifactsAccessToken + $InternalAccessToken ) Import-Module -force $PSScriptRoot/DependencyManagement.psm1 @@ -98,7 +94,7 @@ function GetSdkVersionInfo([string]$sdkUrl) { } } -function ResolveSdkUrl([string]$sdkVersion, [string]$queryString, [bool]$useStableBranding) { +function ResolveSdkUrl([string]$sdkVersion, [bool]$useStableBranding) { if ($useStableBranding) { $sdkStableVersion = ($sdkVersion -split "-")[0] } @@ -111,7 +107,7 @@ function ResolveSdkUrl([string]$sdkVersion, [string]$queryString, [bool]$useStab $containerVersion = $sdkVersion.Replace(".", "-") if ($UseInternalBuild) { - $sdkUrl = "https://dotnetstage.blob.core.windows.net/$containerVersion-internal/Sdk/$sdkVersion/$zipFile$queryString" + $sdkUrl = "https://dotnetstage.blob.core.windows.net/$containerVersion-internal/Sdk/$sdkVersion/$zipFile" } else { $sdkUrl = "https://dotnetbuilds.blob.core.windows.net/public/Sdk/$sdkVersion/$zipFile" @@ -160,7 +156,7 @@ function GetVersionInfoFromBuildId([string]$buildId) { try { New-Item -Path $tempDir -ItemType Directory -Force | Out-Null - $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalArtifactsAccessToken")) + $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalAccessToken")) $headers = @{ "Authorization" = "Basic $base64AccessToken" } @@ -193,7 +189,7 @@ function GetVersionInfoFromBuildId([string]$buildId) { } function GetArtifactUrl([string]$artifactName) { - $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalArtifactsAccessToken")) + $base64AccessToken = [System.Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$InternalAccessToken")) $headers = @{ "Authorization" = "Basic $base64AccessToken" } @@ -237,29 +233,24 @@ if ($UseInternalBuild) { } if ($BuildId) { - if (!$InternalArtifactsAccessToken) { - $InternalArtifactsAccessToken = az account get-access-token --query accessToken --output tsv + if (!$InternalAccessToken) { + $InternalAccessToken = az account get-access-token --query accessToken --output tsv if ($LASTEXITCODE -ne 0) { Write-Error "Failed to obtain access token using Azure CLI" - Write-Error "Please provide 'InternalArtifactsAccessToken' parameter when using 'BuildId' option" + Write-Error "Please provide 'InternalAccessToken' parameter when using 'BuildId' option" exit 1 } } $internalBaseUrl = GetInternalBaseUrl } - - $queryString = "$BlobStorageSasQueryString" -} -else { - $queryString = "" } $sdkVersionInfos = @() if ($Channel) { $sdkFile = "dotnet-sdk-win-x64.zip" - $akaMsUrl = "https://aka.ms/dotnet/$Channel/$sdkFile$queryString" + $akaMsUrl = "https://aka.ms/dotnet/$Channel/$sdkFile" $sdkUrl = Resolve-DotnetProductUrl $akaMsUrl $sdkVersionInfos += GetSdkVersionInfo $sdkUrl @@ -268,7 +259,7 @@ if ($Channel) { foreach ($sdkVersion in $SdkVersions) { $useStableBranding = Get-IsStableBranding -Version $sdkVersion - $sdkUrl = ResolveSdkUrl $sdkVersion $queryString $useStableBranding + $sdkUrl = ResolveSdkUrl $sdkVersion $useStableBranding $sdkVersionInfo = GetSdkVersionInfo $sdkUrl $sdkVersionInfos += $sdkVersionInfo } @@ -317,9 +308,9 @@ if ($UpdateDependencies) { $additionalArgs = @{} - if ($internalBaseUrl -and $InternalArtifactsAccessToken) { + if ($internalBaseUrl -and $InternalAccessToken) { $additionalArgs += @{ InternalBaseUrl = "$internalBaseUrl" } - $additionalArgs += @{ InternalPat = "$InternalArtifactsAccessToken" } + $additionalArgs += @{ InternalAccessToken = "$InternalAccessToken" } } foreach ($versionInfo in $versionInfos) { diff --git a/eng/Set-DotnetVersions.ps1 b/eng/Set-DotnetVersions.ps1 index b0a3837b75..948940371b 100644 --- a/eng/Set-DotnetVersions.ps1 +++ b/eng/Set-DotnetVersions.ps1 @@ -48,14 +48,6 @@ param( [string] $AzdoVariableName, - # SAS query string used to access files in the binary blob container - [string] - $BinarySasQueryString, - - # SAS query string used to access files in the checksum blob container - [string] - $ChecksumSasQueryString, - # File containing checksums for each product asset; used to override the behavior of locating the checksums from blob storage accounts. [string] $ChecksumsFile, @@ -67,7 +59,7 @@ param( # PAT used to access internal AzDO build artifacts [string] - $InternalPat, + $InternalAccessToken, # Base Url for internal AzDO build artifacts [string] @@ -110,14 +102,6 @@ if ($ComputeShas) { $updateDepsArgs += "--compute-shas" } -if ($BinarySasQueryString) { - $updateDepsArgs += "--binary-sas=$BinarySasQueryString" -} - -if ($ChecksumSasQueryString) { - $updateDepsArgs += "--checksum-sas=$ChecksumSasQueryString" -} - if ($ChecksumsFile) { $updateDepsArgs += "--checksums-file=$ChecksumsFile" } @@ -130,8 +114,8 @@ if ($ReleaseState) { $updateDepsArgs += "--release-state=$ReleaseState" } -if ($InternalArtifactsAccessToken) { - $updateDepsArgs += "--internal-pat=$InternalPat" +if ($InternalAccessToken) { + $updateDepsArgs += "--internal-access-token=$InternalAccessToken" } if ($InternalBaseUrl) { diff --git a/eng/pipelines/steps/set-custom-test-variables.yml b/eng/pipelines/steps/set-custom-test-variables.yml index 167ad2dcf9..eb996b503e 100644 --- a/eng/pipelines/steps/set-custom-test-variables.yml +++ b/eng/pipelines/steps/set-custom-test-variables.yml @@ -9,14 +9,14 @@ steps: if ("$(publishRepoPrefix)".Contains("internal/")) { if ($Env:AGENT_OS -eq 'Linux') { - $testRunnerOptions="$testRunnerOptions -e INTERNAL_TESTING='1' -e NUGET_FEED_PASSWORD='$(System.AccessToken)'" + $testRunnerOptions="$testRunnerOptions -e INTERNAL_TESTING='1' -e INTERNAL_ACCESS_TOKEN='$(System.AccessToken)'" } if ($Env:AGENT_OS -eq 'Windows_NT') { # Be sure to use a verbatim string when referencing the environment variables. We don't want the # variables to be resolved in this script. We're generating the script here to be executed by the # test step. - $testInit='$Env:INTERNAL_TESTING=''1'' ; $Env:NUGET_FEED_PASSWORD=''$(System.AccessToken)''' + $testInit='$Env:INTERNAL_TESTING=''1'' ; $Env:INTERNAL_ACCESS_TOKEN=''$(System.AccessToken)''' } } diff --git a/eng/pipelines/steps/update-dotnet-dependencies.yml b/eng/pipelines/steps/update-dotnet-dependencies.yml index 72095339b0..a0a83ed44f 100644 --- a/eng/pipelines/steps/update-dotnet-dependencies.yml +++ b/eng/pipelines/steps/update-dotnet-dependencies.yml @@ -19,7 +19,7 @@ steps: if ("${{ parameters.useInternalBuild }}" -eq "true") { $args["UseInternalBuild"] = $true - $args["BlobStorageSasQueryString"] = "$(dotnetstage-account-sas-read-token)" + $args["InternalAccessToken"] = '$(System.AccessToken)' $args["AzdoVersionsRepoInfoAccessToken"] = "$(dn-bot-devdiv-dnceng-rw-code-pat)" } @@ -43,8 +43,7 @@ steps: } if ("${{ parameters.useInternalBuild }}" -eq "true") { - $args["ChecksumSasQueryString"] = '"$(dotnetchecksumsstage-account-sas-read-token)"' - $args["BinarySasQueryString"] = '"$(dotnetstage-account-sas-read-token)"' + $args["InternalAccessToken"] = '$(System.AccessToken)' } else { $args["ReleaseState"] = $(Get-ProductReleaseState) } diff --git a/eng/update-dependencies/DockerfileShaUpdater.cs b/eng/update-dependencies/DockerfileShaUpdater.cs index 517dcc5f9f..0415487f79 100644 --- a/eng/update-dependencies/DockerfileShaUpdater.cs +++ b/eng/update-dependencies/DockerfileShaUpdater.cs @@ -116,12 +116,12 @@ public DockerfileShaUpdater( return (JObject)variables; }); - if (!string.IsNullOrEmpty(_options.InternalPat)) + if (!string.IsNullOrEmpty(_options.InternalAccessToken)) { s_httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( "Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", - _options.InternalPat)))); + _options.InternalAccessToken)))); } } @@ -314,22 +314,7 @@ private static string GetArch(string[] variableParts) { Trace.TraceInformation($"Downloading '{downloadUrl}'."); return ChecksumHelper.ComputeChecksumShaAsync( - s_httpClient, ApplySasQueryStringIfNecessary(downloadUrl, _options.BinarySasQueryString)); - } - - private static bool IsInternalUrl(string url) - { - return url.Contains("internal"); - } - - private static string ApplySasQueryStringIfNecessary(string url, string sasQueryString) - { - if (IsInternalUrl(url)) - { - return url + sasQueryString; - } - - return url; + s_httpClient, downloadUrl); } private async Task GetDotNetBinaryStorageChecksumsShaAsync(string productDownloadUrl) @@ -345,7 +330,7 @@ private static string ApplySasQueryStringIfNecessary(string url, string sasQuery + shaExt; Trace.TraceInformation($"Downloading '{shaUrl}'."); - using (HttpResponseMessage response = await s_httpClient.GetAsync(ApplySasQueryStringIfNecessary(shaUrl, _options.ChecksumSasQueryString))) + using (HttpResponseMessage response = await s_httpClient.GetAsync(shaUrl)) { if (response.IsSuccessStatusCode) { @@ -468,7 +453,7 @@ private async Task> GetDotnetReleaseChecksums(string async () => { Trace.TraceInformation($"Downloading '{uri}'."); - using (HttpResponseMessage response = await s_httpClient.GetAsync(ApplySasQueryStringIfNecessary(uri, _options.BinarySasQueryString))) + using (HttpResponseMessage response = await s_httpClient.GetAsync(uri)) { if (response.IsSuccessStatusCode) { diff --git a/eng/update-dependencies/NuGetConfigUpdater.cs b/eng/update-dependencies/NuGetConfigUpdater.cs index 9f3f9c5910..ca91b30ac6 100644 --- a/eng/update-dependencies/NuGetConfigUpdater.cs +++ b/eng/update-dependencies/NuGetConfigUpdater.cs @@ -97,7 +97,7 @@ private void UpdatePackageSourceCredentials(string pkgSrcName, XElement configur pkgSourceCreds, () => new XElement(pkgSrcName)); UpdateAddElement(pkgSrcCredsEntry, "Username", "dotnet"); - UpdateAddElement(pkgSrcCredsEntry, "ClearTextPassword", "%NuGetFeedPassword%"); + UpdateAddElement(pkgSrcCredsEntry, "ClearTextPassword", "%InternalAccessToken%"); } else { diff --git a/eng/update-dependencies/Options.cs b/eng/update-dependencies/Options.cs index f4e178c447..da20bd5807 100644 --- a/eng/update-dependencies/Options.cs +++ b/eng/update-dependencies/Options.cs @@ -12,9 +12,7 @@ namespace Dotnet.Docker public class Options { public string InternalBaseUrl { get; } - public string InternalPat { get; } - public string BinarySasQueryString { get; } - public string ChecksumSasQueryString { get; } + public string InternalAccessToken { get; } public bool ComputeChecksums { get; } public string DockerfileVersion { get; } public string Email { get; } @@ -31,13 +29,13 @@ public class Options public string VersionSourceName { get; } public bool UseStableBranding { get; } public bool UpdateOnly => Email == null || Password == null || User == null || TargetBranch == null; - public bool IsInternal => !string.IsNullOrEmpty(InternalBaseUrl) || !string.IsNullOrEmpty(BinarySasQueryString) || !string.IsNullOrEmpty(ChecksumSasQueryString); + public bool IsInternal => !string.IsNullOrEmpty(InternalBaseUrl); public string ChecksumsFile { get; } public ReleaseState? ReleaseState { get; } public Options(string dockerfileVersion, string[] productVersion, string versionSourceName, string email, string password, string user, bool computeShas, bool stableBranding, string binarySas, string checksumSas, string sourceBranch, string targetBranch, string org, - string project, string repo, string checksumsFile, ReleaseState? releaseState, string internalBaseUrl, string internalPat) + string project, string repo, string checksumsFile, ReleaseState? releaseState, string internalBaseUrl, string internalAccessToken) { DockerfileVersion = dockerfileVersion; ProductVersions = productVersion @@ -50,11 +48,9 @@ public Options(string dockerfileVersion, string[] productVersion, string version ComputeChecksums = computeShas; ChecksumsFile = checksumsFile; UseStableBranding = stableBranding; - BinarySasQueryString = binarySas; - ChecksumSasQueryString = checksumSas; SourceBranch = sourceBranch; InternalBaseUrl = internalBaseUrl; - InternalPat = internalPat; + InternalAccessToken = internalAccessToken; // Default TargetBranch to SourceBranch if it's not explicitly provided TargetBranch = string.IsNullOrEmpty(targetBranch) ? sourceBranch : targetBranch; @@ -97,7 +93,7 @@ public static IEnumerable GetCliSymbols() => new Option("--checksums-file", "File containing a list of checksums for each product asset"), new Option("--release-state", "The release state of the product assets"), new Option("--internal-base-url", "Base Url for internal build artifacts"), - new Option("--internal-pat", "PAT for accessing internal build artifacts") + new Option("--internal-access-token", "PAT for accessing internal build artifacts") }; } diff --git a/tests/Microsoft.DotNet.Docker.Tests/Config.cs b/tests/Microsoft.DotNet.Docker.Tests/Config.cs index 7d2fa5762b..2ce7897d33 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/Config.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/Config.cs @@ -32,10 +32,8 @@ public static class Config (Environment.GetEnvironmentVariable("IMAGE_OS_NAMES") ?? string.Empty).Split(",", StringSplitOptions.RemoveEmptyEntries); public static string SourceBranch { get; } = Environment.GetEnvironmentVariable("SOURCE_BRANCH") ?? string.Empty; - public static string SasQueryString { get; } = - Environment.GetEnvironmentVariable("SAS_QUERY_STRING") ?? string.Empty; - public static string NuGetFeedPassword { get; } = - Environment.GetEnvironmentVariable("NUGET_FEED_PASSWORD") ?? string.Empty; + public static string InternalAccessToken { get; } = + Environment.GetEnvironmentVariable("INTERNAL_ACCESS_TOKEN") ?? string.Empty; public static string[] Paths { get; } = Environment.GetEnvironmentVariable("DOCKERFILE_PATHS")? .Split(',', StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty(); diff --git a/tests/Microsoft.DotNet.Docker.Tests/EnvironmentVariableInfo.cs b/tests/Microsoft.DotNet.Docker.Tests/EnvironmentVariableInfo.cs index ef72408d49..00212a6017 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/EnvironmentVariableInfo.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/EnvironmentVariableInfo.cs @@ -48,7 +48,7 @@ public static void Validate( { // If we're validating a product version environment variable for an internal build // we need to trim off the "servicing" or "rtm" part of the version value. - if (variable.IsProductVersion && !string.IsNullOrEmpty(Config.SasQueryString)) + if (variable.IsProductVersion && Config.IsInternal) { int servicingIndex = actualValue.IndexOf("-servicing."); if (servicingIndex != -1) diff --git a/tests/Microsoft.DotNet.Docker.Tests/SdkImageTests.cs b/tests/Microsoft.DotNet.Docker.Tests/SdkImageTests.cs index e67ab12dc4..1cda1cf477 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/SdkImageTests.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/SdkImageTests.cs @@ -327,7 +327,7 @@ private async Task> GetExpectedSdkContentsAsync( httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( "Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", - Config.NuGetFeedPassword)))); + Config.InternalAccessToken)))); await httpClient.DownloadFileAsync(new Uri(sdkUrl), sdkFile); diff --git a/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/ProjectTemplateTestScenario.cs b/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/ProjectTemplateTestScenario.cs index 511bb0dd5d..8122438a10 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/ProjectTemplateTestScenario.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/ProjectTemplateTestScenario.cs @@ -87,12 +87,12 @@ protected string Build(string stageTarget, string[]? customBuildArgs) buildArgs.AddRange(customBuildArgs); } - const string NuGetFeedPasswordVar = "NuGetFeedPassword"; + const string InternalAccessTokenVar = "InternalAccessToken"; - if (!string.IsNullOrEmpty(Config.NuGetFeedPassword)) + if (!string.IsNullOrEmpty(Config.InternalAccessToken)) { - buildArgs.Add(NuGetFeedPasswordVar); - Environment.SetEnvironmentVariable(NuGetFeedPasswordVar, Config.NuGetFeedPassword); + buildArgs.Add(InternalAccessTokenVar); + Environment.SetEnvironmentVariable(InternalAccessTokenVar, Config.InternalAccessToken); } try @@ -107,9 +107,9 @@ protected string Build(string stageTarget, string[]? customBuildArgs) } finally { - if (!string.IsNullOrEmpty(Config.NuGetFeedPassword)) + if (!string.IsNullOrEmpty(Config.InternalAccessToken)) { - Environment.SetEnvironmentVariable(NuGetFeedPasswordVar, null); + Environment.SetEnvironmentVariable(InternalAccessTokenVar, null); } } diff --git a/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/TestDockerfile.cs b/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/TestDockerfile.cs index af5269a0d8..60b183b577 100644 --- a/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/TestDockerfile.cs +++ b/tests/Microsoft.DotNet.Docker.Tests/TestScenarios/TestDockerfile.cs @@ -118,7 +118,7 @@ public static TestDockerfile GetTestProjectDockerfile() $""" FROM {TestDockerfile.BuildStageName} AS {TestDockerfile.AppStageName} ARG rid - ARG NuGetFeedPassword + ARG InternalAccessToken WORKDIR /source/tests COPY tests/*.csproj . RUN dotnet restore -r {FormatArg("rid")} @@ -140,7 +140,7 @@ public static TestDockerfile GetBlazorWasmDockerfile(bool useWasmTools) StringBuilder buildStageBuilder = new( $""" FROM $sdk_image AS {TestDockerfile.BuildStageName} - ARG NuGetFeedPassword + ARG InternalAccessToken ARG port EXPOSE $port """); @@ -208,7 +208,7 @@ private static string GetDefaultBuildStage() $""" FROM $sdk_image AS {TestDockerfile.BuildStageName} ARG rid - ARG NuGetFeedPassword + ARG InternalAccessToken ARG port EXPOSE $port """); diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index de98c166e2..6375c24495 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -31,9 +31,7 @@ param( [ValidateSet("runtime", "runtime-deps", "aspnet", "sdk", "pre-build", "sample", "image-size", "monitor", "aspire-dashboard")] [string[]]$TestCategories = @("runtime", "runtime-deps", "aspnet", "sdk", "monitor", "aspire-dashboard"), - [securestring]$SasQueryString, - - [securestring]$NuGetFeedPassword + [securestring]$InternalAccessToken ) Import-Module -force $PSScriptRoot/../eng/DependencyManagement.psm1 @@ -123,12 +121,9 @@ Try { $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 $env:DOTNET_MULTILEVEL_LOOKUP = '0' - if ($SasQueryString) { - $env:SAS_QUERY_STRING = ConvertFrom-SecureString $SasQueryString -AsPlainText - } - - if ($NuGetFeedPassword) { - $env:NUGET_FEED_PASSWORD = ConvertFrom-SecureString $NuGetFeedPassword -AsPlainText + if ($InternalAccessToken) { + $env:INTERNAL_ACCESS_TOKEN = ConvertFrom-SecureString $InternalAccessToken -AsPlainText + $env:INTERNAL_TESTING = 1 } $testFilter = "" From cfa5c30166e95d25ad435bfae49fb7f5cfd99a5d Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Fri, 4 Oct 2024 15:01:03 -0700 Subject: [PATCH 15/15] Fix a bug in aspnet templates --- eng/dockerfile-templates/aspnet/Dockerfile.linux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/dockerfile-templates/aspnet/Dockerfile.linux b/eng/dockerfile-templates/aspnet/Dockerfile.linux index 18da9e6912..279f29cb18 100644 --- a/eng/dockerfile-templates/aspnet/Dockerfile.linux +++ b/eng/dockerfile-templates/aspnet/Dockerfile.linux @@ -10,7 +10,7 @@ set isFullAzureLinux to isAzureLinux && !isDistroless ^ set isDistrolessAzureLinux to isAzureLinux && isDistroless ^ set baseUrl to VARIABLES[cat("dotnet|", dotnetVersion, "|base-url|", VARIABLES["branch"])] ^ - set isInternal to find(baseUrl, "artifacts.visualstudio.com") ^ + set isInternal to find(baseUrl, "artifacts.visualstudio.com") >= 0 ^ set isRpmInstall to isFullAzureLinux && dotnetVersion = "6.0" ^ set isSingleStage to (isAlpine || isRpmInstall) && !isInternal ^ set runtimeDepsVariant to when(ARGS["is-extra"], "-extra", "") ^