From 3cb13be42f3ff7c5a9c88a7917b3154520e73017 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 15:42:01 +0200 Subject: [PATCH] [main] Update dependencies from dotnet/arcade (#404) Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks From Version 9.0.0-beta.24266.1 -> To Version 9.0.0-beta.24272.5 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 12 +-- .../job/source-index-stage1.yml | 6 +- eng/common/internal/Directory.Build.props | 5 ++ eng/common/internal/Tools.csproj | 1 - eng/common/native/init-compiler.sh | 85 ++++++++++++------- .../templates/steps/get-delegate-sas.yml | 43 ++++++++++ eng/common/tools.ps1 | 6 +- global.json | 2 +- 8 files changed, 117 insertions(+), 43 deletions(-) create mode 100644 eng/common/templates/steps/get-delegate-sas.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c885f3bc..db1dfe76 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -9,18 +9,18 @@ - + https://github.com/dotnet/arcade - ed14da5934ffb536cff8f41f8b5719334524cbed + 2001d73c8ff942331a73300ba61fa6164805b231 - + https://github.com/dotnet/arcade - ed14da5934ffb536cff8f41f8b5719334524cbed + 2001d73c8ff942331a73300ba61fa6164805b231 - + https://github.com/dotnet/arcade - ed14da5934ffb536cff8f41f8b5719334524cbed + 2001d73c8ff942331a73300ba61fa6164805b231 diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index f1938eec..8328e52a 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -1,7 +1,7 @@ parameters: runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20240502.12 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 + sourceIndexUploadPackageVersion: 2.0.0-20240522.1 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -88,4 +88,4 @@ jobs: displayName: "Login to Azure" - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index \ No newline at end of file + displayName: Upload stage1 artifacts to source index diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props index a735fe9a..f1d041c3 100644 --- a/eng/common/internal/Directory.Build.props +++ b/eng/common/internal/Directory.Build.props @@ -1,6 +1,11 @@ + + false + false + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 8fa77e5b..e925952d 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -3,7 +3,6 @@ net472 - false false diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index ccd3a172..62900e12 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -2,7 +2,9 @@ # # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # -# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! +# NOTE: some scripts source this file and rely on stdout being empty, make sure +# to not output *anything* here, unless it is an error message that fails the +# build. if [ -z "$build_arch" ] || [ -z "$compiler" ]; then echo "Usage..." @@ -58,6 +60,26 @@ check_version_exists() { echo "$desired_version" } +__baseOS="$(uname)" +set_compiler_version_from_CC() { + if [ "$__baseOS" = "Darwin" ]; then + # On Darwin, the versions from -version/-dumpversion refer to Xcode + # versions, not llvm versions, so we can't rely on them. + return + fi + + version="$("$CC" -dumpversion)" + if [ -z "$version" ]; then + echo "Error: $CC -dumpversion didn't provide a version" + exit 1 + fi + + # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments. + IFS=. read -r majorVersion minorVersion _ < /dev/null; then - if [ "$(uname)" != "Darwin" ]; then - echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH." - fi - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" - else - echo "No usable version of $compiler found." + if ! command -v "$compiler" > /dev/null; then + echo "Error: No usable version of $compiler found." exit 1 fi + + CC="$(command -v "$compiler" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler" 2> /dev/null)" + set_compiler_version_from_CC else - if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then - if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then - if command -v "$compiler" > /dev/null; then - echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" - else - echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." - exit 1 - fi + if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ] && { [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; }; then + # If a major version was provided explicitly, and it was too old, find a newer compiler instead + if ! command -v "$compiler" > /dev/null; then + echo "Error: Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." + exit 1 fi + + CC="$(command -v "$compiler" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler" 2> /dev/null)" + set_compiler_version_from_CC fi fi else desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" if [ "$desired_version" = "-1" ]; then - echo "Could not find specific version of $compiler: $majorVersion $minorVersion." + echo "Error: Could not find specific version of $compiler: $majorVersion $minorVersion." exit 1 fi fi if [ -z "$CC" ]; then - CC="$(command -v "$compiler$desired_version")" - CXX="$(command -v "$cxxCompiler$desired_version")" - if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi + CC="$(command -v "$compiler$desired_version" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler$desired_version" 2> /dev/null)" + if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler" 2> /dev/null)"; fi + set_compiler_version_from_CC fi else if [ ! -f "$CLR_CC" ]; then - echo "CLR_CC is set but path '$CLR_CC' does not exist" + echo "Error: CLR_CC is set but path '$CLR_CC' does not exist" exit 1 fi CC="$CLR_CC" CXX="$CLR_CXX" + set_compiler_version_from_CC fi if [ -z "$CC" ]; then - echo "Unable to find $compiler." + echo "Error: Unable to find $compiler." exit 1 fi -# Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0. -if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && ([ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]); then - if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then - LDFLAGS="-fuse-ld=lld" +if [ "$__baseOS" != "Darwin" ]; then + # On Darwin, we always want to use the Apple linker. + + # Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0. + if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && { [ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]; }; then + if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then + LDFLAGS="-fuse-ld=lld" + fi fi fi -SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" +SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version" 2> /dev/null)" export CC CXX LDFLAGS SCAN_BUILD_COMMAND diff --git a/eng/common/templates/steps/get-delegate-sas.yml b/eng/common/templates/steps/get-delegate-sas.yml new file mode 100644 index 00000000..c0e8f913 --- /dev/null +++ b/eng/common/templates/steps/get-delegate-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 7fc78a08..0febe696 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -254,7 +254,6 @@ function Retry($downloadBlock, $maxRetries = 5) { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." break } - } } @@ -424,7 +423,6 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] @@ -504,6 +502,10 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath }) + if (!(Test-Path $packagePath)) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild" + throw + } Unzip $packagePath $packageDir } diff --git a/global.json b/global.json index ed721b25..7d2b56ef 100644 --- a/global.json +++ b/global.json @@ -18,7 +18,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24266.1", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24272.5", "MSTest.Sdk": "3.3.1" } }