[main] Update dependencies from dotnet/arcade (#404)

Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.XliffTasks
 From Version 9.0.0-beta.24266.1 -> To Version 9.0.0-beta.24272.5

Co-authored-by: dotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>

eng/Version.Details.xml

@@ -9,18 +9,18 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24266.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24272.5">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ed14da5934ffb536cff8f41f8b5719334524cbed</Sha>
+      <Sha>2001d73c8ff942331a73300ba61fa6164805b231</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24266.1">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.24272.5">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ed14da5934ffb536cff8f41f8b5719334524cbed</Sha>
+      <Sha>2001d73c8ff942331a73300ba61fa6164805b231</Sha>
     </Dependency>
     <!-- Intermediate is necessary for source build. -->
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24266.1">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24272.5">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ed14da5934ffb536cff8f41f8b5719334524cbed</Sha>
+      <Sha>2001d73c8ff942331a73300ba61fa6164805b231</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
   </ToolsetDependencies>

eng/common/core-templates/job/source-index-stage1.yml

@@ -1,7 +1,7 @@
 parameters:
   runAsPublic: false
-  sourceIndexUploadPackageVersion: 2.0.0-20240502.12
-  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
+  sourceIndexUploadPackageVersion: 2.0.0-20240522.1
+  sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
   sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
@@ -88,4 +88,4 @@ jobs:
       displayName: "Login to Azure"
 
     - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
-      displayName: Upload stage1 artifacts to source index
+      displayName: Upload stage1 artifacts to source index

eng/common/internal/Directory.Build.props

@@ -1,6 +1,11 @@
 <!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
 <Project>
 
+  <PropertyGroup>
+    <ImportDirectoryBuildTargets>false</ImportDirectoryBuildTargets>
+    <ImportDirectoryPackagesProps>false</ImportDirectoryPackagesProps>
+  </PropertyGroup>
+
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 
 </Project>

eng/common/internal/Tools.csproj

@@ -3,7 +3,6 @@
 
   <PropertyGroup>
     <TargetFramework>net472</TargetFramework>
-    <ImportDirectoryBuildTargets>false</ImportDirectoryBuildTargets>
     <AutomaticallyUseReferenceAssemblyPackages>false</AutomaticallyUseReferenceAssemblyPackages>
   </PropertyGroup>
   <ItemGroup>

eng/common/native/init-compiler.sh

@@ -2,7 +2,9 @@
 #
 # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables
 #
-# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here!
+# NOTE: some scripts source this file and rely on stdout being empty, make sure
+# to not output *anything* here, unless it is an error message that fails the
+# build.
 
 if [ -z "$build_arch" ] || [ -z "$compiler" ]; then
   echo "Usage..."
@@ -58,6 +60,26 @@ check_version_exists() {
     echo "$desired_version"
 }
 
+__baseOS="$(uname)"
+set_compiler_version_from_CC() {
+    if [ "$__baseOS" = "Darwin" ]; then
+        # On Darwin, the versions from -version/-dumpversion refer to Xcode
+        # versions, not llvm versions, so we can't rely on them.
+        return
+    fi
+
+    version="$("$CC" -dumpversion)"
+    if [ -z "$version" ]; then
+        echo "Error: $CC -dumpversion didn't provide a version"
+        exit 1
+    fi
+
+    # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments.
+    IFS=. read -r majorVersion minorVersion _ <<EOF
+$version
+EOF
+}
+
 if [ -z "$CLR_CC" ]; then
 
     # Set default versions
@@ -74,64 +96,67 @@ if [ -z "$CLR_CC" ]; then
         done
 
         if [ -z "$majorVersion" ]; then
-            if command -v "$compiler" > /dev/null; then
-                if [ "$(uname)" != "Darwin" ]; then
-                    echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH."
-                fi
-                CC="$(command -v "$compiler")"
-                CXX="$(command -v "$cxxCompiler")"
-            else
-                echo "No usable version of $compiler found."
+            if ! command -v "$compiler" > /dev/null; then
+                echo "Error: No usable version of $compiler found."
                 exit 1
             fi
+
+            CC="$(command -v "$compiler" 2> /dev/null)"
+            CXX="$(command -v "$cxxCompiler" 2> /dev/null)"
+            set_compiler_version_from_CC
         else
-            if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
-                if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
-                    if command -v "$compiler" > /dev/null; then
-                        echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
-                        CC="$(command -v "$compiler")"
-                        CXX="$(command -v "$cxxCompiler")"
-                    else
-                        echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
-                        exit 1
-                    fi
+            if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ] && { [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; }; then
+                # If a major version was provided explicitly, and it was too old, find a newer compiler instead
+                if ! command -v "$compiler" > /dev/null; then
+                    echo "Error: Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
+                    exit 1
                 fi
+
+                CC="$(command -v "$compiler" 2> /dev/null)"
+                CXX="$(command -v "$cxxCompiler" 2> /dev/null)"
+                set_compiler_version_from_CC
             fi
         fi
     else
         desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
         if [ "$desired_version" = "-1" ]; then
-            echo "Could not find specific version of $compiler: $majorVersion $minorVersion."
+            echo "Error: Could not find specific version of $compiler: $majorVersion $minorVersion."
             exit 1
         fi
     fi
 
     if [ -z "$CC" ]; then
-        CC="$(command -v "$compiler$desired_version")"
-        CXX="$(command -v "$cxxCompiler$desired_version")"
-        if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi
+        CC="$(command -v "$compiler$desired_version" 2> /dev/null)"
+        CXX="$(command -v "$cxxCompiler$desired_version" 2> /dev/null)"
+        if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler" 2> /dev/null)"; fi
+        set_compiler_version_from_CC
     fi
 else
     if [ ! -f "$CLR_CC" ]; then
-        echo "CLR_CC is set but path '$CLR_CC' does not exist"
+        echo "Error: CLR_CC is set but path '$CLR_CC' does not exist"
         exit 1
     fi
     CC="$CLR_CC"
     CXX="$CLR_CXX"
+    set_compiler_version_from_CC
 fi
 
 if [ -z "$CC" ]; then
-    echo "Unable to find $compiler."
+    echo "Error: Unable to find $compiler."
     exit 1
 fi
 
-# Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0.
-if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && ([ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]); then
-    if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then
-        LDFLAGS="-fuse-ld=lld"
+if [ "$__baseOS" != "Darwin" ]; then
+    # On Darwin, we always want to use the Apple linker.
+
+    # Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0.
+    if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && { [ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]; }; then
+        if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then
+            LDFLAGS="-fuse-ld=lld"
+        fi
     fi
 fi
 
-SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"
+SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version" 2> /dev/null)"
 
 export CC CXX LDFLAGS SCAN_BUILD_COMMAND

eng/common/templates/steps/get-delegate-sas.yml

@@ -0,0 +1,43 @@
+parameters:
+- name: federatedServiceConnection
+  type: string
+- name: outputVariableName
+  type: string
+- name: expiryInHours
+  type: number
+  default: 1
+- name: base64Encode
+  type: boolean
+  default: false
+- name: storageAccount
+  type: string
+- name: container
+  type: string
+- name: permissions
+  type: string
+  default: 'rl'
+
+steps:
+- task: AzureCLI@2
+  displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
+  inputs:
+    azureSubscription: ${{ parameters.federatedServiceConnection }}
+    scriptType: 'pscore'
+    scriptLocation: 'inlineScript'
+    inlineScript: |
+      # Calculate the expiration of the SAS token and convert to UTC
+      $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+      $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+
+      if ($LASTEXITCODE -ne 0) {
+        Write-Error "Failed to generate SAS token."
+        exit 1
+      }
+
+      if ('${{ parameters.base64Encode }}' -eq 'true') {
+        $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
+      }
+
+      Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"

eng/common/tools.ps1

@@ -254,7 +254,6 @@ function Retry($downloadBlock, $maxRetries = 5) {
       Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
       break
     }
-
   }
 }
 
@@ -424,7 +423,6 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
 
     InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
   } else {
-
     if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') {
       $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
       $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
@@ -504,6 +502,10 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
       Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
     })
 
+    if (!(Test-Path $packagePath)) {
+      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild"
+      throw
+    }
     Unzip $packagePath $packageDir
   }
 

global.json

@@ -18,7 +18,7 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24266.1",
+    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24272.5",
     "MSTest.Sdk": "3.3.1"
   }
 }