Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move tools to use LKG ILC #107772

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 18 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Directory.Build.targets
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
<Import Project="$(RepositoryEngineeringDir)Analyzers.targets" />
<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<Import Project="$(RepositoryEngineeringDir)liveBuilds.targets" />
<Import Project="$(RepositoryEngineeringDir)toolAot.targets" />
<Import Project="$(RepositoryEngineeringDir)generators.targets" />
<Import Project="$(RepositoryEngineeringDir)python.targets" />
<Import Project="$(RepositoryEngineeringDir)generatorProjects.targets" Condition="'$(IsGeneratorProject)' == 'true'" />
Expand Down
5 changes: 3 additions & 2 deletions eng/Subsets.props
Original file line number Diff line number Diff line change
Expand Up @@ -357,13 +357,14 @@
$(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj;
$(CoreClrProjectRoot)tools\PdbChecker\PdbChecker.csproj;
$(CoreClrProjectRoot)tools\AssemblyChecker\AssemblyChecker.csproj" Category="clr" Condition="'$(DotNetBuildSourceOnly)' != 'true'"/>
<ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" />
<ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler.Build.Tasks\ILCompiler.Build.Tasks.csproj" Category="clr" Condition="'$(NativeAotSupported)' == 'true'" />
<ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler\ILCompiler.csproj" Category="clr" Condition="'$(NativeAotSupported)' == 'true'" />
<ProjectToBuild Include="$(CoreClrProjectRoot)nativeaot\BuildIntegration\BuildIntegration.proj" Category="clr" Condition="'$(NativeAotSupported)' == 'true'" />

<ProjectToBuild Condition="'$(NativeAotSupported)' == 'true' and ('$(CrossBuild)' == 'true' or '$(BuildArchitecture)' != '$(TargetArchitecture)' or '$(HostOS)' != '$(TargetOS)' or '$(EnableNativeSanitizers)' != '')" Include="$(CoreClrProjectRoot)tools\aot\ILCompiler\ILCompiler_crossarch.csproj" Category="clr" />

<ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2_inbuild.csproj" Category="clr" />
<!-- skip the architectures that don't have LKG runtime packs -->
<ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2_publish.csproj" Condition="'$(NativeAotSupported)' == 'true'" Category="clr" />
Copy link
Member

@am11 am11 Sep 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A few questions:

  1. Was there a reason it was moved below?
  2. This project is also built by src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj (i.e. packs subset), do we need this condition there?
  3. If answer to 2. is "yes", then this is a regression on linux-riscv64 and illumos-x64 etc. where NativeAotSupported is false; i.e. main publishes crossgen2, while PR doesn't.. until the "later PR" is on the horizon, right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(1) No, just accidental due to moving things around
(2) I don't think it needs to be


<ProjectToBuild Condition="'$(TargetOS)' == 'windows' or ('$(TargetOS)' == 'linux' and ('$(TargetArchitecture)' == 'x64' or '$(TargetArchitecture)' == 'arm64')) or '$(TargetOS)' == 'osx'" Include="$(CoreClrProjectRoot)tools\SuperFileCheck\SuperFileCheck.csproj" Category="clr" />

Expand Down
12 changes: 2 additions & 10 deletions eng/liveBuilds.targets
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
<CoreCLRAotSdkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'aotsdk'))</CoreCLRAotSdkDir>
<CoreCLRBuildIntegrationDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'build'))</CoreCLRBuildIntegrationDir>

<Crossgen2Dir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', '$(BuildArchitecture)', 'crossgen2'))</Crossgen2Dir>
<Crossgen2Dir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'crossgen2-published'))</Crossgen2Dir>
<Crossgen2InBuildDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', '$(BuildArchitecture)', 'crossgen2'))</Crossgen2InBuildDir>

<ToolsILLinkDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'ILLink.Tasks', '$(ToolsConfiguration)'))</ToolsILLinkDir>

Expand Down Expand Up @@ -58,15 +59,6 @@
<SingleFileHostSourcePath>$([MSBuild]::NormalizePath('$(CoreCLRArtifactsPath)', 'corehost', 'singlefilehost$(ExeSuffix)'))</SingleFileHostSourcePath>
</PropertyGroup>

<!-- Not a live build, but allow overriding the LKG ILCompiler package with a specific version. -->
<PropertyGroup Condition="'$(UseNativeAotForComponents)' == 'true' and '$(PublishAot)' == 'true' and '$(MicrosoftDotNetILCompilerVersion)' != ''">
<SuppressGenerateILCompilerExplicitPackageReferenceWarning>true</SuppressGenerateILCompilerExplicitPackageReferenceWarning>
</PropertyGroup>
<ItemGroup Condition="'$(UseNativeAotForComponents)' == 'true' and '$(PublishAot)' == 'true' and '$(MicrosoftDotNetILCompilerVersion)' != ''">
<PackageReference Include="Microsoft.DotNet.ILCompiler" Version="$(MicrosoftDotNetILCompilerVersion)" />
<PackageReference Include="runtime.$(ToolsRID).Microsoft.DotNet.ILCompiler" Version="$(MicrosoftDotNetILCompilerVersion)" />
</ItemGroup>

<Target Name="ResolveRuntimeFilesFromLocalBuild">
<Error Condition="!Exists('$(CoreCLRArtifactsPath)') and '$(RuntimeFlavor)' == 'CoreCLR'"
Text="The CoreCLR artifacts path does not exist '$(CoreCLRArtifactsPath)'. The 'clr' subset must be built before building this project. Configuration: '$(CoreCLRConfiguration)'. To use a different configuration, specify the 'RuntimeConfiguration' property." />
Expand Down
14 changes: 1 addition & 13 deletions eng/targetingpacks.targets
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@
NuGetPackageVersion="$(ProductVersion)" />
</ItemGroup>
</Target>

<!-- Use local app host pack for NetCoreAppCurrent. -->
<Target Name="UpdateLocalAppHostPack"
Condition="'$(UseLocalAppHostPack)' == 'true'"
Expand All @@ -167,16 +167,4 @@
Condition="'%(RuntimeFramework.FrameworkName)' == '$(LocalFrameworkOverrideName)'" />
</ItemGroup>
</Target>

<!-- Update the resolved crossgen2 tool to use the app with the same .NET host as the build instead of the built apphost. -->
<Target Name="UpdateLocalCrossgen2Tool"
Condition="'$(UseLocalCrossgen2Pack)' == 'true'"
AfterTargets="ResolveReadyToRunCompilers">
<ItemGroup>
<OriginalCrossgen2Tool Include="@(Crossgen2Tool)" Crossgen2Dll="$([System.IO.Path]::ChangeExtension('%(Identity)', '.dll'))" />
<Crossgen2Tool Remove="@(Crossgen2Tool)" />
<Crossgen2Tool Include="@(OriginalCrossgen2Tool->'%(Crossgen2Dll)')"
DotNetHostPath="$(DotNetTool)" />
</ItemGroup>
</Target>
</Project>
56 changes: 0 additions & 56 deletions eng/testing/tests.singlefile.targets
Original file line number Diff line number Diff line change
Expand Up @@ -38,34 +38,6 @@
<SelfContained>true</SelfContained>
</PropertyGroup>

<!-- Needed for the amd64 -> amd64 musl cross-build to pass the target flag. -->
<Target Name="_FixIlcTargetTriple"
AfterTargets="SetupOSSpecificProps"
Condition="'$(CrossBuild)' == 'true' and '$(HostOS)' != 'windows'">
<!-- Compute CrossCompileRid, and copy the downstream logic as-is. -->
<PropertyGroup>
<CrossCompileRid>$(RuntimeIdentifier)</CrossCompileRid>

<CrossCompileArch />
<CrossCompileArch Condition="$(CrossCompileRid.EndsWith('-x64'))">x86_64</CrossCompileArch>
<CrossCompileArch Condition="$(CrossCompileRid.EndsWith('-arm64')) and '$(_IsApplePlatform)' != 'true'">aarch64</CrossCompileArch>
<CrossCompileArch Condition="$(CrossCompileRid.EndsWith('-arm64')) and '$(_IsApplePlatform)' == 'true'">arm64</CrossCompileArch>

<TargetTriple />
<TargetTriple Condition="'$(CrossCompileArch)' != ''">$(CrossCompileArch)-linux-gnu</TargetTriple>
<TargetTriple Condition="'$(CrossCompileArch)' != '' and ($(CrossCompileRid.StartsWith('linux-musl')) or $(CrossCompileRid.StartsWith('alpine')))">$(CrossCompileArch)-alpine-linux-musl</TargetTriple>
<TargetTriple Condition="'$(CrossCompileArch)' != '' and ($(CrossCompileRid.StartsWith('freebsd')))">$(CrossCompileArch)-unknown-freebsd12</TargetTriple>
</PropertyGroup>

<ItemGroup>
<LinkerArg Include="--target=$(TargetTriple)" Condition="'$(TargetOS)' != 'osx' and '$(TargetTriple)' != ''" />
</ItemGroup>
</Target>

<ItemGroup Condition="'$(NativeAotSupported)' == 'true'">
<CustomLinkerArg Condition="'$(CrossBuild)' == 'true' and '$(_hostArchitecture)' == '$(_targetArchitecture)' and '$(_hostOS)' != 'windows'" Include="--gcc-toolchain=$(ROOTFS_DIR)/usr" />
</ItemGroup>

<PropertyGroup Condition="'$(PublishSingleFile)' == 'true' or '$(TestNativeAot)' == 'true'">
<DefineConstants>$(DefineConstants);SINGLE_FILE_TEST_RUNNER</DefineConstants>
</PropertyGroup>
Expand Down Expand Up @@ -229,34 +201,6 @@
</ItemGroup>
</Target>

<!--
Use init-compiler.sh to locate the compiler toolchain which was resolved for rest of the build.
This target is essentially an override hook which is called before `SetupOSSpecificProps` in
`Microsoft.NETCore.Native.Unix.targets`. Note that the input is `CppCompilerAndLinker`
and the output is `CppLinker`, because from `SetupOSSpecificProps` onwards, we only use `CppLinker`
(when alternative compiler, i.e. gcc, is not selected)
-->
<Target Name="LocateNativeCompiler"
Condition="'$(TestNativeAot)' == 'true' and '$(HostOS)' != 'windows'"
BeforeTargets="SetupOSSpecificProps">
<PropertyGroup>
<CppCompilerAndLinker Condition="'$(CppCompilerAndLinker)' == ''">clang</CppCompilerAndLinker>
</PropertyGroup>

<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(CppCompilerAndLinker)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo &quot;$CC;$LDFLAGS&quot;' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="_CC_LDFLAGS" />
</Exec>

<PropertyGroup>
<CppLinker>$(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';'))))</CppLinker>
<_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1))))</_LDFLAGS>
<LinkerFlavor Condition="$(_LDFLAGS.Contains('lld'))">lld</LinkerFlavor>
</PropertyGroup>
</Target>

<Target Name="PublishTestAsSingleFile"
Condition="'$(IsCrossTargetingBuild)' != 'true'"
AfterTargets="Build"
Expand Down
86 changes: 86 additions & 0 deletions eng/toolAot.targets
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
<Project>
<!-- Settings for tools that publish as AOT -->

<PropertyGroup Condition="'$(AotOrSingleFile)' == 'true' and '$(UseNativeAotForComponents)' == 'true'">
<PublishAot>true</PublishAot>
<!-- Use .dwarf files instead of .dsym files since our symbol exporting may not safely handle folders. -->
<NativeSymbolExt Condition="'$(_IsApplePlatform)' == 'true'">.dwarf</NativeSymbolExt>
<DsymUtilOptions Condition="'$(_IsApplePlatform)' == 'true'">--flat</DsymUtilOptions>
</PropertyGroup>

<PropertyGroup Condition="'$(AotOrSingleFile)' == 'true' and '$(UseNativeAotForComponents)' != 'true'">
<PublishSingleFile>true</PublishSingleFile>
<PublishReadyToRun>true</PublishReadyToRun>
<!-- Disable crossgen on NetBSD, illumos, Solaris, and Haiku for now. This can be revisited when we have full support. -->
<PublishReadyToRun Condition="'$(TargetOS)' == 'netbsd' Or '$(TargetOS)' == 'illumos' Or '$(TargetOS)' == 'solaris' Or '$(TargetOS)' == 'haiku'">false</PublishReadyToRun>
<!-- Disable crossgen on FreeBSD when cross building from Linux. -->
<PublishReadyToRun Condition="'$(TargetOS)' == 'freebsd' and '$(CrossBuild)' == 'true'">false</PublishReadyToRun>
<PublishReadyToRunComposite>$(PublishReadyToRun)</PublishReadyToRunComposite>
</PropertyGroup>

<!-- Allow overriding the LKG ILCompiler package with a specific version. -->
<PropertyGroup Condition="'$(UseNativeAotForComponents)' == 'true' and '$(PublishAot)' == 'true' and '$(MicrosoftDotNetILCompilerVersion)' != ''">
<SuppressGenerateILCompilerExplicitPackageReferenceWarning>true</SuppressGenerateILCompilerExplicitPackageReferenceWarning>
</PropertyGroup>

<ItemGroup Condition="'$(UseNativeAotForComponents)' == 'true' and '$(PublishAot)' == 'true' and '$(MicrosoftDotNetILCompilerVersion)' != ''">
<PackageReference Include="Microsoft.DotNet.ILCompiler" Version="$(MicrosoftDotNetILCompilerVersion)" />
<PackageReference Include="runtime.$(ToolsRID).Microsoft.DotNet.ILCompiler" Version="$(MicrosoftDotNetILCompilerVersion)" />
</ItemGroup>

<!-- Needed for the amd64 -> amd64 musl cross-build to pass the target flag. -->
<Target Name="_FixIlcTargetTriple"
AfterTargets="SetupOSSpecificProps"
Condition="'$(CrossBuild)' == 'true' and '$(HostOS)' != 'windows'">
<!-- Compute CrossCompileRid, and copy the downstream logic as-is. -->
<PropertyGroup>
<CrossCompileRid>$(RuntimeIdentifier)</CrossCompileRid>

<CrossCompileArch />
<CrossCompileArch Condition="$(CrossCompileRid.EndsWith('-x64'))">x86_64</CrossCompileArch>
<CrossCompileArch Condition="$(CrossCompileRid.EndsWith('-arm64')) and '$(_IsApplePlatform)' != 'true'">aarch64</CrossCompileArch>
<CrossCompileArch Condition="$(CrossCompileRid.EndsWith('-arm64')) and '$(_IsApplePlatform)' == 'true'">arm64</CrossCompileArch>

<TargetTriple />
<TargetTriple Condition="'$(CrossCompileArch)' != ''">$(CrossCompileArch)-linux-gnu</TargetTriple>
<TargetTriple Condition="'$(CrossCompileArch)' != '' and ($(CrossCompileRid.StartsWith('linux-musl')) or $(CrossCompileRid.StartsWith('alpine')))">$(CrossCompileArch)-alpine-linux-musl</TargetTriple>
<TargetTriple Condition="'$(CrossCompileArch)' != '' and ($(CrossCompileRid.StartsWith('freebsd')))">$(CrossCompileArch)-unknown-freebsd12</TargetTriple>
</PropertyGroup>

<ItemGroup>
<LinkerArg Include="--target=$(TargetTriple)" Condition="'$(TargetOS)' != 'osx' and '$(TargetTriple)' != ''" />
</ItemGroup>
</Target>

<ItemGroup Condition="'$(NativeAotSupported)' == 'true'">
<CustomLinkerArg Condition="'$(CrossBuild)' == 'true' and '$(_hostArchitecture)' == '$(_targetArchitecture)' and '$(_hostOS)' != 'windows'" Include="--gcc-toolchain=$(ROOTFS_DIR)/usr" />
</ItemGroup>

<Target Name="LocateNativeCompiler"
Condition="'$(UseNativeAotForComponents)' == 'true' and '$(HostOS)' != 'windows'"
BeforeTargets="SetupOSSpecificProps">
<PropertyGroup>
<CppCompilerAndLinker Condition="'$(CppCompilerAndLinker)' == ''">clang</CppCompilerAndLinker>
</PropertyGroup>

<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(CppCompilerAndLinker)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo &quot;$CC;$LDFLAGS&quot;' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="_CC_LDFLAGS" />
</Exec>

<PropertyGroup>
<CppLinker>$(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';'))))</CppLinker>
<_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1))))</_LDFLAGS>
<LinkerFlavor Condition="$(_LDFLAGS.Contains('lld'))">lld</LinkerFlavor>
</PropertyGroup>
</Target>

<!-- Cross-build settings -->
<PropertyGroup>
<LinkerFlavor Condition="'$(CrossBuild)' == 'true' and '$(TargetsLinux)' == 'true'">lld</LinkerFlavor>
<SysRoot Condition="'$(CrossBuild)' == 'true' and '$(HostOS)' != 'windows'">$(ROOTFS_DIR)</SysRoot>
</PropertyGroup>

</Project>
10 changes: 4 additions & 6 deletions src/coreclr/crossgen-corelib.proj
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
<Project Sdk="Microsoft.Build.NoTargets">

<PropertyGroup>
<UseCrossArchCrossgen2 Condition="'$(CrossBuild)' == 'true' or '$(BuildArchitecture)' != '$(TargetArchitecture)' or '$(EnableNativeSanitizers)' != ''">true</UseCrossArchCrossgen2>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="$(CoreClrProjectRoot)/tools/aot/crossgen2/crossgen2_inbuild.csproj" OutputItemType="Crossgen2" />
<ProjectReference Include="$(CoreClrProjectRoot)/tools/dotnet-pgo/dotnet-pgo.csproj" OutputItemType="DotNetPgo" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
Expand Down Expand Up @@ -99,10 +95,12 @@
</ItemGroup>

<PropertyGroup>
<Crossgen2OutputPath>%(Crossgen2.RootDir)%(Crossgen2.Directory)</Crossgen2OutputPath>
<Crossgen2OutputPath>%(Crossgen2.RootDir)/%(Crossgen2.Directory)</Crossgen2OutputPath>
</PropertyGroup>
<ItemGroup>
<Crossgen2Files Include="$(Crossgen2OutputPath)/*.dll;$(Crossgen2OutputPath)/*.so;$(Crossgen2OutputPath)/*.dylib" />
<Crossgen2Files Include="$(Crossgen2OutputPath)/*.dll" />
<Crossgen2Files Include="$(Crossgen2OutputPath)/*.so" />
<Crossgen2Files Include="$(Crossgen2OutputPath)/*.dylib" />
</ItemGroup>
</Target>

Expand Down
Loading
Loading