Build the "in build" crossgen2 into a pack that we can use within the…

… VMR (#110676) * Add a new crossgen2 build variant for a published crossgen2 targetting the host OS * Remove the .NET-hosted in-build crossgen2 and just use the published in-build host-targeting crossgen2. * Extract crossgen2 sfxproj changes from #109710 * Extract the properties and targets that will be shared into a separate props file. * Add a "host crossgen2" project * Add the crossgen2 host package to Subsets.props for the correct scenarios. * Fix crossgen-corelib to work with the new in-build crossgen2 * Publish the in-build crossgen2 as single file to avoid having to adjust the AOT-in-build logic to handle AOT-ing for the host. * AOT on Windows only * Don't set Platform for the shared framework projects. It's not necessary and it breaks the new crossgen2 host package * Forward ShortStack down so we don't build the host crossgen2 when runtime is the last repo. * Add os-only RID to portable RID graph in source-build to enable crossgen2 tasks in the SDK to discover the target OS from the source-build target RID * Publish the host crossgen2 as a "Vertical" visibility asset. * Cleanup crossgen2 infra, add comment about workaround, and adjust publishing to publish packages as packages. * Deduplicate artifacts * Add back the Update, somehow this got lost. * Don't forward down ShortStack * Use new "update" functionality to specify which artifacts are vertical visibility * Revert changes to Signing.props * Remove workaround now that the VMR has a bootstrap SDK with a new enough version. --------- Co-authored-by: Viktor Hofer <[email protected]>
dotnet · Jan 10, 2025 · 71db8b2 · 71db8b2
1 parent 0df31bc
commit 71db8b2
Show file tree

Hide file tree

Showing 12 changed files with 106 additions and 108 deletions.
diff --git a/eng/Publishing.props b/eng/Publishing.props
@@ -8,6 +8,15 @@
     <PlatformName Condition="'$(TargetArchitecture)' != ''">$(TargetArchitecture)</PlatformName>
   </PropertyGroup>
 
+  <!--
+    Mark assets as Vertical visibility when building in the VMR
+  -->
+  <ItemGroup Condition="'$(DotNetBuildOrchestrator)' == 'true'">
+    <Artifact Condition="'$(RuntimeFlavor)' != 'Mono' and '$(OutputRID)' != '$(NETCoreSdkRuntimeIdentifier)'"
+              Update="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.$(NETCoreSdkRuntimeIdentifier).*.nupkg"
+              Visibility="Vertical" />
+  </ItemGroup>
+
   <!--
     Look through the downloaded artifacts to figure out how to ship them. Creates item groups for
     other types of publishing to use.
@@ -203,4 +212,4 @@
     <Error Text="Duplicate RuntimeNupkgFile entries for: %(RuntimeNupkgFile.Identity)" Condition="@(RuntimeNupkgFile->Count()) &gt; 1" />
   </Target>
 
-</Project>
+</Project>
diff --git a/eng/Subsets.props b/eng/Subsets.props
@@ -547,6 +547,13 @@
       <ItemGroup>
         <ProjectToBuild Condition="'$(NativeAotSupported)' == 'true' and '$(RuntimeFlavor)' != 'Mono' and '$(BuildOnlyPgoInstrumentedAssets)' != 'true'" Include="$(InstallerProjectRoot)\pkg\projects\nativeaot-packages.proj" Category="packs" />
       </ItemGroup>
+      <ItemGroup>
+        <!--
+          When we're building in the VMR, we need to provide a crossgen2 that runs on the host machine for downstream repos to use to R2R their code.
+          In non-VMR builds, downstream repos can use the crossgen2 built for the target host SDK from another build leg, but in the VMR we need to provide one to use.
+        -->
+        <ProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono' and '$(DotNetBuildOrchestrator)' == 'true' and '$(OutputRID)' != '$(NETCoreSdkRuntimeIdentifier)'" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Crossgen2.Host.sfxproj" Category="packs" />
+      </ItemGroup>
       <ItemGroup>
         <SharedFrameworkProjectToBuild Condition="'$(BuildMonoAOTCrossCompilerOnly)' != 'true'" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Runtime.sfxproj" />
         <SharedFrameworkProjectToBuild Condition="'$(BuildNativeAOTRuntimePack)' != 'true' and '$(RuntimeFlavor)' == '$(PrimaryRuntimeFlavor)' and '$(TargetsMobile)' != 'true'" Include="$(InstallerProjectRoot)pkg\sfx\bundle\Microsoft.NETCore.App.Bundle.bundleproj" />

diff --git a/src/coreclr/crossgen-corelib.proj b/src/coreclr/crossgen-corelib.proj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.Build.NoTargets">
 
   <ItemGroup>
-    <ProjectReference Include="$(CoreClrProjectRoot)/tools/aot/crossgen2/crossgen2_inbuild.csproj" OutputItemType="Crossgen2" />
+    <ProjectReference Include="$(CoreClrProjectRoot)/tools/aot/crossgen2/crossgen2_inbuild.csproj" OutputItemType="Crossgen2Files" />
     <ProjectReference Include="$(CoreClrProjectRoot)/tools/dotnet-pgo/dotnet-pgo.csproj" OutputItemType="DotNetPgo" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
     <ProjectReference Include="$([MSBuild]::NormalizePath('$(CoreClrProjectRoot)', 'System.Private.CoreLib', 'System.Private.CoreLib.csproj'))" OutputItemType="CoreLib" />
   </ItemGroup>
@@ -92,21 +92,13 @@
     <ItemGroup>
       <Crossgen2Inputs Include="@(CoreLib)" />
       <Crossgen2Inputs Include="$(MergedMibcPath)" />
-    </ItemGroup>
-
-    <PropertyGroup>
-      <Crossgen2OutputPath>%(Crossgen2.RootDir)/%(Crossgen2.Directory)</Crossgen2OutputPath>
-    </PropertyGroup>
-    <ItemGroup>
-      <Crossgen2Files Include="$(Crossgen2OutputPath)/*.dll" />
-      <Crossgen2Files Include="$(Crossgen2OutputPath)/*.so" />
-      <Crossgen2Files Include="$(Crossgen2OutputPath)/*.dylib" />
+      <Crossgen2Inputs Include="@(Crossgen2Files->Metadata('OutputPath'))" />
     </ItemGroup>
   </Target>
 
   <Target Name="InvokeCrossgen"
           DependsOnTargets="PrepareInvokeCrossgen;CreateMergedMibcFile"
-          Inputs="@(Crossgen2Inputs);@(Crossgen2Files)"
+          Inputs="@(Crossgen2Inputs)"
           Outputs="$(CoreLibOutputPath);$(CoreLibNiPdbPath);$(CoreLibPerfMapPath)"
           AfterTargets="Build">
 
@@ -117,7 +109,7 @@
       Text="Generating native image of System.Private.CoreLib for $(OSPlatformConfig). Logging to $(CrossGenCoreLibLog)" />
 
     <PropertyGroup>
-      <CrossGenDllCmd>$(DotNetCli) @(Crossgen2)</CrossGenDllCmd>
+      <CrossGenDllCmd>@(Crossgen2Files->Metadata('OutputPath')->WithMetadataValue('Filename','crossgen2')->WithMetadataValue('Extension','$(ExeSuffix)'))</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) -o:$(CoreLibOutputPath)</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) -r:$([MSBuild]::NormalizePath('$(BinDir)', 'IL', '*.dll'))</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) --targetarch:$(TargetArchitecture)</CrossGenDllCmd>

diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj
@@ -1,9 +1,20 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Publish;PublishItemsOutputGroup">
   <PropertyGroup>
     <CrossHostArch Condition="'$(CrossBuild)' == 'true' or '$(TargetArchitecture)' != '$(BuildArchitecture)' or '$(HostOS)' != '$(TargetOS)' or '$(EnableNativeSanitizers)' != ''">$(BuildArchitecture)</CrossHostArch>
-    <OutputPath>$(RuntimeBinDir)/$(BuildArchitecture)/crossgen2/</OutputPath>
-    <TargetFramework>$(NetCoreAppToolCurrent)</TargetFramework>
-    <UseAppHost>false</UseAppHost>
+    <_IsPublishing>true</_IsPublishing>
+    <RuntimeIdentifier>$(NETCoreSdkRuntimeIdentifier)</RuntimeIdentifier>
+    <PublishDir>$(RuntimeBinDir)$(BuildArchitecture)/crossgen2/</PublishDir>
+    <SelfContained>true</SelfContained>
+    <PublishTrimmed>true</PublishTrimmed>
+    <!--
+      Publish with AOT on Windows as Single file doesn't work well when an app has to include DiaSymReader.Native.
+      Publish single-file elsewhere so we don't need to adjust our "AOT in build" logic to handle targeting the SDK RID.
+    -->
+    <PublishAot Condition="$(NETCoreSdkPortableRuntimeIdentifier.StartsWith('win-'))">true</PublishAot>
+    <PublishSingleFile Condition="'$(PublishAot)' != 'true'">true</PublishSingleFile>
+    <PublishReadyToRun>true</PublishReadyToRun>
+    <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
+    <TrimmerSingleWarn>false</TrimmerSingleWarn>
   </PropertyGroup>
   <Import Project="crossgen2.props" />
 </Project>
diff --git a/src/installer/pkg/Directory.Build.props b/src/installer/pkg/Directory.Build.props
diff --git a/src/installer/pkg/Directory.Build.targets b/src/installer/pkg/Directory.Build.targets
diff --git a/src/installer/pkg/projects/Directory.Build.props b/src/installer/pkg/projects/Directory.Build.props
@@ -2,6 +2,7 @@
   <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props, $(MSBuildThisFileDirectory)..))" />
 
   <PropertyGroup>
+    <Platform>$(TargetArchitecture)</Platform>
     <PackagePlatform>AnyCPU</PackagePlatform>
 
     <SkipPackageFileCheck>true</SkipPackageFileCheck>

diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.Host.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.Host.sfxproj
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Sdk Name="Microsoft.DotNet.SharedFramework.Sdk" />
+
+  <PropertyGroup>
+    <CrossgenPublishProject>$(RepoRoot)src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj</CrossgenPublishProject>
+    <RuntimeIdentifier>$(NETCoreSdkRuntimeIdentifier)</RuntimeIdentifier>
+    <IsShipping>false</IsShipping>
+  </PropertyGroup>
+
+  <Import Project="$(MSBuildThisFileDirectory)Microsoft.NETCore.App.Crossgen2.props" />
+</Project>
diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.props b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.props
@@ -0,0 +1,53 @@
+<Project>
+  <PropertyGroup>
+    <!-- Crossgen is not used for Mono -->
+    <SkipBuild Condition="'$(RuntimeFlavor)' == 'Mono'">true</SkipBuild>
+    <PlatformPackageType>ToolPack</PlatformPackageType>
+    <SharedFrameworkName>$(SharedFrameworkName).Crossgen2</SharedFrameworkName>
+    <OverridePackageId>$(SharedFrameworkName).$(RuntimeIdentifier)</OverridePackageId>
+    <ArchiveName>dotnet-crossgen2</ArchiveName>
+    <SharedFrameworkHostFileNameOverride>crossgen2</SharedFrameworkHostFileNameOverride>
+    <GenerateInstallers>false</GenerateInstallers>
+    <PermitDllAndExeFilesLackingFileVersion>true</PermitDllAndExeFilesLackingFileVersion>
+    <!-- Publishing as single-file or NativeAOT means we can't examine the interior DLLs -->
+    <ShouldVerifyClosure>false</ShouldVerifyClosure>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference
+      Include="$(CrossgenPublishProject)"
+      OutputItemType="_RawCrossgenPublishFiles"
+      ReferenceOutputAssembly="false" />
+  </ItemGroup>
+
+  <Target Name="PublishCrossgen"
+          BeforeTargets="GetFilesToPackage">
+    <ItemGroup>
+      <_CrossgenPublishFiles Include="@(_RawCrossgenPublishFiles->'%(OutputPath)')"
+                             KeepMetadata="REMOVE_ALL" />
+    </ItemGroup>
+    <ItemGroup Condition="'$(NativeAotSupported)' != 'true'">
+      <FilesToPackage Include="@(_CrossgenPublishFiles)"
+                      Exclude="*.pdb;*.h;*.lib"
+                      TargetPath="tools/" />
+    </ItemGroup>
+    <ItemGroup Condition="'$(NativeAotSupported)' == 'true'">
+      <!-- Treat all native aot assets as native runtime assets -->
+      <FilesToPackage Include="@(_CrossgenPublishFiles->Distinct())"
+                          Condition="'%(Extension)' != '.pdb'"
+                          TargetPath="tools/" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="AddCrossgen2SymbolFilesToPackage" BeforeTargets="GetFilesToPackage" DependsOnTargets="PublishCrossgen">
+    <ItemGroup>
+      <_Crossgen2SymbolFilesToPackage Include="@(_CrossgenPublishFiles)" Condition="'%(Extension)' == '.pdb'" />
+      <!-- Symbol files for JIT libraries are placed in a different location for Windows builds -->
+      <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPdbDir)%(FileName).pdb')" Condition="'$(TargetOS)' == 'windows' and '%(FileName)' != 'crossgen2'" />
+      <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPath)%(FileName)%(Extension)$(SymbolsSuffix)')" Condition="'$(TargetOS)' != 'windows' and '%(FileName)' != 'crossgen2'" />
+      <_Crossgen2SymbolFilesToPackage Remove="@(_Crossgen2SymbolFilesToPackage)" Condition="!Exists('%(Identity)')" />
+
+      <_SymbolFilesToPackage Include="@(_Crossgen2SymbolFilesToPackage)" TargetPath="tools/" />
+    </ItemGroup>
+  </Target>
+</Project>
diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj
@@ -2,48 +2,14 @@
   <Sdk Name="Microsoft.DotNet.SharedFramework.Sdk" />
 
   <PropertyGroup>
-    <!-- Crossgen is not used for Mono -->
-    <SkipBuild Condition="'$(RuntimeFlavor)' == 'Mono'">true</SkipBuild>
-    <PlatformPackageType>ToolPack</PlatformPackageType>
-    <SharedFrameworkName>$(SharedFrameworkName).Crossgen2</SharedFrameworkName>
-    <OverridePackageId>$(SharedFrameworkName).$(RuntimeIdentifier)</OverridePackageId>
-    <ArchiveName>dotnet-crossgen2</ArchiveName>
-    <SharedFrameworkHostFileNameOverride>crossgen2</SharedFrameworkHostFileNameOverride>
-    <GenerateInstallers>false</GenerateInstallers>
-    <PermitDllAndExeFilesLackingFileVersion>true</PermitDllAndExeFilesLackingFileVersion>
-    <!-- Publishing as single-file or NativeAOT means we can't examine the interior DLLs -->
-    <ShouldVerifyClosure>false</ShouldVerifyClosure>
+    <CrossgenPublishProject>$(RepoRoot)src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj</CrossgenPublishProject>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.DotNet.Build.Tasks.Archives" Version="$(MicrosoftDotNetBuildTasksArchivesVersion)" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference
-      Include="$(RepoRoot)src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj"
-      OutputItemType="_RawCrossgenPublishFiles"
-      ReferenceOutputAssembly="false" />
-  </ItemGroup>
-
-  <Target Name="PublishCrossgen"
-          BeforeTargets="GetFilesToPackage">
-    <ItemGroup>
-      <_CrossgenPublishFiles Include="@(_RawCrossgenPublishFiles->'%(OutputPath)')"
-                             KeepMetadata="REMOVE_ALL" />
-    </ItemGroup>
-    <ItemGroup Condition="'$(NativeAotSupported)' != 'true'">
-      <FilesToPackage Include="@(_CrossgenPublishFiles)"
-                      Exclude="*.pdb;*.h;*.lib"
-                      TargetPath="tools/" />
-    </ItemGroup>
-    <ItemGroup Condition="'$(NativeAotSupported)' == 'true'">
-      <!-- Treat all native aot assets as native runtime assets -->
-      <FilesToPackage Include="@(_CrossgenPublishFiles->Distinct())"
-                          Condition="'%(Extension)' != '.pdb'"
-                          TargetPath="tools/" />
-    </ItemGroup>
-  </Target>
+  <Import Project="$(MSBuildThisFileDirectory)Microsoft.NETCore.App.Crossgen2.props" />
 
   <Target Name="RunPublishedCrossgen" AfterTargets="PublishCrossgen"
           Condition="'$(TargetOS)' == '$(HostOS)' and '$(TargetArchitecture)' == '$(BuildArchitecture)' and '$(CrossBuild)' != 'true'">
@@ -52,22 +18,4 @@
       <Output TaskParameter="ExitCode" PropertyName="CrossgenExitCode" />
     </Exec>
   </Target>
-
-  <PropertyGroup>
-    <TargetOSComponent>unix</TargetOSComponent>
-    <TargetOSComponent Condition="'$(TargetOS)' == 'windows'">win</TargetOSComponent>
-    <TargetSpec>$(TargetOSComponent)-$(TargetArchitecture)</TargetSpec>
-  </PropertyGroup>
-
-  <Target Name="AddCrossgen2SymbolFilesToPackage" BeforeTargets="GetFilesToPackage" DependsOnTargets="PublishCrossgen">
-    <ItemGroup>
-      <_Crossgen2SymbolFilesToPackage Include="@(_CrossgenPublishFiles)" Condition="'%(Extension)' == '.pdb'" />
-      <!-- Symbol files for JIT libraries are placed in a different location for Windows builds -->
-      <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPdbDir)%(FileName).pdb')" Condition="'$(TargetOS)' == 'windows' and '%(FileName)' != 'crossgen2'" />
-      <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPath)%(FileName)%(Extension)$(SymbolsSuffix)')" Condition="'$(TargetOS)' != 'windows' and '%(FileName)' != 'crossgen2'" />
-      <_Crossgen2SymbolFilesToPackage Remove="@(_Crossgen2SymbolFilesToPackage)" Condition="!Exists('%(Identity)')" />
-
-      <_SymbolFilesToPackage Include="@(_Crossgen2SymbolFilesToPackage)" TargetPath="tools/" />
-    </ItemGroup>
-  </Target>
 </Project>
diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets
@@ -2,16 +2,14 @@
   <Target Name="ResolveReadyToRunCompilers" DependsOnTargets="ResolveRuntimeFilesFromLocalBuild">
     <PropertyGroup>
       <PublishReadyToRunPerfmapFormatVersion Condition="'$(PublishReadyToRunPerfmapFormatVersion)' == ''">1</PublishReadyToRunPerfmapFormatVersion>
-      <Crossgen2Path>$([MSBuild]::NormalizePath('$(Crossgen2InBuildDir)', 'crossgen2.dll'))</Crossgen2Path>
-      <R2RDotnetHostPath>$(DotNetTool)</R2RDotnetHostPath>
+      <Crossgen2Path>$([MSBuild]::NormalizePath('$(Crossgen2InBuildDir)', 'crossgen2$(ExeSuffix)'))</Crossgen2Path>
     </PropertyGroup>
 
     <ItemGroup>
       <Crossgen2Tool Include="$(Crossgen2Path)"
                      TargetArch="$(TargetArchitecture)"
                      TargetOS="$(TargetOS)"
-                     DotnetHostPath="$(R2RDotnetHostPath)"
                      PerfmapFormatVersion="$(PublishReadyToRunPerfmapFormatVersion)"/>
     </ItemGroup>
   </Target>
-</Project>
+</Project>
diff --git a/src/installer/pkg/sfx/installers/dotnet-host.proj b/src/installer/pkg/sfx/installers/dotnet-host.proj
@@ -20,7 +20,7 @@
     <MacOSScriptsTemplateDirectory>osx_scripts/host</MacOSScriptsTemplateDirectory>
     <MacOSPackageDescription>The .NET Shared Host.</MacOSPackageDescription>
     <!-- Enables stable upgrade code - do not change -->
-    <MsiUpgradeCodeSeed>dotnet-host $(MajorVersion).$(MinorVersion) $(Platform)</MsiUpgradeCodeSeed>
+    <MsiUpgradeCodeSeed>dotnet-host $(MajorVersion).$(MinorVersion) $(TargetArchitecture)</MsiUpgradeCodeSeed>
   </PropertyGroup>
 
   <ItemGroup>