Install java via native tools in CI (#54421)

mmitche · web-flow · commit 1aa1c5bab064 · 2024-03-13T00:39:47.000Z
* Install java via native tools in CI
The VMR doesn't have Java installed globally on the machines, but it does have it in the native tool cache. This PR updates the build scripts to use the native tool cache to install Java in CI builds. I've also updated to 11.0.22 (vs. the .3 version from 2019) and the script used to pull this from native tool assets.
diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml
@@ -200,7 +200,8 @@ stages:
         steps:
         - script: ./eng/build.cmd
                   -ci
-                  -prepareMachine
+                  -prepareMchine
+                  -nativeToolsOnMachine
                   -arch x64
                   -all
                   $(_BuildArgs)
@@ -252,6 +253,7 @@ stages:
         - script: ./eng/build.cmd
                   -ci
                   -prepareMachine
+                  -nativeToolsOnMachine
                   -arch x64
                   -pack
                   -all
@@ -737,11 +739,11 @@ stages:
           timeoutInMinutes: 240
           steps:
           # Build the shared framework
-          - script: ./eng/build.cmd -ci -prepareMachine -nobl -all -pack -arch x64
+          - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64
                     /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
             displayName: Build shared fx
           # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step.
-          - script: ./eng/build.cmd -ci -prepareMachine -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
+          - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test
                     -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true
                     /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
             displayName: Run build.cmd helix target
diff --git a/.azure/pipelines/jobs/codesign-xplat.yml b/.azure/pipelines/jobs/codesign-xplat.yml
@@ -13,7 +13,6 @@ jobs:
     jobDisplayName: "Code-sign ${{ parameters.inputName }} packages"
     agentOs: Windows
     installNodeJs: false
-    installJdk: false
     steps:
     - task: DownloadBuildArtifacts@0
       displayName: Download ${{ parameters.inputName }} artifacts
diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml
@@ -68,7 +68,6 @@ parameters:
   container: ''
   enableRichCodeNavigation: ''
   installNodeJs: true
-  installJdk: true # Ignored unless agentOs == Windows.
   timeoutInMinutes: 180
   testRunTitle: $(AgentOsName)-$(BuildConfiguration)
   useHostedUbuntu: true
@@ -147,8 +146,6 @@ jobs:
       - LC_ALL: 'en_US.UTF-8'
       - LANG: 'en_US.UTF-8'
       - LANGUAGE: 'en_US.UTF-8'
-    - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-      - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64
     - ${{ if or(ne(parameters.codeSign, true), ne(variables['System.TeamProject'], 'internal')) }}:
       - _SignType: ''
     - ${{ if and(eq(parameters.codeSign, true), eq(variables['System.TeamProject'], 'internal')) }}:
@@ -202,14 +199,11 @@ jobs:
         displayName: Install Node 20.x
         inputs:
           versionSpec: 20.x
-    - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}:
-      - powershell: ./eng/scripts/InstallJdk.ps1
-        displayName: Install JDK 11
-      - ${{ if eq(parameters.isAzDOTestingJob, true) }}:
-        - powershell: |
-            Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
-            ./eng/scripts/InstallGoogleChrome.ps1
-          displayName: Install Chrome
+    - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}:
+      - powershell: |
+          Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\"
+          ./eng/scripts/InstallGoogleChrome.ps1
+        displayName: Install Chrome
     - ${{ if eq(parameters.agentOs, 'Windows') }}:
       - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools"
         displayName: Add dotnet tools to path
@@ -256,7 +250,7 @@ jobs:
                 ${{ step.env }}
     - ${{ if eq(parameters.steps, '')}}:
       - ${{ if eq(parameters.agentOs, 'Windows') }}:
-        - script: $(BuildDirectory)\build.cmd -ci -prepareMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
+        - script: $(BuildDirectory)\build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
             /p:DotNetSignType=$(_SignType)
           displayName: Run build.cmd
           env:
diff --git a/eng/build.ps1 b/eng/build.ps1
@@ -15,6 +15,9 @@ Sets up CI specific settings and variables.
 .PARAMETER PrepareMachine
 In CI, Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
 
+.PARAMETER NativeToolsOnMachine
+Turns on native tooling handling. On CI machines, promotes native tools listed in global.json to the path.
+
 .PARAMETER Restore
 Run restore.
 
@@ -128,6 +131,7 @@ Online version: https://github.com/dotnet/aspnetcore/blob/main/docs/BuildFromSou
 param(
     [switch]$CI,
     [switch]$PrepareMachine,
+    [switch]$NativeToolsOnMachine,
 
     # Build lifecycle options
     [switch]$Restore,
@@ -312,58 +316,6 @@ $performDesktopBuild = ($BuildInstallers -and $Architecture -ne "arm") -or `
 $performDotnetBuild = $BuildJava -or $BuildManaged -or $BuildNodeJS -or `
     ($All -and -not ($NoBuildJava -and $NoBuildManaged -and $NoBuildNodeJS)) -or `
     ($Projects -and -not ($BuildInstallers -or $specifiedBuildNative))
-$foundJdk = $false
-$javac = Get-Command javac -ErrorAction Ignore -CommandType Application
-$localJdkPath = "$PSScriptRoot\..\.tools\jdk\win-x64\"
-if (Test-Path "$localJdkPath\bin\javac.exe") {
-    $foundJdk = $true
-    Write-Host -f Magenta "Detected JDK in $localJdkPath (via local repo convention)"
-    $env:JAVA_HOME = $localJdkPath
-}
-elseif ($env:JAVA_HOME) {
-    if (-not (Test-Path "${env:JAVA_HOME}\bin\javac.exe")) {
-        Write-Error "The environment variable JAVA_HOME was set, but ${env:JAVA_HOME}\bin\javac.exe does not exist. Remove JAVA_HOME or update it to the correct location for the JDK. See https://www.bing.com/search?q=java_home for details."
-    }
-    else {
-        Write-Host -f Magenta "Detected JDK in ${env:JAVA_HOME} (via JAVA_HOME)"
-        $foundJdk = $true
-    }
-}
-elseif ($javac) {
-    $foundJdk = $true
-    $javaHome = Split-Path -Parent (Split-Path -Parent $javac.Path)
-    $env:JAVA_HOME = $javaHome
-    Write-Host -f Magenta "Detected JDK in $javaHome (via PATH)"
-}
-else {
-    try {
-        $jdkRegistryKeys = @(
-            "HKLM:\SOFTWARE\JavaSoft\JDK",  # for JDK 10+
-            "HKLM:\SOFTWARE\JavaSoft\Java Development Kit"  # fallback for JDK 8
-        )
-        $jdkRegistryKey = $jdkRegistryKeys | Where-Object { Test-Path $_ } | Select-Object -First 1
-        if ($jdkRegistryKey) {
-            $jdkVersion = (Get-Item $jdkRegistryKey | Get-ItemProperty -name CurrentVersion).CurrentVersion
-            $javaHome = (Get-Item $jdkRegistryKey\$jdkVersion | Get-ItemProperty -Name JavaHome).JavaHome
-            if (Test-Path "${javaHome}\bin\javac.exe") {
-                $env:JAVA_HOME = $javaHome
-                Write-Host -f Magenta "Detected JDK $jdkVersion in $env:JAVA_HOME (via registry)"
-                $foundJdk = $true
-            }
-        }
-    }
-    catch {
-        Write-Verbose "Failed to detect Java: $_"
-    }
-}
-
-if ($env:PATH -notlike "*${env:JAVA_HOME}*") {
-    $env:PATH = "$(Join-Path $env:JAVA_HOME bin);${env:PATH}"
-}
-
-if (-not $foundJdk -and $RunBuild -and ($All -or $BuildJava) -and -not $NoBuildJava) {
-    Write-Error "Could not find the JDK. Either run $PSScriptRoot\scripts\InstallJdk.ps1 to install for this repo, or install the JDK globally on your machine (see $PSScriptRoot\..\docs\BuildFromSource.md for details)."
-}
 
 # Initialize global variables need to be set before the import of Arcade is imported
 $restore = $RunRestore
@@ -392,6 +344,61 @@ Remove-Item variable:global:_MSBuildExe -ea Ignore
 # Import Arcade
 . "$PSScriptRoot/common/tools.ps1"
 
+function LocateJava {
+    $foundJdk = $false
+    $javac = Get-Command javac -ErrorAction Ignore -CommandType Application
+    $localJdkPath = "$PSScriptRoot\..\.tools\jdk\win-x64\"
+    if (Test-Path "$localJdkPath\bin\javac.exe") {
+        $foundJdk = $true
+        Write-Host -f Magenta "Detected JDK in $localJdkPath (via local repo convention)"
+        $env:JAVA_HOME = $localJdkPath
+    }
+    elseif ($env:JAVA_HOME) {
+        if (-not (Test-Path "${env:JAVA_HOME}\bin\javac.exe")) {
+            Write-Error "The environment variable JAVA_HOME was set, but ${env:JAVA_HOME}\bin\javac.exe does not exist. Remove JAVA_HOME or update it to the correct location for the JDK. See https://www.bing.com/search?q=java_home for details."
+        }
+        else {
+            Write-Host -f Magenta "Detected JDK in ${env:JAVA_HOME} (via JAVA_HOME)"
+            $foundJdk = $true
+        }
+    }
+    elseif ($javac) {
+        $foundJdk = $true
+        $javaHome = Split-Path -Parent (Split-Path -Parent $javac.Path)
+        $env:JAVA_HOME = $javaHome
+        Write-Host -f Magenta "Detected JDK in $javaHome (via PATH)"
+    }
+    else {
+        try {
+            $jdkRegistryKeys = @(
+                "HKLM:\SOFTWARE\JavaSoft\JDK",  # for JDK 10+
+                "HKLM:\SOFTWARE\JavaSoft\Java Development Kit"  # fallback for JDK 8
+            )
+            $jdkRegistryKey = $jdkRegistryKeys | Where-Object { Test-Path $_ } | Select-Object -First 1
+            if ($jdkRegistryKey) {
+                $jdkVersion = (Get-Item $jdkRegistryKey | Get-ItemProperty -name CurrentVersion).CurrentVersion
+                $javaHome = (Get-Item $jdkRegistryKey\$jdkVersion | Get-ItemProperty -Name JavaHome).JavaHome
+                if (Test-Path "${javaHome}\bin\javac.exe") {
+                    $env:JAVA_HOME = $javaHome
+                    Write-Host -f Magenta "Detected JDK $jdkVersion in $env:JAVA_HOME (via registry)"
+                    $foundJdk = $true
+                }
+            }
+        }
+        catch {
+            Write-Verbose "Failed to detect Java: $_"
+        }
+    }
+
+    if ($env:PATH -notlike "*${env:JAVA_HOME}*") {
+        $env:PATH = "$(Join-Path $env:JAVA_HOME bin);${env:PATH}"
+    }
+
+    if (-not $foundJdk -and $RunBuild -and ($All -or $BuildJava) -and -not $NoBuildJava) {
+        Write-Error "Could not find the JDK. Either run $PSScriptRoot\scripts\InstallJdk.ps1 to install for this repo, or install the JDK globally on your machine (see $PSScriptRoot\..\docs\BuildFromSource.md for details)."
+    }
+}
+
 # Add default .binlog location if not already on the command line. tools.ps1 does not handle this; it just checks
 # $BinaryLog, $CI and $ExcludeCIBinarylog values for an error case. But tools.ps1 provides a nice function to help.
 if ($BinaryLog) {
@@ -424,6 +431,17 @@ try {
     $tmpRestore = $restore
     $restore = $true
 
+    # Initialize the native tools before locating java.
+    if ($NativeToolsOnMachine) {
+        $env:NativeToolsOnMachine=$true
+        # Do not promote native tools except in cases where -NativeToolsOnMachine is passed.
+        # Currently the JDK is laid out in an incorrect pattern: https://github.com/dotnet/dnceng/issues/2185
+        InitializeNativeTools
+    }
+
+    # Locate java, now that we may have java available after initializing native tools.
+    LocateJava
+
     $toolsetBuildProj = InitializeToolset
 
     $restore = $tmpRestore
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
@@ -296,7 +296,7 @@ function InstallDotNet([string] $dotnetRoot,
     if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" }
     if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" }
     $runtimePath = $runtimePath + "\" + $version
-  
+
     $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'"
 
     if (Test-Path $runtimePath) {
@@ -608,7 +608,7 @@ function InitializeBuildTool() {
     } else {
       $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework
     }
-    
+
     $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework }
   } elseif ($msbuildEngine -eq "vs") {
     try {
diff --git a/eng/configure-toolset.sh b/eng/configure-toolset.sh
@@ -5,3 +5,6 @@
 if [ "${DotNetBuildFromSource:-false}" = false ]; then
     use_installed_dotnet_cli="false"
 fi
+
+# Working around issue https://github.com/dotnet/arcade/issues/2673
+DisableNativeToolsetInstalls=true
diff --git a/eng/scripts/InstallJdk.ps1 b/eng/scripts/InstallJdk.ps1
@@ -23,7 +23,7 @@ $javacExe = "$installDir\bin\javac.exe"
 $tempDir = "$repoRoot\obj"
 if (-not $JdkVersion) {
     $globalJson = Get-Content "$repoRoot\global.json" | ConvertFrom-Json
-    $JdkVersion = $globalJson.tools.jdk
+    $JdkVersion = $globalJson.'native-tools'.jdk
 }
 
 if (Test-Path $javacExe) {
@@ -40,12 +40,13 @@ Remove-Item -Force -Recurse $tempDir -ErrorAction Ignore | out-null
 mkdir $tempDir -ea Ignore | out-null
 mkdir $installDir -ea Ignore | out-null
 Write-Host "Starting download of JDK ${JdkVersion}"
-& $PSScriptRoot\Download.ps1 "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/java/jdk-${JdkVersion}_windows-x64_bin.zip" "$tempDir/jdk.zip"
+& $PSScriptRoot\Download.ps1 "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/java/microsoft-jdk-${JdkVersion}-windows-x64.zip" "$tempDir/jdk.zip"
 Write-Host "Done downloading JDK ${JdkVersion}"
 Expand-Archive "$tempDir/jdk.zip" -d "$tempDir/jdk/"
 Write-Host "Expanded JDK to $tempDir"
 Write-Host "Installing JDK to $installDir"
-Move-Item "$tempDir/jdk/jdk-${JdkVersion}/*" $installDir
+# The name of the file directory within the zip is based on the version, but may contain a +N for build number.
+Move-Item "$(Get-ChildItem -Path "$tempDir/jdk" | Select-Object -First 1)/*" $installDir
 Write-Host "Done installing JDK to $installDir"
 
 if ($env:TF_BUILD) {
diff --git a/global.json b/global.json
@@ -12,8 +12,6 @@
         "$(MicrosoftNETCoreBrowserDebugHostTransportVersion)"
       ]
     },
-    "Git": "2.22.0",
-    "jdk": "11.0.3",
     "vs": {
       "version": "17.2",
       "components": [
@@ -25,6 +23,9 @@
     },
     "xcopy-msbuild": "17.1.0"
   },
+  "native-tools": {
+    "jdk": "11.0.22"
+  },
   "msbuild-sdks": {
     "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24159.1",
     "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24159.1"
