From bb92d76a173d61cc7819c9744bc3da32a02e3e71 Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Wed, 28 Feb 2024 09:17:48 -0800 Subject: [PATCH 01/12] utils: support building android SDKs on Windows This adds the ability to build the Android SDKs as part of the Windows build. --- utils/build.ps1 | 354 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 286 insertions(+), 68 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index bfac3272df009..d036ccdb1a35a 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -32,6 +32,12 @@ The debug information format for C/C++ code: dwarf or codeview. .PARAMETER SwiftDebugFormat The debug information format for Swift code: dwarf or codeview. +.PARAMETER AndroidAPILevel +The API Level to target when building the Android SDKs + +.PARAMETER AndroidSDKs +An array of architectures for which the Android Swift SDK should be built. + .PARAMETER WindowsSDKs An array of architectures for which the Windows Swift SDK should be built. @@ -45,6 +51,9 @@ The toolchain snapshot to build the early components with. .PARAMETER PinnedSHA256 The SHA256 for the pinned toolchain. +.PARAMETER AndroidNDKVersion +The version number of the Android NDK to be used. + .PARAMETER WinSDKVersion The version number of the Windows SDK to be used. Overrides the value resolved by the Visual Studio command prompt. @@ -96,11 +105,14 @@ param( [string] $ImageRoot = "S:", [string] $CDebugFormat = "dwarf", [string] $SwiftDebugFormat = "dwarf", + [string] $AndroidAPILevel = 28, + [string[]] $AndroidSDKs = @("aarch64","armv7","i686","x86_64"), [string[]] $WindowsSDKs = @("X64","X86","Arm64"), [string] $ProductVersion = "0.0.0", [string] $PinnedBuild = "", [string] $PinnedSHA256 = "", [string] $PythonVersion = "3.9.10", + [string] $AndroidNDKVersion = "r26b", [string] $WinSDKVersion = "", [switch] $SkipBuild = $false, [switch] $SkipRedistInstall = $false, @@ -166,6 +178,7 @@ if (-not (Test-Path $python)) { } # Work around limitations of cmd passing in array arguments via powershell.exe -File +if ($AndroidSDKs.Length -eq 1) { $AndroidSDKs = $AndroidSDKs[0].Split(",") } if ($WindowsSDKs.Length -eq 1) { $WindowsSDKs = $WindowsSDKs[0].Split(",") } if ($Test.Length -eq 1) { $Test = $Test[0].Split(",") } @@ -219,6 +232,58 @@ $ArchARM64 = @{ ToolchainInstallRoot = "$BinaryCache\arm64\toolchains\$ProductVersion+Asserts"; } +$AndroidARM64 = @{ + AndroidArchABI = "arm64-v8a"; + BinaryDir = "bin64a"; + CMakeName = "aarch64"; + LLVMName = "aarch64"; + LLVMTarget = "aarch64-unknown-linux-android$AndroidAPILevel"; + BuildID = 400; + BinaryCache = "$BinaryCache\aarch64"; + PlatformInstallRoot = "$BinaryCache\arm64\Android.platform"; + SDKInstallRoot = "$BinaryCache\arm64\Android.platform\Developer\SDKs\Android.sdk"; + XCTestInstallRoot = "$BinaryCache\arm64\Android.platform\Developer\Library\XCTest-development"; +} + +$AndroidARMv7 = @{ + AndroidArchABI = "armeabi-v7a"; + BinaryDir = "bina"; + CMakeName = "armv7-a"; + LLVMName = "armv7"; + LLVMTarget = "armv7a-unknown-linux-androideabi"; + BuildID = 500; + BinaryCache = "$BinaryCache\armv7"; + PlatformInstallRoot = "$BinaryCache\armv7\Android.platform"; + SDKInstallRoot = "$BinaryCache\armv7\Android.platform\Developer\SDKs\Android.sdk"; + XCTestInstallRoot = "$BinaryCache\armv7\Android.platform\Developer\Library\XCTest-development"; +} + +$AndroidX86 = @{ + AndroidArchABI = "x86"; + BinaryDir = "bin"; + CMakeName = "i686"; + LLVMName = "i686"; + LLVMTarget = "i686-unknown-linux-android"; + BuildID = 600; + BinaryCache = "$BinaryCache\i686"; + PlatformInstallRoot = "$BinaryCache\x86\Android.platform"; + SDKInstallRoot = "$BinaryCache\x86\Android.platform\Developer\SDKs\Android.sdk"; + XCTestInstallRoot = "$BinaryCache\x86\Android.platform\Developer\Library\XCTest-development"; +} + +$AndroidX64 = @{ + AndroidArchABI = "x86_64"; + BinaryDir = "bin64"; + CMakeName = "x86_64"; + LLVMName = "x86_64"; + LLVMTarget = "x86_64-unknown-linux-android"; + BuildID = 700; + BinaryCache = "$BinaryCache\x86_64"; + PlatformInstallRoot = "$BinaryCache\x64\Android.platform"; + SDKInstallRoot = "$BinaryCache\x64\Android.platform\Developer\SDKs\Android.sdk"; + XCTestInstallRoot = "$BinaryCache\x64\Android.platform\Developer\Library\XCTest-development"; +} + $HostArch = switch ($HostArchName) { "AMD64" { $ArchX64 } "ARM64" { $ArchARM64 } @@ -251,6 +316,10 @@ function Get-InstallDir($Arch) { return "$ImageRoot\$ProgramFilesName\Swift" } +function Get-HostSwiftSDK() { + return ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) +} + $NugetRoot = "$BinaryCache\nuget" $PinnedToolchain = [IO.Path]::GetFileNameWithoutExtension($PinnedBuild) @@ -261,6 +330,15 @@ $LibraryRoot = "$ImageRoot\Library" $HostArch.ToolchainInstallRoot = "$(Get-InstallDir $HostArch)\Toolchains\$ProductVersion+Asserts" # Resolve the architectures received as argument +$AndroidSDKArchs = @($AndroidSDKs | ForEach-Object { + switch ($_) { + "aarch64" { $AndroidARM64 } + "armv7" { $AndroidARMv7 } + "i686" { $AndroidX86 } + "x86_64" { $AndroidX64 } + default { throw "Unknown architecture $_" } + } +}) $WindowsSDKArchs = @($WindowsSDKs | ForEach-Object { switch ($_) { "X64" { $ArchX64 } @@ -552,6 +630,17 @@ function Fetch-Dependencies { Download-Python $BuildArchName } + if ($AndroidSDKArchs.count -gt 0) { + $NDKURL = "https://dl.google.com/android/repository/android-ndk-r26b-windows.zip" + $NDKHash = "A478D43D4A45D0D345CDA6BE50D79642B92FB175868D9DC0DFC86181D80F691E" + DownloadAndVerify $NDKURL "$BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip" $NDKHash + + # TODO(compnerd) stamp/validate that we need to re-extract + Write-Output "Extracting Android NDK $AndroidNDKVersion ..." + New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache | Out-Null + Expand-Archive -Path $BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip -Destination $BinaryCache -Force + } + if ($WinSDKVersion) { try { # Check whether VsDevShell can already resolve the requested Windows SDK Version @@ -696,12 +785,30 @@ function Build-CMakeProject { TryAdd-KeyValue $Defines CMAKE_SYSTEM_PROCESSOR $Arch.CMakeName } + # TODO(compnerd) clean up these and formalise this selection + if ($Platform -eq "Android") { + $env:Path = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja;${env:Path}" + TryAdd-KeyValue $Defines CMAKE_C_COMPILER "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\bin\clang.exe" + TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\bin\clang++.exe" + TryAdd-KeyValue $Defines CMAKE_MAKE_PROGRAM "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" + TryAdd-KeyValue $Defines CMAKE_ANDROID_API "$AndroidAPILevel" + TryAdd-KeyValue $Defines CMAKE_ANDROID_ARCH_ABI $Arch.AndroidArchABI + TryAdd-KeyValue $Defines CMAKE_ANDROID_NDK "S:\b\android-ndk-r26b" + TryAdd-KeyValue $Defines CMAKE_C_COMPILER_WORKS YES + TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER_WORKS YES + } + TryAdd-KeyValue $Defines CMAKE_BUILD_TYPE Release TryAdd-KeyValue $Defines CMAKE_MT "mt" $CFlags = @() - if ($Platform -eq "Windows") { - $CFlags = @("/GS-", "/Gw", "/Gy", "/Oi", "/Oy", "/Zc:inline") + switch ($Platform) { + Windows { + $CFlags = @("/GS-", "/Gw", "/Gy", "/Oi", "/Oy", "/Zc:inline") + } + Android { + $CFlags = @("--sysroot=S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\sysroot") + } } $CXXFlags = @() @@ -712,7 +819,7 @@ function Build-CMakeProject { if ($UseMSVCCompilers.Contains("C") -Or $UseMSVCCompilers.Contains("CXX") -Or $UseBuiltCompilers.Contains("C") -Or $UseBuiltCompilers.Contains("CXX") -Or $UsePinnedCompilers.Contains("C") -Or $UsePinnedCompilers.Contains("CXX")) { - if ($DebugInfo) { + if ($DebugInfo -and $Platform -eq "Windows") { Append-FlagsDefine $Defines CMAKE_MSVC_DEBUG_INFORMATION_FORMAT Embedded Append-FlagsDefine $Defines CMAKE_POLICY_CMP0141 NEW # Add additional linker flags for generating the debug info. @@ -736,23 +843,27 @@ function Build-CMakeProject { Append-FlagsDefine $Defines CMAKE_CXX_FLAGS $CXXFlags } if ($UsePinnedCompilers.Contains("ASM") -Or $UseBuiltCompilers.Contains("ASM")) { + $Driver = if ($Platform -eq "Windows") { "clang-cl.exe" } else { "clang.exe" } if ($UseBuiltCompilers.Contains("ASM")) { - TryAdd-KeyValue $Defines CMAKE_ASM_COMPILER ([IO.Path]::Combine($CompilersBinaryCache, "bin", "clang-cl.exe")) + TryAdd-KeyValue $Defines CMAKE_ASM_COMPILER ([IO.Path]::Combine($CompilersBinaryCache, "bin", $Driver)) } else { - TryAdd-KeyValue $Defines CMAKE_ASM_COMPILER (Join-Path -Path (Get-PinnedToolchainTool) -ChildPath "clang-cl.exe") + TryAdd-KeyValue $Defines CMAKE_ASM_COMPILER (Join-Path -Path (Get-PinnedToolchainTool) -ChildPath $Driver) } Append-FlagsDefine $Defines CMAKE_ASM_FLAGS "--target=$($Arch.LLVMTarget)" - TryAdd-KeyValue $Defines CMAKE_ASM_COMPILE_OPTIONS_MSVC_RUNTIME_LIBRARY_MultiThreadedDLL "/MD" + if ($Platform -eq "Windows") { + TryAdd-KeyValue $Defines CMAKE_ASM_COMPILE_OPTIONS_MSVC_RUNTIME_LIBRARY_MultiThreadedDLL "/MD" + } } if ($UsePinnedCompilers.Contains("C") -Or $UseBuiltCompilers.Contains("C")) { + $Driver = if ($Platform -eq "Windows") { "clang-cl.exe" } else { "clang.exe" } if ($UseBuiltCompilers.Contains("C")) { - TryAdd-KeyValue $Defines CMAKE_C_COMPILER ([IO.Path]::Combine($CompilersBinaryCache, "bin", "clang-cl.exe")) + TryAdd-KeyValue $Defines CMAKE_C_COMPILER ([IO.Path]::Combine($CompilersBinaryCache, "bin", $Driver)) } else { - TryAdd-KeyValue $Defines CMAKE_C_COMPILER (Join-Path -Path (Get-PinnedToolchainTool) -ChildPath "clang-cl.exe") + TryAdd-KeyValue $Defines CMAKE_C_COMPILER (Join-Path -Path (Get-PinnedToolchainTool) -ChildPath $Driver) } TryAdd-KeyValue $Defines CMAKE_C_COMPILER_TARGET $Arch.LLVMTarget - if (-not (Test-CMakeAtLeast -Major 3 -Minor 26 -Patch 3)) { + if (-not (Test-CMakeAtLeast -Major 3 -Minor 26 -Patch 3) -and $Platform -eq "Windows") { # Workaround for https://github.com/ninja-build/ninja/issues/2280 TryAdd-KeyValue $Defines CMAKE_CL_SHOWINCLUDES_PREFIX "Note: including file: " } @@ -763,14 +874,15 @@ function Build-CMakeProject { Append-FlagsDefine $Defines CMAKE_C_FLAGS $CFlags } if ($UsePinnedCompilers.Contains("CXX") -Or $UseBuiltCompilers.Contains("CXX")) { + $Driver = if ($Platform -eq "Windows") { "clang-cl.exe" } else { "clang++.exe" } if ($UseBuiltCompilers.Contains("CXX")) { - TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER ([IO.Path]::Combine($CompilersBinaryCache, "bin", "clang-cl.exe")) + TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER ([IO.Path]::Combine($CompilersBinaryCache, "bin", $Driver)) } else { - TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER (Join-Path -Path (Get-PinnedToolchainTool) -ChildPath "clang-cl.exe") + TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER (Join-Path -Path (Get-PinnedToolchainTool) -ChildPath $Driver) } TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER_TARGET $Arch.LLVMTarget - if (-not (Test-CMakeAtLeast -Major 3 -Minor 26 -Patch 3)) { + if (-not (Test-CMakeAtLeast -Major 3 -Minor 26 -Patch 3) -and $Platform -eq "Windows") { # Workaround for https://github.com/ninja-build/ninja/issues/2280 TryAdd-KeyValue $Defines CMAKE_CL_SHOWINCLUDES_PREFIX "Note: including file: " } @@ -790,7 +902,10 @@ function Build-CMakeProject { } else { TryAdd-KeyValue $Defines CMAKE_Swift_COMPILER (Join-Path -Path (Get-PinnedToolchainTool) -ChildPath "swiftc.exe") } - TryAdd-KeyValue $Defines CMAKE_Swift_COMPILER_TARGET $Arch.LLVMTarget + if (-not ($Platform -eq "Windows")) { + TryAdd-KeyValue $Defines CMAKE_Swift_COMPILER_WORKS = "YES" + } + TryAdd-KeyValue $Defines CMAKE_Swift_COMPILER_TARGET $Arch.LLVMTarget.Replace("$AndroidAPILevel", "") if ($UseBuiltCompilers.Contains("Swift")) { if ($SwiftSDK -ne "") { $SwiftArgs += @("-sdk", $SwiftSDK) @@ -798,9 +913,28 @@ function Build-CMakeProject { $RuntimeBinaryCache = Get-TargetProjectBinaryCache $Arch Runtime $SwiftResourceDir = "${RuntimeBinaryCache}\lib\swift" + switch ($Platform) { + Windows { + $SwiftArgs += @( + "-vfsoverlay", "$RuntimeBinaryCache\stdlib\windows-vfs-overlay.yaml", + "-strict-implicit-module-context", + "-Xcc", "-Xclang", "-Xcc", "-fbuiltin-headers-in-system-modules" + ) + } + Android { + $SwiftArgs += @("-sdk", "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\sysroot") + $SwiftArgs += @( + "-Xclang-linker", "-target", + "-Xclang-linker", $Arch.LLVMTarget, + "-Xclang-linker", "--sysroot", + "-Xclang-linker", "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\sysroot", + "-Xclang-linker", "-resource-dir", + "-Xclang-linker", "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\lib\clang\17" + ) + } + } $SwiftArgs += @("-resource-dir", "$SwiftResourceDir") - $SwiftArgs += @("-L", "$SwiftResourceDir\windows") - $SwiftArgs += @("-vfsoverlay", "$RuntimeBinaryCache\stdlib\windows-vfs-overlay.yaml", "-strict-implicit-module-context", "-Xcc", "-Xclang", "-Xcc", "-fbuiltin-headers-in-system-modules") + $SwiftArgs += @("-L", "$SwiftResourceDir\$($Platform.ToString().ToLowerInvariant())") } } else { $SwiftArgs += @("-sdk", (Get-PinnedToolchainSDK)) @@ -808,19 +942,26 @@ function Build-CMakeProject { # Debug Information if ($DebugInfo) { - if ($SwiftDebugFormat -eq "dwarf") { - $SwiftArgs += @("-g", "-Xlinker", "/DEBUG:DWARF", "-use-ld=lld-link") + if ($Platform -eq "Windows") { + if ($SwiftDebugFormat -eq "dwarf") { + $SwiftArgs += @("-g", "-Xlinker", "/DEBUG:DWARF", "-use-ld=lld-link") + } else { + $SwiftArgs += @("-g", "-debug-info-format=codeview", "-Xlinker", "-debug") + } } else { - $SwiftArgs += @("-g", "-debug-info-format=codeview", "-Xlinker", "-debug") + $SwiftArgs += @("-g") } } else { $SwiftArgs += "-gnone" } - $SwiftArgs += @("-Xlinker", "/INCREMENTAL:NO") - # Swift Requries COMDAT folding and de-duplication - $SwiftArgs += @("-Xlinker", "/OPT:REF") - $SwiftArgs += @("-Xlinker", "/OPT:ICF") + if ($Platform -eq "Windows") { + $SwiftArgs += @("-Xlinker", "/INCREMENTAL:NO") + # Swift Requries COMDAT folding and de-duplication + $SwiftArgs += @("-Xlinker", "/OPT:REF") + $SwiftArgs += @("-Xlinker", "/OPT:ICF") + } + Append-FlagsDefine $Defines CMAKE_Swift_FLAGS $SwiftArgs # Workaround CMake 3.26+ enabling `-wmo` by default on release builds @@ -1160,7 +1301,7 @@ function Build-LLVM([Platform]$Platform, $Arch) { -Platform $Platform ` -UseMSVCCompilers C,CXX ` -Defines @{ - CMAKE_SYSTEM_NAME = if ($Platform -eq "Windows") { "Windows" } else { "Android" }; + CMAKE_SYSTEM_NAME = $Platform.ToString(); LLVM_HOST_TRIPLE = $Arch.LLVMTarget; } } @@ -1178,7 +1319,8 @@ function Build-ZLib([Platform]$Platform, $Arch) { -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; - CMAKE_SYSTEM_NAME = if ($Platform -eq "Windows") { "Windows" } else { "Android" }; + CMAKE_POSITION_INDEPENDENT_CODE = "YES"; + CMAKE_SYSTEM_NAME = $Platform.ToString(); INSTALL_BIN_DIR = "$LibraryRoot\zlib-1.3.1\usr\bin\$Platform\$ArchName"; INSTALL_LIB_DIR = "$LibraryRoot\zlib-1.3.1\usr\lib\$Platform\$ArchName"; } @@ -1199,7 +1341,7 @@ function Build-XML2([Platform]$Platform, $Arch) { BUILD_SHARED_LIBS = "NO"; CMAKE_INSTALL_BINDIR = "bin/$Platform/$ArchName"; CMAKE_INSTALL_LIBDIR = "lib/$Platform/$ArchName"; - CMAKE_SYSTEM_NAME = if ($Platform -eq "Windows") { "Windows" } else { "Android" }; + CMAKE_SYSTEM_NAME = $Platform.ToString(); LIBXML2_WITH_ICONV = "NO"; LIBXML2_WITH_ICU = "NO"; LIBXML2_WITH_LZMA = "NO"; @@ -1232,7 +1374,7 @@ function Build-CURL([Platform]$Platform, $Arch) { BUILD_SHARED_LIBS = "NO"; BUILD_TESTING = "NO"; CMAKE_INSTALL_LIBDIR = "lib/$Platform/$ArchName"; - CMAKE_SYSTEM_NAME = if ($Platform -eq "Windows") { "Windows" } else { "Android" }; + CMAKE_SYSTEM_NAME = $Platform.ToString(); BUILD_CURL_EXE = "NO"; CURL_CA_BUNDLE = "none"; CURL_CA_FALLBACK = "NO"; @@ -1328,6 +1470,7 @@ function Build-ICU([Platform]$Platform, $Arch) { } else { $BuildToolsDefines = @{ BUILD_TOOLS = "NO"; + BUILD_DATA = if ($Platform -eq "Android") { "NO" } else { "YES" }; ICU_TOOLS_DIR = "$($BuildArch.BinaryCache)\windows\icu-69.1" } } @@ -1342,13 +1485,23 @@ function Build-ICU([Platform]$Platform, $Arch) { -BuildTargets default ` -Defines ($BuildToolsDefines + @{ BUILD_SHARED_LIBS = "NO"; - CMAKE_SYSTEM_NAME = if ($Platform -eq "Windows") { "Windows" } else { "Android" }; + CMAKE_SYSTEM_NAME = $Platform.ToString(); + CMAKE_POSITION_INDEPENDENT_CODE = "YES"; CMAKE_INSTALL_BINDIR = "bin/$Platform/$ArchName"; CMAKE_INSTALL_LIBDIR = "lib/$Platform/$ArchName"; }) } function Build-Runtime([Platform]$Platform, $Arch) { + $PlatformDefines = @{} + if ($Platform -eq "Android") { + $PlatformDefines += @{ + LLVM_ENABLE_LIBCXX = "YES"; + SWIFT_USE_LINKER = "lld"; + } + } + + Isolate-EnvVars { $env:Path = "$($HostArch.BinaryCache)\cmark-gfm-0.29.0.gfm.13\src;$(Get-PinnedToolchainRuntime);${env:Path}" @@ -1364,13 +1517,13 @@ function Build-Runtime([Platform]$Platform, $Arch) { -InstallTo "$($Arch.SDKInstallRoot)\usr" ` -Arch $Arch ` -Platform $Platform ` - -CacheScript $SourceCache\swift\cmake\caches\Runtime-Windows-$($Arch.LLVMName).cmake ` + -CacheScript $SourceCache\swift\cmake\caches\Runtime-$Platform-$($Arch.LLVMName).cmake ` -UseBuiltCompilers C,CXX,Swift ` -BuildTargets default ` - -Defines @{ - CMAKE_Swift_COMPILER_TARGET = $Arch.LLVMTarget; + -Defines ($PlatformDefines + @{ + CMAKE_Swift_COMPILER_TARGET = $Arch.LLVMTarget.Replace("$AndroidAPILevel", ""); CMAKE_Swift_COMPILER_WORKS = "YES"; - CMAKE_SYSTEM_NAME = if ($Platform -eq "Windows") { "Windows" } else { "Android" }; + CMAKE_SYSTEM_NAME = $Platform.ToString(); LLVM_DIR = "$(Get-TargetProjectBinaryCache $Arch LLVM)\lib\cmake\llvm"; SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY = "YES"; SWIFT_ENABLE_EXPERIMENTAL_CXX_INTEROP = "YES"; @@ -1382,8 +1535,8 @@ function Build-Runtime([Platform]$Platform, $Arch) { SWIFT_NATIVE_SWIFT_TOOLS_PATH = (Join-Path -Path $CompilersBinaryCache -ChildPath "bin"); SWIFT_PATH_TO_LIBDISPATCH_SOURCE = "$SourceCache\swift-corelibs-libdispatch"; SWIFT_PATH_TO_STRING_PROCESSING_SOURCE = "$SourceCache\swift-experimental-string-processing"; - CMAKE_SHARED_LINKER_FLAGS = @("/INCREMENTAL:NO", "/OPT:REF", "/OPT:ICF"); - } + CMAKE_SHARED_LINKER_FLAGS = if ($Platform -eq "Windows") { @("/INCREMENTAL:NO", "/OPT:REF", "/OPT:ICF") } else { @() }; + }) } Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'DEFAULT_USE_RUNTIME': 'MD' } }), encoding='utf-8'))" ` @@ -1402,6 +1555,7 @@ function Build-Dispatch([Platform]$Platform, $Arch, [switch]$Test = $false) { -UseBuiltCompilers C,CXX,Swift ` -BuildTargets $Targets ` -Defines @{ + CMAKE_SYSTEM_NAME = $Platform.ToString(); ENABLE_SWIFT = "YES"; } } @@ -1435,19 +1589,42 @@ function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { -UseBuiltCompilers ASM,C,Swift ` -BuildTargets $Targets ` -Defines (@{ + BUILD_TOOLS = if ($Platform -eq "Windows") { "YES" } else { "NO" }; + CMAKE_SYSTEM_NAME = $Platform.ToString(); # Turn off safeseh for lld as it has safeseh enabled by default # and fails with an ICU data object file icudt69l_dat.obj. This # matters to X86 only. CMAKE_Swift_FLAGS = if ($Arch -eq $ArchX86) { @("-Xlinker", "/SAFESEH:NO") } else { "" }; CURL_DIR = "$LibraryRoot\curl-8.5.0\usr\lib\$Platform\$ShortArch\cmake\CURL"; - ICU_DATA_LIBRARY_RELEASE = "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\sicudt69.lib"; - ICU_I18N_LIBRARY_RELEASE = "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\sicuin69.lib"; + ICU_DATA_LIBRARY_RELEASE = if ($Platform -eq "Windows") { + "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\sicudt69.lib" + } else { + "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\libicudt69.a" + }; + ICU_I18N_LIBRARY_RELEASE = if ($Platform -eq "Windows") { + "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\sicuin69.lib" + } else { + "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\libicuin69.a" + }; ICU_ROOT = "$LibraryRoot\icu-69.1\usr"; - ICU_UC_LIBRARY_RELEASE = "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\sicuuc69.lib"; - LIBXML2_LIBRARY = "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\libxml2s.lib"; + ICU_INCLUDE_DIR = "$LibraryRoot\icu-69.1\usr\include"; + ICU_UC_LIBRARY_RELEASE = if ($Platform -eq "Windows") { + "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\sicuuc69.lib"; + } else { + "$LibraryRoot\icu-69.1\usr\lib\$Platform\$ShortArch\libicuuc69.a" + }; + LIBXML2_LIBRARY = if ($Platform -eq "Windows") { + "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\libxml2s.lib"; + } else { + "$LibraryRoot\libxml2-2.11.5\usr\lib\$Platform\$ShortArch\libxml2.a"; + }; LIBXML2_INCLUDE_DIR = "$LibraryRoot\libxml2-2.11.5\usr\include\libxml2"; - LIBXML2_DEFINITIONS = "/DLIBXML_STATIC"; - ZLIB_LIBRARY = "$LibraryRoot\zlib-1.3.1\usr\lib\$Platform\$ShortArch\zlibstatic.lib"; + LIBXML2_DEFINITIONS = "-DLIBXML_STATIC"; + ZLIB_LIBRARY = if ($Platform -eq "Windows") { + "$LibraryRoot\zlib-1.3.1\usr\lib\$Platform\$ShortArch\zlibstatic.lib" + } else { + "$LibraryRoot\zlib-1.3.1\usr\lib\$Platform\$ShortArch\libz.a" + }; ZLIB_INCLUDE_DIR = "$LibraryRoot\zlib-1.3.1\usr\include"; dispatch_DIR = "$DispatchBinaryCache\cmake\modules"; } + $TestingDefines) @@ -1484,11 +1661,13 @@ function Build-XCTest([Platform]$Platform, $Arch, [switch]$Test = $false) { -UseBuiltCompilers Swift ` -BuildTargets $Targets ` -Defines (@{ + CMAKE_BUILD_WITH_INSTALL_RPATH = "YES"; + CMAKE_SYSTEM_NAME = $Platform.ToString(); dispatch_DIR = "$DispatchBinaryCache\cmake\modules"; Foundation_DIR = "$FoundationBinaryCache\cmake\modules"; } + $TestingDefines) - $PList = [IO.Path]::Combine($Arch.BinaryCache, "${Platform}.platform".ToLower(), "Info.plist") + $PList = Join-Path -Path $Arch.PlatformInstallRoot -ChildPath "Info.plist" Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" ` -OutFile "$PList" } @@ -1515,28 +1694,35 @@ function Install-Platform([Platform]$Platform, $Arch) { Copy-File "$($Arch.SDKInstallRoot)\usr\share\*.*" $SDKInstallRoot\usr\share\ # Copy SDK libs, placing them in an arch-specific directory - $WindowsLibSrc = "$($Arch.SDKInstallRoot)\usr\lib\swift\windows" - $WindowsLibDst = "$SDKInstallRoot\usr\lib\swift\windows" + $PlatformLibSrc = "$($Arch.SDKInstallRoot)\usr\lib\swift\$($Platform.ToString().ToLowerInvariant())" + $PlatformLibDst = "$SDKInstallRoot\usr\lib\swift\$($Platform.ToString().ToLowerInvariant())" - Copy-File "$WindowsLibSrc\*.lib" "$WindowsLibDst\$($Arch.LLVMName)\" - Copy-File "$WindowsLibSrc\$($Arch.LLVMName)\*.lib" "$WindowsLibDst\$($Arch.LLVMName)\" + switch ($Platform) { + Windows { + Copy-File "$PlatformLibSrc\*.lib" "$PlatformLibDst\$($Arch.LLVMName)\" + Copy-File "$PlatformLibSrc\$($Arch.LLVMName)\*.lib" "$PlatformLibDst\$($Arch.LLVMName)\" + } + Android { + Copy-File "$PlatformLibSrc\*.so" "$PlatformLibDst\$($Arch.LLVMName)\" + } + } # Copy well-structured SDK modules - Copy-Directory "$WindowsLibSrc\*.swiftmodule" "$WindowsLibDst\" + Copy-Directory "$PlatformLibSrc\*.swiftmodule" "$PlatformLibDst\" # Copy files from the arch subdirectory, including "*.swiftmodule" which need restructuring - Get-ChildItem -Recurse "$WindowsLibSrc\$($Arch.LLVMName)" | ForEach-Object { + Get-ChildItem -Recurse "$PlatformLibSrc\$($Arch.LLVMName)" | ForEach-Object { if (".swiftmodule", ".swiftdoc", ".swiftinterface" -contains $_.Extension) { - $DstDir = "$WindowsLibDst\$($_.BaseName).swiftmodule" + $DstDir = "$PlatformLibDst\$($_.BaseName).swiftmodule" Copy-File $_.FullName "$DstDir\$($Arch.LLVMTarget)$($_.Extension)" } else { - Copy-File $_.FullName "$WindowsLibDst\$($Arch.LLVMName)\" + Copy-File $_.FullName "$PlatformLibDst\$($Arch.LLVMName)\" } } # Copy the CxxShim module foreach ($Source in ("libcxxshim.h", "libcxxshim.modulemap", "libcxxstdlibshim.h")) { - Copy-File "$WindowsLibSrc\$Source" "$WindowsLibDst" + Copy-File "$PlatformLibSrc\$Source" "$PlatformLibDst" } # Copy plist files (same across architectures) @@ -1551,11 +1737,11 @@ function Install-Platform([Platform]$Platform, $Arch) { Copy-File "$($Arch.XCTestInstallRoot)\usr\lib\swift\windows\XCTest.lib" "$XCTestInstallRoot\usr\lib\swift\windows\$($Arch.LLVMName)\" } default { - Copy-File "$($Arch.XCTestInstallRoot)\usr\lib\libXCTest.so" "$XCTestInstallRoot\usr\lib\$($Arch.BinaryDir)\" + Copy-File "$($Arch.XCTestInstallRoot)\usr\lib\swift\$($Platform.ToString().ToLowerInvariant())\libXCTest.so" "$XCTestInstallRoot\usr\lib\$($Arch.BinaryDir)\" } } - Copy-File "$($Arch.XCTestInstallRoot)\usr\lib\swift\$($Platform.ToString().ToLower())\$($Arch.LLVMName)\XCTest.swiftmodule" "$XCTestInstallRoot\usr\lib\swift\$($Platform.ToString().ToLower())\XCTest.swiftmodule\$($Arch.LLVMTarget).swiftmodule" - Copy-File "$($Arch.XCTestInstallRoot)\usr\lib\swift\$($Platform.ToString().ToLower())\$($Arch.LLVMName)\XCTest.swiftdoc" "$XCTestInstallRoot\usr\lib\swift\$($Platform.ToString().ToLower())\XCTest.swiftmodule\$($Arch.LLVMTarget).swiftdoc" + Copy-File "$($Arch.XCTestInstallRoot)\usr\lib\swift\$($Platform.ToString().ToLowerInvariant())\$($Arch.LLVMName)\XCTest.swiftmodule" "$XCTestInstallRoot\usr\lib\swift\$($Platform.ToString().ToLowerInvariant())\XCTest.swiftmodule\$($Arch.LLVMTarget).swiftmodule" + Copy-File "$($Arch.XCTestInstallRoot)\usr\lib\swift\$($Platform.ToString().ToLowerInvariant())\$($Arch.LLVMName)\XCTest.swiftdoc" "$XCTestInstallRoot\usr\lib\swift\$($Platform.ToString().ToLowerInvariant())\XCTest.swiftmodule\$($Arch.LLVMTarget).swiftdoc" } function Build-SQLite($Arch) { @@ -1611,8 +1797,9 @@ function Build-System($Arch) { -Bin (Get-HostProjectBinaryCache System) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "YES"; @@ -1630,8 +1817,9 @@ function Build-ToolsSupportCore($Arch) { -Bin (Get-HostProjectBinaryCache ToolsSupportCore) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "YES"; @@ -1666,9 +1854,10 @@ function Build-LLBuild($Arch, [switch]$Test = $false) { -Bin (Get-HostProjectBinaryCache LLBuild) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseMSVCCompilers CXX ` -UseBuiltCompilers Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets $Targets ` -Defines ($TestingDefines + @{ BUILD_SHARED_LIBS = "YES"; @@ -1684,8 +1873,9 @@ function Build-Yams($Arch) { -Src $SourceCache\Yams ` -Bin (Get-HostProjectBinaryCache Yams) ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; @@ -1699,8 +1889,9 @@ function Build-ArgumentParser($Arch) { -Bin (Get-HostProjectBinaryCache ArgumentParser) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "YES"; @@ -1714,8 +1905,9 @@ function Build-Driver($Arch) { -Bin (Get-HostProjectBinaryCache Driver) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,CXX,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "YES"; @@ -1739,8 +1931,9 @@ function Build-Crypto($Arch) { -Src $SourceCache\swift-crypto ` -Bin (Get-HostProjectBinaryCache Crypto) ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; @@ -1753,8 +1946,9 @@ function Build-Collections($Arch) { -Bin (Get-HostProjectBinaryCache Collections) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "YES"; @@ -1767,7 +1961,7 @@ function Build-ASN1($Arch) { -Bin (Get-HostProjectBinaryCache ASN1) ` -Arch $Arch ` -UseBuiltCompilers C,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; @@ -1779,8 +1973,9 @@ function Build-Certificates($Arch) { -Src $SourceCache\swift-certificates ` -Bin (Get-HostProjectBinaryCache Certificates) ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; @@ -1801,8 +1996,9 @@ function Build-PackageManager($Arch) { -Bin (Get-HostProjectBinaryCache PackageManager) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "YES"; @@ -1828,8 +2024,9 @@ function Build-Markdown($Arch) { -Bin (Get-HostProjectBinaryCache Markdown) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; @@ -1844,9 +2041,10 @@ function Build-Format($Arch) { -Bin (Get-HostProjectBinaryCache Format) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseMSVCCompilers C ` -UseBuiltCompilers Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "YES"; @@ -1858,12 +2056,13 @@ function Build-Format($Arch) { } function Build-IndexStoreDB($Arch) { - $SDKInstallRoot = ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) + $SDKInstallRoot = (Get-HostSwiftSDK); Build-CMakeProject ` -Src $SourceCache\indexstore-db ` -Bin (Get-HostProjectBinaryCache IndexStoreDB) ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,CXX,Swift ` -SwiftSDK $SDKInstallRoot ` -BuildTargets default ` @@ -1880,8 +2079,9 @@ function Build-SourceKitLSP($Arch) { -Bin (Get-HostProjectBinaryCache SourceKitLSP) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` + -Platform Windows ` -UseBuiltCompilers C,Swift ` - -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -SwiftSDK (Get-HostSwiftSDK) ` -BuildTargets default ` -Defines @{ SwiftSyntax_DIR = (Get-HostProjectCMakeModules Compilers); @@ -1918,7 +2118,7 @@ function Install-HostToolchain() { function Build-Inspect() { $OutDir = Join-Path -Path $HostArch.BinaryCache -ChildPath swift-inspect - $SDKInstallRoot = ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + $SDKInstallRoot = (Get-HostSwiftSDK) ` Isolate-EnvVars { $env:SWIFTCI_USE_LOCAL_DEPS=1 @@ -2056,6 +2256,20 @@ if (-not $SkipBuild) { Invoke-BuildStep Build-Foundation Windows $Arch Invoke-BuildStep Build-XCTest Windows $Arch } + + foreach ($Arch in $AndroidSDKArchs) { + Invoke-BuildStep Build-ZLib Android $Arch + Invoke-BuildStep Build-XML2 Android $Arch + Invoke-BuildStep Build-CURL Android $Arch + Invoke-BuildStep Build-ICU Android $Arch + Invoke-BuildStep Build-LLVM Android $Arch + + # Build platform: SDK, Redist and XCTest + Invoke-BuildStep Build-Runtime Android $Arch + Invoke-BuildStep Build-Dispatch Android $Arch + Invoke-BuildStep Build-Foundation Android $Arch + Invoke-BuildStep Build-XCTest Android $Arch + } } if (-not $ToBatch) { @@ -2070,6 +2284,10 @@ if (-not $ToBatch) { foreach ($Arch in $WindowsSDKArchs) { Install-Platform Windows $Arch } + + foreach ($Arch in $AndroidSDKArchs) { + Install-Platform Android $Arch + } } if (-not $SkipBuild) { From 63172291b7dc54c6252253250bde43af48fa80f7 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Fri, 19 Apr 2024 11:01:53 -0700 Subject: [PATCH 02/12] [build utils] do not extract the android NDK when it's up to date --- utils/build.ps1 | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index d036ccdb1a35a..c2dbe9ae3aab3 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -584,6 +584,32 @@ function Fetch-Dependencies { } } + function Extract-ZipFile { + param ( + [string]$ZipFileName, + [string]$BinaryCache, + [string]$ExtractPath + ) + + $zipFilePath = Join-Path -Path $BinaryCache -ChildPath $ZipFileName + $extractedPath = Join-Path -Path $BinaryCache -ChildPath $ExtractPath + + # Check if the extracted directory already exists and is up to date. + if (Test-Path $extractedPath) { + $zipLastWriteTime = (Get-Item $zipFilePath).LastWriteTime + $extractedLastWriteTime = (Get-Item $extractedPath).LastWriteTime + # Compare the last write times + if ($zipLastWriteTime -le $extractedLastWriteTime) { + Write-Output "'$ZipFileName' is already extracted and up to date." + return + } + } + + Write-Output "Extracting '$ZipFileName' ..." + New-Item -ItemType Directory -ErrorAction Ignore -Path $BinaryCache | Out-Null + Expand-Archive -Path $zipFilePath -DestinationPath $BinaryCache -Force + } + $WiXVersion = "4.0.4" $WiXURL = "https://www.nuget.org/api/v2/package/wix/$WiXVersion" $WiXHash = "A9CA12214E61BB49430A8C6E5E48AC5AE6F27DC82573B5306955C4D35F2D34E2" @@ -635,10 +661,7 @@ function Fetch-Dependencies { $NDKHash = "A478D43D4A45D0D345CDA6BE50D79642B92FB175868D9DC0DFC86181D80F691E" DownloadAndVerify $NDKURL "$BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip" $NDKHash - # TODO(compnerd) stamp/validate that we need to re-extract - Write-Output "Extracting Android NDK $AndroidNDKVersion ..." - New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache | Out-Null - Expand-Archive -Path $BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip -Destination $BinaryCache -Force + Extract-ZipFile -ZipFileName "android-ndk-$AndroidNDKVersion-windows.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-$AndroidNDKVersion" } if ($WinSDKVersion) { From 81d1fdde3ff39d2915cfdba4736b92a3268866bc Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Thu, 25 Apr 2024 14:53:00 -0700 Subject: [PATCH 03/12] [android][build] clean up the android path handling --- utils/build.ps1 | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index c2dbe9ae3aab3..8a4208e1a8923 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -300,6 +300,11 @@ $IsCrossCompiling = $HostArchName -ne $BuildArchName $TimingData = New-Object System.Collections.Generic.List[System.Object] +function Get-AndroidNDKPath { + $androidNDKPath = Join-Path -Path $BinaryCache -ChildPath "android-ndk-$AndroidNDKVersion" + return $androidNDKPath +} + function Get-InstallDir($Arch) { if ($Arch -eq $HostArch) { $ProgramFilesName = "Program Files" @@ -808,15 +813,23 @@ function Build-CMakeProject { TryAdd-KeyValue $Defines CMAKE_SYSTEM_PROCESSOR $Arch.CMakeName } - # TODO(compnerd) clean up these and formalise this selection if ($Platform -eq "Android") { - $env:Path = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja;${env:Path}" - TryAdd-KeyValue $Defines CMAKE_C_COMPILER "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\bin\clang.exe" - TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\bin\clang++.exe" - TryAdd-KeyValue $Defines CMAKE_MAKE_PROGRAM "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" + if (Test-Path "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin") { + $env:Path = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja;${env:Path}" + TryAdd-KeyValue $Defines CMAKE_MAKE_PROGRAM "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" + } elseif (Test-Path "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin") { + $env:Path = "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin;C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja;${env:Path}" + TryAdd-KeyValue $Defines CMAKE_MAKE_PROGRAM "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" + } else { + throw "Missing CMake and Ninja in the visual studio installation that are needed to build Android" + } + $androidNDKPath = Get-AndroidNDKPath + TryAdd-KeyValue $Defines CMAKE_C_COMPILER (Join-Path -Path $androidNDKPath -ChildPath "toolchains\llvm\prebuilt\windows-x86_64\bin\clang.exe") + TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER (Join-Path -Path $androidNDKPath -ChildPath "toolchains\llvm\prebuilt\windows-x86_64\bin\clang++.exe") TryAdd-KeyValue $Defines CMAKE_ANDROID_API "$AndroidAPILevel" TryAdd-KeyValue $Defines CMAKE_ANDROID_ARCH_ABI $Arch.AndroidArchABI - TryAdd-KeyValue $Defines CMAKE_ANDROID_NDK "S:\b\android-ndk-r26b" + TryAdd-KeyValue $Defines CMAKE_ANDROID_NDK "$androidNDKPath" + TryAdd-KeyValue $Defines SWIFT_ANDROID_NDK_PATH "$androidNDKPath" TryAdd-KeyValue $Defines CMAKE_C_COMPILER_WORKS YES TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER_WORKS YES } @@ -830,7 +843,8 @@ function Build-CMakeProject { $CFlags = @("/GS-", "/Gw", "/Gy", "/Oi", "/Oy", "/Zc:inline") } Android { - $CFlags = @("--sysroot=S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\sysroot") + $androidNDKPath = Get-AndroidNDKPath + $CFlags = @("--sysroot=$androidNDKPath\toolchains\llvm\prebuilt\windows-x86_64\sysroot") } } @@ -945,14 +959,15 @@ function Build-CMakeProject { ) } Android { - $SwiftArgs += @("-sdk", "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\sysroot") + $androidNDKPath = Get-AndroidNDKPath + $SwiftArgs += @("-sdk", "$androidNDKPath\toolchains\llvm\prebuilt\windows-x86_64\sysroot") $SwiftArgs += @( "-Xclang-linker", "-target", "-Xclang-linker", $Arch.LLVMTarget, "-Xclang-linker", "--sysroot", - "-Xclang-linker", "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\sysroot", + "-Xclang-linker", "$androidNDKPath\toolchains\llvm\prebuilt\windows-x86_64\sysroot", "-Xclang-linker", "-resource-dir", - "-Xclang-linker", "S:\b\android-ndk-r26b\toolchains\llvm\prebuilt\windows-x86_64\lib\clang\17" + "-Xclang-linker", "$androidNDKPath\toolchains\llvm\prebuilt\windows-x86_64\lib\clang\17" ) } } From 7a5a0aad22f1cb13b820681e48a50fbc34caca98 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Mon, 29 Apr 2024 15:53:58 -0700 Subject: [PATCH 04/12] [android] misc stdlib build fixes --- stdlib/cmake/modules/AddSwiftStdlib.cmake | 22 +++++++++++++------ .../Concurrency/DispatchGlobalExecutor.inc | 4 ++++ .../swift-reflection-test/CMakeLists.txt | 1 + 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/stdlib/cmake/modules/AddSwiftStdlib.cmake b/stdlib/cmake/modules/AddSwiftStdlib.cmake index d95ce397e7726..aa17bf5e42783 100644 --- a/stdlib/cmake/modules/AddSwiftStdlib.cmake +++ b/stdlib/cmake/modules/AddSwiftStdlib.cmake @@ -541,13 +541,16 @@ function(_add_target_variant_link_flags) list(APPEND link_libraries "dl" "log") # We need to add the math library, which is linked implicitly by libc++ list(APPEND result "-lm") - if(NOT "${SWIFT_ANDROID_NDK_PATH}" STREQUAL "") - if("${SWIFT_ANDROID_NDK_PATH}" MATCHES "r26") - file(GLOB RESOURCE_DIR ${SWIFT_SDK_ANDROID_ARCH_${LFLAGS_ARCH}_PATH}/../lib/clang/*) - else() - file(GLOB RESOURCE_DIR ${SWIFT_SDK_ANDROID_ARCH_${LFLAGS_ARCH}_PATH}/../lib64/clang/*) + if(NOT CMAKE_HOST_SYSTEM MATCHES Windows) + # The Android resource dir is specified from build.ps1 on windows. + if(NOT "${SWIFT_ANDROID_NDK_PATH}" STREQUAL "") + if("${SWIFT_ANDROID_NDK_PATH}" MATCHES "r26") + file(GLOB RESOURCE_DIR ${SWIFT_SDK_ANDROID_ARCH_${LFLAGS_ARCH}_PATH}/../lib/clang/*) + else() + file(GLOB RESOURCE_DIR ${SWIFT_SDK_ANDROID_ARCH_${LFLAGS_ARCH}_PATH}/../lib64/clang/*) + endif() + list(APPEND result "-resource-dir=${RESOURCE_DIR}") endif() - list(APPEND result "-resource-dir=${RESOURCE_DIR}") endif() # link against the custom C++ library @@ -3062,9 +3065,14 @@ function(add_swift_target_executable name) # it tries to build swift-backtrace it fails because *the compiler* # refers to a libswiftCore.so that can't be found. + if(SWIFTEXE_TARGET_NOSWIFTRT) + set(NOSWIFTRT_KEYWORD "NOSWIFTRT") + else() + set(NOSWIFTRT_KEYWORD "") + endif() _add_swift_target_executable_single( ${VARIANT_NAME} - ${SWIFTEXE_TARGET_NOSWIFTRT_keyword} + ${NOSWIFTRT_KEYWORD} ${SWIFTEXE_TARGET_SOURCES} DEPENDS ${SWIFTEXE_TARGET_DEPENDS_with_suffix} diff --git a/stdlib/public/Concurrency/DispatchGlobalExecutor.inc b/stdlib/public/Concurrency/DispatchGlobalExecutor.inc index 8010d123dd57e..a2d0bb8a12abe 100644 --- a/stdlib/public/Concurrency/DispatchGlobalExecutor.inc +++ b/stdlib/public/Concurrency/DispatchGlobalExecutor.inc @@ -42,6 +42,9 @@ struct MinimalDispatchObjectHeader { int Opaque1; void *Linkage; }; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wgnu-offsetof-extensions" static_assert( offsetof(Job, metadata) == offsetof(MinimalDispatchObjectHeader, VTable), "Job Metadata field must match location of Dispatch VTable field."); @@ -49,6 +52,7 @@ static_assert(offsetof(Job, SchedulerPrivate[Job::DispatchLinkageIndex]) == offsetof(MinimalDispatchObjectHeader, Linkage), "Dispatch Linkage field must match Job " "SchedulerPrivate[DispatchLinkageIndex]."); +#pragma clang diagnostic pop /// The function passed to dispatch_async_f to execute a job. static void __swift_run_job(void *_job) { diff --git a/stdlib/tools/swift-reflection-test/CMakeLists.txt b/stdlib/tools/swift-reflection-test/CMakeLists.txt index d66bf30273746..2dd113998e4fd 100644 --- a/stdlib/tools/swift-reflection-test/CMakeLists.txt +++ b/stdlib/tools/swift-reflection-test/CMakeLists.txt @@ -1,4 +1,5 @@ add_swift_target_executable(swift-reflection-test BUILD_WITH_STDLIB + NOSWIFTRT swift-reflection-test.c overrides.c LINK_LIBRARIES From 7815f843a34888d7492fa58c2b14d4ee296ae58c Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Mon, 29 Apr 2024 15:53:00 -0700 Subject: [PATCH 05/12] [android] do not build the builtin float overlay for android (NDK's clang module map doesn't have it) --- stdlib/public/ClangOverlays/CMakeLists.txt | 28 ++++++++++++---------- utils/build.ps1 | 3 +++ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/stdlib/public/ClangOverlays/CMakeLists.txt b/stdlib/public/ClangOverlays/CMakeLists.txt index cb9316b6ace64..f53382567e45e 100644 --- a/stdlib/public/ClangOverlays/CMakeLists.txt +++ b/stdlib/public/ClangOverlays/CMakeLists.txt @@ -1,17 +1,19 @@ -add_swift_target_library(swift_Builtin_float - ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} - IS_SDK_OVERLAY +if(NOT DEFINED SWIFT_BUILD_CLANG_OVERLAYS_SKIP_BUILTIN_FLOAT OR NOT SWIFT_BUILD_CLANG_OVERLAYS_SKIP_BUILTIN_FLOAT) + add_swift_target_library(swift_Builtin_float + ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} + IS_SDK_OVERLAY - GYB_SOURCES - float.swift.gyb + GYB_SOURCES + float.swift.gyb - SWIFT_COMPILE_FLAGS - ${SWIFT_RUNTIME_SWIFT_COMPILE_FLAGS} - ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} - -Xfrontend -previous-module-installname-map-file -Xfrontend "${SWIFT_SOURCE_DIR}/stdlib/linker-support/previous-module-installname.json" + SWIFT_COMPILE_FLAGS + ${SWIFT_RUNTIME_SWIFT_COMPILE_FLAGS} + ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS} + -Xfrontend -previous-module-installname-map-file -Xfrontend "${SWIFT_SOURCE_DIR}/stdlib/linker-support/previous-module-installname.json" - LINK_FLAGS - ${SWIFT_RUNTIME_SWIFT_LINK_FLAGS} + LINK_FLAGS + ${SWIFT_RUNTIME_SWIFT_LINK_FLAGS} - INSTALL_IN_COMPONENT stdlib - MACCATALYST_BUILD_FLAVOR zippered) + INSTALL_IN_COMPONENT stdlib + MACCATALYST_BUILD_FLAVOR zippered) +endif() diff --git a/utils/build.ps1 b/utils/build.ps1 index 8a4208e1a8923..6f240924ecbb3 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -832,6 +832,9 @@ function Build-CMakeProject { TryAdd-KeyValue $Defines SWIFT_ANDROID_NDK_PATH "$androidNDKPath" TryAdd-KeyValue $Defines CMAKE_C_COMPILER_WORKS YES TryAdd-KeyValue $Defines CMAKE_CXX_COMPILER_WORKS YES + # The current Android NDK ships with Clang 17, + # which doesn't provide the _Builtin_float module. + TryAdd-KeyValue $Defines SWIFT_BUILD_CLANG_OVERLAYS_SKIP_BUILTIN_FLOAT YES } TryAdd-KeyValue $Defines CMAKE_BUILD_TYPE Release From 5905dc9ef3b5e4b0df4669c983a006cace483f04 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Tue, 14 May 2024 12:14:08 -0700 Subject: [PATCH 06/12] more cross-arch build.ps1 android fixes --- cmake/caches/Runtime-Android-i686.cmake | 24 +++++++++++++++++++++++ cmake/caches/Runtime-Android-x86_64.cmake | 24 +++++++++++++++++++++++ utils/build.ps1 | 8 ++++---- 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 cmake/caches/Runtime-Android-i686.cmake create mode 100644 cmake/caches/Runtime-Android-x86_64.cmake diff --git a/cmake/caches/Runtime-Android-i686.cmake b/cmake/caches/Runtime-Android-i686.cmake new file mode 100644 index 0000000000000..66d3063709924 --- /dev/null +++ b/cmake/caches/Runtime-Android-i686.cmake @@ -0,0 +1,24 @@ + +set(SWIFT_HOST_VARIANT_SDK ANDROID CACHE STRING "") +set(SWIFT_HOST_VARIANT_ARCH i686 CACHE STRING "") + +# NOTE(compnerd) disable the tools, we are trying to build just the standard +# library. +set(SWIFT_INCLUDE_TOOLS NO CACHE BOOL "") + +# NOTE(compnerd) cannot build tests since the tests require the toolchain +set(SWIFT_INCLUDE_TESTS NO CACHE BOOL "") + +# NOTE(compnerd) cannot build docs since that requires perl +set(SWIFT_INCLUDE_DOCS NO CACHE BOOL "") + +# NOTE(compnerd) these are part of the toolchain, not the runtime. +set(SWIFT_BUILD_SOURCEKIT NO CACHE BOOL "") + +# NOTE(compnerd) build with the compiler specified, not a just built compiler. +set(SWIFT_BUILD_RUNTIME_WITH_HOST_COMPILER YES CACHE BOOL "") + +set(SWIFT_SDK_ANDROID_ARCHITECTURES i686 CACHE STRING "") + +# NOTE(compnerd) this is lollipop, which seems to still have decent usage. +set(SWIFT_ANDROID_API_LEVEL 21 CACHE STRING "") diff --git a/cmake/caches/Runtime-Android-x86_64.cmake b/cmake/caches/Runtime-Android-x86_64.cmake new file mode 100644 index 0000000000000..8780313aaa3ca --- /dev/null +++ b/cmake/caches/Runtime-Android-x86_64.cmake @@ -0,0 +1,24 @@ + +set(SWIFT_HOST_VARIANT_SDK ANDROID CACHE STRING "") +set(SWIFT_HOST_VARIANT_ARCH x86_64 CACHE STRING "") + +# NOTE(compnerd) disable the tools, we are trying to build just the standard +# library. +set(SWIFT_INCLUDE_TOOLS NO CACHE BOOL "") + +# NOTE(compnerd) cannot build tests since the tests require the toolchain +set(SWIFT_INCLUDE_TESTS NO CACHE BOOL "") + +# NOTE(compnerd) cannot build docs since that requires perl +set(SWIFT_INCLUDE_DOCS NO CACHE BOOL "") + +# NOTE(compnerd) these are part of the toolchain, not the runtime. +set(SWIFT_BUILD_SOURCEKIT NO CACHE BOOL "") + +# NOTE(compnerd) build with the compiler specified, not a just built compiler. +set(SWIFT_BUILD_RUNTIME_WITH_HOST_COMPILER YES CACHE BOOL "") + +set(SWIFT_SDK_ANDROID_ARCHITECTURES x86_64 CACHE STRING "") + +# NOTE(compnerd) this is lollipop, which seems to still have decent usage. +set(SWIFT_ANDROID_API_LEVEL 21 CACHE STRING "") diff --git a/utils/build.ps1 b/utils/build.ps1 index 6f240924ecbb3..1de48298aab98 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -250,7 +250,7 @@ $AndroidARMv7 = @{ BinaryDir = "bina"; CMakeName = "armv7-a"; LLVMName = "armv7"; - LLVMTarget = "armv7a-unknown-linux-androideabi"; + LLVMTarget = "armv7-unknown-linux-androideabi$AndroidAPILevel"; BuildID = 500; BinaryCache = "$BinaryCache\armv7"; PlatformInstallRoot = "$BinaryCache\armv7\Android.platform"; @@ -263,7 +263,7 @@ $AndroidX86 = @{ BinaryDir = "bin"; CMakeName = "i686"; LLVMName = "i686"; - LLVMTarget = "i686-unknown-linux-android"; + LLVMTarget = "i686-unknown-linux-android$AndroidAPILevel"; BuildID = 600; BinaryCache = "$BinaryCache\i686"; PlatformInstallRoot = "$BinaryCache\x86\Android.platform"; @@ -276,7 +276,7 @@ $AndroidX64 = @{ BinaryDir = "bin64"; CMakeName = "x86_64"; LLVMName = "x86_64"; - LLVMTarget = "x86_64-unknown-linux-android"; + LLVMTarget = "x86_64-unknown-linux-android$AndroidAPILevel"; BuildID = 700; BinaryCache = "$BinaryCache\x86_64"; PlatformInstallRoot = "$BinaryCache\x64\Android.platform"; @@ -1630,7 +1630,7 @@ function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { -UseBuiltCompilers ASM,C,Swift ` -BuildTargets $Targets ` -Defines (@{ - BUILD_TOOLS = if ($Platform -eq "Windows") { "YES" } else { "NO" }; + FOUNDATION_BUILD_TOOLS = if ($Platform -eq "Windows") { "YES" } else { "NO" }; CMAKE_SYSTEM_NAME = $Platform.ToString(); # Turn off safeseh for lld as it has safeseh enabled by default # and fails with an ICU data object file icudt69l_dat.obj. This From 9db8d925230d5f17620a2178403f5b4b712f9823 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Tue, 14 May 2024 12:37:47 -0700 Subject: [PATCH 07/12] review fixes --- utils/build.ps1 | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 1de48298aab98..e66cc26a491ed 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -596,13 +596,13 @@ function Fetch-Dependencies { [string]$ExtractPath ) - $zipFilePath = Join-Path -Path $BinaryCache -ChildPath $ZipFileName - $extractedPath = Join-Path -Path $BinaryCache -ChildPath $ExtractPath + $source = Join-Path -Path $BinaryCache -ChildPath $ZipFileName + $destination = Join-Path -Path $BinaryCache -ChildPath $ExtractPath # Check if the extracted directory already exists and is up to date. - if (Test-Path $extractedPath) { - $zipLastWriteTime = (Get-Item $zipFilePath).LastWriteTime - $extractedLastWriteTime = (Get-Item $extractedPath).LastWriteTime + if (Test-Path $destination) { + $zipLastWriteTime = (Get-Item $source).LastWriteTime + $extractedLastWriteTime = (Get-Item $destination).LastWriteTime # Compare the last write times if ($zipLastWriteTime -le $extractedLastWriteTime) { Write-Output "'$ZipFileName' is already extracted and up to date." @@ -612,7 +612,7 @@ function Fetch-Dependencies { Write-Output "Extracting '$ZipFileName' ..." New-Item -ItemType Directory -ErrorAction Ignore -Path $BinaryCache | Out-Null - Expand-Archive -Path $zipFilePath -DestinationPath $BinaryCache -Force + Expand-Archive -Path $source -DestinationPath $BinaryCache -Force } $WiXVersion = "4.0.4" @@ -814,12 +814,11 @@ function Build-CMakeProject { } if ($Platform -eq "Android") { - if (Test-Path "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin") { - $env:Path = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja;${env:Path}" - TryAdd-KeyValue $Defines CMAKE_MAKE_PROGRAM "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" - } elseif (Test-Path "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin") { - $env:Path = "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin;C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja;${env:Path}" - TryAdd-KeyValue $Defines CMAKE_MAKE_PROGRAM "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" + $vsWherePath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $vsInstallPath = & $vsWherePath -latest -property installationPath + if (Test-Path "${vsInstallPath}\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin") { + $env:Path = "${vsInstallPath}\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin;${vsInstallPath}\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja;${env:Path}" + TryAdd-KeyValue $Defines CMAKE_MAKE_PROGRAM "${vsInstallPath}\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja\ninja.exe" } else { throw "Missing CMake and Ninja in the visual studio installation that are needed to build Android" } @@ -846,8 +845,7 @@ function Build-CMakeProject { $CFlags = @("/GS-", "/Gw", "/Gy", "/Oi", "/Oy", "/Zc:inline") } Android { - $androidNDKPath = Get-AndroidNDKPath - $CFlags = @("--sysroot=$androidNDKPath\toolchains\llvm\prebuilt\windows-x86_64\sysroot") + $CFlags = @("--sysroot=$(Get-AndroidNDKPath)\toolchains\llvm\prebuilt\windows-x86_64\sysroot") } } From c032c9b0341ce3ae2f569b2fbcb40e0c1f31e8b1 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Thu, 6 Jun 2024 09:32:40 -0700 Subject: [PATCH 08/12] [android] add additional checks for build.ps1 android build --- utils/build.ps1 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utils/build.ps1 b/utils/build.ps1 index e66cc26a491ed..e39e29bc3bea6 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -344,6 +344,11 @@ $AndroidSDKArchs = @($AndroidSDKs | ForEach-Object { default { throw "Unknown architecture $_" } } }) +if ($AndroidSDKArchs.count -gt 0) { + if ($HostArch -ne $ArchX64) { + throw "Unsupported host architecture for building android SDKs" + } +} $WindowsSDKArchs = @($WindowsSDKs | ForEach-Object { switch ($_) { "X64" { $ArchX64 } @@ -662,6 +667,10 @@ function Fetch-Dependencies { } if ($AndroidSDKArchs.count -gt 0) { + # Only a specific NDK version is supported right now. + if ($AndroidNDKVersion -ne "r26b") { + throw "Unsupported Android NDK version" + } $NDKURL = "https://dl.google.com/android/repository/android-ndk-r26b-windows.zip" $NDKHash = "A478D43D4A45D0D345CDA6BE50D79642B92FB175868D9DC0DFC86181D80F691E" DownloadAndVerify $NDKURL "$BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip" $NDKHash From 4cfd5f10b9f3dd156063d0b7e33717ba02279b1b Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Thu, 6 Jun 2024 10:32:35 -0700 Subject: [PATCH 09/12] [android] only build android SDKs when -Android is passed, or -AndroidSDKs are specified when running build.ps1 --- utils/build.ps1 | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index e39e29bc3bea6..404af19ad3c3f 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -35,6 +35,9 @@ The debug information format for Swift code: dwarf or codeview. .PARAMETER AndroidAPILevel The API Level to target when building the Android SDKs +.PARAMETER Android +When set, build android SDKs. + .PARAMETER AndroidSDKs An array of architectures for which the Android Swift SDK should be built. @@ -106,7 +109,7 @@ param( [string] $CDebugFormat = "dwarf", [string] $SwiftDebugFormat = "dwarf", [string] $AndroidAPILevel = 28, - [string[]] $AndroidSDKs = @("aarch64","armv7","i686","x86_64"), + [string[]] $AndroidSDKs = @(), [string[]] $WindowsSDKs = @("X64","X86","Arm64"), [string] $ProductVersion = "0.0.0", [string] $PinnedBuild = "", @@ -114,6 +117,7 @@ param( [string] $PythonVersion = "3.9.10", [string] $AndroidNDKVersion = "r26b", [string] $WinSDKVersion = "", + [switch] $Android = $false, [switch] $SkipBuild = $false, [switch] $SkipRedistInstall = $false, [switch] $SkipPackaging = $false, @@ -177,11 +181,20 @@ if (-not (Test-Path $python)) { } } +if ($Android -and ($AndroidSDKs.Length -eq 0)) { + # Enable all android SDKs by default. + $AndroidSDKs = @("aarch64","armv7","i686","x86_64") +} # Work around limitations of cmd passing in array arguments via powershell.exe -File if ($AndroidSDKs.Length -eq 1) { $AndroidSDKs = $AndroidSDKs[0].Split(",") } if ($WindowsSDKs.Length -eq 1) { $WindowsSDKs = $WindowsSDKs[0].Split(",") } if ($Test.Length -eq 1) { $Test = $Test[0].Split(",") } +if ($AndroidSDKs.Length -gt 0) { + # Always enable android when one of the SDKs is specified. + $Android = $true +} + if ($Test -contains "*") { # Explicitly don't include llbuild yet since tests are known to fail on Windows $Test = @("swift", "dispatch", "foundation", "xctest") @@ -344,7 +357,7 @@ $AndroidSDKArchs = @($AndroidSDKs | ForEach-Object { default { throw "Unknown architecture $_" } } }) -if ($AndroidSDKArchs.count -gt 0) { +if ($Android) { if ($HostArch -ne $ArchX64) { throw "Unsupported host architecture for building android SDKs" } @@ -666,7 +679,7 @@ function Fetch-Dependencies { Download-Python $BuildArchName } - if ($AndroidSDKArchs.count -gt 0) { + if ($Android) { # Only a specific NDK version is supported right now. if ($AndroidNDKVersion -ne "r26b") { throw "Unsupported Android NDK version" From eac1fe4cf0bcbb4b6528ea615f0ecd06fb27c5ad Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Thu, 6 Jun 2024 11:12:21 -0700 Subject: [PATCH 10/12] Drop unecessary CMAKE_SYSTEM_NAME for dispatch, foundation and xctest build invocations --- utils/build.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 404af19ad3c3f..d76bed660c4b3 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -1616,7 +1616,6 @@ function Build-Dispatch([Platform]$Platform, $Arch, [switch]$Test = $false) { -UseBuiltCompilers C,CXX,Swift ` -BuildTargets $Targets ` -Defines @{ - CMAKE_SYSTEM_NAME = $Platform.ToString(); ENABLE_SWIFT = "YES"; } } @@ -1651,7 +1650,6 @@ function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { -BuildTargets $Targets ` -Defines (@{ FOUNDATION_BUILD_TOOLS = if ($Platform -eq "Windows") { "YES" } else { "NO" }; - CMAKE_SYSTEM_NAME = $Platform.ToString(); # Turn off safeseh for lld as it has safeseh enabled by default # and fails with an ICU data object file icudt69l_dat.obj. This # matters to X86 only. @@ -1723,7 +1721,6 @@ function Build-XCTest([Platform]$Platform, $Arch, [switch]$Test = $false) { -BuildTargets $Targets ` -Defines (@{ CMAKE_BUILD_WITH_INSTALL_RPATH = "YES"; - CMAKE_SYSTEM_NAME = $Platform.ToString(); dispatch_DIR = "$DispatchBinaryCache\cmake\modules"; Foundation_DIR = "$FoundationBinaryCache\cmake\modules"; } + $TestingDefines) From 30dbf3a4aa5ef29f4729d50fb5cfd0b5fb7d0602 Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Thu, 9 May 2024 07:43:00 -0700 Subject: [PATCH 11/12] do not add ObjectIdentifier+DebugDescription.swift for android builds --- stdlib/public/core/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/public/core/CMakeLists.txt b/stdlib/public/core/CMakeLists.txt index 4d7646ffd428f..1f62b9ad93321 100644 --- a/stdlib/public/core/CMakeLists.txt +++ b/stdlib/public/core/CMakeLists.txt @@ -262,8 +262,8 @@ if(SWIFT_STDLIB_ENABLE_VECTOR_TYPES) list(APPEND SWIFTLIB_EMBEDDED_GYB_SOURCES SIMDConcreteOperations.swift.gyb SIMDVectorTypes.swift.gyb) endif() -# Freestanding and Linux builds both have failures to resolve. -if(NOT BOOTSTRAPPING_MODE STREQUAL "OFF" AND NOT SWIFT_FREESTANDING_FLAVOR AND NOT SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX") +# Freestanding and Linux/Android builds both have failures to resolve. +if(NOT BOOTSTRAPPING_MODE STREQUAL "OFF" AND NOT SWIFT_FREESTANDING_FLAVOR AND NOT SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX" AND NOT SWIFT_HOST_VARIANT_SDK STREQUAL "ANDROID") list(APPEND SWIFTLIB_SOURCES ObjectIdentifier+DebugDescription.swift) endif() From 6381a77eb46baa8c0eca31162f2ba6b48cd5a46d Mon Sep 17 00:00:00 2001 From: Alex Lorenz Date: Thu, 6 Jun 2024 13:33:59 -0700 Subject: [PATCH 12/12] [android] disable Synchronization for android armv7 builds --- utils/build.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index d76bed660c4b3..a264020e4f525 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -1592,7 +1592,8 @@ function Build-Runtime([Platform]$Platform, $Arch) { SWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED = "YES"; SWIFT_ENABLE_EXPERIMENTAL_OBSERVATION = "YES"; SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING = "YES"; - SWIFT_ENABLE_SYNCHRONIZATION = "YES"; + # FIXME: re-enable after https://github.com/apple/swift/issues/74186 is fixed. + SWIFT_ENABLE_SYNCHRONIZATION = if (($Platform -eq "Android") -and ($Arch -eq $AndroidARMv7)) { "NO" } else { "YES" }; SWIFT_NATIVE_SWIFT_TOOLS_PATH = (Join-Path -Path $CompilersBinaryCache -ChildPath "bin"); SWIFT_PATH_TO_LIBDISPATCH_SOURCE = "$SourceCache\swift-corelibs-libdispatch"; SWIFT_PATH_TO_STRING_PROCESSING_SOURCE = "$SourceCache\swift-experimental-string-processing";