From 8c71c5b4d17bfbb63e1d81420b96c14c850a9dc7 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Fri, 12 Oct 2018 16:16:25 -0700 Subject: [PATCH 1/6] Add option to build ARM64 on Linux --- build.psm1 | 32 +++++++++++++++++++------ src/libpsl-native/arm64.toolchain.cmake | 17 +++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 src/libpsl-native/arm64.toolchain.cmake diff --git a/build.psm1 b/build.psm1 index 3f63aff..e9e9aa4 100644 --- a/build.psm1 +++ b/build.psm1 @@ -391,7 +391,8 @@ cmd.exe /C cd /d "$location" "&" "$vcvarsallbatPath" "$Arch" "&" "$cmakePath" "$ function Start-BuildNativeUnixBinaries { param ( - [switch] $BuildLinuxArm + [switch] $BuildLinuxArm, + [switch] $BuildLinuxArm64 ) if (-not $Environment.IsLinux -and -not $Environment.IsMacOS) { @@ -399,8 +400,8 @@ function Start-BuildNativeUnixBinaries { return } - if ($BuildLinuxArm -and -not $Environment.IsUbuntu) { - throw "Cross compiling for linux-arm is only supported on Ubuntu environment" + if (($BuildLinuxArm -or $BuildLinuxArm64) -and -not $Environment.IsUbuntu) { + throw "Cross compiling for linux-arm/linux-arm64 are only supported on Ubuntu environment" } # Verify we have all tools in place to do the build @@ -413,6 +414,10 @@ function Start-BuildNativeUnixBinaries { foreach ($Dependency in 'arm-linux-gnueabihf-gcc', 'arm-linux-gnueabihf-g++') { $precheck = $precheck -and (precheck $Dependency "Build dependency '$Dependency' not found. Run 'Start-PSBootstrap'.") } + } elseif ($BuildLinuxArm64) { + foreach ($Dependency in 'aarch64-linux-gnu-gcc', 'aarch64-linux-gnu-g++') { + $precheck = $precheck -and (precheck $Dependency "Build dependency '$Dependency' not found. Run 'Start-PSBootstrap'.") + } } # Abort if any precheck failed @@ -439,6 +444,10 @@ function Start-BuildNativeUnixBinaries { Start-NativeExecution { cmake -DCMAKE_TOOLCHAIN_FILE="./arm.toolchain.cmake" . } Start-NativeExecution { make -j } } + elseif ($BuildLinuxArm64) { + Start-NativeExecution { cmake -DCMAKE_TOOLCHAIN_FILE="./arm64.toolchain.cmake" . } + Start-NativeExecution { make -j } + } else { Start-NativeExecution { cmake -DCMAKE_BUILD_TYPE=Debug . } Start-NativeExecution { make -j } @@ -610,6 +619,7 @@ function PlaceUnixBinaries $RuntimePathLinux = New-Item -ItemType Directory -Path (Join-Path $PackageRoot -ChildPath 'runtimes/linux-x64/native') -Force $RuntimePathLinuxARM = New-Item -ItemType Directory -Path (Join-Path $PackageRoot -ChildPath 'runtimes/linux-arm/native') -Force + $RuntimePathLinuxARM64 = New-Item -ItemType Directory -Path (Join-Path $PackageRoot -ChildPath 'runtimes/linux-arm64/native') -Force $RuntimePathOSX = New-Item -ItemType Directory -Path (Join-Path $PackageRoot -ChildPath 'runtimes/osx/native') -Force Copy-Item "$BinFolderLinux\*" -Destination $RuntimePathLinux -Verbose @@ -725,6 +735,7 @@ function Start-PSBuild { "osx-x64", "linux-x64", "linux-arm", + "linux-arm64", "win-arm", "win-arm64")] [string]$Runtime, @@ -739,8 +750,8 @@ function Start-PSBuild { [string]$ReleaseTag ) - if ($Runtime -eq "linux-arm" -and -not $Environment.IsUbuntu) { - throw "Cross compiling for linux-arm is only supported on Ubuntu environment" + if (($Runtime -eq "linux-arm" -or $Runtime -eq "linux-arm64") -and -not $Environment.IsUbuntu) { + throw "Cross compiling for linux-arm/linux-arm64 are only supported on Ubuntu environment" } if ("win-arm","win-arm64" -contains $Runtime -and -not $Environment.IsWindows) { @@ -1037,6 +1048,7 @@ function New-PSOptions { "osx-x64", "linux-x64", "linux-arm", + "linux-arm64", "win-arm", "win-arm64")] [string]$Runtime, @@ -1860,6 +1872,7 @@ function Start-PSBootstrap { [switch]$NoSudo, [switch]$BuildWindowsNative, [switch]$BuildLinuxArm, + [switch]$BuildLinuxArm64, [switch]$Force ) @@ -1883,8 +1896,8 @@ function Start-PSBootstrap { Pop-Location } - if ($BuildLinuxArm -and -not $Environment.IsUbuntu) { - Write-Error "Cross compiling for linux-arm is only supported on Ubuntu environment" + if (($BuildLinuxArm -or $BuildLinuxArm64) -and -not $Environment.IsUbuntu) { + Write-Error "Cross compiling for linux-arm/linux-arm64 are only supported on Ubuntu environment" return } @@ -1896,6 +1909,8 @@ function Start-PSBootstrap { if ($BuildLinuxArm) { $Deps += "gcc-arm-linux-gnueabihf", "g++-arm-linux-gnueabihf" + } elseif ($BuildLinuxArm64) { + $Deps += "gcc-aarch64-linux-gnu", "g++-aarch64-linux-gnu" } # .NET Core required runtime libraries @@ -2464,6 +2479,7 @@ function Start-CrossGen { "osx-x64", "linux-x64", "linux-arm", + "linux-arm64", "win-arm", "win-arm64")] [string] @@ -2526,6 +2542,8 @@ function Start-CrossGen { } } elseif ($Runtime -eq "linux-arm") { throw "crossgen is not available for 'linux-arm'" + } elseif ($Runtime -eq "linux-arm64") { + throw "crossgen is not available for 'linux-arm64'" } elseif ($Environment.IsLinux) { "linux-x64" } elseif ($Environment.IsMacOS) { diff --git a/src/libpsl-native/arm64.toolchain.cmake b/src/libpsl-native/arm64.toolchain.cmake new file mode 100644 index 0000000..2bd02df --- /dev/null +++ b/src/libpsl-native/arm64.toolchain.cmake @@ -0,0 +1,17 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++ -fstack-protector-strong -fpie -DFORTIFY_SOURCE=2 -O2) +set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-z,relro,-z,now") +set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc) + +add_compile_options(-g) + +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) +set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) From 1046066349338336de8d8867494cbd8053831a8e Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL)" Date: Mon, 3 Dec 2018 20:03:39 -0800 Subject: [PATCH 2/6] use spectre enabled libraries --- .vsts-ci/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index 3d28029..eac4987 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -36,7 +36,7 @@ phases: condition: succeeded() - powershell: | Invoke-WebRequest "https://aka.ms/vs/15/release/vs_BuildTools.exe" -OutFile vs_BuildTools.exe -UseBasicParsing - Start-Process -FilePath 'vs_BuildTools.exe' -ArgumentList '--quiet', '--norestart', '--locale en-US', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM64', '--includeRecommended', '--add Microsoft.VisualStudio.Workload.VCTools', '--add Microsoft.VisualStudio.Component.Windows10SDK.16299.Desktop.arm', '--add Microsoft.VisualStudio.Component.VC.ATL', '--add Microsoft.VisualStudio.Component.VC.ATLMFC', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM64' -Wait + Start-Process -FilePath 'vs_BuildTools.exe' -ArgumentList '--quiet', '--norestart', '--locale en-US', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM64', '--includeRecommended', '--add Microsoft.VisualStudio.Workload.VCTools', '--add Microsoft.VisualStudio.Component.Windows10SDK.16299.Desktop.arm', '--add Microsoft.VisualStudio.Component.VC.ATL.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATLMFC.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM64.Spectre' -Wait Remove-Item .\vs_BuildTools.exe Remove-Item -Force -Recurse 'C:\Program Files (x86)\Microsoft Visual Studio\Installer' $vsPath = ${Env:ProgramFiles(x86)} + '\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin' From 3c092bb0b47fa15b46e8837234ca199f913345a3 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL)" Date: Mon, 3 Dec 2018 20:09:08 -0800 Subject: [PATCH 3/6] add build support for linux-arm64 --- .vsts-ci/linux.yml | 2 ++ tools/releaseBuild/PowershellNative.ps1 | 13 ++++++++++++- tools/releaseBuild/build.json | 12 ++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 596d9e8..53e857d 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -18,6 +18,8 @@ phases: matrix: Linux ARM Native: buildName: ubuntu.16.04 + Linux ARM64 Native: + buildName: ubuntu.16.04 Linux Native: buildName: centos.7 Linux Alpine: diff --git a/tools/releaseBuild/PowershellNative.ps1 b/tools/releaseBuild/PowershellNative.ps1 index 89411e3..2a963cb 100644 --- a/tools/releaseBuild/PowershellNative.ps1 +++ b/tools/releaseBuild/PowershellNative.ps1 @@ -6,7 +6,7 @@ param ( [Parameter(Mandatory, ParameterSetName = 'Build')] - [ValidateSet('x64', 'x86', 'x64_arm', 'x64_arm64', 'linux-x64', 'osx', 'linux-arm', 'linux-musl-x64')] + [ValidateSet('x64', 'x86', 'x64_arm', 'x64_arm64', 'linux-x64', 'osx', 'linux-arm', 'linux-arm64', 'linux-musl-x64')] [string] $Arch, @@ -62,6 +62,17 @@ end { Write-Verbose -Verbose "Skipping artifact upload since this is a PR." } } + elseif ($Arch -eq 'linux-arm64') { + Start-PSBootstrap -BuildLinuxArm64 + Start-BuildNativeUnixBinaries -BuildLinuxArm64 + + if ($env:BUILD_REASON -ne 'PullRequest') { + $buildOutputPath = Join-Path $RepoRoot "src/powershell-unix" + Compress-Archive -Path $buildOutputPath/libpsl-native.* -DestinationPath "$TargetLocation/$Arch-symbols.zip" -Verbose + } else { + Write-Verbose -Verbose "Skipping artifact upload since this is a PR." + } + } else { Write-Verbose "Starting Start-PSBootstrap" -Verbose Start-PSBootstrap -BuildWindowsNative diff --git a/tools/releaseBuild/build.json b/tools/releaseBuild/build.json index 4520857..6497eb7 100644 --- a/tools/releaseBuild/build.json +++ b/tools/releaseBuild/build.json @@ -86,6 +86,18 @@ "BinaryBucket": "release", "EnableFeature": [ "ArtifactAsFolder" ] }, + { + "Name": "ubuntu.16.04", + "RepoDestinationPath": "/PowerShellNative", + "BuildCommand": "/PowerShellNative/tools/releaseBuild/PowershellNative.ps1 -RepoRoot _RepoDestinationPath_ -TargetLocation _DockerVolume_ -Arch linux-arm64 -Configuration Release", + "AdditionalContextFiles": [ + "./tools/releaseBuild/PowershellNative.ps1" + ], + "DockerFile": "./tools/releaseBuild/images/Ubuntu16.04/Dockerfile", + "DockerImageName": "ps-ubunutu-16-04", + "BinaryBucket": "release", + "EnableFeature": [ "ArtifactAsFolder" ] + }, { "Name": "centos.7", "RepoDestinationPath": "/PowerShellNative", From a314eb96068217063c1585825c71f221e547a244 Mon Sep 17 00:00:00 2001 From: "Steve Lee (POWERSHELL)" Date: Mon, 3 Dec 2018 20:56:40 -0800 Subject: [PATCH 4/6] use spectre runtime libraries --- .vsts-ci/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index eac4987..ac07bdb 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -36,7 +36,7 @@ phases: condition: succeeded() - powershell: | Invoke-WebRequest "https://aka.ms/vs/15/release/vs_BuildTools.exe" -OutFile vs_BuildTools.exe -UseBasicParsing - Start-Process -FilePath 'vs_BuildTools.exe' -ArgumentList '--quiet', '--norestart', '--locale en-US', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM64', '--includeRecommended', '--add Microsoft.VisualStudio.Workload.VCTools', '--add Microsoft.VisualStudio.Component.Windows10SDK.16299.Desktop.arm', '--add Microsoft.VisualStudio.Component.VC.ATL.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATLMFC.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM64.Spectre' -Wait + Start-Process -FilePath 'vs_BuildTools.exe' -ArgumentList '--quiet', '--norestart', '--locale en-US', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM64', '--includeRecommended', '--add Microsoft.VisualStudio.Workload.VCTools', '--add Microsoft.VisualStudio.Component.Windows10SDK.16299.Desktop.arm', '--add Microsoft.VisualStudio.Component.VC.ATL.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATLMFC.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM64.Spectre', '--add Microsoft.VisualStudio.Component.VC.Runtimes.ARM.Spectre', '--add Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre', '--add Microsoft.VisualStudio.Component.VC.Runtimes.x86.x64.Spectre' -Wait Remove-Item .\vs_BuildTools.exe Remove-Item -Force -Recurse 'C:\Program Files (x86)\Microsoft Visual Studio\Installer' $vsPath = ${Env:ProgramFiles(x86)} + '\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin' From 088345734922d5d3cc6ede3c0b32558f19f90c35 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Tue, 4 Dec 2018 11:44:01 -0800 Subject: [PATCH 5/6] address Aditya's feedback --- .vsts-ci/linux.yml | 4 ++-- build.psm1 | 10 ++++++++++ tools/releaseBuild/build.json | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 53e857d..886f748 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -17,9 +17,9 @@ phases: parallel: 2 matrix: Linux ARM Native: - buildName: ubuntu.16.04 + buildName: ubuntu.16.04-arm Linux ARM64 Native: - buildName: ubuntu.16.04 + buildName: ubuntu.16.04-arm64 Linux Native: buildName: centos.7 Linux Alpine: diff --git a/build.psm1 b/build.psm1 index 8a2b572..ac91a03 100644 --- a/build.psm1 +++ b/build.psm1 @@ -501,6 +501,10 @@ function Start-BuildPowerShellNativePackage [ValidateScript({Test-Path $_ -PathType Leaf})] [string] $LinuxARMZipPath, + [Parameter(Mandatory = $true)] + [ValidateScript({Test-Path $_ -PathType Leaf})] + [string] $LinuxARM64ZipPath, + [Parameter(Mandatory = $true)] [ValidateScript({Test-Path $_ -PathType Leaf})] [string] $LinuxAlpineZipPath, @@ -545,6 +549,7 @@ function Start-BuildPowerShellNativePackage Expand-Archive -Path $LinuxZipPath -DestinationPath $BinFolderLinux -Force Expand-Archive -Path $LinuxAlpineZipPath -DestinationPath $BinFolderLinuxAlpine -Force Expand-Archive -Path $LinuxARMZipPath -DestinationPath $BinFolderLinuxARM -Force + Expand-Archive -Path $LinuxARM64ZipPath -DestinationPath $BinFolderLinuxARM64 -Force Expand-Archive -Path $macOSZipPath -DestinationPath $BinFolderMacOS -Force Expand-Archive -Path $psrpZipPath -DestinationPath $BinFolderPSRP -Force @@ -615,6 +620,10 @@ function PlaceUnixBinaries [ValidateScript({Test-Path $_ -PathType Container})] $BinFolderLinuxARM, + [Parameter(Mandatory = $true)] + [ValidateScript({Test-Path $_ -PathType Container})] + $BinFolderLinuxARM64, + [Parameter(Mandatory = $true)] [ValidateScript({Test-Path $_ -PathType Container})] $BinFolderLinuxAlpine, @@ -636,6 +645,7 @@ function PlaceUnixBinaries Copy-Item "$BinFolderLinux\*" -Destination $RuntimePathLinux -Verbose Copy-Item "$BinFolderLinuxARM\*" -Destination $RuntimePathLinuxARM -Verbose + Copy-Item "$BinFolderLinuxARM64\*" -Destination $RuntimePathLinuxARM64 -Verbose Copy-Item "$BinFolderLinuxAlpine\*" -Destination $RuntimePathLinuxAlpine -Verbose Copy-Item "$BinFolderOSX\*" -Destination $RuntimePathOSX -Verbose diff --git a/tools/releaseBuild/build.json b/tools/releaseBuild/build.json index 6497eb7..8188770 100644 --- a/tools/releaseBuild/build.json +++ b/tools/releaseBuild/build.json @@ -75,7 +75,7 @@ ], "Linux": [ { - "Name": "ubuntu.16.04", + "Name": "ubuntu.16.04-arm", "RepoDestinationPath": "/PowerShellNative", "BuildCommand": "/PowerShellNative/tools/releaseBuild/PowershellNative.ps1 -RepoRoot _RepoDestinationPath_ -TargetLocation _DockerVolume_ -Arch linux-arm -Configuration Release", "AdditionalContextFiles": [ @@ -87,7 +87,7 @@ "EnableFeature": [ "ArtifactAsFolder" ] }, { - "Name": "ubuntu.16.04", + "Name": "ubuntu.16.04-arm64", "RepoDestinationPath": "/PowerShellNative", "BuildCommand": "/PowerShellNative/tools/releaseBuild/PowershellNative.ps1 -RepoRoot _RepoDestinationPath_ -TargetLocation _DockerVolume_ -Arch linux-arm64 -Configuration Release", "AdditionalContextFiles": [ From 1febed2f20a98fe2c420a47b45d66cd58c389258 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 4 Dec 2018 17:22:15 -0800 Subject: [PATCH 6/6] Add Linux ARM64 to the nuget package --- build.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index ac91a03..3a62bdb 100644 --- a/build.psm1 +++ b/build.psm1 @@ -538,6 +538,7 @@ function Start-BuildPowerShellNativePackage $BinFolderARM64 = Join-Path $tempExtractionPath "ARM64" $BinFolderLinux = Join-Path $tempExtractionPath "Linux" $BinFolderLinuxARM = Join-Path $tempExtractionPath "LinuxARM" + $BinFolderLinuxARM64 = Join-Path $tempExtractionPath "LinuxARM64" $BinFolderLinuxAlpine = Join-Path $tempExtractionPath "LinuxAlpine" $BinFolderMacOS = Join-Path $tempExtractionPath "MacOS" $BinFolderPSRP = Join-Path $tempExtractionPath "PSRP" @@ -555,7 +556,7 @@ function Start-BuildPowerShellNativePackage PlaceWindowsNativeBinaries -PackageRoot $PackageRoot -BinFolderX64 $BinFolderX64 -BinFolderX86 $BinFolderX86 -BinFolderARM $BinFolderARM -BinFolderARM64 $BinFolderARM64 - PlaceUnixBinaries -PackageRoot $PackageRoot -BinFolderLinux $BinFolderLinux -BinFolderLinuxARM $BinFolderLinuxARM -BinFolderOSX $BinFolderMacOS -BinFolderPSRP $BinFolderPSRP -BinFolderLinuxAlpine $BinFolderLinuxAlpine + PlaceUnixBinaries -PackageRoot $PackageRoot -BinFolderLinux $BinFolderLinux -BinFolderLinuxARM $BinFolderLinuxARM -BinFolderLinuxARM64 $BinFolderLinuxARM64 -BinFolderOSX $BinFolderMacOS -BinFolderPSRP $BinFolderPSRP -BinFolderLinuxAlpine $BinFolderLinuxAlpine $Nuspec = @'