From 73069c3f5bb81544127eb13be030f8e30132781b Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 5 Feb 2019 01:00:23 -0800 Subject: [PATCH 01/25] Add bootstrapping code to install dotnet --- build.ps1 | 9 ++++++++- build.psm1 | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index 2473146e2..e0382f6cf 100644 --- a/build.ps1 +++ b/build.ps1 @@ -31,7 +31,10 @@ param( [switch] $Test, [Parameter(ParameterSetName='Test')] - [switch] $InProcess + [switch] $InProcess, + + [Parameter(ParameterSetName='Bootstrap')] + [switch] $Bootstrap ) END { @@ -58,6 +61,10 @@ END { } Start-ScriptAnalyzerBuild @buildArgs } + "Bootstrap" { + Install-DotNet + return + } "Test" { Test-ScriptAnalyzer -InProcess:$InProcess return diff --git a/build.psm1 b/build.psm1 index f9f216554..3a83762b8 100644 --- a/build.psm1 +++ b/build.psm1 @@ -270,3 +270,43 @@ function Get-TestFailures $results = [xml](Get-Content $logPath) $results.SelectNodes(".//test-case[@result='Failure']") } + +# BOOTSTRAPPING CODE FOR INSTALLING DOTNET +# install dotnet cli tools based on the version mentioned in global.json +function Install-Dotnet +{ + [CmdletBinding(SupportsShouldProcess=$true)] + param ( [Parameter()][Switch]$Force ) + $json = Get-Content (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json + $version = $json.sdk.Version + try { + Push-Location $PSScriptRoot + $installScriptName = Receive-DotnetInstallScript + If ( $PSCmdlet.ShouldProcess("$installScriptName for $version")) { + $installScriptPath = Join-Path . $installScriptName + & "${installScriptPath}" -c release -v $version + } + } + finally { + if ( Test-Path $installScriptName ) { + Remove-Item $installScriptName + } + Pop-Location + } +} + +function Receive-DotnetInstallScript +{ + if ( $IsWindows ) { + $installScriptName = "dotnet-install.ps1" + } + else { + $installScriptName = "dotnet-install.sh" + } + $null = Invoke-WebRequest -Uri "https://dot.net/v1/${installScriptName}" -OutFile "${installScriptName}" + if ( ! $IsWindows ) { + chmod +x $installScriptName + } + return $installScriptName +} + From b10fa9c59feb8d748ea57a37b7af0e8e869cd049 Mon Sep 17 00:00:00 2001 From: James Truher Date: Wed, 6 Feb 2019 14:41:59 -0800 Subject: [PATCH 02/25] fix logic for downloading proper dotnet installer. --- build.psm1 | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/build.psm1 b/build.psm1 index 3a83762b8..c532ce54f 100644 --- a/build.psm1 +++ b/build.psm1 @@ -297,16 +297,14 @@ function Install-Dotnet function Receive-DotnetInstallScript { - if ( $IsWindows ) { - $installScriptName = "dotnet-install.ps1" - } - else { + $installScriptName = "dotnet-install.ps1" + + if ( (Test-Path Variable:IsWindows) -and -not $IsWindows ) { $installScriptName = "dotnet-install.sh" } $null = Invoke-WebRequest -Uri "https://dot.net/v1/${installScriptName}" -OutFile "${installScriptName}" - if ( ! $IsWindows ) { + if ( (Test-Path Variable:IsWindows) -and -not $IsWindows ) { chmod +x $installScriptName } return $installScriptName } - From a3379fc737157f9978586029d2b604bd57ff14a7 Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 7 Feb 2019 16:43:13 -0800 Subject: [PATCH 03/25] add a version checker for installed dotnet Also harden logic for running the installation script --- build.psm1 | 48 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/build.psm1 b/build.psm1 index c532ce54f..af8ad1ab4 100644 --- a/build.psm1 +++ b/build.psm1 @@ -277,24 +277,40 @@ function Install-Dotnet { [CmdletBinding(SupportsShouldProcess=$true)] param ( [Parameter()][Switch]$Force ) - $json = Get-Content (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json + $json = Get-Content -raw (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json $version = $json.sdk.Version + if ( Test-DotnetInstallation -version $version ) { + Write-Verbose -Verbose "dotnet version '$version' already installed" + return + } try { Push-Location $PSScriptRoot - $installScriptName = Receive-DotnetInstallScript + $installScriptPath = Receive-DotnetInstallScript If ( $PSCmdlet.ShouldProcess("$installScriptName for $version")) { - $installScriptPath = Join-Path . $installScriptName & "${installScriptPath}" -c release -v $version } } + catch { + throw $_ + } finally { - if ( Test-Path $installScriptName ) { - Remove-Item $installScriptName + if ( Test-Path $installScriptPath ) { + Remove-Item $installScriptPath } Pop-Location } } +function Test-DotnetInstallation +{ + param ( $version ) + $installedVersions = dotnet --list-sdks | Foreach-Object { $_.Split()[0] } + if ( $installedVersions -contains $version ) { + return $true + } + return $false +} + function Receive-DotnetInstallScript { $installScriptName = "dotnet-install.ps1" @@ -302,9 +318,27 @@ function Receive-DotnetInstallScript if ( (Test-Path Variable:IsWindows) -and -not $IsWindows ) { $installScriptName = "dotnet-install.sh" } - $null = Invoke-WebRequest -Uri "https://dot.net/v1/${installScriptName}" -OutFile "${installScriptName}" + $uri = "https://dot.net/v1/${installScriptName}" + # enable Tls12 for the request + # -SslProtocol parameter for Invoke-WebRequest wasn't in PSv3 + $securityProtocol = [System.Net.ServicePointManager]::SecurityProtocol + $tls12 = [System.Net.SecurityProtocolType]::Tls12 + try { + if ( ([System.Net.ServicePointManager]::SecurityProtocol -band $tls12) -eq 0 ) { + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor $tls12 + } + $null = Invoke-WebRequest -Uri ${uri} -OutFile "${installScriptName}" + } + finally { + [System.Net.ServicePointManager]::SecurityProtocol = $securityProtocol + } if ( (Test-Path Variable:IsWindows) -and -not $IsWindows ) { chmod +x $installScriptName } - return $installScriptName + $installScript = Get-Item $installScriptName -ErrorAction Stop + if ( -not $installScript ) { + throw "Download failure of ${uri}" + } + + return $installScript.FullName } From d2b4ecab407de58973b4bc7b14402b5d8da11c42 Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 5 Feb 2019 01:00:23 -0800 Subject: [PATCH 04/25] Add bootstrapping code to install dotnet --- build.ps1 | 9 ++++++++- build.psm1 | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index 2473146e2..e0382f6cf 100644 --- a/build.ps1 +++ b/build.ps1 @@ -31,7 +31,10 @@ param( [switch] $Test, [Parameter(ParameterSetName='Test')] - [switch] $InProcess + [switch] $InProcess, + + [Parameter(ParameterSetName='Bootstrap')] + [switch] $Bootstrap ) END { @@ -58,6 +61,10 @@ END { } Start-ScriptAnalyzerBuild @buildArgs } + "Bootstrap" { + Install-DotNet + return + } "Test" { Test-ScriptAnalyzer -InProcess:$InProcess return diff --git a/build.psm1 b/build.psm1 index d283eef00..cf4be90e5 100644 --- a/build.psm1 +++ b/build.psm1 @@ -271,3 +271,43 @@ function Get-TestFailures $results = [xml](Get-Content $logPath) $results.SelectNodes(".//test-case[@result='Failure']") } + +# BOOTSTRAPPING CODE FOR INSTALLING DOTNET +# install dotnet cli tools based on the version mentioned in global.json +function Install-Dotnet +{ + [CmdletBinding(SupportsShouldProcess=$true)] + param ( [Parameter()][Switch]$Force ) + $json = Get-Content (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json + $version = $json.sdk.Version + try { + Push-Location $PSScriptRoot + $installScriptName = Receive-DotnetInstallScript + If ( $PSCmdlet.ShouldProcess("$installScriptName for $version")) { + $installScriptPath = Join-Path . $installScriptName + & "${installScriptPath}" -c release -v $version + } + } + finally { + if ( Test-Path $installScriptName ) { + Remove-Item $installScriptName + } + Pop-Location + } +} + +function Receive-DotnetInstallScript +{ + if ( $IsWindows ) { + $installScriptName = "dotnet-install.ps1" + } + else { + $installScriptName = "dotnet-install.sh" + } + $null = Invoke-WebRequest -Uri "https://dot.net/v1/${installScriptName}" -OutFile "${installScriptName}" + if ( ! $IsWindows ) { + chmod +x $installScriptName + } + return $installScriptName +} + From 7f3ddb3060ba33c797225c4b6bc68d20e90b1ca5 Mon Sep 17 00:00:00 2001 From: James Truher Date: Mon, 11 Feb 2019 17:04:52 -0800 Subject: [PATCH 05/25] Handle missing dotnet, and install it. --- build.psm1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index 1309598f0..af4100c12 100644 --- a/build.psm1 +++ b/build.psm1 @@ -306,7 +306,12 @@ function Install-Dotnet function Test-DotnetInstallation { param ( $version ) - $installedVersions = dotnet --list-sdks | Foreach-Object { $_.Split()[0] } + try { + $installedVersions = dotnet --list-sdks | Foreach-Object { $_.Split()[0] } + } + catch { + $installedVersions = @() + } if ( $installedVersions -contains $version ) { return $true } From ecc4b3f85cc20171bf598796b31790339fcabf50 Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 12 Feb 2019 14:06:49 -0800 Subject: [PATCH 06/25] Add logic for checking the appropriate version of the Cli tools before starting to build --- build.psm1 | 158 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 137 insertions(+), 21 deletions(-) diff --git a/build.psm1 b/build.psm1 index af4100c12..f9654b98b 100644 --- a/build.psm1 +++ b/build.psm1 @@ -21,7 +21,7 @@ function Publish-File # attempt to get the users module directory function Get-UserModulePath { - if ( $IsCoreCLR -and ! $IsWindows ) + if ( $IsCoreCLR -and -not $IsWindows ) { $platformType = "System.Management.Automation.Platform" -as [Type] if ( $platformType ) { @@ -104,7 +104,7 @@ function Start-DocumentationBuild throw "Cannot find markdown documentation folder." } Import-Module platyPS - if ( ! (Test-Path $outputDocsPath)) { + if ( -not (Test-Path $outputDocsPath)) { $null = New-Item -Type Directory -Path $outputDocsPath -Force } $null = New-ExternalHelp -Path $markdownDocsPath -OutputPath $outputDocsPath -Force @@ -126,7 +126,22 @@ function Start-ScriptAnalyzerBuild [switch]$Documentation ) + BEGIN { + # don't allow the build to be started unless we have the proper Cli version + if ( -not (Test-SuitableDotnet) ) { + $requiredVersion = Get-GlobalJsonSdkVersion + throw "No suitable dotnet CLI found, requires version '$requiredVersion'" + } + } END { + + # Build docs either when -Documentation switch is being specified or the first time in a clean repo + $documentationFileExists = Test-Path (Join-Path $PSScriptRoot 'out\PSScriptAnalyzer\en-us\Microsoft.Windows.PowerShell.ScriptAnalyzer.dll-Help.xml') + if ( $Documentation -or -not $documentationFileExists ) + { + Start-DocumentationBuild + } + if ( $All ) { # Build all the versions of the analyzer @@ -136,13 +151,6 @@ function Start-ScriptAnalyzerBuild return } - $documentationFileExists = Test-Path (Join-Path $PSScriptRoot 'out\PSScriptAnalyzer\en-us\Microsoft.Windows.PowerShell.ScriptAnalyzer.dll-Help.xml') - # Build docs either when -Documentation switch is being specified or the first time in a clean repo - if ( $Documentation -or -not $documentationFileExists ) - { - Start-DocumentationBuild - } - if ($PSVersion -ge 6) { $framework = 'netstandard2.0' } @@ -277,17 +285,25 @@ function Get-TestFailures function Install-Dotnet { [CmdletBinding(SupportsShouldProcess=$true)] - param ( [Parameter()][Switch]$Force ) + param ( + [Parameter()][Switch]$Force, + [Parameter()]$version = $( Get-GlobalJsonSdkVersion ) + ) - $json = Get-Content -raw (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json - $version = $json.sdk.Version - if ( Test-DotnetInstallation -version $version ) { + if ( Test-DotnetInstallation -requestedversion $version ) { Write-Verbose -Verbose "dotnet version '$version' already installed" - return + if ( $Force ) { + Write-Verbose -Verbose "Installing again" + } + else { + return + } } + try { Push-Location $PSScriptRoot $installScriptPath = Receive-DotnetInstallScript + $installScriptName = [System.IO.Path]::GetFileName($installScriptPath) If ( $PSCmdlet.ShouldProcess("$installScriptName for $version")) { & "${installScriptPath}" -c release -v $version } @@ -303,19 +319,119 @@ function Install-Dotnet } } -function Test-DotnetInstallation -{ - param ( $version ) +function Get-GlobalJsonSdkVersion { + $json = Get-Content -raw (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json + $version = $json.sdk.Version + ConvertTo-PortableVersion $version +} + +# we don't have semantic version in earlier versions of PowerShell, so we need to +# create something that we can use +function ConvertTo-PortableVersion { + param ( [string[]]$strVersion ) + if ( -not $strVersion ) { + return (ConvertTo-PortableVersion "0.0.0-0") + } + foreach ( $v in $strVersion ) { + $ver, $pre = $v.split("-",2) + try { + [int]$major, [int]$minor, [int]$patch = $ver.Split(".") + } + catch { + Write-Warning "Cannot convert '$v' to portable version" + continue + } + $h = @{ + Major = $major + Minor = $minor + Patch = $patch + } + if ( $pre ) { + $h['PrereleaseLabel'] = $pre + } + else { + $h['PrereleaseLabel'] = [String]::Empty + } + $customObject = [pscustomobject]$h + Add-Member -inputobject $customObject -Type ScriptMethod -Name ToString -Force -Value { + $str = "{0}.{1}.{2}" -f $this.Major,$this.Minor,$this.Patch + if ( $this.PrereleaseLabel ) { + $str += "-{0}" -f $this.PrereleaseLabel + } + return $str + } + Add-Member -inputobject $customObject -Type ScriptMethod -Name IsContainedIn -Value { + param ( [object[]]$collection ) + foreach ( $object in $collection ) { + if ( + $this.Major -eq $object.Major -and + $this.Minor -eq $object.Minor -and + $this.Patch -eq $object.Patch -and + $this.PrereleaseLabel -eq $object.PrereleaseLabel + ) { + return $true + } + } + return $false + } + $customObject + } +} + +# see https://docs.microsoft.com/en-us/dotnet/core/tools/global-json for rules +# on how version checks are done +function Test-SuitableDotnet { + param ( + $availableVersions = $( Get-InstalledCliVersion), + $requiredVersion = $( Get-GlobalJsonSdkVersion ) + ) + if ( $requiredVersion -is [String] -or $requiredVersion -is [Version] ) { + $requiredVersion = ConvertTo-PortableVersion "$requiredVersion" + } + # if we have what was requested, we can use it + if ( $RequiredVersion.IsContainedIn($availableVersions)) { + return $true + } + # if we had found a match, we would have returned $true above + # exact match required for 2.1.100 through 2.1.201 + if ( $RequiredVersion.Major -eq 2 -and $RequiredVersion.Minor -eq 1 -and $RequiredVersion.Patch -ge 100 -and $RequiredVersion.Patch -le 201 ) { + return $false + } + # we need to check each available version for something that's useable + foreach ( $version in $availableVersions ) { + # major/minor numbers don't match - keep looking + if ( $version.Major -ne $requiredVersion.Major -or $version.Minor -ne $requiredVersion.Minor ) { + continue + } + $requiredPatch = $requiredVersion.Patch + $possiblePatch = $version.Patch + if ( $requiredPatch -gt $possiblePath ) { + continue + } + if ( ($requiredPatch - $possiblePatch) -ge 100 ) { + continue + } + return $true + } + return $false +} + +# these are mockable functions for testing +function Get-InstalledCLIVersion { try { $installedVersions = dotnet --list-sdks | Foreach-Object { $_.Split()[0] } } catch { $installedVersions = @() } - if ( $installedVersions -contains $version ) { - return $true - } - return $false + return (ConvertTo-PortableVersion $installedVersions) +} + +function Test-DotnetInstallation +{ + param ( $requestedVersion = $( Get-GlobalJsonSdkVersion ) ) + $installedVersions = Get-InstalledCLIVersion + return (Test-SuitableDotnet -availableVersions $installedVersions -requiredVersion $requestedVersion ) } function Receive-DotnetInstallScript From 2df5451de8ab8ba5a80f6596e2c812be19dd6644 Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 12 Feb 2019 15:49:30 -0800 Subject: [PATCH 07/25] Update appveyor.psm1 to use build script bootstrapping for installing dotnet CLI update build script to handle WMF4 better (theoretically) --- build.psm1 | 12 +++++++++++- tools/appveyor.psm1 | 34 ++++------------------------------ 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/build.psm1 b/build.psm1 index f9654b98b..cdd7a1af3 100644 --- a/build.psm1 +++ b/build.psm1 @@ -419,7 +419,17 @@ function Test-SuitableDotnet { # these are mockable functions for testing function Get-InstalledCLIVersion { try { - $installedVersions = dotnet --list-sdks | Foreach-Object { $_.Split()[0] } + # earlier versions of dotnet do not support --list-sdks, so we'll check the output + # and use dotnet --version as a fallback + + $sdkList = dotnet --list-sdks 2>&1 + $sdkList = "Unknown option" + if ( $sdkList -match "Unknown option" ) { + $installedVersions = dotnet --version + } + else { + $installedVersions = $sdkList | Foreach-Object { $_.Split()[0] } + } } catch { $installedVersions = @() diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index d9cbf3663..c444c8d93 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -31,36 +31,10 @@ function Invoke-AppVeyorInstall { Install-Module -Name platyPS -Force -Scope CurrentUser -RequiredVersion $platyPSVersion } - # the legacy WMF4 image only has the old preview SDKs of dotnet - $globalDotJson = Get-Content (Join-Path $PSScriptRoot '..\global.json') -Raw | ConvertFrom-Json - $requiredDotNetCoreSDKVersion = $globalDotJson.sdk.version - if ($PSVersionTable.PSVersion.Major -gt 4) { - $requiredDotNetCoreSDKVersionPresent = (dotnet --list-sdks) -match $requiredDotNetCoreSDKVersion - } - else { - # WMF 4 image has old SDK that does not have --list-sdks parameter - $requiredDotNetCoreSDKVersionPresent = (dotnet --version).StartsWith($requiredDotNetCoreSDKVersion) - } - if (-not $requiredDotNetCoreSDKVersionPresent) { - Write-Verbose -Verbose "Installing required .Net CORE SDK $requiredDotNetCoreSDKVersion" - $originalSecurityProtocol = [Net.ServicePointManager]::SecurityProtocol - try { - [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 - if ($IsLinux -or $isMacOS) { - Invoke-WebRequest 'https://dot.net/v1/dotnet-install.sh' -OutFile dotnet-install.sh - bash dotnet-install.sh --version $requiredDotNetCoreSDKVersion - [System.Environment]::SetEnvironmentVariable('PATH', "/home/appveyor/.dotnet$([System.IO.Path]::PathSeparator)$PATH") - } - else { - Invoke-WebRequest 'https://dot.net/v1/dotnet-install.ps1' -OutFile dotnet-install.ps1 - .\dotnet-install.ps1 -Version $requiredDotNetCoreSDKVersion - } - } - finally { - [Net.ServicePointManager]::SecurityProtocol = $originalSecurityProtocol - Remove-Item .\dotnet-install.* - } - } + Write-Verbose -Verbose "Installing required .Net CORE SDK $requiredDotNetCoreSDKVersion" + # the build script sorts out the problems of WMF4 and earlier versions of dotnet CLI + $buildScriptDir = (Resolve-Path "$PSScriptRoot/..").Path + & "$buildScriptDir/build.ps1" -bootstrap } # Implements AppVeyor 'test_script' step From 4d2bf72ea87c9b4b0ad0303bb5ac31dfe6b4e9f2 Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 12 Feb 2019 16:29:35 -0800 Subject: [PATCH 08/25] make the hunt for the dotnet executable more generic and try harder to find the exe --- build.psm1 | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/build.psm1 b/build.psm1 index cdd7a1af3..cdd8fbe06 100644 --- a/build.psm1 +++ b/build.psm1 @@ -130,7 +130,8 @@ function Start-ScriptAnalyzerBuild # don't allow the build to be started unless we have the proper Cli version if ( -not (Test-SuitableDotnet) ) { $requiredVersion = Get-GlobalJsonSdkVersion - throw "No suitable dotnet CLI found, requires version '$requiredVersion'" + $foundVersion = Get-InstalledCLIVersion + throw "No suitable dotnet CLI found, requires version '$requiredVersion' found only '$foundVersion'" } } END { @@ -201,7 +202,7 @@ function Start-ScriptAnalyzerBuild try { Push-Location $projectRoot/Rules Write-Progress "Building ScriptAnalyzer for PSVersion '$PSVersion' using framework '$framework' and configuration '$Configuration'" - $buildOutput = dotnet build --framework $framework --configuration "$config" + $buildOutput = & $script:dotnetExe build --framework $framework --configuration "$config" if ( $LASTEXITCODE -ne 0 ) { throw "$buildOutput" } } catch { @@ -285,7 +286,7 @@ function Get-TestFailures function Install-Dotnet { [CmdletBinding(SupportsShouldProcess=$true)] - param ( + param ( [Parameter()][Switch]$Force, [Parameter()]$version = $( Get-GlobalJsonSdkVersion ) ) @@ -363,7 +364,7 @@ function ConvertTo-PortableVersion { Add-Member -inputobject $customObject -Type ScriptMethod -Name IsContainedIn -Value { param ( [object[]]$collection ) foreach ( $object in $collection ) { - if ( + if ( $this.Major -eq $object.Major -and $this.Minor -eq $object.Minor -and $this.Patch -eq $object.Patch -and @@ -422,10 +423,10 @@ function Get-InstalledCLIVersion { # earlier versions of dotnet do not support --list-sdks, so we'll check the output # and use dotnet --version as a fallback - $sdkList = dotnet --list-sdks 2>&1 + $sdkList = & $script:dotnetExe --list-sdks 2>&1 $sdkList = "Unknown option" if ( $sdkList -match "Unknown option" ) { - $installedVersions = dotnet --version + $installedVersions = & $script:dotnetExe --version } else { $installedVersions = $sdkList | Foreach-Object { $_.Split()[0] } @@ -475,3 +476,28 @@ function Receive-DotnetInstallScript return $installScript.FullName } + +function Get-DotnetExe +{ + $discoveredDotNet = Get-Command -CommandType Application dotnet + if ( $discoveredDotNet ) { + $discoveredDotNet | Select-Object -First 1 | Foreach-Object { $_.Source } + return + } + # it's not in the path, try harder to find it + # check the usual places + if ( ! (test-path variable:IsWindows) -or $IsWindows ) { + $dotnetHuntPath = "$HOME\AppData\Local\Microsoft\dotnet\dotnet.exe" + if ( test-path $dotnetHuntPath ) { + return $dotnetHuntPath + } + } + else { + $dotnetHuntPath = "$HOME/.dotnet/dotnet" + if ( test-path $dotnetHuntPath ) { + return $dotnetHuntPath + } + } + throw "Could not find dotnet executable" +} +$script:dotnetExe = Get-DotnetExe From d9833c47d953a87413e8fc9fd1bfc329c342871d Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 12 Feb 2019 16:47:36 -0800 Subject: [PATCH 09/25] Fix typo when checking for usable versions of dotnet --- build.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index cdd8fbe06..d190b83db 100644 --- a/build.psm1 +++ b/build.psm1 @@ -386,6 +386,7 @@ function Test-SuitableDotnet { $availableVersions = $( Get-InstalledCliVersion), $requiredVersion = $( Get-GlobalJsonSdkVersion ) ) + if ( $requiredVersion -is [String] -or $requiredVersion -is [Version] ) { $requiredVersion = ConvertTo-PortableVersion "$requiredVersion" } @@ -406,7 +407,7 @@ function Test-SuitableDotnet { } $requiredPatch = $requiredVersion.Patch $possiblePatch = $version.Patch - if ( $requiredPatch -gt $possiblePath ) { + if ( $requiredPatch -gt $possiblePatch ) { continue } if ( ($requiredPatch - $possiblePatch) -ge 100 ) { From 4909995fd44fb7f58171d670d96919f25406543d Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 12 Feb 2019 17:05:14 -0800 Subject: [PATCH 10/25] use -version rather than ambiguous -v when installing dotnet --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index d190b83db..3f735b161 100644 --- a/build.psm1 +++ b/build.psm1 @@ -306,7 +306,7 @@ function Install-Dotnet $installScriptPath = Receive-DotnetInstallScript $installScriptName = [System.IO.Path]::GetFileName($installScriptPath) If ( $PSCmdlet.ShouldProcess("$installScriptName for $version")) { - & "${installScriptPath}" -c release -v $version + & "${installScriptPath}" -c release -version $version } } catch { From b3d23cfb9ddff36b27e7cbfee5ca56e46f895613 Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 12 Feb 2019 17:23:53 -0800 Subject: [PATCH 11/25] Improve error message when hunting for cli version Use output of dotnet --version if there are problems with --list-sdks --- build.psm1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.psm1 b/build.psm1 index 3f735b161..2a0b02c3b 100644 --- a/build.psm1 +++ b/build.psm1 @@ -423,7 +423,6 @@ function Get-InstalledCLIVersion { try { # earlier versions of dotnet do not support --list-sdks, so we'll check the output # and use dotnet --version as a fallback - $sdkList = & $script:dotnetExe --list-sdks 2>&1 $sdkList = "Unknown option" if ( $sdkList -match "Unknown option" ) { @@ -434,7 +433,8 @@ function Get-InstalledCLIVersion { } } catch { - $installedVersions = @() + Write-Verbose -Verbose "$_" + $installedVersions = & $script:dotnetExe --version } return (ConvertTo-PortableVersion $installedVersions) } From 020c0865e4172996a204eb98f1002e398ab758ed Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 14 Feb 2019 10:32:10 -0800 Subject: [PATCH 12/25] ignore errors when first attempting to find dotnet executable set failure of finding dotnet as a warning, we will attempt to install it during bootstrap this handles the case of a blank slate where dotnet has never been installed --- build.psm1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.psm1 b/build.psm1 index 2a0b02c3b..519e40213 100644 --- a/build.psm1 +++ b/build.psm1 @@ -480,7 +480,7 @@ function Receive-DotnetInstallScript function Get-DotnetExe { - $discoveredDotNet = Get-Command -CommandType Application dotnet + $discoveredDotNet = Get-Command -CommandType Application dotnet -ErrorAction SilentlyContinue if ( $discoveredDotNet ) { $discoveredDotNet | Select-Object -First 1 | Foreach-Object { $_.Source } return @@ -499,6 +499,7 @@ function Get-DotnetExe return $dotnetHuntPath } } - throw "Could not find dotnet executable" + Write-Warning "Could not find dotnet executable" + return [String]::Empty } $script:dotnetExe = Get-DotnetExe From 34835b883ef5909c457beab2b503b95b0f8d8dcd Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 14 Feb 2019 10:56:39 -0800 Subject: [PATCH 13/25] Improve logic for handling a system where dotnet has never been installed --- build.psm1 | 33 +++++++++++++++++++++++++++++---- tools/appveyor.psm1 | 3 ++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/build.psm1 b/build.psm1 index 519e40213..526dd9c3d 100644 --- a/build.psm1 +++ b/build.psm1 @@ -128,10 +128,13 @@ function Start-ScriptAnalyzerBuild BEGIN { # don't allow the build to be started unless we have the proper Cli version + # this will not actually install dotnet if it's already present, but it will + # install the proper version + Install-Dotnet if ( -not (Test-SuitableDotnet) ) { $requiredVersion = Get-GlobalJsonSdkVersion $foundVersion = Get-InstalledCLIVersion - throw "No suitable dotnet CLI found, requires version '$requiredVersion' found only '$foundVersion'" + Write-Warning "No suitable dotnet CLI found, requires version '$requiredVersion' found only '$foundVersion'" } } END { @@ -202,6 +205,9 @@ function Start-ScriptAnalyzerBuild try { Push-Location $projectRoot/Rules Write-Progress "Building ScriptAnalyzer for PSVersion '$PSVersion' using framework '$framework' and configuration '$Configuration'" + if ( -not $script:DotnetExe ) { + $script:dotnetExe = Get-DotnetExe + } $buildOutput = & $script:dotnetExe build --framework $framework --configuration "$config" if ( $LASTEXITCODE -ne 0 ) { throw "$buildOutput" } } @@ -308,6 +314,11 @@ function Install-Dotnet If ( $PSCmdlet.ShouldProcess("$installScriptName for $version")) { & "${installScriptPath}" -c release -version $version } + # this may be the first time that dotnet has been installed, + # set up the executable variable + if ( -not $script:DotnetExe ) { + $script:DotnetExe = Get-DotnetExe + } } catch { throw $_ @@ -336,7 +347,10 @@ function ConvertTo-PortableVersion { foreach ( $v in $strVersion ) { $ver, $pre = $v.split("-",2) try { - [int]$major, [int]$minor, [int]$patch = $ver.Split(".") + [int]$major, [int]$minor, [int]$patch, $unused = $ver.Split(".",4) + if ( -not $pre ) { + $pre = $unused + } } catch { Write-Warning "Cannot convert '$v' to portable version" @@ -420,6 +434,11 @@ function Test-SuitableDotnet { # these are mockable functions for testing function Get-InstalledCLIVersion { + # dotnet might not have been installed _ever_, so just return 0.0.0.0 + if ( -not $script:DotnetExe ) { + Write-Warning "Dotnet executable not found" + return (ConvertTo-PortableVersion 0.0.0) + } try { # earlier versions of dotnet do not support --list-sdks, so we'll check the output # and use dotnet --version as a fallback @@ -482,20 +501,26 @@ function Get-DotnetExe { $discoveredDotNet = Get-Command -CommandType Application dotnet -ErrorAction SilentlyContinue if ( $discoveredDotNet ) { - $discoveredDotNet | Select-Object -First 1 | Foreach-Object { $_.Source } - return + $dotnetFoundPath = $discoveredDotNet | Select-Object -First 1 | Foreach-Object { $_.Source } + Write-Verbose -Verbose "Found dotnet here: $dotnetFoundPath" + $script:DotnetExe = $dotnetFoundPath + return $dotnetFoundPath } # it's not in the path, try harder to find it # check the usual places if ( ! (test-path variable:IsWindows) -or $IsWindows ) { $dotnetHuntPath = "$HOME\AppData\Local\Microsoft\dotnet\dotnet.exe" + Write-Verbose -Verbose "checking $dotnetHuntPath" if ( test-path $dotnetHuntPath ) { + $script:DotnetExe = $dotnetHuntPath return $dotnetHuntPath } } else { $dotnetHuntPath = "$HOME/.dotnet/dotnet" + Write-Verbose -Verbose "checking $dotnetHuntPath" if ( test-path $dotnetHuntPath ) { + $script:DotnetExe = $dotnetHuntPath return $dotnetHuntPath } } diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index c444c8d93..51547cc86 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -31,8 +31,9 @@ function Invoke-AppVeyorInstall { Install-Module -Name platyPS -Force -Scope CurrentUser -RequiredVersion $platyPSVersion } - Write-Verbose -Verbose "Installing required .Net CORE SDK $requiredDotNetCoreSDKVersion" # the build script sorts out the problems of WMF4 and earlier versions of dotnet CLI + Write-Verbose -Verbose "Installing required .Net CORE SDK" + Write-Verbose "& $buildScriptDir/build.ps1 -bootstrap" $buildScriptDir = (Resolve-Path "$PSScriptRoot/..").Path & "$buildScriptDir/build.ps1" -bootstrap } From e3752cc657d06808016abec0d23b7ffee7a410fb Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 14 Feb 2019 12:53:46 -0800 Subject: [PATCH 14/25] fine tune messages emitted during the hunt for dotnet --- build.psm1 | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/build.psm1 b/build.psm1 index 526dd9c3d..6371f76cf 100644 --- a/build.psm1 +++ b/build.psm1 @@ -206,9 +206,9 @@ function Start-ScriptAnalyzerBuild Push-Location $projectRoot/Rules Write-Progress "Building ScriptAnalyzer for PSVersion '$PSVersion' using framework '$framework' and configuration '$Configuration'" if ( -not $script:DotnetExe ) { - $script:dotnetExe = Get-DotnetExe + $script:DotnetExe = Get-DotnetExe } - $buildOutput = & $script:dotnetExe build --framework $framework --configuration "$config" + $buildOutput = & $script:DotnetExe build --framework $framework --configuration "$config" if ( $LASTEXITCODE -ne 0 ) { throw "$buildOutput" } } catch { @@ -442,10 +442,10 @@ function Get-InstalledCLIVersion { try { # earlier versions of dotnet do not support --list-sdks, so we'll check the output # and use dotnet --version as a fallback - $sdkList = & $script:dotnetExe --list-sdks 2>&1 + $sdkList = & $script:DotnetExe --list-sdks 2>&1 $sdkList = "Unknown option" if ( $sdkList -match "Unknown option" ) { - $installedVersions = & $script:dotnetExe --version + $installedVersions = & $script:DotnetExe --version } else { $installedVersions = $sdkList | Foreach-Object { $_.Split()[0] } @@ -453,7 +453,7 @@ function Get-InstalledCLIVersion { } catch { Write-Verbose -Verbose "$_" - $installedVersions = & $script:dotnetExe --version + $installedVersions = & $script:DotnetExe --version } return (ConvertTo-PortableVersion $installedVersions) } @@ -501,16 +501,15 @@ function Get-DotnetExe { $discoveredDotNet = Get-Command -CommandType Application dotnet -ErrorAction SilentlyContinue if ( $discoveredDotNet ) { - $dotnetFoundPath = $discoveredDotNet | Select-Object -First 1 | Foreach-Object { $_.Source } Write-Verbose -Verbose "Found dotnet here: $dotnetFoundPath" - $script:DotnetExe = $dotnetFoundPath - return $dotnetFoundPath + $script:DotnetExe = $discoveredDotNet + return $discoveredDotNet } # it's not in the path, try harder to find it # check the usual places if ( ! (test-path variable:IsWindows) -or $IsWindows ) { $dotnetHuntPath = "$HOME\AppData\Local\Microsoft\dotnet\dotnet.exe" - Write-Verbose -Verbose "checking $dotnetHuntPath" + Write-Verbose -Verbose "checking Windows $dotnetHuntPath" if ( test-path $dotnetHuntPath ) { $script:DotnetExe = $dotnetHuntPath return $dotnetHuntPath @@ -518,13 +517,14 @@ function Get-DotnetExe } else { $dotnetHuntPath = "$HOME/.dotnet/dotnet" - Write-Verbose -Verbose "checking $dotnetHuntPath" + Write-Verbose -Verbose "checking non-Windows $dotnetHuntPath" if ( test-path $dotnetHuntPath ) { $script:DotnetExe = $dotnetHuntPath return $dotnetHuntPath } } + Write-Warning "Could not find dotnet executable" return [String]::Empty } -$script:dotnetExe = Get-DotnetExe +$script:DotnetExe = Get-DotnetExe From 8a7ea478a25e5442254c45f6002325bd3dd868a0 Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 14 Feb 2019 13:48:44 -0800 Subject: [PATCH 15/25] additional logic for finding dotnet executable --- build.psm1 | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/build.psm1 b/build.psm1 index 6371f76cf..bf8910c25 100644 --- a/build.psm1 +++ b/build.psm1 @@ -499,11 +499,13 @@ function Receive-DotnetInstallScript function Get-DotnetExe { - $discoveredDotNet = Get-Command -CommandType Application dotnet -ErrorAction SilentlyContinue - if ( $discoveredDotNet ) { - Write-Verbose -Verbose "Found dotnet here: $dotnetFoundPath" - $script:DotnetExe = $discoveredDotNet - return $discoveredDotNet + $discoveredDotnet = Get-Command -CommandType Application dotnet -ErrorAction SilentlyContinu + if ( $discoveredDotnet ) { + # it's possible that there are multiples. Take the highest version we find + $latestDotnet = $discoveredDotNet | Sort-object { [version](& $_ --version) } | Select-Object -Last 1 + Write-Verbose -Verbose "Found dotnet here: $latestDotnet" + $script:DotnetExe = $latestDotnet + return $latestDotnet } # it's not in the path, try harder to find it # check the usual places From c19ae0ad2b802fc7485df7dbc4e04e27de99b306 Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 14 Feb 2019 14:12:17 -0800 Subject: [PATCH 16/25] harden search for dotnet Sometimes you can't even invoke dotnet if the version is too low --- build.psm1 | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/build.psm1 b/build.psm1 index bf8910c25..8be622fd1 100644 --- a/build.psm1 +++ b/build.psm1 @@ -502,13 +502,19 @@ function Get-DotnetExe $discoveredDotnet = Get-Command -CommandType Application dotnet -ErrorAction SilentlyContinu if ( $discoveredDotnet ) { # it's possible that there are multiples. Take the highest version we find - $latestDotnet = $discoveredDotNet | Sort-object { [version](& $_ --version) } | Select-Object -Last 1 - Write-Verbose -Verbose "Found dotnet here: $latestDotnet" - $script:DotnetExe = $latestDotnet - return $latestDotnet - } - # it's not in the path, try harder to find it - # check the usual places + # the problem is that invoking dotnet on a version which is lower than the specified + # version in global.json will produce an error, so we can only take the dotnet which executes + $latestDotnet = $discoveredDotNet | + Where-Object { try { & $_ --version } catch { } } | + Sort-Object { [version](& $_ --version) } | + Select-Object -Last 1 + if ( $latestDotnet ) { + Write-Verbose -Verbose "Found dotnet here: $latestDotnet" + $script:DotnetExe = $latestDotnet + return $latestDotnet + } + } + # it's not in the path, try harder to find it by checking some usual places if ( ! (test-path variable:IsWindows) -or $IsWindows ) { $dotnetHuntPath = "$HOME\AppData\Local\Microsoft\dotnet\dotnet.exe" Write-Verbose -Verbose "checking Windows $dotnetHuntPath" From a3e81e132c3749c33860396e07c64d5faf762bf6 Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 14 Feb 2019 15:57:05 -0800 Subject: [PATCH 17/25] Attempt to quieten build output. Throw away stderr of dotnet execution under some case and emit it in others. Change ToString of PortableVersion to enable better sorts --- build.psm1 | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/build.psm1 b/build.psm1 index 8be622fd1..535ff8ebd 100644 --- a/build.psm1 +++ b/build.psm1 @@ -208,7 +208,7 @@ function Start-ScriptAnalyzerBuild if ( -not $script:DotnetExe ) { $script:DotnetExe = Get-DotnetExe } - $buildOutput = & $script:DotnetExe build --framework $framework --configuration "$config" + $buildOutput = & $script:DotnetExe build --framework $framework --configuration "$config" 2>&1 if ( $LASTEXITCODE -ne 0 ) { throw "$buildOutput" } } catch { @@ -368,8 +368,11 @@ function ConvertTo-PortableVersion { $h['PrereleaseLabel'] = [String]::Empty } $customObject = [pscustomobject]$h + # we do this so we can get an approximate sort, since this implements a pseudo-version + # type in script, we need a way to find the highest version of dotnet, it's not a great solution + # but it will work in most cases. Add-Member -inputobject $customObject -Type ScriptMethod -Name ToString -Force -Value { - $str = "{0}.{1}.{2}" -f $this.Major,$this.Minor,$this.Patch + $str = "{0:0000}.{1:0000}.{2:0000}.{3:0000}" -f $this.Major,$this.Minor,$this.Patch if ( $this.PrereleaseLabel ) { $str += "-{0}" -f $this.PrereleaseLabel } @@ -445,7 +448,7 @@ function Get-InstalledCLIVersion { $sdkList = & $script:DotnetExe --list-sdks 2>&1 $sdkList = "Unknown option" if ( $sdkList -match "Unknown option" ) { - $installedVersions = & $script:DotnetExe --version + $installedVersions = & $script:DotnetExe --version 2>$null } else { $installedVersions = $sdkList | Foreach-Object { $_.Split()[0] } @@ -453,7 +456,7 @@ function Get-InstalledCLIVersion { } catch { Write-Verbose -Verbose "$_" - $installedVersions = & $script:DotnetExe --version + $installedVersions = & $script:DotnetExe --version 2>$null } return (ConvertTo-PortableVersion $installedVersions) } @@ -505,8 +508,8 @@ function Get-DotnetExe # the problem is that invoking dotnet on a version which is lower than the specified # version in global.json will produce an error, so we can only take the dotnet which executes $latestDotnet = $discoveredDotNet | - Where-Object { try { & $_ --version } catch { } } | - Sort-Object { [version](& $_ --version) } | + Where-Object { try { & $_ --version 2>$null } catch { } } | + Sort-Object { $pv = ConvertTo-PortableVersion (& $_ --version 2>$null ); "$pv" } | Select-Object -Last 1 if ( $latestDotnet ) { Write-Verbose -Verbose "Found dotnet here: $latestDotnet" From 16ae243603cfa28c984f1997ed06535366b20ff9 Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 14 Feb 2019 16:14:43 -0800 Subject: [PATCH 18/25] Fix tostring method for portable version --- build.psm1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.psm1 b/build.psm1 index 535ff8ebd..a9baa1367 100644 --- a/build.psm1 +++ b/build.psm1 @@ -372,7 +372,7 @@ function ConvertTo-PortableVersion { # type in script, we need a way to find the highest version of dotnet, it's not a great solution # but it will work in most cases. Add-Member -inputobject $customObject -Type ScriptMethod -Name ToString -Force -Value { - $str = "{0:0000}.{1:0000}.{2:0000}.{3:0000}" -f $this.Major,$this.Minor,$this.Patch + $str = "{0:0000}.{1:0000}.{2:0000}" -f $this.Major,$this.Minor,$this.Patch if ( $this.PrereleaseLabel ) { $str += "-{0}" -f $this.PrereleaseLabel } @@ -509,7 +509,7 @@ function Get-DotnetExe # version in global.json will produce an error, so we can only take the dotnet which executes $latestDotnet = $discoveredDotNet | Where-Object { try { & $_ --version 2>$null } catch { } } | - Sort-Object { $pv = ConvertTo-PortableVersion (& $_ --version 2>$null ); "$pv" } | + Sort-Object { $pv = ConvertTo-PortableVersion (& $_ --version 2>$null); "$pv" } | Select-Object -Last 1 if ( $latestDotnet ) { Write-Verbose -Verbose "Found dotnet here: $latestDotnet" From 1747530e9a5c26c731d4854c3208331d61405afe Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 14 Feb 2019 18:11:12 -0800 Subject: [PATCH 19/25] add -Raw flag to Get-GlobalJsonSdkVersion Sometimes we need the raw version so the installation script works, but sometimes we need to be able to compare it with other versions. --- build.psm1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build.psm1 b/build.psm1 index a9baa1367..ff57dabe6 100644 --- a/build.psm1 +++ b/build.psm1 @@ -294,7 +294,7 @@ function Install-Dotnet [CmdletBinding(SupportsShouldProcess=$true)] param ( [Parameter()][Switch]$Force, - [Parameter()]$version = $( Get-GlobalJsonSdkVersion ) + [Parameter()]$version = $( Get-GlobalJsonSdkVersion -Raw ) ) if ( Test-DotnetInstallation -requestedversion $version ) { @@ -332,9 +332,15 @@ function Install-Dotnet } function Get-GlobalJsonSdkVersion { + param ( [switch]$Raw ) $json = Get-Content -raw (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json $version = $json.sdk.Version - ConvertTo-PortableVersion $version + if ( $Raw ) { + return $version + } + else { + ConvertTo-PortableVersion $version + } } # we don't have semantic version in earlier versions of PowerShell, so we need to From 264652dbe73a85ca60c19e18e0a90ecee58716d4 Mon Sep 17 00:00:00 2001 From: James Truher Date: Fri, 15 Feb 2019 09:09:59 -0800 Subject: [PATCH 20/25] Attempt to harden the upload code Also add additional verbose output to improve debuggability print response from upload --- tools/appveyor.psm1 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index 51547cc86..4c56e95b7 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -50,10 +50,14 @@ function Invoke-AppveyorTest { $modulePath = $env:PSModulePath.Split([System.IO.Path]::PathSeparator) | Where-Object { Test-Path $_} | Select-Object -First 1 Copy-Item "${CheckoutPath}\out\PSScriptAnalyzer" "$modulePath\" -Recurse -Force - $testResultsFile = ".\TestResults.xml" + $testResultsPath = Join-Path ${CheckoutPath} TestResults.xml $testScripts = "${CheckoutPath}\Tests\Engine","${CheckoutPath}\Tests\Rules","${CheckoutPath}\Tests\Documentation" - $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsFile -PassThru - (New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}", (Resolve-Path $testResultsFile)) + $uploadUrl = "https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}" + $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsPath -PassThru + Write-Verbose -Verbose "Uploading test results '$testResultsPath' to '${uploadUrl}'" + $response = (New-Object 'System.Net.WebClient').UploadFile("$uploadUrl" , $testResultsPath) + $responseString = [System.Text.Encoding]::ASCII.GetString($response) + Write-Verbose -Verbose ("Response: ({0} bytes) ${responseString}" -f $response.Count) if ($testResults.FailedCount -gt 0) { throw "$($testResults.FailedCount) tests failed." } From cd1cbb9deaf912e34568bafdab6bafaba70cde9b Mon Sep 17 00:00:00 2001 From: James Truher Date: Fri, 15 Feb 2019 15:18:12 -0800 Subject: [PATCH 21/25] upgrade pester version to 4.4.4 --- tools/appveyor.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index 4c56e95b7..7f8444138 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -5,7 +5,7 @@ $ErrorActionPreference = 'Stop' # Implements the AppVeyor 'install' step and installs the required versions of Pester, platyPS and the .Net Core SDK if needed. function Invoke-AppVeyorInstall { - $requiredPesterVersion = '4.4.1' + $requiredPesterVersion = '4.4.4' $pester = Get-Module Pester -ListAvailable | Where-Object { $_.Version -eq $requiredPesterVersion } if ($null -eq $pester) { if ($null -eq (Get-Module -ListAvailable PowershellGet)) { @@ -70,6 +70,7 @@ function Invoke-AppveyorFinish { Add-Type -AssemblyName 'System.IO.Compression.FileSystem' [System.IO.Compression.ZipFile]::CreateFromDirectory((Join-Path $pwd 'out'), $zipFile) @( + (Get-ChildItem TestResults.xml) # You can add other artifacts here (Get-ChildItem $zipFile) ) | ForEach-Object { Push-AppveyorArtifact $_.FullName } From fb8013ae4895b0fe6faaec0a94da4f8520fcaab8 Mon Sep 17 00:00:00 2001 From: James Truher Date: Fri, 15 Feb 2019 16:17:44 -0800 Subject: [PATCH 22/25] Force the testsuite TestFixture to be named 'Pester' to get through appveyors test result recognition --- tools/appveyor.psm1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index 7f8444138..bbb750831 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -55,6 +55,14 @@ function Invoke-AppveyorTest { $uploadUrl = "https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}" $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsPath -PassThru Write-Verbose -Verbose "Uploading test results '$testResultsPath' to '${uploadUrl}'" + # there seems to be a problem where sometimes the test-suite name is not set to Pester + # which causes appveyor to not recognise it as test results + # attempt to force it to look right + $x = [xml](Get-Content $testResultsPath) + if ( $x."test-results"."test-suite".name -ne "Pester" ) { + $x."test-results"."test-suite".name = "Pester" + $x.Save($testResultsPath) + } $response = (New-Object 'System.Net.WebClient').UploadFile("$uploadUrl" , $testResultsPath) $responseString = [System.Text.Encoding]::ASCII.GetString($response) Write-Verbose -Verbose ("Response: ({0} bytes) ${responseString}" -f $response.Count) From 44a370b011abc11bf5d8bbed75477a95d1cc51e0 Mon Sep 17 00:00:00 2001 From: James Truher Date: Sat, 16 Feb 2019 13:05:46 -0800 Subject: [PATCH 23/25] Emit env:LANG to output before executing tests We've had trouble with LANG being set to something that causes problems, so make sure we report what it is. Removed a bit of the verbosity which was added for debugging --- build.psm1 | 2 -- tools/appveyor.psm1 | 14 +++----------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/build.psm1 b/build.psm1 index ff57dabe6..4bc0978d4 100644 --- a/build.psm1 +++ b/build.psm1 @@ -298,7 +298,6 @@ function Install-Dotnet ) if ( Test-DotnetInstallation -requestedversion $version ) { - Write-Verbose -Verbose "dotnet version '$version' already installed" if ( $Force ) { Write-Verbose -Verbose "Installing again" } @@ -518,7 +517,6 @@ function Get-DotnetExe Sort-Object { $pv = ConvertTo-PortableVersion (& $_ --version 2>$null); "$pv" } | Select-Object -Last 1 if ( $latestDotnet ) { - Write-Verbose -Verbose "Found dotnet here: $latestDotnet" $script:DotnetExe = $latestDotnet return $latestDotnet } diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index bbb750831..373e44292 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -47,6 +47,7 @@ function Invoke-AppveyorTest { ) Write-Verbose -Verbose ("Running tests on PowerShell version " + $PSVersionTable.PSVersion) + Write-Verbose -Verbose "Language set to '${env:LANG}'" $modulePath = $env:PSModulePath.Split([System.IO.Path]::PathSeparator) | Where-Object { Test-Path $_} | Select-Object -First 1 Copy-Item "${CheckoutPath}\out\PSScriptAnalyzer" "$modulePath\" -Recurse -Force @@ -55,17 +56,7 @@ function Invoke-AppveyorTest { $uploadUrl = "https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}" $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsPath -PassThru Write-Verbose -Verbose "Uploading test results '$testResultsPath' to '${uploadUrl}'" - # there seems to be a problem where sometimes the test-suite name is not set to Pester - # which causes appveyor to not recognise it as test results - # attempt to force it to look right - $x = [xml](Get-Content $testResultsPath) - if ( $x."test-results"."test-suite".name -ne "Pester" ) { - $x."test-results"."test-suite".name = "Pester" - $x.Save($testResultsPath) - } - $response = (New-Object 'System.Net.WebClient').UploadFile("$uploadUrl" , $testResultsPath) - $responseString = [System.Text.Encoding]::ASCII.GetString($response) - Write-Verbose -Verbose ("Response: ({0} bytes) ${responseString}" -f $response.Count) + [byte[]]$response = (New-Object 'System.Net.WebClient').UploadFile("$uploadUrl" , $testResultsPath) if ($testResults.FailedCount -gt 0) { throw "$($testResults.FailedCount) tests failed." } @@ -78,6 +69,7 @@ function Invoke-AppveyorFinish { Add-Type -AssemblyName 'System.IO.Compression.FileSystem' [System.IO.Compression.ZipFile]::CreateFromDirectory((Join-Path $pwd 'out'), $zipFile) @( + # add test results as an artifact (Get-ChildItem TestResults.xml) # You can add other artifacts here (Get-ChildItem $zipFile) From 58e9216caa27b9d35dc446eeb26d93f2d4a2f9f9 Mon Sep 17 00:00:00 2001 From: James Truher Date: Mon, 18 Feb 2019 22:15:05 -0800 Subject: [PATCH 24/25] Create tests for build module Fix some errors in the module discovered by the tests --- BuildModule.tests.ps1 | 91 +++++++++++++++++++++++++++++++++++++++++++ build.psm1 | 37 ++++++++++++------ 2 files changed, 116 insertions(+), 12 deletions(-) create mode 100644 BuildModule.tests.ps1 diff --git a/BuildModule.tests.ps1 b/BuildModule.tests.ps1 new file mode 100644 index 000000000..f809ec40d --- /dev/null +++ b/BuildModule.tests.ps1 @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# these are tests for the build module + +import-module -force "./build.psm1" +Describe "Build Module Tests" { + Context "Global.json" { + BeforeAll { + $globalJson = Get-Content (Join-Path $PSScriptRoot global.json) | ConvertFrom-Json + $expectedVersion = $globalJson.sdk.version + $result = Get-GlobalJsonSdkVersion + } + $propertyTestcases = @{ Name = "Major"; Type = "System.Int32" }, + @{ Name = "Minor"; Type = "System.Int32" }, + @{ Name = "Patch"; Type = "System.Int32" }, + @{ Name = "PrereleaseLabel"; Type = "System.String" } + It "Get-GlobalJsonSdkVersion returns a portable version object with property '' with type ''" -TestCases $propertyTestcases { + param ( $Name, $Type ) + $result.psobject.properties[$Name] | Should -BeOfType [System.Management.Automation.PSNoteProperty] + $result.psobject.properties[$Name].TypeNameOfValue | Should -Be $Type + } + It "Can retrieve the version from global.json" { + $result = Get-GlobalJsonSdkVersion + $resultString = "{0}.{1}.{2}" -f $result.Major,$result.Minor,$result.Patch + if ( $result.prereleasestring ) { $resultString += "-" + $result.prereleasestring } + $resultString | Should -Be $expectedVersion + } + } + Context "Test-SuiteableDotnet" { + It "Test-SuitableDotnet should return true when the expected version matches the installed version" { + Test-SuitableDotnet -availableVersions 2.1.2 -requiredVersion 2.1.2 | Should -Be $true + } + It "Test-SuitableDotnet should return true when the expected version matches the available versions" { + Test-SuitableDotnet -availableVersions "2.1.1","2.1.2","2.1.3" -requiredVersion 2.1.2 | Should -Be $true + } + It "Test-SuitableDotnet should return false when the expected version does not match an available" { + Test-SuitableDotnet -availableVersions "2.2.100","2.2.300" -requiredVersion 2.2.200 | Should -Be $false + } + It "Test-SuitableDotnet should return false when the expected version does not match an available" { + Test-SuitableDotnet -availableVersions "2.2.100","2.2.300" -requiredVersion 2.2.105 | Should -Be $false + } + It "Test-SuitableDotnet should return true when the expected version matches an available" { + Test-SuitableDotnet -availableVersions "2.2.150","2.2.300" -requiredVersion 2.2.105 | Should -Be $true + } + It "Test-SuitableDotnet should return false when the expected version does not match an available" { + Test-SuitableDotnet -availableVersions "2.2.400","2.2.401","2.2.405" -requiredVersion "2.2.410" | Should -Be $false + } + } + + Context "Test-DotnetInstallation" { + BeforeAll { + $availableVersions = ConvertTo-PortableVersion -strVersion "2.2.400","2.2.401","2.2.405" + $foundVersion = ConvertTo-PortableVersion -strVersion 2.2.402 + $missingVersion = ConvertTo-PortableVersion -strVersion 2.2.410 + } + + It "Test-DotnetInstallation finds a good version" { + Mock Get-InstalledCLIVersion { return $availableVersions } + Mock Get-GlobalJSonSdkVersion { return $foundVersion } + $result = Test-DotnetInstallation -requestedVersion (Get-GlobalJsonSdkVersion) -installedVersions (Get-InstalledCLIVersion) + Assert-MockCalled "Get-InstalledCLIVersion" -Times 1 + Assert-MockCalled "Get-GlobalJsonSdkVersion" -Times 1 + $result | Should -Be $true + } + + It "Test-DotnetInstallation cannot find a good version should return false" { + Mock Get-InstalledCLIVersion { return $availableVersions } + Mock Get-GlobalJSonSdkVersion { return $missingVersion } + $result = Test-DotnetInstallation -requestedVersion (Get-GlobalJsonSdkVersion) -installedVersions (Get-InstalledCLIVersion) + Assert-MockCalled "Get-InstalledCLIVersion" -Times 1 + Assert-MockCalled "Get-GlobalJsonSdkVersion" -Times 1 + $result | Should -Be $false + } + } + + Context "Receive-DotnetInstallScript" { + Mock -ModuleName Build Receive-File { new-item -type file TestDrive:/dotnet-install.sh } + It "Downloads the proper file" { + try { + push-location TestDrive: + Receive-DotnetInstallScript -forceNonWindows + "TestDrive:/dotnet-install.sh" | Should -Exist + } + finally { + Pop-Location + } + } + + } +} diff --git a/build.psm1 b/build.psm1 index 4bc0978d4..9e2283d36 100644 --- a/build.psm1 +++ b/build.psm1 @@ -412,6 +412,9 @@ function Test-SuitableDotnet { if ( $requiredVersion -is [String] -or $requiredVersion -is [Version] ) { $requiredVersion = ConvertTo-PortableVersion "$requiredVersion" } + + $availableVersionList = $availableVersions | ForEach-Object { if ( $_ -is [string] -or $_ -is [version] ) { ConvertTo-PortableVersion $_ } else { $_ } } + $availableVersions = $availableVersionList # if we have what was requested, we can use it if ( $RequiredVersion.IsContainedIn($availableVersions)) { return $true @@ -429,13 +432,13 @@ function Test-SuitableDotnet { } $requiredPatch = $requiredVersion.Patch $possiblePatch = $version.Patch + if ( $requiredPatch -gt $possiblePatch ) { continue } - if ( ($requiredPatch - $possiblePatch) -ge 100 ) { - continue + if ( [math]::Abs(($requiredPatch - $possiblePatch)) -lt 100 ) { + return $true } - return $true } return $false } @@ -468,19 +471,16 @@ function Get-InstalledCLIVersion { function Test-DotnetInstallation { - param ( $requestedVersion = $( Get-GlobalJsonSdkVersion ) ) - $installedVersions = Get-InstalledCLIVersion + param ( + $requestedVersion = $( Get-GlobalJsonSdkVersion ), + $installedVersions = $( Get-InstalledCLIVersion ) + ) return (Test-SuitableDotnet -availableVersions $installedVersions -requiredVersion $requestedVersion ) } -function Receive-DotnetInstallScript -{ - $installScriptName = "dotnet-install.ps1" +function Receive-File { + param ( [Parameter(Mandatory,Position=0)]$uri ) - if ( (Test-Path Variable:IsWindows) -and -not $IsWindows ) { - $installScriptName = "dotnet-install.sh" - } - $uri = "https://dot.net/v1/${installScriptName}" # enable Tls12 for the request # -SslProtocol parameter for Invoke-WebRequest wasn't in PSv3 $securityProtocol = [System.Net.ServicePointManager]::SecurityProtocol @@ -501,7 +501,20 @@ function Receive-DotnetInstallScript if ( -not $installScript ) { throw "Download failure of ${uri}" } + return $installScript +} + +function Receive-DotnetInstallScript +{ + param ( [switch]$forceNonWindows ) + $installScriptName = "dotnet-install.ps1" + + if ( ((Test-Path Variable:IsWindows) -and -not $IsWindows) -or $forceNonWindows ) { + $installScriptName = "dotnet-install.sh" + } + $uri = "https://dot.net/v1/${installScriptName}" + $installScript = Receive-File -Uri $uri return $installScript.FullName } From 8dcfa8181c6880acebf1a1150ae94cacdf8c6f40 Mon Sep 17 00:00:00 2001 From: James Truher Date: Tue, 19 Feb 2019 12:25:28 -0800 Subject: [PATCH 25/25] Add additional tests Change logic when downloading the dotnet install script Also enforce en_US.UTF-8 language to ensure that appveyor can see our test results --- BuildModule.tests.ps1 | 66 +++++++++++++++++++++++++++++++++++++++++-- build.psm1 | 19 +++++++++++-- tools/appveyor.psm1 | 2 ++ 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/BuildModule.tests.ps1 b/BuildModule.tests.ps1 index f809ec40d..a9d6afb3a 100644 --- a/BuildModule.tests.ps1 +++ b/BuildModule.tests.ps1 @@ -75,11 +75,12 @@ Describe "Build Module Tests" { } Context "Receive-DotnetInstallScript" { + Mock -ModuleName Build Receive-File { new-item -type file TestDrive:/dotnet-install.sh } - It "Downloads the proper file" { + It "Downloads the proper non-Windows file" { try { push-location TestDrive: - Receive-DotnetInstallScript -forceNonWindows + Receive-DotnetInstallScript -platform NonWindows "TestDrive:/dotnet-install.sh" | Should -Exist } finally { @@ -87,5 +88,66 @@ Describe "Build Module Tests" { } } + Mock -ModuleName Build Receive-File { new-item -type file TestDrive:/dotnet-install.ps1 } + It "Downloads the proper file Windows file" { + try { + push-location TestDrive: + Receive-DotnetInstallScript -platform "Windows" + "TestDrive:/dotnet-install.ps1" | Should -Exist + } + finally { + Pop-Location + } + } + + } + + Context "Test result functions" { + BeforeAll { + $xmlFile = @' + + + + + + + + + + + + + + Expected 2, but got 1. + at <ScriptBlock>, /tmp/bad.tests.ps1: line 3 +3: It "a failing test" { 1 | Should -Be 2 } + + + + + + + + + +'@ + + $xmlFile | out-file TESTDRIVE:/results.xml + $results = Get-TestResults -logfile TESTDRIVE:/results.xml + $failures = Get-TestFailures -logfile TESTDRIVE:/results.xml + } + + It "Get-TestResults finds 2 results" { + $results.Count | Should -Be 2 + } + It "Get-TestResults finds 1 pass" { + @($results | ?{ $_.result -eq "Success" }).Count |Should -Be 1 + } + It "Get-TestResults finds 1 failure" { + @($results | ?{ $_.result -eq "Failure" }).Count |Should -Be 1 + } + It "Get-TestFailures finds 1 failure" { + $failures.Count | Should -Be 1 + } } } diff --git a/build.psm1 b/build.psm1 index 9e2283d36..e9837479d 100644 --- a/build.psm1 +++ b/build.psm1 @@ -506,12 +506,25 @@ function Receive-File { function Receive-DotnetInstallScript { - param ( [switch]$forceNonWindows ) - $installScriptName = "dotnet-install.ps1" + # param '$platform' is a hook to enable forcing download of a specific + # install script, generally it should not be used except in testing. + param ( $platform = "" ) - if ( ((Test-Path Variable:IsWindows) -and -not $IsWindows) -or $forceNonWindows ) { + # if $platform has been set, it has priority + # if it's not set to Windows or NonWindows, it will be ignored + if ( $platform -eq "Windows" ) { + $installScriptName = "dotnet-install.ps1" + } + elseif ( $platform -eq "NonWindows" ) { + $installScriptName = "dotnet-install.sh" + } + elseif ( ((Test-Path Variable:IsWindows) -and -not $IsWindows) ) { + # if the variable IsWindows exists and it is set to false $installScriptName = "dotnet-install.sh" } + else { # the default case - we're running on a Windows system + $installScriptName = "dotnet-install.ps1" + } $uri = "https://dot.net/v1/${installScriptName}" $installScript = Receive-File -Uri $uri diff --git a/tools/appveyor.psm1 b/tools/appveyor.psm1 index 373e44292..6367dda7d 100644 --- a/tools/appveyor.psm1 +++ b/tools/appveyor.psm1 @@ -46,6 +46,8 @@ function Invoke-AppveyorTest { $CheckoutPath ) + # enforce the language to utf-8 to avoid issues + $env:LANG = "en_US.UTF-8" Write-Verbose -Verbose ("Running tests on PowerShell version " + $PSVersionTable.PSVersion) Write-Verbose -Verbose "Language set to '${env:LANG}'"