Add integration testing module with simple tests to verify PSES starts and stops (#944)

* Add integration testing module as a tool
* Add Pester execution and depedency to build
* Add simple start and stop integration tests as Pester tests

Author: rjmholt

.vsts-ci/templates/ci-general.yml

@@ -14,6 +14,11 @@ steps:
       testRunner: VSTest
       testResultsFiles: '**/*.trx'
     condition: succeededOrFailed()
+  - task: PublishTestResults@2
+    inputs:
+      testRunner: NUnit
+      testResultsFiles: '**/TestResults.xml'
+    condition: succeededOrFailed()
   - task: PublishBuildArtifacts@1
     inputs:
       ArtifactName: PowerShellEditorServices

PowerShellEditorServices.build.ps1

@@ -18,11 +18,11 @@ param(
 
 #Requires -Modules @{ModuleName="InvokeBuild";ModuleVersion="3.2.1"}
 
-$script:IsCIBuild = $env:TF_BUILD -ne $null
 $script:IsUnix = $PSVersionTable.PSEdition -and $PSVersionTable.PSEdition -eq "Core" -and !$IsWindows
 $script:TargetPlatform = "netstandard2.0"
 $script:TargetFrameworksParam = "/p:TargetFrameworks=`"$script:TargetPlatform`""
 $script:RequiredSdkVersion = (Get-Content (Join-Path $PSScriptRoot 'global.json') | ConvertFrom-Json).sdk.version
+$script:MinimumPesterVersion = '4.7'
 $script:NugetApiUriBase = 'https://www.nuget.org/api/v2/package'
 $script:ModuleBinPath = "$PSScriptRoot/module/PowerShellEditorServices/bin/"
 $script:VSCodeModuleBinPath = "$PSScriptRoot/module/PowerShellEditorServices.VSCode/bin/"
@@ -330,12 +330,17 @@ task Build {
     exec { & $script:dotnetExe build -c $Configuration .\src\PowerShellEditorServices.VSCode\PowerShellEditorServices.VSCode.csproj $script:TargetFrameworksParam }
 }
 
+task BuildPsesClientModule SetupDotNet,{
+    Write-Verbose 'Building PsesPsClient testing module'
+    & $PSScriptRoot/tools/PsesPsClient/build.ps1 -DotnetExe $script:dotnetExe
+}
+
 function DotNetTestFilter {
     # Reference https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests
     if ($TestFilter) { @("--filter",$TestFilter) } else { "" }
 }
 
-task Test TestServer,TestProtocol
+task Test TestServer,TestProtocol,TestPester
 
 task TestServer {
     Set-Location .\test\PowerShellEditorServices.Test\
@@ -373,6 +378,28 @@ task TestHost {
     exec { & $script:dotnetExe test -f $script:TestRuntime.Core (DotNetTestFilter) }
 }
 
+task TestPester Build,BuildPsesClientModule,EnsurePesterInstalled,{
+    $testParams = @{}
+    if ($env:TF_BUILD)
+    {
+        $testParams += @{
+            OutputFormat = 'NUnitXml'
+            OutputFile = 'TestResults.xml'
+        }
+    }
+    $result = Invoke-Pester "$PSScriptRoot/test/Pester/" @testParams -PassThru
+
+    if ($result.FailedCount -gt 0)
+    {
+        throw "$($result.FailedCount) tests failed."
+    }
+}
+
+task EnsurePesterInstalled -If (-not (Get-Module Pester -ListAvailable | Where-Object Version -ge $script:MinimumPesterVersion)) {
+    Write-Warning "Required Pester version not found, installing Pester to current user scope"
+    Install-Module -Scope CurrentUser Pester -Force -SkipPublisherCheck
+}
+
 task LayoutModule -After Build {
     # Copy Third Party Notices.txt to module folder
     Copy-Item -Force -Path "$PSScriptRoot\Third Party Notices.txt" -Destination $PSScriptRoot\module\PowerShellEditorServices

test/Pester/EditorServices.Integration.Tests.ps1

@@ -0,0 +1,86 @@
+
+$script:ExceptionRegex = [regex]::new('\s*Exception: (.*)$', 'Compiled,Multiline,IgnoreCase')
+function ReportLogErrors
+{
+    param(
+        [Parameter()][string]$LogPath,
+
+        [Parameter()][ref]<#[int]#>$FromIndex = 0,
+
+        [Parameter()][string[]]$IgnoreException = @()
+    )
+
+    $logEntries = Parse-PsesLog $LogPath |
+        Where-Object Index -ge $FromIndex.Value
+
+    # Update the index to the latest in the log
+    $FromIndex.Value = ($FromIndex.Value,$errorLogs.Index | Measure-Object -Maximum).Maximum
+
+    $errorLogs = $logEntries |
+        Where-Object LogLevel -eq Error |
+        Where-Object {
+            $match = $script:ExceptionRegex.Match($_.Message.Data)
+
+            (-not $match) -or ($match.Groups[1].Value.Trim() -notin $IgnoreException)
+        }
+
+    if ($errorLogs)
+    {
+        $errorLogs | ForEach-Object { Write-Error "ERROR from PSES log: $($_.Message.Data)" }
+    }
+}
+
+Describe "Loading and running PowerShellEditorServices" {
+    BeforeAll {
+        Import-Module -Force "$PSScriptRoot/../../module/PowerShellEditorServices"
+        Import-Module -Force "$PSScriptRoot/../../tools/PsesPsClient/out/PsesPsClient"
+        Import-Module -Force "$PSScriptRoot/../../tools/PsesLogAnalyzer"
+
+        $logIdx = 0
+        $psesServer = Start-PsesServer
+        $client = Connect-PsesServer -PipeName $psesServer.SessionDetails.languageServicePipeName
+    }
+
+    # This test MUST be first
+    It "Starts and responds to an initialization request" {
+        $request = Send-LspInitializeRequest -Client $client
+        $response = Get-LspResponse -Client $client -Id $request.Id
+        $response.Id | Should -BeExactly $request.Id
+
+        ReportLogErrors -LogPath $psesServer.LogPath -FromIndex ([ref]$logIdx)
+    }
+
+    # This test MUST be last
+    It "Shuts down the process properly" {
+        $request = Send-LspShutdownRequest -Client $client
+        $response = Get-LspResponse -Client $client -Id $request.Id
+        $response.Id | Should -BeExactly $request.Id
+        $response.Result | Should -BeNull
+        # TODO: The server seems to stay up waiting for the debug connection
+        # $psesServer.PsesProcess.HasExited | Should -BeTrue
+
+        # We close the process here rather than in an AfterAll
+        # since errors can occur and we want to test for them.
+        # Naturally this depends on Pester executing tests in order.
+
+        # We also have to dispose of everything properly,
+        # which means we have to use these cascading try/finally statements
+        try
+        {
+            $psesServer.PsesProcess.Kill()
+        }
+        finally
+        {
+            try
+            {
+                $psesServer.PsesProcess.Dispose()
+            }
+            finally
+            {
+                $client.Dispose()
+            }
+        }
+
+        ReportLogErrors -LogPath $psesServer.LogPath -FromIndex ([ref]$logIdx)
+    }
+}