From 7e8224119f0ac333387cfbf2c032ec16d73d18a8 Mon Sep 17 00:00:00 2001 From: "John L. Singleton" Date: Fri, 6 Jan 2023 18:00:37 -0500 Subject: [PATCH 1/2] skip summary report flag --- scripts/matrix_testing/CreateMatrixTestReport.ps1 | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/matrix_testing/CreateMatrixTestReport.ps1 b/scripts/matrix_testing/CreateMatrixTestReport.ps1 index 3af69e6353..7789308b0b 100644 --- a/scripts/matrix_testing/CreateMatrixTestReport.ps1 +++ b/scripts/matrix_testing/CreateMatrixTestReport.ps1 @@ -123,6 +123,12 @@ param( [string] $ReportDir = (Get-Location), + # Skip summary report -- used for Linux hosts that don't support + # the OLE database stuff. + [Parameter(Mandatory = $false)] + [switch] + $SkipSummaryReport, + # Tells the script to use the sytem tmp directory instead of the rule # directory. [Parameter(Mandatory = $false)] @@ -245,6 +251,7 @@ else { Write-Host "Loaded $($queriesToCheck.Count) Queries." } + # # Step 2: Verify All the Required CLI Tools are Installed # @@ -420,6 +427,8 @@ foreach ($r in $REPORT) { [PSCustomObject]$r | Export-CSV -Path $reportOutputFile -Append -NoTypeInformation } -# write out a summary -Write-Host "Writing summary report to $summaryReportOutputFile" -Create-Summary-Report -DataFile $reportOutputFile -OutputFile $summaryReportOutputFile \ No newline at end of file +if (-not $SkipSummaryReport){ + # write out a summary + Write-Host "Writing summary report to $summaryReportOutputFile" + Create-Summary-Report -DataFile $reportOutputFile -OutputFile $summaryReportOutputFile +} \ No newline at end of file From 10c3d662ebb2c61827285318bf687f061d7e43df Mon Sep 17 00:00:00 2001 From: "John L. Singleton" Date: Tue, 17 Jan 2023 14:27:53 -0500 Subject: [PATCH 2/2] wrk --- .../matrix_testing/CreateMatrixTestReport.ps1 | 25 ++++--------------- scripts/matrix_testing/NewDatabaseForRule.ps1 | 8 ++++-- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/scripts/matrix_testing/CreateMatrixTestReport.ps1 b/scripts/matrix_testing/CreateMatrixTestReport.ps1 index 7789308b0b..3a15148066 100644 --- a/scripts/matrix_testing/CreateMatrixTestReport.ps1 +++ b/scripts/matrix_testing/CreateMatrixTestReport.ps1 @@ -297,8 +297,7 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel "RULE" = $CurrentRuleName; "QUERY" = $CurrentQueryName; "COMPILE_PASS" = $false; - "EXTRACTOR_PASS" = $false; - "EXTRACTOR_ERRORS" = ""; + "COMPILE_ERROR_OUTPUT" = ""; "TEST_PASS" = $false ; "TEST_DIFFERENCE" = ""; } @@ -323,6 +322,7 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel } catch { Write-Host -ForegroundColor ([ConsoleColor]4) "FAILED" + $row["COMPILE_ERROR_OUTPUT"] = $_ return $row # although it is unlikely to succeed with the next rule skipping to the next rule # ensures all of the rules will be reported in the @@ -330,25 +330,11 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel } $row["COMPILE_PASS"] = $true - Write-Host "Validating extractor results..." -NoNewline - - try { - $diagnostics = Execute-QueryAndDecodeAsJson -DatabasePath $db -QueryPath $diagnostic_query - }catch { - Write-Host -ForegroundColor ([ConsoleColor]4) $_Exception.Message - return $row - } - - if ( $diagnostics.'#select'.tuples.Length -eq 0 ) { - $row["EXTRACTOR_PASS"] = $true - Write-Host -ForegroundColor ([ConsoleColor]2) "OK" - } else { - Write-Host -ForegroundColor ([ConsoleColor]4) "FAILED" - $row["EXTRACTOR_ERRORS"] = $diagnostics | ConvertTo-Json -Depth 100 - } - + Write-Host "Checking expected output..." + # Dragons below 🐉🐉🐉 + # # Note this technique uses so-called "wizard" settings to make it possible # to compare hand compiled databases using qltest. The relative paths and # other options are required to be set as below (especially the detail about @@ -388,7 +374,6 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel Write-Host "Standard Out Buffered to: $stdOut" Write-Host "Standard Error Buffered to: $stdErr" - $procDetails = Start-Process -FilePath "codeql" -PassThru -NoNewWindow -Wait -ArgumentList "test run $qlRefFile --dataset=`"$datasetRelPath`"" -RedirectStandardOutput $stdOut -RedirectStandardError $stdErr if (-Not $procDetails.ExitCode -eq 0) { diff --git a/scripts/matrix_testing/NewDatabaseForRule.ps1 b/scripts/matrix_testing/NewDatabaseForRule.ps1 index 6dfd6fa86d..d771afc010 100644 --- a/scripts/matrix_testing/NewDatabaseForRule.ps1 +++ b/scripts/matrix_testing/NewDatabaseForRule.ps1 @@ -39,10 +39,14 @@ function New-Database-For-Rule { Write-Host "codeql database create -l cpp -s $RuleTestDir --command='$BUILD_COMMAND' $DB_PATH" - $procDetails = Start-Process -FilePath "codeql" -PassThru -NoNewWindow -Wait -ArgumentList "database create -l cpp -s $RuleTestDir --command=`"$BUILD_COMMAND`" $DB_PATH" + $stdOut = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid()) + + $procDetails = Start-Process -FilePath "codeql" -RedirectStandardOutput $stdOut -PassThru -NoNewWindow -Wait -ArgumentList "database create -l cpp -s $RuleTestDir --command=`"$BUILD_COMMAND`" $DB_PATH" + + Get-Content $stdOut | Out-String | Write-Host if (-Not $procDetails.ExitCode -eq 0) { - throw "Database creation failed." + throw Get-Content $stdOut | Out-String } return $DB_PATH