Skip to content

Commit d74f0f2

Browse files
authored
Merge pull request #166 from jsinglet/jsinglet/matrix-testing
Matrix Testing Updates
2 parents e5fdac4 + 8374019 commit d74f0f2

File tree

2 files changed

+23
-25
lines changed

2 files changed

+23
-25
lines changed

scripts/matrix_testing/CreateMatrixTestReport.ps1

Lines changed: 17 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,12 @@ param(
123123
[string]
124124
$ReportDir = (Get-Location),
125125

126+
# Skip summary report -- used for Linux hosts that don't support
127+
# the OLE database stuff.
128+
[Parameter(Mandatory = $false)]
129+
[switch]
130+
$SkipSummaryReport,
131+
126132
# Tells the script to use the sytem tmp directory instead of the rule
127133
# directory.
128134
[Parameter(Mandatory = $false)]
@@ -245,6 +251,7 @@ else {
245251
Write-Host "Loaded $($queriesToCheck.Count) Queries."
246252
}
247253

254+
248255
#
249256
# Step 2: Verify All the Required CLI Tools are Installed
250257
#
@@ -290,8 +297,7 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
290297
"RULE" = $CurrentRuleName;
291298
"QUERY" = $CurrentQueryName;
292299
"COMPILE_PASS" = $false;
293-
"EXTRACTOR_PASS" = $false;
294-
"EXTRACTOR_ERRORS" = "";
300+
"COMPILE_ERROR_OUTPUT" = "";
295301
"TEST_PASS" = $false ;
296302
"TEST_DIFFERENCE" = "";
297303
}
@@ -316,32 +322,19 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
316322
}
317323
catch {
318324
Write-Host -ForegroundColor ([ConsoleColor]4) "FAILED"
325+
$row["COMPILE_ERROR_OUTPUT"] = $_
319326

320327
return $row # although it is unlikely to succeed with the next rule skipping to the next rule
321328
# ensures all of the rules will be reported in the
322329
# output.
323330
}
324331

325332
$row["COMPILE_PASS"] = $true
326-
Write-Host "Validating extractor results..." -NoNewline
327-
328-
try {
329-
$diagnostics = Execute-QueryAndDecodeAsJson -DatabasePath $db -QueryPath $diagnostic_query
330-
}catch {
331-
Write-Host -ForegroundColor ([ConsoleColor]4) $_Exception.Message
332-
return $row
333-
}
334-
335-
if ( $diagnostics.'#select'.tuples.Length -eq 0 ) {
336-
$row["EXTRACTOR_PASS"] = $true
337-
Write-Host -ForegroundColor ([ConsoleColor]2) "OK"
338-
} else {
339-
Write-Host -ForegroundColor ([ConsoleColor]4) "FAILED"
340-
$row["EXTRACTOR_ERRORS"] = $diagnostics | ConvertTo-Json -Depth 100
341-
}
342-
333+
343334
Write-Host "Checking expected output..."
344335

336+
# Dragons below 🐉🐉🐉
337+
#
345338
# Note this technique uses so-called "wizard" settings to make it possible
346339
# to compare hand compiled databases using qltest. The relative paths and
347340
# other options are required to be set as below (especially the detail about
@@ -381,7 +374,6 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
381374
Write-Host "Standard Out Buffered to: $stdOut"
382375
Write-Host "Standard Error Buffered to: $stdErr"
383376

384-
385377
$procDetails = Start-Process -FilePath "codeql" -PassThru -NoNewWindow -Wait -ArgumentList "test run $qlRefFile --dataset=`"$datasetRelPath`"" -RedirectStandardOutput $stdOut -RedirectStandardError $stdErr
386378

387379
if (-Not $procDetails.ExitCode -eq 0) {
@@ -420,6 +412,8 @@ foreach ($r in $REPORT) {
420412
[PSCustomObject]$r | Export-CSV -Path $reportOutputFile -Append -NoTypeInformation
421413
}
422414

423-
# write out a summary
424-
Write-Host "Writing summary report to $summaryReportOutputFile"
425-
Create-Summary-Report -DataFile $reportOutputFile -OutputFile $summaryReportOutputFile
415+
if (-not $SkipSummaryReport){
416+
# write out a summary
417+
Write-Host "Writing summary report to $summaryReportOutputFile"
418+
Create-Summary-Report -DataFile $reportOutputFile -OutputFile $summaryReportOutputFile
419+
}

scripts/matrix_testing/NewDatabaseForRule.ps1

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,14 @@ function New-Database-For-Rule {
3939

4040
Write-Host "codeql database create -l cpp -s $RuleTestDir --command='$BUILD_COMMAND' $DB_PATH"
4141

42-
$procDetails = Start-Process -FilePath "codeql" -PassThru -NoNewWindow -Wait -ArgumentList "database create -l cpp -s $RuleTestDir --command=`"$BUILD_COMMAND`" $DB_PATH"
42+
$stdOut = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid())
43+
44+
$procDetails = Start-Process -FilePath "codeql" -RedirectStandardOutput $stdOut -PassThru -NoNewWindow -Wait -ArgumentList "database create -l cpp -s $RuleTestDir --command=`"$BUILD_COMMAND`" $DB_PATH"
45+
46+
Get-Content $stdOut | Out-String | Write-Host
4347

4448
if (-Not $procDetails.ExitCode -eq 0) {
45-
throw "Database creation failed."
49+
throw Get-Content $stdOut | Out-String
4650
}
4751

4852
return $DB_PATH

0 commit comments

Comments
 (0)