Merge pull request #171 from github/jsinglet/slash-fixes

Cross Platform Automation Fixes + Additional Matrix Features + Additional Automation

Author: jsinglet

.github/workflows/dispatch-matrix-check.yml

@@ -0,0 +1,31 @@
+name: 🤖 Run Matrix Check 
+
+on:
+  pull_request:
+    branches:
+      - "**"
+  workflow_dispatch:
+
+jobs:
+  dispatch-matrix-check:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Dispatch Matrix Testing Job
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          token: ${{ secrets.RELEASE_ENGINEERING_TOKEN }}
+          repository: github/codeql-coding-standards-release-engineering
+          event-type: matrix-test
+          client-payload: '{"pr": "${{ github.event.number  }}"}'
+
+
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: '🤖 Beep Boop! Matrix Testing for this PR has been initiated. Please check back later for results. <br><br> :bulb: If you do not hear back from me please check my status! **I will report even if this PR does not contain files eligible for matrix testing.**'
+            })

.github/workflows/dispatch-release-performance-check.yml

@@ -0,0 +1,35 @@
+name: 🏁 Run Release Performance Check
+
+on:
+  issue_comment:
+    types: [created]
+    branches:
+      - main
+      - "rc/**"
+      - next
+
+jobs:
+  dispatch-matrix-check:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Dispatch Performance Testing Job
+        if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/test-performance') }}
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          token: ${{ secrets.RELEASE_ENGINEERING_TOKEN }}
+          repository: github/codeql-coding-standards-release-engineering
+          event-type: performance-test
+          client-payload: '{"pr": "${{ github.event.number  }}"}'
+
+
+      - uses: actions/github-script@v6
+        if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/test-performance') }}
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: '🏁 Beep Boop! Performance testing for this PR has been initiated. Please check back later for results. Note that the query package generation step must complete before testing  will start so it might be a minute. <br><br> :bulb: If you do not hear back from me please check my status! **I will report even if I fail!**'
+            })

change_notes/2023-02-10-refactor-test-cases.md

@@ -26,7 +26,7 @@ where
     c.getAnArgument() = flag and
     flag =
       [
-        "-Ofast", "-ffast-math", "-fgnu-keywords", "-fno-signed-zeroes", "-menable-unsafe-fp-math",
+        "-Ofast", "-ffast-math", "-fgnu-keywords", "-fno-signed-zeroes", "-fno-signed-zeros", "-menable-unsafe-fp-math",
         "-menable-no-nans", "-menable-no-infs", "-menable-unsafe-fp-math", "-ffinite-math-only",
         "-ffloat-store"
       ]

cpp/autosar/src/rules/A1-1-3/UncompliantOptimizationOptionMustBeDisabledInCompiler.ql

@@ -0,0 +1 @@
+-Wall 

cpp/autosar/test/rules/A0-1-1/options.clang

@@ -1,9 +1,9 @@
-
 // A POD class
 struct A {
   int f;
   int f2;
 };
+
 // A non-POD class, with a trivial compiler generated constructor
 struct B {
   int f;

cpp/autosar/test/rules/A0-1-1/test.cpp

@@ -0,0 +1 @@
+rules/A1-1-2/CompilerWarningLevelNotInCompliance.ql

cpp/autosar/test/rules/A1-1-2.1/CompilerWarningLevelNotInCompliance.expected

@@ -0,0 +1 @@
+@foo

cpp/autosar/test/rules/A1-1-2.1/CompilerWarningLevelNotInCompliance.qlref

@@ -0,0 +1 @@
+@foo

cpp/autosar/test/rules/A1-1-2/foo renamed to cpp/autosar/test/rules/A1-1-2.1/foo

@@ -0,0 +1 @@
+rules/A1-1-2/CompilerWarningLevelNotInCompliance.ql

cpp/autosar/test/rules/A1-1-2.1/options.clang

@@ -0,0 +1 @@
+-Wcast-function-type

cpp/autosar/test/rules/A1-1-2.1/options.gcc

@@ -0,0 +1 @@
+-Wcast-function-type

cpp/autosar/test/rules/A1-1-2/responsefile.cpp renamed to cpp/autosar/test/rules/A1-1-2.1/responsefile.cpp

@@ -0,0 +1 @@
+| test.cpp:0:0:0:0 | test.cpp | No warning-level options were used in the compilation of 'test.cpp'. |

cpp/autosar/test/rules/A1-1-2.2/CompilerWarningLevelNotInCompliance.expected

@@ -0,0 +1 @@
+rules/A1-1-2/CompilerWarningLevelNotInCompliance.ql

cpp/autosar/test/rules/A1-1-2.2/CompilerWarningLevelNotInCompliance.qlref

@@ -0,0 +1 @@
+-w

cpp/autosar/test/rules/A1-1-2/Wcast-function-type.cpp renamed to cpp/autosar/test/rules/A1-1-2.2/Wcast-function-type.cpp

@@ -1 +0,0 @@
-| test.cpp:0:0:0:0 | test.cpp | No warning-level options were used in the compilation of 'test.cpp'. |

cpp/autosar/test/rules/A1-1-2.2/options.clang

@@ -0,0 +1 @@
+-Wall

cpp/autosar/test/rules/A1-1-2.2/options.gcc

@@ -0,0 +1 @@
+-Wall

cpp/autosar/test/rules/A1-1-2.3/CompilerWarningLevelNotInCompliance.expected

@@ -0,0 +1,5 @@
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-Ofast'. |
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-ffast-math'. |
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-ffinite-math-only'. |
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-ffloat-store'. |
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-fgnu-keywords'. |

cpp/autosar/test/rules/A1-1-2.3/CompilerWarningLevelNotInCompliance.qlref

@@ -0,0 +1,5 @@
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-Ofast'. |
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-ffast-math'. |
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-ffinite-math-only'. |
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-ffloat-store'. |
+| test.cpp:0:0:0:0 | test.cpp | File compiled with uncompliant optimization flag '-fgnu-keywords'. |

cpp/autosar/test/rules/A1-1-2.3/options.clang

@@ -0,0 +1 @@
+-Ofast -ffast-math -fgnu-keywords -fno-signed-zeros -ffinite-math-only -ffloat-store

cpp/autosar/test/rules/A1-1-2.3/options.gcc

@@ -0,0 +1 @@
+-Ofast -ffast-math -fgnu-keywords -fno-signed-zeros -ffinite-math-only -ffloat-store

cpp/autosar/test/rules/A1-1-2/test.cpp renamed to cpp/autosar/test/rules/A1-1-2.3/test.cpp

@@ -1,4 +1,4 @@
-$Functions  = @( Get-ChildItem -Path $PSScriptRoot\*.ps1 -ErrorAction SilentlyContinue )
+$Functions  = @( Get-ChildItem -Path $PSScriptRoot/*.ps1 -ErrorAction SilentlyContinue )
 
 foreach($i in $Functions){
     . $i.FullName 

cpp/autosar/test/rules/A1-1-2/CompilerWarningLevelNotInCompliance.expected

@@ -0,0 +1,20 @@
+function Get-ATestDirectory {
+    param(
+        [Parameter(Mandatory)] 
+        [object]
+        $RuleObject,
+        [Parameter(Mandatory)] 
+        [ValidateSet('c', 'cpp')]
+        [string]
+        $Language
+    )
+
+    $ruleDir = Get-TestDirectory -RuleObject $RuleObject -Language $Language
+
+    # return value MUST include the explicit test directory 
+    $dirs = @($ruleDir)
+
+    $dirs +=  (Get-Item "$($ruleDir).*" | ForEach-Object { $_.FullName })
+
+    $dirs 
+}

cpp/autosar/test/rules/A1-1-2/Wextra.cpp

@@ -0,0 +1,17 @@
+function Get-LanguageForPath {
+    param([Parameter(Mandatory)] 
+        [string]
+        $Path)
+
+    $parts = $Path -split '/'
+
+    $Language = $parts[0]
+
+    foreach($L in $AVAILABLE_LANGUAGES){
+        if($Language -eq $L){
+            return $L
+        }
+    }
+
+    throw "Unsupported Language: $Language"
+}

cpp/autosar/test/rules/A1-1-2/options.clang

@@ -4,5 +4,5 @@ function Get-Packages {
         [string]
         $Language)
 
-    return Get-ChildItem (Join-Path (Get-RepositoryRoot) "\rule_packages\$Language\*.json")
+    return Get-ChildItem (Join-Path (Get-RepositoryRoot) "/rule_packages/$Language/*.json")
 }

cpp/autosar/test/rules/A1-1-2/options.gcc

@@ -0,0 +1,85 @@
+# takes paths like this: 
+# c/cert/src/rules/DCL39-C/InformationLeakageAcrossTrustBoundariesC.ql
+# c/common/test/rules/informationleakageacrossboundaries/InformationLeakageAcrossBoundaries.expected
+# c/common/test/rules/informationleakageacrossboundaries/InformationLeakageAcrossBoundaries.ql
+# c/common/test/rules/informationleakageacrossboundaries/arrays.c
+# c/common/test/rules/informationleakageacrossboundaries/interprocedural.c
+# c/common/test/rules/informationleakageacrossboundaries/multilayer.c
+# c/common/test/rules/informationleakageacrossboundaries/test.c
+# c/misra/src/rules/RULE-18-8/VariableLengthArrayTypesUsed.ql
+# c/misra/src/rules/RULE-8-12/ValueImplicitEnumerationConstantNotUnique.ql
+# c/misra/test/rules/RULE-18-8/VariableLengthArrayTypesUsed.expected
+# c/misra/test/rules/RULE-18-8/test.c
+# c/misra/test/rules/RULE-8-12/ValueImplicitEnumerationConstantNotUnique.expected
+# c/misra/test/rules/RULE-8-12/test.c
+# cpp/cert/src/rules/DCL55-CPP/InformationLeakageAcrossTrustBoundaries.ql
+# cpp/common/test/rules/informationleakageacrossboundaries/InformationLeakageAcrossBoundaries.expected
+# cpp/common/test/rules/informationleakageacrossboundaries/InformationLeakageAcrossBoundaries.ql
+# cpp/common/test/rules/informationleakageacrossboundaries/InformationLeakageAcrossTrustBoundaries.expected
+# cpp/common/test/rules/informationleakageacrossboundaries/arrays.cpp
+# cpp/common/test/rules/informationleakageacrossboundaries/inheritance.cpp
+# cpp/common/test/rules/informationleakageacrossboundaries/interprocedural.cpp
+# cpp/common/test/rules/informationleakageacrossboundaries/multilayer.cpp
+# cpp/common/test/rules/informationleakageacrossboundaries/test.cpp
+
+# And produces one or more rules for it. It does this by loading every rule
+# and computing the test directory for it. This test directory is then 
+# used to see if a) it is a substring of the supplied path or if b) it 
+# is a substring of the path once the substitution `/src/` -> `/test/` is
+# applied  
+
+function Get-RuleForPath {
+    param([Parameter(Mandatory)] 
+        [string]
+        $Path,
+        [ValidateSet('c', 'cpp')]
+        [string]
+        $Language
+        ) 
+
+    # load all the queries for all languages 
+    $allQueries = @()
+    $queriesToCheck = @()
+
+    # load all the queries 
+    foreach ($s in $AVAILABLE_SUITES) {
+        $allQueries += Get-RulesInSuite -Suite $s -Language $Language
+    }
+
+    $modifiedPathWithReplacement = Join-Path (Resolve-Path . -Relative) $Path 
+    # replace "src" with "test" to make it match up
+    $sep = [IO.Path]::DirectorySeparatorChar
+    $modifiedPathWithReplacement = $modifiedPathWithReplacement.Replace( ($sep + "src" + $sep + "rules"), ($sep + "test" + $sep + "rules"))    
+    $modifiedPath = Join-Path (Resolve-Path . -Relative) $Path 
+
+    
+    $matchingRules = @()
+
+    # for each query, create the test directory 
+    foreach($q in $allQueries){
+
+        # get test directory
+        $testDirs = (Get-ATestDirectory -RuleObject $q -Language $Language)
+        foreach($testDirectory in $testDirs){
+            # resolve path to be compatible 
+            $testPath = Join-Path (Resolve-Path . -Relative) $testDirectory
+
+            # see if the TEST directory is a substring of the full path 
+            if($modifiedPath.StartsWith($testPath)){
+                $matchingRules += $q 
+                continue 
+            }
+
+            if($modifiedPathWithReplacement.StartsWith($testPath)){
+                $matchingRules += $q 
+                continue 
+            }
+        }
+    }
+
+    if($matchingRules.Count -gt 0){
+        return $matchingRules
+    }
+
+    throw "Path does not appear to be part of a rule."
+}

cpp/autosar/test/rules/A1-1-3/UncompliantOptimizationOptionMustBeDisabledInCompiler.expected.clang

@@ -6,8 +6,12 @@ function Get-RulesInPackageAndSuite {
         $Package,
         [Parameter(Mandatory)] 
         [string]
-        $Suite
-    )
+        $Suite,
+        [Parameter(Mandatory)] 
+        [ValidateSet('c', 'cpp')]
+        [string]
+        $Language
+        )
 
     $rulesInPackage = @()
 
@@ -30,6 +34,7 @@ function Get-RulesInPackageAndSuite {
         $queries | Add-Member -NotePropertyName __memberof_suite -NotePropertyValue $Suite
         $queries | Add-Member -NotePropertyName __memberof_package -NotePropertyValue $Package.BaseName
         $queries | Add-Member -NotePropertyName __memberof_rule -NotePropertyValue $n.Name 
+        $queries | Add-Member -NotePropertyName __memberof_language -NotePropertyValue $Language 
 
         $rulesInPackage += $queries 
     }

cpp/autosar/test/rules/A1-1-3/UncompliantOptimizationOptionMustBeDisabledInCompiler.expected.gcc

@@ -13,7 +13,7 @@ function Get-RulesInSuite {
 
     foreach ($p in Get-Packages -Language $Language) {
         Write-Host "Reading package: [$Language/$Suite/$($p.BaseName)]"        
-        $tmpQueries += Get-RulesInPackageAndSuite -Package $p -Suite $Suite 
+        $tmpQueries += Get-RulesInPackageAndSuite -Package $p -Suite $Suite -Language $Language
     }
 
     return $tmpQueries

cpp/autosar/test/rules/A1-1-3/options.clang

@@ -9,6 +9,14 @@ fi
 
 echo "Setting Local Branch Version to $1."
 
+# update the qlpacks 
 find . -name 'qlpack.yml' | grep -v './codeql_modules' | grep -v './scripts' | xargs sed -i "s/^version.*$/version: ${1}/"
 
+# update the documentation. 
+
+find docs -name 'user_manual.md' | xargs sed -i "s/code-scanning-cpp-query-pack-anon-.*\.zip\`/code-scanning-cpp-query-pack-anon-${1}.zip\`/"
+find docs -name 'user_manual.md' | xargs sed -i "s/supported_rules_list_.*\.csv\`/supported_rules_list_${1}.csv\`/"
+find docs -name 'user_manual.md' | xargs sed -i "s/supported_rules_list_.*\.md\`/upported_rules_list_${1}.md\`/"
+find docs -name 'user_manual.md' | xargs sed -i "s/user_manual_.*\.md\`/user_manual_${1}.md\`/"
+
 echo "Done."            

cpp/autosar/test/rules/A1-1-3/options.gcc

@@ -16,16 +16,16 @@ param(
 
     # The compiler to use. Valid values are 'clang' and 'arm-clang'.
     [Parameter(Mandatory)] 
-    [ValidateSet('clang', 'armclang', 'tiarmclang')]
+    [ValidateSet('clang', 'armclang', 'tiarmclang', 'gcc')]
     [string]
     $Configuration
 )
 
-Import-Module -Name "$PSScriptRoot\..\PSCodingStandards\CodingStandards"
+Import-Module -Name "$PSScriptRoot/../PSCodingStandards/CodingStandards"
 
-. "$PSScriptRoot\NewDatabaseForRule.ps1"
-. "$PSScriptRoot\Config.ps1"
-. "$PSScriptRoot\Get-CompilerExecutable.ps1"
+. "$PSScriptRoot/NewDatabaseForRule.ps1"
+. "$PSScriptRoot/Config.ps1"
+. "$PSScriptRoot/Get-CompilerExecutable.ps1"
 
 #
 # Verify All the Required CLI Tools are Installed