Merge branch 'main' into knewbury01/Declarations5

Author: knewbury01

.github/workflows/code-scanning-pack-gen.yml

@@ -86,7 +86,7 @@ jobs:
           codeql query compile --search-path c --search-path cpp --threads 0 c
 
           cd ..
-          zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
+          zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas
 
       - name: Upload GHAS Query Pack
         uses: actions/upload-artifact@v2

.github/workflows/tooling-unit-tests.yml

@@ -0,0 +1,91 @@
+name: 🧰 Tooling unit tests
+
+on:
+  push:
+    branches:
+      - main
+      - "rc/**"
+      - next
+  pull_request:
+    branches:
+      - main
+      - "rc/**"
+      - next
+
+jobs:
+  prepare-supported-codeql-env-matrix:
+    name: Prepare supported CodeQL environment matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.export-supported-codeql-env-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Export supported CodeQL environment matrix
+        id: export-supported-codeql-env-matrix
+        run: |
+          echo "::set-output name=matrix::$(
+            jq --compact-output '.supported_environment | {include: .}' supported_codeql_configs.json
+          )"
+
+  analysis-report-tests:
+    name: Run analysis report tests
+    needs: prepare-supported-codeql-env-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.prepare-supported-codeql-env-matrix.outputs.matrix) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Install Python dependencies
+        run: pip install -r scripts/reports/requirements.txt
+
+      - name: Cache CodeQL
+        id: cache-codeql
+        uses: actions/cache@v2.1.3
+        with:
+          path: ${{ github.workspace }}/codeql_home
+          key: codeql-home-${{ matrix.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}
+
+      - name: Install CodeQL
+        if: steps.cache-codeql.outputs.cache-hit != 'true'
+        uses: ./.github/actions/install-codeql
+        with:
+          codeql-cli-version: ${{ matrix.codeql_cli }}
+          codeql-stdlib-version: ${{ matrix.codeql_standard_library }}
+          codeql-home: ${{ github.workspace }}/codeql_home
+          add-to-path: false
+
+      - name: Run PyTest
+        env:
+          CODEQL_HOME: ${{ github.workspace }}/codeql_home
+        run: |
+          PATH=$PATH:$CODEQL_HOME/codeql
+          pytest scripts/reports/analysis_report_test.py
+
+  recategorization-tests:
+    name: Run Guideline Recategorization tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Install Python dependencies
+        run: pip install -r scripts/guideline_recategorization/requirements.txt
+
+      - name: Run PyTest
+        run: |
+          pytest scripts/guideline_recategorization/recategorize_test.py

.github/workflows/validate-coding-standards.yml

@@ -28,6 +28,15 @@ jobs:
         with:
           python-version: "3.9"
 
+      - name: Install CodeQL
+        run: |
+          VERSION="v$( jq -r '.supported_environment | .[0] | .codeql_cli' supported_codeql_configs.json)"
+          gh extensions install github/gh-codeql
+          gh codeql set-version "$VERSION"
+          gh codeql install-stub
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
       - name: Install generate_package_files.py dependencies
         run: pip install -r scripts/requirements.txt
 
@@ -49,14 +58,14 @@ jobs:
 
       - name: Validate Package Files (CPP)
         run: |
-          find rule_packages/cpp -name \*.json -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python scripts/generate_rules/generate_package_files.py cpp
+          find rule_packages/cpp -name \*.json -exec basename {} .json \; | xargs python scripts/generate_rules/generate_package_files.py cpp
           git diff
           git diff --compact-summary
           git diff --quiet
 
       - name: Validate Package Files (C)
         run: |
-          find rule_packages/c -name \*.json -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python scripts/generate_rules/generate_package_files.py c
+          find rule_packages/c -name \*.json -exec basename {} .json \; | xargs python scripts/generate_rules/generate_package_files.py c
           git diff
           git diff --compact-summary
           git diff --quiet
@@ -68,25 +77,26 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Fetch CodeQL
+      - name: Install CodeQL
         run: |
-          TAG="v$( jq -r '.supported_environment | .[0] | .codeql_cli' supported_codeql_configs.json)"
-          gh release download $TAG --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip
-          unzip -q codeql-linux64.zip
+          VERSION="v$( jq -r '.supported_environment | .[0] | .codeql_cli' supported_codeql_configs.json)"
+          gh extensions install github/gh-codeql
+          gh codeql set-version "$VERSION"
+          gh codeql install-stub
         env:
           GITHUB_TOKEN: ${{ github.token }}
 
       - name: Validate CodeQL Format (CPP)
         run: |
-          find cpp -name \*.ql -or -name \*.qll -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql/codeql query format --in-place
+          find cpp -name \*.ql -or -name \*.qll -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql query format --in-place
 
           git diff
           git diff --compact-summary
           git diff --quiet
 
       - name: Validate CodeQL Format (C)
         run: |
-          find c -name \*.ql -or -name \*.qll -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql/codeql query format --in-place
+          find c -name \*.ql -or -name \*.qll -print0 | xargs -0 --max-procs "$XARGS_MAX_PROCS" codeql query format --in-place
 
           git diff
           git diff --compact-summary

.vscode/tasks.json

@@ -223,6 +223,7 @@
                 "Iterators",
                 "Lambdas",
                 "Language1",
+                "Language2",
                 "Literals",
                 "Loops",
                 "Macros",

c/cert/src/qlpack.yml

@@ -1,4 +1,4 @@
 name: cert-c-coding-standards
-version: 2.12.0-dev
+version: 2.13.0-dev
 suites: codeql-suites
 libraryPathDependencies: common-c-coding-standards

c/cert/test/qlpack.yml

@@ -1,4 +1,4 @@
 name: cert-c-coding-standards-tests
-version: 2.12.0-dev
+version: 2.13.0-dev
 libraryPathDependencies: cert-c-coding-standards
 extractor: cpp

c/common/src/qlpack.yml

@@ -1,3 +1,3 @@
 name: common-c-coding-standards
-version: 2.12.0-dev
+version: 2.13.0-dev
 libraryPathDependencies: common-cpp-coding-standards

c/common/test/qlpack.yml

@@ -1,4 +1,4 @@
 name: common-c-coding-standards-tests
-version: 2.12.0-dev
+version: 2.13.0-dev
 libraryPathDependencies: common-c-coding-standards
 extractor: cpp

c/misra/src/qlpack.yml

@@ -1,4 +1,4 @@
 name: misra-c-coding-standards
-version: 2.12.0-dev
+version: 2.13.0-dev
 suites: codeql-suites
 libraryPathDependencies: common-c-coding-standards

c/misra/src/rules/DIR-4-2/UsageOfAssemblyLanguageShouldBeDocumented.ql

@@ -0,0 +1,22 @@
+/**
+ * @id c/misra/usage-of-assembly-language-should-be-documented
+ * @name DIR-4-2: All usage of assembly language should be documented
+ * @description Assembly language is not portable and should be documented.
+ * @kind problem
+ * @precision very-high
+ * @problem.severity warning
+ * @tags external/misra/id/dir-4-2
+ *       maintainability
+ *       readability
+ *       external/misra/obligation/advisory
+ */
+
+import cpp
+import codingstandards.c.misra
+import codingstandards.cpp.rules.usageofassemblernotdocumented.UsageOfAssemblerNotDocumented
+
+class UsageOfAssemblyLanguageShouldBeDocumentedQuery extends UsageOfAssemblerNotDocumentedSharedQuery {
+  UsageOfAssemblyLanguageShouldBeDocumentedQuery() {
+    this = Language2Package::usageOfAssemblyLanguageShouldBeDocumentedQuery()
+  }
+}

c/misra/src/rules/RULE-1-4/EmergentLanguageFeaturesUsed.ql

@@ -0,0 +1,21 @@
+/**
+ * @id c/misra/emergent-language-features-used
+ * @name RULE-1-4: Emergent language features shall not be used
+ * @description Emergent language features may have unpredictable behavior and should not be used.
+ * @kind problem
+ * @precision very-high
+ * @problem.severity warning
+ * @tags external/misra/id/rule-1-4
+ *       maintainability
+ *       readability
+ *       external/misra/obligation/required
+ */
+
+import cpp
+import codingstandards.c.misra
+import codingstandards.cpp.Emergent
+
+from C11::EmergentLanguageFeature ef
+where not isExcluded(ef, Language2Package::emergentLanguageFeaturesUsedQuery())
+select ef, "Usage of emergent language feature."
+

c/misra/src/rules/RULE-21-19/ValuesReturnedByLocaleSettingUsedAsPtrToConst.ql

@@ -4,7 +4,7 @@
  * @description The pointers returned by the Standard Library functions localeconv, getenv,
  *              setlocale or, strerror shall only be used as if they have pointer to const-qualified
  *              type.
- * @kind problem
+ * @kind path-problem
  * @precision very-high
  * @problem.severity error
  * @tags external/misra/id/rule-21-19

c/misra/test/qlpack.yml

@@ -1,4 +1,4 @@
 name: misra-c-coding-standards-tests
-version: 2.12.0-dev
+version: 2.13.0-dev
 libraryPathDependencies: misra-c-coding-standards
 extractor: cpp

c/misra/test/rules/DIR-4-2/UsageOfAssemblyLanguageShouldBeDocumented.testref

@@ -0,0 +1 @@
+cpp/common/test/rules/usageofassemblernotdocumented/UsageOfAssemblerNotDocumented.ql

c/misra/test/rules/RULE-1-4/EmergentLanguageFeaturesUsed.expected

@@ -0,0 +1,15 @@
+| test.c:1:1:1:21 | #include <stdalign.h> | Usage of emergent language feature. |
+| test.c:2:1:2:22 | #include <stdatomic.h> | Usage of emergent language feature. |
+| test.c:3:1:3:24 | #include <stdnoreturn.h> | Usage of emergent language feature. |
+| test.c:4:1:4:20 | #include <threads.h> | Usage of emergent language feature. |
+| test.c:6:1:6:49 | #define MACRO(x) _Generic((x), int : 0, long : 1) | Usage of emergent language feature. |
+| test.c:7:1:7:32 | #define __STDC_WANT_LIB_EXT1__ 1 | Usage of emergent language feature. |
+| test.c:9:16:9:17 | f0 | Usage of emergent language feature. |
+| test.c:12:26:12:40 | atomic_new_type | Usage of emergent language feature. |
+| test.c:17:15:17:15 | i | Usage of emergent language feature. |
+| test.c:19:3:19:10 | alignas(...) | Usage of emergent language feature. |
+| test.c:20:3:20:9 | alignas(...) | Usage of emergent language feature. |
+| test.c:21:11:21:23 | alignof(int) | Usage of emergent language feature. |
+| test.c:22:12:22:23 | alignof(int) | Usage of emergent language feature. |
+| test.c:24:27:24:28 | i3 | Usage of emergent language feature. |
+| test.c:25:28:25:29 | i4 | Usage of emergent language feature. |

c/misra/test/rules/RULE-1-4/EmergentLanguageFeaturesUsed.qlref

@@ -0,0 +1 @@
+rules/RULE-1-4/EmergentLanguageFeaturesUsed.ql

c/misra/test/rules/RULE-1-4/test.c

@@ -0,0 +1,26 @@
+#include <stdalign.h>    //NON_COMPLIANT
+#include <stdatomic.h>   //NON_COMPLIANT
+#include <stdnoreturn.h> //NON_COMPLIANT
+#include <threads.h>     //NON_COMPLIANT
+
+#define MACRO(x) _Generic((x), int : 0, long : 1) // NON_COMPLIANT
+#define __STDC_WANT_LIB_EXT1__ 1                  // NON_COMPLIANT
+
+_Noreturn void f0(); // NON_COMPLIANT
+
+typedef int new_type;                     // COMPLIANT
+typedef _Atomic new_type atomic_new_type; // NON_COMPLIANT
+
+void f(int p) {
+  int i0 = _Generic(p, int : 0, long : 1); // NON_COMPLIANT[FALSE_NEGATIVE]
+
+  _Atomic int i; // NON_COMPLIANT
+
+  _Alignas(4) int i1;    // NON_COMPLIANT
+  alignas(4) int i2;     // NON_COMPLIANT
+  int a = _Alignof(int); // NON_COMPLIANT
+  int a1 = alignof(int); // NON_COMPLIANT
+
+  static thread_local int i3;  // NON_COMPLIANT
+  static _Thread_local int i4; // NON_COMPLIANT
+}

change_notes/2022-11-07-add-guideline-recategorization-scripts.md

@@ -0,0 +1,2 @@
+- Add the Python scripts under `scripts/guideline_recategorization` and the JSON schemas under `schemas`.
+- Add the Python scripts under `scripts/shared` relied upon by the analysis report generation.

change_notes/2022-11-21-null-sizet-not-cstdio.md

@@ -0,0 +1,3 @@
+ - `M27-0-1`
+   - `CstdioTypesUsed.ql` - Exclude `size_t` from this rule, as it can be provided by headers other than `<cstdio>`.
+   - `CstdioMacrosUsed.ql` - Exclude `NULL` from this rule, as it can be provided by headers other than `<cstdio>`.

cpp/autosar/src/qlpack.yml

@@ -1,4 +1,4 @@
 name: autosar-cpp-coding-standards
-version: 2.12.0-dev
+version: 2.13.0-dev
 suites: codeql-suites
 libraryPathDependencies: common-cpp-coding-standards

cpp/autosar/src/rules/A18-5-8/UnnecessaryUseOfDynamicStorage.ql

@@ -64,13 +64,13 @@ class MakeSharedOrUnique extends FunctionCall, CandidateFunctionLocalHeapAllocat
  * An `AllocationExpr` that allocates heap memory, where the memory is freed on at least one path
  * through the enclosing function.
  */
-class AllocationExprFunctionLocal extends AllocationExpr, CandidateFunctionLocalHeapAllocationExpr {
+class AllocationExprFunctionLocal extends CandidateFunctionLocalHeapAllocationExpr instanceof AllocationExpr {
   AllocationExprFunctionLocal() {
     this.getSizeBytes() < 1024 and
     TaintTracking::localExprTaint(this, any(DeallocationExpr de).getFreedExpr())
   }
 
-  override int getHeapSizeBytes() { result = this.getSizeBytes() }
+  override int getHeapSizeBytes() { result = super.getSizeBytes() }
 
   DeallocationExpr getADeallocation() { TaintTracking::localExprTaint(this, result.getFreedExpr()) }
 

cpp/autosar/src/rules/M27-0-1/CstdioMacrosUsed.ql

@@ -22,7 +22,7 @@ from MacroInvocation mi
 where
   not isExcluded(mi, BannedLibrariesPackage::cstdioMacrosUsedQuery()) and
   mi.getMacroName() in [
-      "BUFSIZ", "EOF", "FILENAME_MAX", "FOPEN_MAX", "L_tmpnam", "NULL", "TMP_MAX", "_IOFBF",
-      "IOLBF", "_IONBF", "SEEK_CUR", "SEEK_END", "SEEK_SET"
+      "BUFSIZ", "EOF", "FILENAME_MAX", "FOPEN_MAX", "L_tmpnam", "TMP_MAX", "_IOFBF", "IOLBF",
+      "_IONBF", "SEEK_CUR", "SEEK_END", "SEEK_SET"
     ]
 select mi, "Use of <cstdio> macro '" + mi.getMacroName() + "'."

cpp/autosar/src/rules/M27-0-1/CstdioTypesUsed.ql

@@ -22,5 +22,5 @@ from TypeMention tm, UserType ut
 where
   not isExcluded(tm, BannedLibrariesPackage::cstdioTypesUsedQuery()) and
   ut = tm.getMentionedType() and
-  ut.hasGlobalOrStdName(["FILE", "fpos_t", "size_t"])
+  ut.hasGlobalOrStdName(["FILE", "fpos_t"])
 select tm, "Use of <cstdio> type '" + ut.getQualifiedName() + "'."

cpp/autosar/src/rules/M7-4-1/UsageOfAssemblerNotDocumented.ql

@@ -17,10 +17,10 @@
 
 import cpp
 import codingstandards.cpp.autosar
+import codingstandards.cpp.rules.usageofassemblernotdocumented.UsageOfAssemblerNotDocumented
 
-from AsmStmt a
-where
-  not isExcluded(a, BannedLibrariesPackage::usageOfAssemblerNotDocumentedQuery()) and
-  not exists(Comment c | c.getCommentedElement() = a) and
-  not a.isAffectedByMacro()
-select a, "Use of assembler is not documented."
+class UsageOfAssemblerNotDocumentedQuery extends UsageOfAssemblerNotDocumentedSharedQuery {
+  UsageOfAssemblerNotDocumentedQuery() {
+    this = BannedLibrariesPackage::usageOfAssemblerNotDocumentedQuery()
+  }
+}

cpp/autosar/test/qlpack.yml

@@ -1,4 +1,4 @@
 name: autosar-cpp-coding-standards-tests
-version: 2.12.0-dev
+version: 2.13.0-dev
 libraryPathDependencies: autosar-cpp-coding-standards
 extractor: cpp

cpp/autosar/test/rules/M27-0-1/CstdioMacrosUsed.expected

@@ -7,4 +7,3 @@
 | test.cpp:30:29:30:37 | FOPEN_MAX | Use of <cstdio> macro 'FOPEN_MAX'. |
 | test.cpp:41:14:41:16 | EOF | Use of <cstdio> macro 'EOF'. |
 | test.cpp:50:24:50:31 | SEEK_SET | Use of <cstdio> macro 'SEEK_SET'. |
-| test.cpp:60:10:60:13 | NULL | Use of <cstdio> macro 'NULL'. |

cpp/autosar/test/rules/M27-0-1/CstdioTypesUsed.expected

@@ -1,7 +1,4 @@
 | test.cpp:4:8:4:11 | type mention | Use of <cstdio> type 'std::FILE'. |
 | test.cpp:6:8:6:13 | type mention | Use of <cstdio> type 'std::fpos_t'. |
-| test.cpp:20:18:20:23 | type mention | Use of <cstdio> type 'size_t'. |
-| test.cpp:21:18:21:23 | type mention | Use of <cstdio> type 'size_t'. |
 | test.cpp:34:3:34:6 | type mention | Use of <cstdio> type 'FILE'. |
 | test.cpp:36:3:36:8 | type mention | Use of <cstdio> type 'fpos_t'. |
-| test.cpp:50:14:50:19 | type mention | Use of <cstdio> type 'size_t'. |