Modernize qlpacks

.codeqlmanifest.json

@@ -1 +1 @@
-{ "provide": [ "codeql_modules/*/.codeqlmanifest.json", "cpp/.codeqlmanifest.json", "c/.codeqlmanifest.json"] }
+{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml", "scripts/generate_modules/queries/qlpack.yml" ] }

.github/actions/install-codeql-packs/action.yml

@@ -0,0 +1,25 @@
+name: Install CodeQL library pack dependencies
+description: |
+  Downloads any necessary CodeQL library packs needed by packs in the repo.
+inputs:
+  cli_path:
+    description: |
+      The path to the CodeQL CLI directory.
+    required: false
+
+  mode:
+    description: |
+      The `--mode` option to `codeql pack install`.
+    required: true
+    default: verify
+
+runs:
+  using: composite
+  steps:
+    - name: Install CodeQL library packs
+      shell: bash
+      env:
+        CODEQL_CLI: ${{ inputs.cli_path }}
+      run: |
+        PATH=$PATH:$CODEQL_CLI
+        python scripts/install-packs.py --mode ${{ inputs.mode }}

.github/workflows/code-scanning-pack-gen.yml

@@ -59,6 +59,11 @@ jobs:
           codeql-home: ${{ github.workspace }}/codeql_home
           add-to-path: false
 
+      - name: Install CodeQL packs
+        uses: ./.github/actions/install-codeql-packs
+        with:
+          cli_path: ${{ github.workspace }}/codeql_home/codeql
+
       - name: Checkout external help files
         continue-on-error: true
         id: checkout-external-help-files
@@ -82,8 +87,8 @@ jobs:
         run: |
           PATH=$PATH:$CODEQL_HOME/codeql
 
-          codeql query compile --search-path cpp --threads 0 cpp
-          codeql query compile --search-path c --search-path cpp --threads 0 c
+          codeql query compile --threads 0 cpp
+          codeql query compile --threads 0 c
 
           cd ..
           zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas

.github/workflows/codeql_unit_tests.yml

@@ -47,6 +47,9 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+
+      - name: Install Python dependencies
+        run: pip install -r scripts/requirements.txt
 
       - name: Cache CodeQL
         id: cache-codeql
@@ -66,11 +69,16 @@ jobs:
           codeql-home: ${{ github.workspace }}/codeql_home
           add-to-path: false
 
+      - name: Install CodeQL packs
+        uses: ./.github/actions/install-codeql-packs
+        with:
+          cli_path: ${{ github.workspace }}/codeql_home/codeql
+
       - name: Pre-Compile Queries
         id: pre-compile-queries
         run: |
-         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --search-path cpp --threads 0 cpp
-         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --search-path c --search-path cpp --threads 0 c
+         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 cpp
+         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 c
 
 
       - name: Run test suites
@@ -122,18 +130,11 @@ jobs:
               os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
               test_report_file = open(test_report_path, 'w')
               files_to_close.append(test_report_file)
-              if "${{ matrix.language }}".casefold() == "c".casefold():
-                # c tests require cpp -- but we don't want c things on the cpp
-                # path in case of design errors. 
-                cpp_language_root = Path(workspace, 'cpp')
-                procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={cpp_language_root}', f'--search-path={language_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
-              else:
-                procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={language_root}', f'--search-path={language_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
+              procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
 
             for p in procs:
-              p.wait()
+              _, err = p.communicate()
               if p.returncode != 0:
-                _, err = p.communicate()
                 if p.returncode == 122:
                   # Failed because a test case failed, so just print the regular output.
                   # This will allow us to proceed to validate-test-results, which will fail if

.github/workflows/standard_library_upgrade_tests.yml

@@ -116,7 +116,7 @@ jobs:
             stdlib_path = os.path.join(codeql_home, 'codeql-stdlib')
             cpp_test_root = Path(stdlib_path, 'cpp/ql/test')
             print(f"Executing tests found (recursively) in the directory '{cpp_test_root}'")
-            cp = subprocess.run([codeql_bin, "test", "run", "--format=json", f'--search-path={stdlib_path}', cpp_test_root], stdout=test_report_file, stderr=subprocess.PIPE)
+            cp = subprocess.run([codeql_bin, "test", "run", "--format=json", cpp_test_root], stdout=test_report_file, stderr=subprocess.PIPE)
             if cp.returncode != 0:
               print_error_and_fail(f"Failed to run tests with return code {cp.returncode} and error {cp.stderr}")
 

.github/workflows/tooling-unit-tests.yml

@@ -64,6 +64,11 @@ jobs:
           codeql-home: ${{ github.workspace }}/codeql_home
           add-to-path: false
 
+      - name: Install CodeQL packs
+        uses: ./.github/actions/install-codeql-packs
+        with:
+          cli_path: ${{ github.workspace }}/codeql_home/codeql
+
       - name: Run PyTest
         env:
           CODEQL_HOME: ${{ github.workspace }}/codeql_home

.github/workflows/verify-standard-library-dependencies.yml

@@ -0,0 +1,79 @@
+name: Verify Standard Library Dependencies
+
+# Run this workflow every time the "supported_codeql_configs.json" file or a "qlpack.yml" file is changed
+on:
+  pull_request:
+    branches:
+      - main
+      - "rc/**"
+      - next
+    paths:
+      - "supported_codeql_configs.json"
+      - "**/qlpack.yml"
+  workflow_dispatch:
+
+jobs:
+  prepare-matrix:
+    name: Prepare CodeQL configuration matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.export-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Export unit test matrix
+        id: export-matrix
+        run: |
+          echo "::set-output name=matrix::$(
+            jq --compact-output \
+            '.supported_environment | map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}' \
+            supported_codeql_configs.json
+          )"
+
+  verify-dependencies:
+    name: Verify dependencies
+    needs: prepare-matrix
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJSON(needs.prepare-matrix.outputs.matrix)}}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup Python 3
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Cache CodeQL
+        id: cache-codeql
+        uses: actions/cache@v2.1.3
+        with:
+          # A list of files, directories, and wildcard patterns to cache and restore
+          path: ${{github.workspace}}/codeql_home
+          # An explicit key for restoring and saving the cache
+          key: codeql-home-${{matrix.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library}}
+
+      - name: Install CodeQL
+        if: steps.cache-codeql.outputs.cache-hit != 'true'
+        uses: ./.github/actions/install-codeql
+        with:
+          codeql-cli-version: ${{matrix.codeql_cli}}
+          codeql-stdlib-version: ${{matrix.codeql_standard_library}}
+          codeql-home: ${{ github.workspace }}/codeql_home
+
+      - name: Verify dependencies
+        shell: bash
+        env:
+          CLI_PATH: ${{ github.workspace }}/codeql_home/codeql
+          STDLIB_PATH: ${{ github.workspace }}/codeql_home/codeql-stdlib
+        run: |
+          PATH=$PATH:$CLI_PATH
+          ls $STDLIB_PATH
+          pip install -r scripts/requirements.txt
+          python3 scripts/verify-standard-library-version.py --codeql-repo $STDLIB_PATH --mode verify
+

.gitignore

@@ -20,3 +20,6 @@
 # C/C++ build artifacts
 *.o
 /databases/
+
+# CodeQL build artifacts
+**/.codeql/**

c/cert/src/codeql-pack.lock.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0

c/cert/src/codeql-suites/cert-default.qls

@@ -1,5 +1,5 @@
 - description: CERT C 2016 (Default)
-- qlpack: cert-c-coding-standards
+- qlpack: codeql/cert-c-coding-standards
 - include:
     kind:
     - problem

c/cert/src/qlpack.yml

@@ -1,4 +1,8 @@
-name: cert-c-coding-standards
+name: codeql/cert-c-coding-standards
 version: 2.14.0-dev
+description: CERT C 2016
 suites: codeql-suites
-libraryPathDependencies: common-c-coding-standards
+license: MIT
+dependencies:
+  codeql/common-c-coding-standards: '*'
+  codeql/cpp-all: 0.2.3

c/cert/test/codeql-pack.lock.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0

c/cert/test/qlpack.yml

@@ -1,4 +1,6 @@
-name: cert-c-coding-standards-tests
+name: codeql/cert-c-coding-standards-tests
 version: 2.14.0-dev
-libraryPathDependencies: cert-c-coding-standards
-extractor: cpp
+extractor: cpp
+license: MIT
+dependencies:
+  codeql/cert-c-coding-standards: '*'

c/common/src/codeql-pack.lock.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0

c/common/src/qlpack.yml

@@ -1,3 +1,6 @@
-name: common-c-coding-standards
+name: codeql/common-c-coding-standards
 version: 2.14.0-dev
-libraryPathDependencies: common-cpp-coding-standards
+license: MIT
+dependencies:
+  codeql/common-cpp-coding-standards: '*'
+  codeql/cpp-all: 0.2.3

c/common/test/codeql-pack.lock.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0

c/common/test/qlpack.yml

@@ -1,4 +1,6 @@
-name: common-c-coding-standards-tests
+name: codeql/common-c-coding-standards-tests
 version: 2.14.0-dev
-libraryPathDependencies: common-c-coding-standards
 extractor: cpp
+license: MIT
+dependencies:
+  codeql/common-c-coding-standards: '*'

c/misra/src/codeql-pack.lock.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0

c/misra/src/codeql-suites/misra-default.qls

@@ -1,5 +1,5 @@
 - description: MISRA C 2012 (Default)
-- qlpack: misra-c-coding-standards
+- qlpack: codeql/misra-c-coding-standards
 - include:
     kind:
     - problem

c/misra/src/qlpack.yml

@@ -1,4 +1,8 @@
-name: misra-c-coding-standards
+name: codeql/misra-c-coding-standards
 version: 2.14.0-dev
+description: MISRA C 2012
 suites: codeql-suites
-libraryPathDependencies: common-c-coding-standards
+license: MIT
+dependencies:
+  codeql/common-c-coding-standards: '*'
+  codeql/cpp-all: 0.2.3

c/misra/test/codeql-pack.lock.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0

c/misra/test/qlpack.yml

@@ -1,4 +1,6 @@
-name: misra-c-coding-standards-tests
+name: codeql/misra-c-coding-standards-tests
 version: 2.14.0-dev
-libraryPathDependencies: misra-c-coding-standards
-extractor: cpp
+extractor: cpp
+license: MIT
+dependencies:
+  codeql/misra-c-coding-standards: '*'

cpp/autosar/src/codeql-pack.lock.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0

cpp/autosar/src/codeql-suites/autosar-advisory.qls

@@ -1,5 +1,5 @@
 - description: AUTOSAR C++14 Guidelines 20-11 (Advisory)
-- qlpack: autosar-cpp-coding-standards
+- qlpack: codeql/autosar-cpp-coding-standards
 - include:
     kind:
     - problem

cpp/autosar/src/codeql-suites/autosar-audit.qls

@@ -1,5 +1,5 @@
 - description: AUTOSAR C++14 Guidelines 20-11 (Audit)
-- qlpack: autosar-cpp-coding-standards
+- qlpack: codeql/autosar-cpp-coding-standards
 - include:
     kind:
     - problem

cpp/autosar/src/codeql-suites/autosar-default.qls

@@ -1,5 +1,5 @@
 - description: AUTOSAR C++14 Guidelines 20-11 (Default)
-- qlpack: autosar-cpp-coding-standards
+- qlpack: codeql/autosar-cpp-coding-standards
 - include:
     kind:
     - problem

cpp/autosar/src/codeql-suites/autosar-required.qls

@@ -1,5 +1,5 @@
 - description: AUTOSAR C++14 Guidelines 20-11 (Required)
-- qlpack: autosar-cpp-coding-standards
+- qlpack: codeql/autosar-cpp-coding-standards
 - include:
     kind:
     - problem

cpp/autosar/src/codeql-suites/autosar-single-translation-unit.qls

@@ -1,5 +1,5 @@
 - description: AUTOSAR C++14 Guidelines 20-11 (Single Translation Unit)
-- qlpack: autosar-cpp-coding-standards
+- qlpack: codeql/autosar-cpp-coding-standards
 - include:
     kind:
     - problem

cpp/autosar/src/qlpack.yml

@@ -1,4 +1,8 @@
-name: autosar-cpp-coding-standards
+name: codeql/autosar-cpp-coding-standards
 version: 2.14.0-dev
+description: AUTOSAR C++14 Guidelines 20-11
 suites: codeql-suites
-libraryPathDependencies: common-cpp-coding-standards
+license: MIT
+dependencies:
+  codeql/common-cpp-coding-standards: '*'
+  codeql/cpp-all: 0.2.3

cpp/autosar/test/codeql-pack.lock.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0