Merge branch 'main' into jeongsoolee09/a0-1-2_and_a0-1-4

Author: lcartey

.codeqlmanifest.json

@@ -1 +1 @@
-{ "provide": [ "codeql_modules/*/.codeqlmanifest.json", "cpp/.codeqlmanifest.json", "c/.codeqlmanifest.json"] }
+{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml", "scripts/generate_modules/queries/qlpack.yml" ] }

.github/actions/install-codeql-packs/action.yml

@@ -0,0 +1,25 @@
+name: Install CodeQL library pack dependencies
+description: |
+  Downloads any necessary CodeQL library packs needed by packs in the repo.
+inputs:
+  cli_path:
+    description: |
+      The path to the CodeQL CLI directory.
+    required: false
+
+  mode:
+    description: |
+      The `--mode` option to `codeql pack install`.
+    required: true
+    default: verify
+
+runs:
+  using: composite
+  steps:
+    - name: Install CodeQL library packs
+      shell: bash
+      env:
+        CODEQL_CLI: ${{ inputs.cli_path }}
+      run: |
+        PATH=$PATH:$CODEQL_CLI
+        python scripts/install-packs.py --mode ${{ inputs.mode }}

.github/workflows/code-scanning-pack-gen.yml

@@ -59,6 +59,11 @@ jobs:
           codeql-home: ${{ github.workspace }}/codeql_home
           add-to-path: false
 
+      - name: Install CodeQL packs
+        uses: ./.github/actions/install-codeql-packs
+        with:
+          cli_path: ${{ github.workspace }}/codeql_home/codeql
+
       - name: Checkout external help files
         continue-on-error: true
         id: checkout-external-help-files
@@ -82,8 +87,8 @@ jobs:
         run: |
           PATH=$PATH:$CODEQL_HOME/codeql
 
-          codeql query compile --search-path cpp --threads 0 cpp
-          codeql query compile --search-path c --search-path cpp --threads 0 c
+          codeql query compile --threads 0 cpp
+          codeql query compile --threads 0 c
 
           cd ..
           zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas

.github/workflows/codeql_unit_tests.yml

@@ -47,6 +47,9 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          
+      - name: Install Python dependencies
+        run: pip install -r scripts/requirements.txt
 
       - name: Cache CodeQL
         id: cache-codeql
@@ -66,11 +69,15 @@ jobs:
           codeql-home: ${{ github.workspace }}/codeql_home
           add-to-path: false
 
+      - name: Install CodeQL packs
+        uses: ./.github/actions/install-codeql-packs
+        with:
+          cli_path: ${{ github.workspace }}/codeql_home/codeql
+
       - name: Pre-Compile Queries
         id: pre-compile-queries
         run: |
-         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --search-path cpp --threads 0 cpp
-         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --search-path c --search-path cpp --threads 0 c
+         ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 ${{ matrix.language }}
 
 
       - name: Run test suites
@@ -122,18 +129,11 @@ jobs:
               os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
               test_report_file = open(test_report_path, 'w')
               files_to_close.append(test_report_file)
-              if "${{ matrix.language }}".casefold() == "c".casefold():
-                # c tests require cpp -- but we don't want c things on the cpp
-                # path in case of design errors. 
-                cpp_language_root = Path(workspace, 'cpp')
-                procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={cpp_language_root}', f'--search-path={language_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
-              else:
-                procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={language_root}', f'--search-path={language_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
+              procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
 
             for p in procs:
-              p.wait()
+              _, err = p.communicate()
               if p.returncode != 0:
-                _, err = p.communicate()
                 if p.returncode == 122:
                   # Failed because a test case failed, so just print the regular output.
                   # This will allow us to proceed to validate-test-results, which will fail if

.github/workflows/dispatch-matrix-check.yml

@@ -1,7 +1,8 @@
 name: 🤖 Run Matrix Check 
 
 on:
-  pull_request:
+  pull_request_target:
+    types: [synchronize,opened]
     branches:
       - "**"
   workflow_dispatch:
@@ -11,7 +12,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+      - name: Test Variables
+        shell: pwsh
+        run: |          
+          Write-Host "Running as: ${{github.actor}}"    
+    
       - name: Dispatch Matrix Testing Job
+        if: ${{ contains(fromJSON('["jsinglet", "mbaluda", "lcartey", "rvermeulen", "ravikprasad", "jeongsoolee09", "hohn", "knewbury01", "kraiouchkine"]'), github.actor) }}
         uses: peter-evans/repository-dispatch@v2
         with:
           token: ${{ secrets.RELEASE_ENGINEERING_TOKEN }}
@@ -21,6 +28,7 @@ jobs:
 
 
       - uses: actions/github-script@v6
+        if: ${{ contains(fromJSON('["jsinglet", "mbaluda", "lcartey", "rvermeulen", "ravikprasad", "jeongsoolee09", "hohn", "knewbury01", "kraiouchkine"]'), github.actor) }}
         with:
           script: |
             github.rest.issues.createComment({

.github/workflows/dispatch-release-performance-check.yml

@@ -13,14 +13,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+      - name: Test Variables
+        shell: pwsh
+        run: |          
+          Write-Host "Running as: ${{github.actor}}"    
+
+          $actor = "${{github.actor}}"
+
+          $acl = @("jsinglet","mbaluda", "lcartey", "rvermeulen", "ravikprasad", "jeongsoolee09", "hohn", "knewbury01", "kraiouchkine") 
+
+          if(-not ($actor -in $acl)){
+            throw "Refusing to run workflow for user not in acl."
+          }
+
       - name: Dispatch Performance Testing Job
         if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/test-performance') }}
         uses: peter-evans/repository-dispatch@v2
         with:
           token: ${{ secrets.RELEASE_ENGINEERING_TOKEN }}
           repository: github/codeql-coding-standards-release-engineering
           event-type: performance-test
-          client-payload: '{"pr": "${{ github.event.number  }}"}'
+          client-payload: '{"pr": "${{ github.event.issue.number  }}"}'
 
 
       - uses: actions/github-script@v6

.github/workflows/standard_library_upgrade_tests.yml

@@ -116,7 +116,7 @@ jobs:
             stdlib_path = os.path.join(codeql_home, 'codeql-stdlib')
             cpp_test_root = Path(stdlib_path, 'cpp/ql/test')
             print(f"Executing tests found (recursively) in the directory '{cpp_test_root}'")
-            cp = subprocess.run([codeql_bin, "test", "run", "--format=json", f'--search-path={stdlib_path}', cpp_test_root], stdout=test_report_file, stderr=subprocess.PIPE)
+            cp = subprocess.run([codeql_bin, "test", "run", "--format=json", cpp_test_root], stdout=test_report_file, stderr=subprocess.PIPE)
             if cp.returncode != 0:
               print_error_and_fail(f"Failed to run tests with return code {cp.returncode} and error {cp.stderr}")
 

.github/workflows/tooling-unit-tests.yml

@@ -64,6 +64,11 @@ jobs:
           codeql-home: ${{ github.workspace }}/codeql_home
           add-to-path: false
 
+      - name: Install CodeQL packs
+        uses: ./.github/actions/install-codeql-packs
+        with:
+          cli_path: ${{ github.workspace }}/codeql_home/codeql
+
       - name: Run PyTest
         env:
           CODEQL_HOME: ${{ github.workspace }}/codeql_home

.github/workflows/verify-standard-library-dependencies.yml

@@ -0,0 +1,79 @@
+name: Verify Standard Library Dependencies
+
+# Run this workflow every time the "supported_codeql_configs.json" file or a "qlpack.yml" file is changed
+on:
+  pull_request:
+    branches:
+      - main
+      - "rc/**"
+      - next
+    paths:
+      - "supported_codeql_configs.json"
+      - "**/qlpack.yml"
+  workflow_dispatch:
+
+jobs:
+  prepare-matrix:
+    name: Prepare CodeQL configuration matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.export-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Export unit test matrix
+        id: export-matrix
+        run: |
+          echo "::set-output name=matrix::$(
+            jq --compact-output \
+            '.supported_environment | map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}' \
+            supported_codeql_configs.json
+          )"
+
+  verify-dependencies:
+    name: Verify dependencies
+    needs: prepare-matrix
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJSON(needs.prepare-matrix.outputs.matrix)}}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup Python 3
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Cache CodeQL
+        id: cache-codeql
+        uses: actions/cache@v2.1.3
+        with:
+          # A list of files, directories, and wildcard patterns to cache and restore
+          path: ${{github.workspace}}/codeql_home
+          # An explicit key for restoring and saving the cache
+          key: codeql-home-${{matrix.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library}}
+
+      - name: Install CodeQL
+        if: steps.cache-codeql.outputs.cache-hit != 'true'
+        uses: ./.github/actions/install-codeql
+        with:
+          codeql-cli-version: ${{matrix.codeql_cli}}
+          codeql-stdlib-version: ${{matrix.codeql_standard_library}}
+          codeql-home: ${{ github.workspace }}/codeql_home
+
+      - name: Verify dependencies
+        shell: bash
+        env:
+          CLI_PATH: ${{ github.workspace }}/codeql_home/codeql
+          STDLIB_PATH: ${{ github.workspace }}/codeql_home/codeql-stdlib
+        run: |
+          PATH=$PATH:$CLI_PATH
+          ls $STDLIB_PATH
+          pip install -r scripts/requirements.txt
+          python3 scripts/verify-standard-library-version.py --codeql-repo $STDLIB_PATH --mode verify
+

.gitignore

@@ -20,3 +20,6 @@
 # C/C++ build artifacts
 *.o
 /databases/
+
+# CodeQL build artifacts
+**/.codeql/**

.vscode/tasks.json

@@ -193,6 +193,11 @@
                 "Classes",
                 "Comments",
                 "Contracts1",
+                "Contracts2",
+                "Contracts3",
+                "Contracts4",
+                "Contracts5",
+                "Contracts6",
                 "Concurrency",
                 "Concurrency",
                 "Concurrency1",

c/.codeqlmanifest.json

@@ -0,0 +1,6 @@
+---
+dependencies:
+  codeql/cpp-all:
+    version: 0.2.3
+compiled: false
+lockVersion: 1.0.0

c/cert/src/codeql-pack.lock.yml

@@ -1,5 +1,5 @@
 - description: CERT C 2016 (Default)
-- qlpack: cert-c-coding-standards
+- qlpack: codeql/cert-c-coding-standards
 - include:
     kind:
     - problem

c/cert/src/codeql-suites/cert-default.qls

@@ -1,4 +1,8 @@
-name: cert-c-coding-standards
-version: 2.13.0-dev
+name: codeql/cert-c-coding-standards
+version: 2.14.0-dev
+description: CERT C 2016
 suites: codeql-suites
-libraryPathDependencies: common-c-coding-standards
+license: MIT
+dependencies:
+  codeql/common-c-coding-standards: '*'
+  codeql/cpp-all: 0.2.3