From dfc95c16f9b966cd7f5da7772e74c8933af724dc Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 14 Jul 2022 11:49:59 +0200 Subject: [PATCH 01/37] Modernize qlpacks and update to CodeQL 2.8.5 --- .codeqlmanifest.json | 2 +- .github/actions/action.yml | 25 ++++++ .../actions/install-codeql-packs/action.yml | 25 ++++++ .github/workflows/code-scanning-pack-gen.yml | 9 ++- .github/workflows/codeql_unit_tests.yml | 20 +++-- .../standard_library_upgrade_tests.yml | 2 +- .../verify-standard-library-dependencies.yml | 79 +++++++++++++++++++ .gitignore | 3 + c/.codeqlmanifest.json | 3 - c/cert/src/codeql-pack.lock.yml | 6 ++ c/cert/src/qlpack.yml | 4 +- c/cert/test/codeql-pack.lock.yml | 6 ++ c/cert/test/qlpack.yml | 7 +- c/common/src/codeql-pack.lock.yml | 6 ++ c/common/src/qlpack.yml | 6 +- c/common/test/codeql-pack.lock.yml | 6 ++ c/common/test/qlpack.yml | 5 +- c/misra/src/codeql-pack.lock.yml | 6 ++ c/misra/src/qlpack.yml | 6 +- c/misra/test/codeql-pack.lock.yml | 6 ++ c/misra/test/qlpack.yml | 7 +- cpp/.codeqlmanifest.json | 3 - cpp/autosar/src/codeql-pack.lock.yml | 6 ++ cpp/autosar/src/qlpack.yml | 6 +- cpp/autosar/test/codeql-pack.lock.yml | 6 ++ cpp/autosar/test/qlpack.yml | 5 +- cpp/cert/src/codeql-pack.lock.yml | 6 ++ cpp/cert/src/qlpack.yml | 6 +- cpp/cert/test/codeql-pack.lock.yml | 6 ++ cpp/cert/test/qlpack.yml | 5 +- cpp/common/src/codeql-pack.lock.yml | 6 ++ cpp/common/src/qlpack.yml | 5 +- cpp/common/test/codeql-pack.lock.yml | 6 ++ cpp/common/test/qlpack.yml | 5 +- cpp/misra/src/codeql-pack.lock.yml | 6 ++ cpp/misra/src/qlpack.yml | 6 +- cpp/misra/test/codeql-pack.lock.yml | 6 ++ cpp/misra/test/qlpack.yml | 5 +- cpp/report/src/codeql-pack.lock.yml | 6 ++ cpp/report/src/qlpack.yml | 5 +- docs/development_handbook.md | 13 ++- scripts/get_workspace_packs.py | 14 ++++ scripts/install-packs.py | 23 ++++++ scripts/verify-standard-library-version.py | 68 ++++++++++++++++ supported_codeql_configs.json | 6 +- 45 files changed, 409 insertions(+), 59 deletions(-) create mode 100644 .github/actions/action.yml create mode 100644 .github/actions/install-codeql-packs/action.yml create mode 100644 .github/workflows/verify-standard-library-dependencies.yml delete mode 100644 c/.codeqlmanifest.json create mode 100644 c/cert/src/codeql-pack.lock.yml create mode 100644 c/cert/test/codeql-pack.lock.yml create mode 100644 c/common/src/codeql-pack.lock.yml create mode 100644 c/common/test/codeql-pack.lock.yml create mode 100644 c/misra/src/codeql-pack.lock.yml create mode 100644 c/misra/test/codeql-pack.lock.yml delete mode 100644 cpp/.codeqlmanifest.json create mode 100644 cpp/autosar/src/codeql-pack.lock.yml create mode 100644 cpp/autosar/test/codeql-pack.lock.yml create mode 100644 cpp/cert/src/codeql-pack.lock.yml create mode 100644 cpp/cert/test/codeql-pack.lock.yml create mode 100644 cpp/common/src/codeql-pack.lock.yml create mode 100644 cpp/common/test/codeql-pack.lock.yml create mode 100644 cpp/misra/src/codeql-pack.lock.yml create mode 100644 cpp/misra/test/codeql-pack.lock.yml create mode 100644 cpp/report/src/codeql-pack.lock.yml create mode 100644 scripts/get_workspace_packs.py create mode 100644 scripts/install-packs.py create mode 100644 scripts/verify-standard-library-version.py diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json index ac468d9838..d6277f5949 100644 --- a/.codeqlmanifest.json +++ b/.codeqlmanifest.json @@ -1 +1 @@ -{ "provide": [ "codeql_modules/*/.codeqlmanifest.json", "cpp/.codeqlmanifest.json", "c/.codeqlmanifest.json"] } +{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml" ] } diff --git a/.github/actions/action.yml b/.github/actions/action.yml new file mode 100644 index 0000000000..2e6d5f1a2e --- /dev/null +++ b/.github/actions/action.yml @@ -0,0 +1,25 @@ +name: Install CodeQL library pack dependencies +description: | + Downloads any necessary CodeQL library packs needed by packs in the repo. +inputs: + cli_path: + description: | + The path to the CodeQL CLI directory. + required: false + + mode: + description: | + The `--mode` option to `codeql pack install`. + required: true + default: verify + +runs: + using: composite + steps: + - name: Install CodeQL library packs + shell: bash + env: + CODEQL_CLI: ${{ inputs.cli_path }} + run: | + PATH=$PATH:$CODEQL_CLI + python scripts/install-packs.py --mode ${{ inputs.mode }} diff --git a/.github/actions/install-codeql-packs/action.yml b/.github/actions/install-codeql-packs/action.yml new file mode 100644 index 0000000000..2e6d5f1a2e --- /dev/null +++ b/.github/actions/install-codeql-packs/action.yml @@ -0,0 +1,25 @@ +name: Install CodeQL library pack dependencies +description: | + Downloads any necessary CodeQL library packs needed by packs in the repo. +inputs: + cli_path: + description: | + The path to the CodeQL CLI directory. + required: false + + mode: + description: | + The `--mode` option to `codeql pack install`. + required: true + default: verify + +runs: + using: composite + steps: + - name: Install CodeQL library packs + shell: bash + env: + CODEQL_CLI: ${{ inputs.cli_path }} + run: | + PATH=$PATH:$CODEQL_CLI + python scripts/install-packs.py --mode ${{ inputs.mode }} diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index 0814e059e8..7ffa250669 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -59,6 +59,11 @@ jobs: codeql-home: ${{ github.workspace }}/codeql_home add-to-path: false + - name: Install CodeQL packs + uses: ./.github/actions/install-codeql-packs + with: + cli_path: ${{ github.workspace }}/codeql_home/codeql + - name: Checkout external help files continue-on-error: true id: checkout-external-help-files @@ -82,8 +87,8 @@ jobs: run: | PATH=$PATH:$CODEQL_HOME/codeql - codeql query compile --search-path cpp --threads 0 cpp - codeql query compile --search-path c --search-path cpp --threads 0 c + codeql query compile --threads 0 cpp + codeql query compile --threads 0 c cd .. zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports diff --git a/.github/workflows/codeql_unit_tests.yml b/.github/workflows/codeql_unit_tests.yml index 1a2374d19d..8fad3c3292 100644 --- a/.github/workflows/codeql_unit_tests.yml +++ b/.github/workflows/codeql_unit_tests.yml @@ -66,11 +66,16 @@ jobs: codeql-home: ${{ github.workspace }}/codeql_home add-to-path: false + - name: Install CodeQL packs + uses: ./.github/actions/install-codeql-packs + with: + cli_path: ${{ github.workspace }}/codeql_home/codeql + - name: Pre-Compile Queries id: pre-compile-queries run: | - ${{ github.workspace }}/codeql_home/codeql/codeql query compile --search-path cpp --threads 0 cpp - ${{ github.workspace }}/codeql_home/codeql/codeql query compile --search-path c --search-path cpp --threads 0 c + ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 cpp + ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 c - name: Run test suites @@ -122,18 +127,11 @@ jobs: os.makedirs(os.path.dirname(test_report_path), exist_ok=True) test_report_file = open(test_report_path, 'w') files_to_close.append(test_report_file) - if "${{ matrix.language }}".casefold() == "c".casefold(): - # c tests require cpp -- but we don't want c things on the cpp - # path in case of design errors. - cpp_language_root = Path(workspace, 'cpp') - procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={cpp_language_root}', f'--search-path={language_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE)) - else: - procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={language_root}', f'--search-path={language_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE)) + procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE)) for p in procs: - p.wait() + _, err = p.communicate() if p.returncode != 0: - _, err = p.communicate() if p.returncode == 122: # Failed because a test case failed, so just print the regular output. # This will allow us to proceed to validate-test-results, which will fail if diff --git a/.github/workflows/standard_library_upgrade_tests.yml b/.github/workflows/standard_library_upgrade_tests.yml index 0a4e58dbd3..4f7c2de75c 100644 --- a/.github/workflows/standard_library_upgrade_tests.yml +++ b/.github/workflows/standard_library_upgrade_tests.yml @@ -116,7 +116,7 @@ jobs: stdlib_path = os.path.join(codeql_home, 'codeql-stdlib') cpp_test_root = Path(stdlib_path, 'cpp/ql/test') print(f"Executing tests found (recursively) in the directory '{cpp_test_root}'") - cp = subprocess.run([codeql_bin, "test", "run", "--format=json", f'--search-path={stdlib_path}', cpp_test_root], stdout=test_report_file, stderr=subprocess.PIPE) + cp = subprocess.run([codeql_bin, "test", "run", "--format=json", cpp_test_root], stdout=test_report_file, stderr=subprocess.PIPE) if cp.returncode != 0: print_error_and_fail(f"Failed to run tests with return code {cp.returncode} and error {cp.stderr}") diff --git a/.github/workflows/verify-standard-library-dependencies.yml b/.github/workflows/verify-standard-library-dependencies.yml new file mode 100644 index 0000000000..1be00ef43e --- /dev/null +++ b/.github/workflows/verify-standard-library-dependencies.yml @@ -0,0 +1,79 @@ +name: Verify Standard Library Dependencies + +# Run this workflow every time the "supported_codeql_configs.json" file or a "qlpack.yml" file is changed +on: + pull_request: + branches: + - main + - "rc/**" + - next + paths: + - "supported_codeql_configs.json" + - "qlpack.yml" + workflow_dispatch: + +jobs: + prepare-matrix: + name: Prepare CodeQL configuration matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.export-matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Export unit test matrix + id: export-matrix + run: | + echo "::set-output name=matrix::$( + jq --compact-output \ + '.supported_environment | map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}' \ + supported_codeql_configs.json + )" + + verify-dependencies: + name: Verify dependencies + needs: prepare-matrix + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{fromJSON(needs.prepare-matrix.outputs.matrix)}} + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup Python 3 + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - name: Cache CodeQL + id: cache-codeql + uses: actions/cache@v2.1.3 + with: + # A list of files, directories, and wildcard patterns to cache and restore + path: ${{github.workspace}}/codeql_home + # An explicit key for restoring and saving the cache + key: codeql-home-${{matrix.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library}} + + - name: Install CodeQL + if: steps.cache-codeql.outputs.cache-hit != 'true' + uses: ./.github/actions/install-codeql + with: + codeql-cli-version: ${{matrix.codeql_cli}} + codeql-stdlib-version: ${{matrix.codeql_standard_library}} + codeql-home: ${{ github.workspace }}/codeql_home + + - name: Verify dependencies + shell: bash + env: + CLI_PATH: ${{ github.workspace }}/codeql_home/codeql + STDLIB_PATH: ${{ github.workspace }}/codeql_home/codeql-stdlib + run: | + PATH=$PATH:$CLI_PATH + ls $STDLIB_PATH + pip install -r scripts/requirements.txt + python3 scripts/verify-standard-library-version.py --codeql-repo $STDLIB_PATH --mode verify + diff --git a/.gitignore b/.gitignore index 5466e33c8f..360134b51c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ # C/C++ build artifacts *.o /databases/ + +# CodeQL build artifacts +**/.codeql/** diff --git a/c/.codeqlmanifest.json b/c/.codeqlmanifest.json deleted file mode 100644 index 384848fdd1..0000000000 --- a/c/.codeqlmanifest.json +++ /dev/null @@ -1,3 +0,0 @@ -{ "provide": [ - "*/src/qlpack.yml", - "*/test/qlpack.yml" ] } diff --git a/c/cert/src/codeql-pack.lock.yml b/c/cert/src/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/c/cert/src/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/c/cert/src/qlpack.yml b/c/cert/src/qlpack.yml index eed7d5a341..ce775072e3 100644 --- a/c/cert/src/qlpack.yml +++ b/c/cert/src/qlpack.yml @@ -1,4 +1,6 @@ name: cert-c-coding-standards version: 2.8.0-dev suites: codeql-suites -libraryPathDependencies: common-c-coding-standards \ No newline at end of file +dependencies: + codeql/common-c-coding-standards: '*' + codeql/cpp-all: 0.0.13 diff --git a/c/cert/test/codeql-pack.lock.yml b/c/cert/test/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/c/cert/test/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/c/cert/test/qlpack.yml b/c/cert/test/qlpack.yml index c8d8bcc665..bd8995b986 100644 --- a/c/cert/test/qlpack.yml +++ b/c/cert/test/qlpack.yml @@ -1,4 +1,5 @@ -name: cert-c-coding-standards-tests +name: codeql/cert-c-coding-standards-tests version: 2.8.0-dev -libraryPathDependencies: cert-c-coding-standards -extractor: cpp \ No newline at end of file +extractor: cpp +dependencies: + codeql/cert-c-coding-standards: '*' diff --git a/c/common/src/codeql-pack.lock.yml b/c/common/src/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/c/common/src/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/c/common/src/qlpack.yml b/c/common/src/qlpack.yml index 4b9b0fd2e1..b93e61c8ad 100644 --- a/c/common/src/qlpack.yml +++ b/c/common/src/qlpack.yml @@ -1,3 +1,5 @@ -name: common-c-coding-standards +name: codeql/common-c-coding-standards version: 2.8.0-dev -libraryPathDependencies: common-cpp-coding-standards +dependencies: + codeql/common-cpp-coding-standards: '*' + codeql/cpp-all: 0.0.13 diff --git a/c/common/test/codeql-pack.lock.yml b/c/common/test/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/c/common/test/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/c/common/test/qlpack.yml b/c/common/test/qlpack.yml index 15401c7d35..ec7387fb99 100644 --- a/c/common/test/qlpack.yml +++ b/c/common/test/qlpack.yml @@ -1,4 +1,5 @@ -name: common-c-coding-standards-tests +name: codeql/common-c-coding-standards-tests version: 2.8.0-dev -libraryPathDependencies: common-c-coding-standards extractor: cpp +dependencies: + codeql/common-c-coding-standards: '*' diff --git a/c/misra/src/codeql-pack.lock.yml b/c/misra/src/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/c/misra/src/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/c/misra/src/qlpack.yml b/c/misra/src/qlpack.yml index 025314afb2..cb9b81dbc5 100644 --- a/c/misra/src/qlpack.yml +++ b/c/misra/src/qlpack.yml @@ -1,4 +1,6 @@ -name: misra-c-coding-standards +name: codeql/misra-c-coding-standards version: 2.8.0-dev suites: codeql-suites -libraryPathDependencies: common-c-coding-standards +dependencies: + codeql/common-c-coding-standards: '*' + codeql/cpp-all: 0.0.13 diff --git a/c/misra/test/codeql-pack.lock.yml b/c/misra/test/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/c/misra/test/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/c/misra/test/qlpack.yml b/c/misra/test/qlpack.yml index e496bada24..642e48fdf8 100644 --- a/c/misra/test/qlpack.yml +++ b/c/misra/test/qlpack.yml @@ -1,4 +1,5 @@ -name: misra-c-coding-standards-tests +name: codeql/misra-c-coding-standards-tests version: 2.8.0-dev -libraryPathDependencies: misra-c-coding-standards -extractor: cpp \ No newline at end of file +extractor: cpp +dependencies: + codeql/misra-c-coding-standards: '*' diff --git a/cpp/.codeqlmanifest.json b/cpp/.codeqlmanifest.json deleted file mode 100644 index 384848fdd1..0000000000 --- a/cpp/.codeqlmanifest.json +++ /dev/null @@ -1,3 +0,0 @@ -{ "provide": [ - "*/src/qlpack.yml", - "*/test/qlpack.yml" ] } diff --git a/cpp/autosar/src/codeql-pack.lock.yml b/cpp/autosar/src/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/autosar/src/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/autosar/src/qlpack.yml b/cpp/autosar/src/qlpack.yml index 734829a2f5..34949a9d49 100644 --- a/cpp/autosar/src/qlpack.yml +++ b/cpp/autosar/src/qlpack.yml @@ -1,4 +1,6 @@ -name: autosar-cpp-coding-standards +name: codeql/autosar-cpp-coding-standards version: 2.8.0-dev suites: codeql-suites -libraryPathDependencies: common-cpp-coding-standards +dependencies: + codeql/common-cpp-coding-standards: '*' + codeql/cpp-all: 0.0.13 diff --git a/cpp/autosar/test/codeql-pack.lock.yml b/cpp/autosar/test/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/autosar/test/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/autosar/test/qlpack.yml b/cpp/autosar/test/qlpack.yml index 1cae2388c4..dcc32f21ff 100644 --- a/cpp/autosar/test/qlpack.yml +++ b/cpp/autosar/test/qlpack.yml @@ -1,4 +1,5 @@ -name: autosar-cpp-coding-standards-tests +name: codeql/autosar-cpp-coding-standards-tests version: 2.8.0-dev -libraryPathDependencies: autosar-cpp-coding-standards extractor: cpp +dependencies: + codeql/autosar-cpp-coding-standards: '*' diff --git a/cpp/cert/src/codeql-pack.lock.yml b/cpp/cert/src/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/cert/src/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/cert/src/qlpack.yml b/cpp/cert/src/qlpack.yml index 0eff915b6c..2120913d86 100644 --- a/cpp/cert/src/qlpack.yml +++ b/cpp/cert/src/qlpack.yml @@ -1,4 +1,6 @@ -name: cert-cpp-coding-standards +name: codeql/cert-cpp-coding-standards version: 2.8.0-dev suites: codeql-suites -libraryPathDependencies: common-cpp-coding-standards +dependencies: + codeql/cpp-all: 0.0.13 + codeql/common-cpp-coding-standards: '*' diff --git a/cpp/cert/test/codeql-pack.lock.yml b/cpp/cert/test/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/cert/test/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/cert/test/qlpack.yml b/cpp/cert/test/qlpack.yml index 7305b286f2..b65f6e59ec 100644 --- a/cpp/cert/test/qlpack.yml +++ b/cpp/cert/test/qlpack.yml @@ -1,4 +1,5 @@ -name: cert-cpp-coding-standards-tests +name: codeql/cert-cpp-coding-standards-tests version: 2.8.0-dev -libraryPathDependencies: cert-cpp-coding-standards extractor: cpp +dependencies: + codeql/cert-cpp-coding-standards: '*' diff --git a/cpp/common/src/codeql-pack.lock.yml b/cpp/common/src/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/common/src/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/common/src/qlpack.yml b/cpp/common/src/qlpack.yml index a2dd5bcff1..ac8d433906 100644 --- a/cpp/common/src/qlpack.yml +++ b/cpp/common/src/qlpack.yml @@ -1,3 +1,4 @@ -name: common-cpp-coding-standards +name: codeql/common-cpp-coding-standards version: 2.8.0-dev -libraryPathDependencies: codeql-cpp +dependencies: + codeql/cpp-all: 0.0.13 diff --git a/cpp/common/test/codeql-pack.lock.yml b/cpp/common/test/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/common/test/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/common/test/qlpack.yml b/cpp/common/test/qlpack.yml index ce5b6c0943..fb42d63ab3 100644 --- a/cpp/common/test/qlpack.yml +++ b/cpp/common/test/qlpack.yml @@ -1,4 +1,5 @@ -name: common-cpp-coding-standards-tests +name: codeql/common-cpp-coding-standards-tests version: 2.8.0-dev -libraryPathDependencies: common-cpp-coding-standards extractor: cpp +dependencies: + codeql/common-cpp-coding-standards: '*' diff --git a/cpp/misra/src/codeql-pack.lock.yml b/cpp/misra/src/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/misra/src/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/misra/src/qlpack.yml b/cpp/misra/src/qlpack.yml index d300163d7d..ce43905c4f 100644 --- a/cpp/misra/src/qlpack.yml +++ b/cpp/misra/src/qlpack.yml @@ -1,3 +1,5 @@ -name: misra-cpp-coding-standards +name: codeql/misra-cpp-coding-standards version: 2.8.0-dev -libraryPathDependencies: common-cpp-coding-standards +dependencies: + codeql/common-cpp-coding-standards: '*' + codeql/cpp-all: 0.0.13 diff --git a/cpp/misra/test/codeql-pack.lock.yml b/cpp/misra/test/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/misra/test/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/misra/test/qlpack.yml b/cpp/misra/test/qlpack.yml index f4e8b89b67..e8117d58e9 100644 --- a/cpp/misra/test/qlpack.yml +++ b/cpp/misra/test/qlpack.yml @@ -1,4 +1,5 @@ -name: misra-cpp-coding-standards-tests +name: codeql/misra-cpp-coding-standards-tests version: 2.8.0-dev -libraryPathDependencies: misra-cpp-coding-standards extractor: cpp +dependencies: + codeql/misra-cpp-coding-standards: '*' \ No newline at end of file diff --git a/cpp/report/src/codeql-pack.lock.yml b/cpp/report/src/codeql-pack.lock.yml new file mode 100644 index 0000000000..421c52af64 --- /dev/null +++ b/cpp/report/src/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.0.13 +compiled: false +lockVersion: 1.0.0 diff --git a/cpp/report/src/qlpack.yml b/cpp/report/src/qlpack.yml index e7ea756cfd..41e6570fd4 100644 --- a/cpp/report/src/qlpack.yml +++ b/cpp/report/src/qlpack.yml @@ -1,3 +1,4 @@ -name: report-cpp-coding-standards +name: codeql/report-cpp-coding-standards version: 2.8.0-dev -libraryPathDependencies: codeql-cpp +dependencies: + codeql/cpp-all: 0.0.13 diff --git a/docs/development_handbook.md b/docs/development_handbook.md index 898fed7542..6144ead0d4 100644 --- a/docs/development_handbook.md +++ b/docs/development_handbook.md @@ -32,6 +32,7 @@ | 0.24.0 | 2022-07-05 | Remco Vermeulen | Update release process to include steps for external help files. | | 0.25.0 | 2022-07-22 | Jeroen Ketema | Document the existence and purpose of the `next` branch. | | 0.26.0 | 2022-08-10 | Remco Vermeulen | Address incorrect package file generation command. This was missing the required language argument. | +| 0.27.0 | 2022-08-31 | David Bartolomeo | Add section on installing QL dependencies and update CLI commands to account for the migration to CodeQL packs. ## Scope of work @@ -332,6 +333,12 @@ A query **must** include: All public predicates, classes, modules and files should be documented with QLDoc. All QLDoc should follow the [QLDoc style guide](https://github.com/github/codeql/blob/main/docs/qldoc-style-guide.md). +### Installing QL dependencies + +All of our query and library packs depend on the standard CodeQL library for C++, `codeql/cpp-all`. This dependency is specified in the `qlpack.yml` file for each of our packs. Before compiling, running, or testing any of our queries or libraries, you must download the proper dependencies by running `python3 scripts/install-packs.py`. This will download the appropriate version of the standard library from the public package registry, installing it in a cache in your `~/.codeql` directory. When compiling queries or running tests, the QL compiler will pick up the appropriate dependencies from this cache without any need to specify an additional library search path on the command line. + +Because the downloaded packs are cached, it is only necessary to run `install-packs.py` once each time we upgrade to a new standard library version. It does not hurt to run it more often; if all necessary packs are already in the download cache, then it will complete quickly without trying to download anything. + ### Unit testing Every query which implements a rule **must** include: @@ -345,11 +352,10 @@ During query development in VS Code, the unit tests can be run using the [testin Unit tests can also be run on the command line using the CodeQL CLI. With an appropriate CodeQL CLI (as specified in the `supported_codeql_configs.json` at the root of the repository), you can run the following from the root of the repository: ``` -codeql test run --show-extractor-output --search-path . path/to/test/directory +codeql test run --show-extractor-output path/to/test/directory ``` * `--show-extractor-output` - this shows the output from the extractor. It is most useful when the test fails because the file is not valid C++, where the extractor output will include the compilation failure. This is not shown in VS Code. -* `--search-path .` - this allows the CodeQL CLI to discover all the QL packs within our repository. * `path/to/test/directory` - this can be a qlref file (like `cpp/autosar/test/rules/A15-2-2/`), a rule directory (`cpp/autosar/test/rules/A15-2-2/`) or a test qlpack (`cpp/autosar/test/`). For more details on running unit tests with the CodeQL CLI see the [Testing custom queries](https://codeql.github.com/docs/codeql-cli/testing-custom-queries/) help topic. @@ -668,7 +674,6 @@ ls cpp/cert/src/$(cat cpp/cert/test/rules/EXP52-CPP/DoNotRelyOnSideEffectsInDecl # Run a test. See # https://github.com/github/codeql-coding-standards/blob/main/development_handbook.md#unit-testing codeql test run --show-extractor-output \ - --search-path . \ cpp/cert/test/rules/EXP52-CPP/DoNotRelyOnSideEffectsInDeclTypeOperand.qlref # Get a db error? Applying the recommended fix @@ -686,7 +691,7 @@ codeql test run --show-extractor-output \ # If the expected output is not yet present, it is printed as a diff: mv cpp/cert/test/rules/EXP52-CPP/DoNotRelyOnSideEffectsInDeclTypeOperand.expected foo -codeql test run --show-extractor-output --search-path . \ +codeql test run --show-extractor-output \ cpp/cert/test/rules/EXP52-CPP/DoNotRelyOnSideEffectsInDeclTypeOperand.qlref # The actual output can be accepted via codeql test accept (which moves some files): diff --git a/scripts/get_workspace_packs.py b/scripts/get_workspace_packs.py new file mode 100644 index 0000000000..fc9054c641 --- /dev/null +++ b/scripts/get_workspace_packs.py @@ -0,0 +1,14 @@ +import glob +import json +import os + +def get_workspace_packs(root): + # Find the packs by globbing using the 'provide' patterns in the manifest. + os.chdir(root) + with open('.codeqlmanifest.json') as manifest_file: + manifest = json.load(manifest_file) + packs = [] + for pattern in manifest['provide']: + packs.extend(glob.glob(pattern, recursive=True)) + + return packs diff --git a/scripts/install-packs.py b/scripts/install-packs.py new file mode 100644 index 0000000000..a0286e2228 --- /dev/null +++ b/scripts/install-packs.py @@ -0,0 +1,23 @@ +import argparse +import os +import subprocess +import get_workspace_packs + +parser = argparse.ArgumentParser(description="Install CodeQL library pack dependencies.") +parser.add_argument('--mode', required=False, choices=['use-lock', 'update', 'verify', 'no-lock'], default="use-lock", help="Installation mode, identical to the `--mode` argument to `codeql pack install`") +parser.add_argument('--codeql', required=False, default='codeql', help="Path to the `codeql` executable.") +args = parser.parse_args() + +# Find the root of the repo +root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +packs = get_workspace_packs.get_workspace_packs(root) + +# Find the CodeQL packs in the repo. This can also return packs outside of the repo, if those packs +# are installed in a sibling directory to the CLI. +for pack in packs: + pack_path = os.path.join(root, pack) + # Run `codeql pack install` to install dependencies. + command = [args.codeql, 'pack', 'install', '--mode', args.mode, pack_path] + print(f'Running `{" ".join(command)}`') + subprocess.check_call(command) diff --git a/scripts/verify-standard-library-version.py b/scripts/verify-standard-library-version.py new file mode 100644 index 0000000000..0b46068e1d --- /dev/null +++ b/scripts/verify-standard-library-version.py @@ -0,0 +1,68 @@ +import argparse +import json +import os +import subprocess +import yaml +import get_workspace_packs + +def get_codeql_packs(codeql_repo, codeql): + command = [codeql, 'resolve', 'qlpacks', '--additional-packs', codeql_repo, '--format', 'json'] + print(f'Running `{" ".join(command)}`') + packs_json = subprocess.check_output(command) + print(packs_json) + packs = json.loads(packs_json) + return packs + +parser = argparse.ArgumentParser(description='Ensure that CodeQL library pack dependency versions match the supported configuration.') +parser.add_argument('--codeql-repo', required=True, help='Path to checkout of `github/codeql` repo at desired branch.') +parser.add_argument('--mode', required=False, choices=['verify', 'update'], default='verify', help="`verify` to fail on mismatch; `update` to change `qlpack.lock.yml` files to use new version.") +parser.add_argument('--codeql', required=False, default='codeql', help='Path to the `codeql` executable.') +args = parser.parse_args() + +# Find the root of the repo +root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Get the packs for the repo's workspace. +workspace_packs = get_workspace_packs.get_workspace_packs(root) + +# Get the packs from the `codeql` repo checkout. +codeql_packs = get_codeql_packs(args.codeql_repo, args.codeql) + +failed = False +for pack in workspace_packs: + pack_path = os.path.join(root, pack) + + print(f"Scanning dependencies of '{pack_path}'...") + + # Read our pack's configuration file. + with open(pack_path) as pack_file: + pack_yaml = yaml.safe_load(pack_file) + + updated = False + if 'dependencies' in pack_yaml: + dependencies = pack_yaml['dependencies'] + for ref_name in dependencies: + ref_version = dependencies[ref_name] + if ref_name in codeql_packs: + # Found this reference in the `codeql` repo. The version of the reference should match + # the version of that pack in the `codeql` repo. + lib_path = codeql_packs[ref_name][0] + lib_path = os.path.join(lib_path, 'qlpack.yml') + with open(lib_path) as lib_file: + lib_yaml = yaml.safe_load(lib_file) + lib_version = lib_yaml['version'] + if ref_version != lib_version: + print(f"Mismatched versions for '{ref_name}', referenced from '{pack_path}'. " + + f"referenced version is '{ref_version}', but should be '{lib_version}'.") + if args.mode == 'verify': + failed = True # Report an error at the end. + else: + pack_yaml['dependencies'][ref_name] = lib_version + updated = True # Update our pack in-place. + + if updated: + print(f"Updating '{pack_path}'...") + with open(pack_path, 'w', newline='\n') as pack_file: # Always use LF even on Windows + yaml.safe_dump(pack_yaml, pack_file, sort_keys=False) + +exit(1 if failed else 0) diff --git a/supported_codeql_configs.json b/supported_codeql_configs.json index a0ad42a349..bba3112bae 100644 --- a/supported_codeql_configs.json +++ b/supported_codeql_configs.json @@ -1,9 +1,9 @@ { "supported_environment": [ { - "codeql_cli": "2.7.6", - "codeql_standard_library": "codeql-cli/v2.7.6", - "codeql_cli_bundle": "codeql-bundle-20220120" + "codeql_cli": "2.8.5", + "codeql_standard_library": "codeql-cli/v2.8.5", + "codeql_cli_bundle": "codeql-bundle-20220401" } ], "supported_language" : [ From 3b7a5e0702636968061c30ef153051622f49b28c Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 14 Jul 2022 15:27:51 +0200 Subject: [PATCH 02/37] Update to CodeQL 2.9.4 --- c/cert/src/codeql-pack.lock.yml | 2 +- c/cert/src/qlpack.yml | 2 +- c/cert/test/codeql-pack.lock.yml | 2 +- c/common/src/codeql-pack.lock.yml | 2 +- c/common/src/qlpack.yml | 2 +- c/common/test/codeql-pack.lock.yml | 2 +- c/misra/src/codeql-pack.lock.yml | 2 +- c/misra/src/qlpack.yml | 2 +- c/misra/test/codeql-pack.lock.yml | 2 +- cpp/autosar/src/codeql-pack.lock.yml | 2 +- cpp/autosar/src/qlpack.yml | 2 +- cpp/autosar/test/codeql-pack.lock.yml | 2 +- cpp/cert/src/codeql-pack.lock.yml | 2 +- cpp/cert/src/qlpack.yml | 2 +- cpp/cert/test/codeql-pack.lock.yml | 2 +- cpp/common/src/codeql-pack.lock.yml | 2 +- cpp/common/src/qlpack.yml | 2 +- cpp/common/test/codeql-pack.lock.yml | 2 +- cpp/misra/src/codeql-pack.lock.yml | 2 +- cpp/misra/src/qlpack.yml | 2 +- cpp/misra/test/codeql-pack.lock.yml | 2 +- cpp/report/src/codeql-pack.lock.yml | 2 +- cpp/report/src/qlpack.yml | 2 +- supported_codeql_configs.json | 6 +++--- 24 files changed, 26 insertions(+), 26 deletions(-) diff --git a/c/cert/src/codeql-pack.lock.yml b/c/cert/src/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/c/cert/src/codeql-pack.lock.yml +++ b/c/cert/src/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/c/cert/src/qlpack.yml b/c/cert/src/qlpack.yml index ce775072e3..644ac974d6 100644 --- a/c/cert/src/qlpack.yml +++ b/c/cert/src/qlpack.yml @@ -3,4 +3,4 @@ version: 2.8.0-dev suites: codeql-suites dependencies: codeql/common-c-coding-standards: '*' - codeql/cpp-all: 0.0.13 + codeql/cpp-all: 0.2.3 diff --git a/c/cert/test/codeql-pack.lock.yml b/c/cert/test/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/c/cert/test/codeql-pack.lock.yml +++ b/c/cert/test/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/c/common/src/codeql-pack.lock.yml b/c/common/src/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/c/common/src/codeql-pack.lock.yml +++ b/c/common/src/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/c/common/src/qlpack.yml b/c/common/src/qlpack.yml index b93e61c8ad..2d3953f63a 100644 --- a/c/common/src/qlpack.yml +++ b/c/common/src/qlpack.yml @@ -2,4 +2,4 @@ name: codeql/common-c-coding-standards version: 2.8.0-dev dependencies: codeql/common-cpp-coding-standards: '*' - codeql/cpp-all: 0.0.13 + codeql/cpp-all: 0.2.3 diff --git a/c/common/test/codeql-pack.lock.yml b/c/common/test/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/c/common/test/codeql-pack.lock.yml +++ b/c/common/test/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/c/misra/src/codeql-pack.lock.yml b/c/misra/src/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/c/misra/src/codeql-pack.lock.yml +++ b/c/misra/src/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/c/misra/src/qlpack.yml b/c/misra/src/qlpack.yml index cb9b81dbc5..17586fc09f 100644 --- a/c/misra/src/qlpack.yml +++ b/c/misra/src/qlpack.yml @@ -3,4 +3,4 @@ version: 2.8.0-dev suites: codeql-suites dependencies: codeql/common-c-coding-standards: '*' - codeql/cpp-all: 0.0.13 + codeql/cpp-all: 0.2.3 diff --git a/c/misra/test/codeql-pack.lock.yml b/c/misra/test/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/c/misra/test/codeql-pack.lock.yml +++ b/c/misra/test/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/autosar/src/codeql-pack.lock.yml b/cpp/autosar/src/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/autosar/src/codeql-pack.lock.yml +++ b/cpp/autosar/src/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/autosar/src/qlpack.yml b/cpp/autosar/src/qlpack.yml index 34949a9d49..56abc6f259 100644 --- a/cpp/autosar/src/qlpack.yml +++ b/cpp/autosar/src/qlpack.yml @@ -3,4 +3,4 @@ version: 2.8.0-dev suites: codeql-suites dependencies: codeql/common-cpp-coding-standards: '*' - codeql/cpp-all: 0.0.13 + codeql/cpp-all: 0.2.3 diff --git a/cpp/autosar/test/codeql-pack.lock.yml b/cpp/autosar/test/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/autosar/test/codeql-pack.lock.yml +++ b/cpp/autosar/test/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/cert/src/codeql-pack.lock.yml b/cpp/cert/src/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/cert/src/codeql-pack.lock.yml +++ b/cpp/cert/src/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/cert/src/qlpack.yml b/cpp/cert/src/qlpack.yml index 2120913d86..f2e265f4a3 100644 --- a/cpp/cert/src/qlpack.yml +++ b/cpp/cert/src/qlpack.yml @@ -2,5 +2,5 @@ name: codeql/cert-cpp-coding-standards version: 2.8.0-dev suites: codeql-suites dependencies: - codeql/cpp-all: 0.0.13 + codeql/cpp-all: 0.2.3 codeql/common-cpp-coding-standards: '*' diff --git a/cpp/cert/test/codeql-pack.lock.yml b/cpp/cert/test/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/cert/test/codeql-pack.lock.yml +++ b/cpp/cert/test/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/common/src/codeql-pack.lock.yml b/cpp/common/src/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/common/src/codeql-pack.lock.yml +++ b/cpp/common/src/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/common/src/qlpack.yml b/cpp/common/src/qlpack.yml index ac8d433906..7bcd661037 100644 --- a/cpp/common/src/qlpack.yml +++ b/cpp/common/src/qlpack.yml @@ -1,4 +1,4 @@ name: codeql/common-cpp-coding-standards version: 2.8.0-dev dependencies: - codeql/cpp-all: 0.0.13 + codeql/cpp-all: 0.2.3 diff --git a/cpp/common/test/codeql-pack.lock.yml b/cpp/common/test/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/common/test/codeql-pack.lock.yml +++ b/cpp/common/test/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/misra/src/codeql-pack.lock.yml b/cpp/misra/src/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/misra/src/codeql-pack.lock.yml +++ b/cpp/misra/src/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/misra/src/qlpack.yml b/cpp/misra/src/qlpack.yml index ce43905c4f..44d2b10c43 100644 --- a/cpp/misra/src/qlpack.yml +++ b/cpp/misra/src/qlpack.yml @@ -2,4 +2,4 @@ name: codeql/misra-cpp-coding-standards version: 2.8.0-dev dependencies: codeql/common-cpp-coding-standards: '*' - codeql/cpp-all: 0.0.13 + codeql/cpp-all: 0.2.3 diff --git a/cpp/misra/test/codeql-pack.lock.yml b/cpp/misra/test/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/misra/test/codeql-pack.lock.yml +++ b/cpp/misra/test/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/report/src/codeql-pack.lock.yml b/cpp/report/src/codeql-pack.lock.yml index 421c52af64..b0f02a9d1f 100644 --- a/cpp/report/src/codeql-pack.lock.yml +++ b/cpp/report/src/codeql-pack.lock.yml @@ -1,6 +1,6 @@ --- dependencies: codeql/cpp-all: - version: 0.0.13 + version: 0.2.3 compiled: false lockVersion: 1.0.0 diff --git a/cpp/report/src/qlpack.yml b/cpp/report/src/qlpack.yml index 41e6570fd4..f34d8fe4cd 100644 --- a/cpp/report/src/qlpack.yml +++ b/cpp/report/src/qlpack.yml @@ -1,4 +1,4 @@ name: codeql/report-cpp-coding-standards version: 2.8.0-dev dependencies: - codeql/cpp-all: 0.0.13 + codeql/cpp-all: 0.2.3 diff --git a/supported_codeql_configs.json b/supported_codeql_configs.json index bba3112bae..269af480e3 100644 --- a/supported_codeql_configs.json +++ b/supported_codeql_configs.json @@ -1,9 +1,9 @@ { "supported_environment": [ { - "codeql_cli": "2.8.5", - "codeql_standard_library": "codeql-cli/v2.8.5", - "codeql_cli_bundle": "codeql-bundle-20220401" + "codeql_cli": "2.9.4", + "codeql_standard_library": "codeql-cli/v2.9.4", + "codeql_cli_bundle": "codeql-bundle-20220615" } ], "supported_language" : [ From 8c2be2889c8b6eaf6c3a6742120842a6f16a8b38 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 15 Jul 2022 19:34:28 +0200 Subject: [PATCH 03/37] Standardize on Python 3.9 per the user manual --- .github/actions/action.yml | 25 ------------------- .../verify-standard-library-dependencies.yml | 4 +-- 2 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 .github/actions/action.yml diff --git a/.github/actions/action.yml b/.github/actions/action.yml deleted file mode 100644 index 2e6d5f1a2e..0000000000 --- a/.github/actions/action.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Install CodeQL library pack dependencies -description: | - Downloads any necessary CodeQL library packs needed by packs in the repo. -inputs: - cli_path: - description: | - The path to the CodeQL CLI directory. - required: false - - mode: - description: | - The `--mode` option to `codeql pack install`. - required: true - default: verify - -runs: - using: composite - steps: - - name: Install CodeQL library packs - shell: bash - env: - CODEQL_CLI: ${{ inputs.cli_path }} - run: | - PATH=$PATH:$CODEQL_CLI - python scripts/install-packs.py --mode ${{ inputs.mode }} diff --git a/.github/workflows/verify-standard-library-dependencies.yml b/.github/workflows/verify-standard-library-dependencies.yml index 1be00ef43e..0b05736ddb 100644 --- a/.github/workflows/verify-standard-library-dependencies.yml +++ b/.github/workflows/verify-standard-library-dependencies.yml @@ -45,9 +45,9 @@ jobs: uses: actions/checkout@v2 - name: Setup Python 3 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: "3.x" + python-version: "3.9" - name: Cache CodeQL id: cache-codeql From 4d698bc7d52468b2242a23dc6caed95fd0431257 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 14 Jul 2022 20:35:49 +0200 Subject: [PATCH 04/37] Bump codeql submodule --- codeql_modules/codeql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeql_modules/codeql b/codeql_modules/codeql index a1cdf256ad..4551af90f6 160000 --- a/codeql_modules/codeql +++ b/codeql_modules/codeql @@ -1 +1 @@ -Subproject commit a1cdf256ad6b7c3e9984db9069671647e5f47921 +Subproject commit 4551af90f61a8d5f5c1c88a036595b5919a6c98e From a1750a6e97da0b8fd0403cbebc78b1e7216a543d Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 14 Jul 2022 13:13:14 +0200 Subject: [PATCH 05/37] Update test expectations for new flow paths --- ...OwnedPointerValueStoredInUnrelatedSmartPointer.expected | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cpp/common/test/rules/ownedpointervaluestoredinunrelatedsmartpointer/OwnedPointerValueStoredInUnrelatedSmartPointer.expected b/cpp/common/test/rules/ownedpointervaluestoredinunrelatedsmartpointer/OwnedPointerValueStoredInUnrelatedSmartPointer.expected index e291147d15..a4f85ecb72 100644 --- a/cpp/common/test/rules/ownedpointervaluestoredinunrelatedsmartpointer/OwnedPointerValueStoredInUnrelatedSmartPointer.expected +++ b/cpp/common/test/rules/ownedpointervaluestoredinunrelatedsmartpointer/OwnedPointerValueStoredInUnrelatedSmartPointer.expected @@ -11,21 +11,14 @@ edges | test.cpp:3:14:3:15 | v1 | test.cpp:6:31:6:33 | call to get | | test.cpp:3:14:3:15 | v1 | test.cpp:7:28:7:29 | v2 | | test.cpp:4:13:4:14 | v1 | test.cpp:7:28:7:29 | v2 | -| test.cpp:5:27:5:28 | v1 | ../../includes/standard-library/memory.h:76:17:76:19 | ptr | | test.cpp:5:27:5:29 | call to shared_ptr | test.cpp:6:31:6:33 | call to get | -| test.cpp:6:31:6:33 | call to get | ../../includes/standard-library/memory.h:76:17:76:19 | ptr | -| test.cpp:6:31:6:33 | call to get | ../../includes/standard-library/memory.h:76:17:76:19 | ptr | -| test.cpp:7:28:7:29 | v2 | ../../includes/standard-library/memory.h:76:17:76:19 | ptr | -| test.cpp:7:28:7:29 | v2 | ../../includes/standard-library/memory.h:76:17:76:19 | ptr | | test.cpp:8:8:8:14 | 0 | test.cpp:9:28:9:29 | v2 | -| test.cpp:9:28:9:29 | v2 | ../../includes/standard-library/memory.h:76:17:76:19 | ptr | | test.cpp:10:8:10:17 | new | test.cpp:11:28:11:29 | v2 | | test.cpp:10:8:10:17 | new | test.cpp:11:28:11:29 | v2 | | test.cpp:10:8:10:17 | new | test.cpp:12:28:12:29 | v2 | | test.cpp:11:28:11:29 | ref arg v2 | test.cpp:12:28:12:29 | v2 | | test.cpp:11:28:11:29 | v2 | ../../includes/standard-library/memory.h:76:17:76:19 | ptr | | test.cpp:11:28:11:29 | v2 | test.cpp:11:28:11:29 | ref arg v2 | -| test.cpp:12:28:12:29 | v2 | ../../includes/standard-library/memory.h:76:17:76:19 | ptr | | test.cpp:16:13:16:22 | new | test.cpp:17:27:17:28 | v1 | | test.cpp:16:13:16:22 | new | test.cpp:17:27:17:28 | v1 | | test.cpp:16:13:16:22 | new | test.cpp:19:6:19:7 | v1 | From 487c3712e9c39cca05b5ed63b975690c77f0b7d3 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 15 Jul 2022 17:46:05 +0200 Subject: [PATCH 06/37] Fix braced initialization detection in A8-5-3 --- ...-fix-A8-5-3-braced-initialization-detection.md | 3 +++ .../A8-5-3/AvoidAutoWithBracedInitialization.ql | 2 +- cpp/autosar/test/rules/A8-5-3/test.cpp | 15 ++++++++------- 3 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 change_notes/2022-06-01-fix-A8-5-3-braced-initialization-detection.md diff --git a/change_notes/2022-06-01-fix-A8-5-3-braced-initialization-detection.md b/change_notes/2022-06-01-fix-A8-5-3-braced-initialization-detection.md new file mode 100644 index 0000000000..ec48fda73a --- /dev/null +++ b/change_notes/2022-06-01-fix-A8-5-3-braced-initialization-detection.md @@ -0,0 +1,3 @@ +- `A8-5-3` - `AvoidAutoWithBracedInitialization.ql`: + - Fix regression where `auto x{0}` was no longer detected as a braced initialization with type `auto` with the latest CodeQL versions. + - No longer falsely detect cases where braced initialization was not used, but where the inferred type would be `std::initializer_list`. diff --git a/cpp/autosar/src/rules/A8-5-3/AvoidAutoWithBracedInitialization.ql b/cpp/autosar/src/rules/A8-5-3/AvoidAutoWithBracedInitialization.ql index b62b8785c3..bcb64dba8d 100644 --- a/cpp/autosar/src/rules/A8-5-3/AvoidAutoWithBracedInitialization.ql +++ b/cpp/autosar/src/rules/A8-5-3/AvoidAutoWithBracedInitialization.ql @@ -21,5 +21,5 @@ from Variable v where not isExcluded(v, InitializationPackage::avoidAutoWithBracedInitializationQuery()) and v.getTypeWithAuto().getUnspecifiedType() instanceof AutoType and - v.getType().getUnspecifiedType().(Class).hasQualifiedName("std", "initializer_list") + v.getInitializer().isBraced() select v, "Variable " + v.getName() + " of type auto uses braced initialization." diff --git a/cpp/autosar/test/rules/A8-5-3/test.cpp b/cpp/autosar/test/rules/A8-5-3/test.cpp index c17c8241a1..7baa9aa487 100644 --- a/cpp/autosar/test/rules/A8-5-3/test.cpp +++ b/cpp/autosar/test/rules/A8-5-3/test.cpp @@ -1,11 +1,12 @@ #include void test() { - auto a1(1); // COMPLIANT - auto a2{1}; // NON_COMPLIANT - auto a3 = 1; // COMPLIANT - auto a4 = {1}; // NON_COMPLIANT - int a5 = {1}; // COMPLIANT - const auto a6(1); // COMPLIANT - const auto a7{1}; // NON_COMPLIANT + auto a1(1); // COMPLIANT + auto a2{1}; // NON_COMPLIANT + auto a3 = 1; // COMPLIANT + auto a4 = {1}; // NON_COMPLIANT + int a5 = {1}; // COMPLIANT + const auto a6(1); // COMPLIANT + const auto a7{1}; // NON_COMPLIANT + auto a8 = std::initializer_list(); // COMPLIANT } \ No newline at end of file From 5c63296d7c0707dd44fa21e577f1ff40e3394a44 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 15 Jul 2022 17:36:22 +0200 Subject: [PATCH 07/37] Update symbol location in expected test output --- change_notes/2022-07-15-fix-A7-3-1-location-reporting.md | 2 ++ .../A7-3-1/DefinitionNotConsideredForUnqualifiedLookup.expected | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 change_notes/2022-07-15-fix-A7-3-1-location-reporting.md diff --git a/change_notes/2022-07-15-fix-A7-3-1-location-reporting.md b/change_notes/2022-07-15-fix-A7-3-1-location-reporting.md new file mode 100644 index 0000000000..b310d1c010 --- /dev/null +++ b/change_notes/2022-07-15-fix-A7-3-1-location-reporting.md @@ -0,0 +1,2 @@ +- `A7-3-1` - `DefinitionNotConsideredForUnqualifiedLookup.ql` + - The locations reported for names occurring in using-declarations has improved in the latest CodeQL versions. diff --git a/cpp/autosar/test/rules/A7-3-1/DefinitionNotConsideredForUnqualifiedLookup.expected b/cpp/autosar/test/rules/A7-3-1/DefinitionNotConsideredForUnqualifiedLookup.expected index a549240c66..ea0f998533 100644 --- a/cpp/autosar/test/rules/A7-3-1/DefinitionNotConsideredForUnqualifiedLookup.expected +++ b/cpp/autosar/test/rules/A7-3-1/DefinitionNotConsideredForUnqualifiedLookup.expected @@ -1 +1 @@ -| test.cpp:42:6:42:7 | declaration of f1 | Definition for 'f1' is not available for unqualified lookup because it is declared after $@ | test.cpp:39:1:39:13 | using f1 | using-declaration | +| test.cpp:42:6:42:7 | declaration of f1 | Definition for 'f1' is not available for unqualified lookup because it is declared after $@ | test.cpp:39:12:39:13 | using f1 | using-declaration | From e85a16f5f137f2d1d50fc894bdfff802f53f8b36 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 15 Jul 2022 17:33:59 +0200 Subject: [PATCH 08/37] Fix M0-1-4 to ignore compiler-generated accesses --- change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md | 3 +++ cpp/autosar/src/rules/M0-1-4/SingleUsePODVariable.qll | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md diff --git a/change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md b/change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md new file mode 100644 index 0000000000..8d3a2045f2 --- /dev/null +++ b/change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md @@ -0,0 +1,3 @@ + - `M0-1-4` - `SingleUsePODVariable.ql` + - This rule no longer considers compiler-generated access to a variable when determining if the + variable has a single use. diff --git a/cpp/autosar/src/rules/M0-1-4/SingleUsePODVariable.qll b/cpp/autosar/src/rules/M0-1-4/SingleUsePODVariable.qll index 765848a63c..c750bb130c 100644 --- a/cpp/autosar/src/rules/M0-1-4/SingleUsePODVariable.qll +++ b/cpp/autosar/src/rules/M0-1-4/SingleUsePODVariable.qll @@ -10,8 +10,9 @@ int getUseCount(Variable v) { // We enforce that it's a POD type variable, so if it has an initializer it is explicit (if v.hasInitializer() then initializers = 1 else initializers = 0) and result = - initializers + count(v.getAnAccess()) + - count(UserProvidedConstructorFieldInit cfi | cfi.getTarget() = v) + initializers + + count(VariableAccess access | access = v.getAnAccess() and not access.isCompilerGenerated()) + + count(UserProvidedConstructorFieldInit cfi | cfi.getTarget() = v) ) } @@ -23,7 +24,9 @@ Element getSingleUse(Variable v) { or result = any(UserProvidedConstructorFieldInit cfi | cfi.getTarget() = v) or - result = v.getAnAccess() + exists(VariableAccess access | + access = v.getAnAccess() and not access.isCompilerGenerated() and result = access + ) ) } From 498f72c447d37ce526b3db27714fe20041db498b Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 14 Jul 2022 20:39:03 +0200 Subject: [PATCH 09/37] Bump codeql submodule --- codeql_modules/codeql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeql_modules/codeql b/codeql_modules/codeql index 4551af90f6..28fe7a7660 160000 --- a/codeql_modules/codeql +++ b/codeql_modules/codeql @@ -1 +1 @@ -Subproject commit 4551af90f61a8d5f5c1c88a036595b5919a6c98e +Subproject commit 28fe7a76603ab7ef884ca35115b63104ecb699a7 From 6bd668aff989872c5c59597fc3f845db5445b8fa Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 31 Aug 2022 17:17:41 +0200 Subject: [PATCH 10/37] Update c/cert/src/qlpack.yml Co-authored-by: Jeroen Ketema <93738568+jketema@users.noreply.github.com> --- c/cert/src/qlpack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/cert/src/qlpack.yml b/c/cert/src/qlpack.yml index 644ac974d6..b13e62decc 100644 --- a/c/cert/src/qlpack.yml +++ b/c/cert/src/qlpack.yml @@ -1,4 +1,4 @@ -name: cert-c-coding-standards +name: codeql/cert-c-coding-standards version: 2.8.0-dev suites: codeql-suites dependencies: From 41ce0199b0a051d603ac87f56bf49b056404c498 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 31 Aug 2022 17:21:20 +0200 Subject: [PATCH 11/37] Update change notes --- change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md | 3 --- .../2022-06-01-fix-A8-5-3-braced-initialization-detection.md | 3 --- change_notes/2022-07-15-fix-A7-3-1-location-reporting.md | 2 -- 3 files changed, 8 deletions(-) delete mode 100644 change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md delete mode 100644 change_notes/2022-06-01-fix-A8-5-3-braced-initialization-detection.md delete mode 100644 change_notes/2022-07-15-fix-A7-3-1-location-reporting.md diff --git a/change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md b/change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md deleted file mode 100644 index 8d3a2045f2..0000000000 --- a/change_notes/2022-05-04-compiler-generated-fp-M0-1-4.md +++ /dev/null @@ -1,3 +0,0 @@ - - `M0-1-4` - `SingleUsePODVariable.ql` - - This rule no longer considers compiler-generated access to a variable when determining if the - variable has a single use. diff --git a/change_notes/2022-06-01-fix-A8-5-3-braced-initialization-detection.md b/change_notes/2022-06-01-fix-A8-5-3-braced-initialization-detection.md deleted file mode 100644 index ec48fda73a..0000000000 --- a/change_notes/2022-06-01-fix-A8-5-3-braced-initialization-detection.md +++ /dev/null @@ -1,3 +0,0 @@ -- `A8-5-3` - `AvoidAutoWithBracedInitialization.ql`: - - Fix regression where `auto x{0}` was no longer detected as a braced initialization with type `auto` with the latest CodeQL versions. - - No longer falsely detect cases where braced initialization was not used, but where the inferred type would be `std::initializer_list`. diff --git a/change_notes/2022-07-15-fix-A7-3-1-location-reporting.md b/change_notes/2022-07-15-fix-A7-3-1-location-reporting.md deleted file mode 100644 index b310d1c010..0000000000 --- a/change_notes/2022-07-15-fix-A7-3-1-location-reporting.md +++ /dev/null @@ -1,2 +0,0 @@ -- `A7-3-1` - `DefinitionNotConsideredForUnqualifiedLookup.ql` - - The locations reported for names occurring in using-declarations has improved in the latest CodeQL versions. From b4b935d3f2917b3090c3172bab280a822ba6a5a9 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 31 Aug 2022 18:42:07 +0200 Subject: [PATCH 12/37] Update change notes --- change_notes/2022-08-31-update-to-CodeQL-2.9.4.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 change_notes/2022-08-31-update-to-CodeQL-2.9.4.md diff --git a/change_notes/2022-08-31-update-to-CodeQL-2.9.4.md b/change_notes/2022-08-31-update-to-CodeQL-2.9.4.md new file mode 100644 index 0000000000..443352baa9 --- /dev/null +++ b/change_notes/2022-08-31-update-to-CodeQL-2.9.4.md @@ -0,0 +1,8 @@ +- `M0-1-4` - `SingleUsePODVariable.ql` + - This rule no longer considers compiler-generated access to a variable when determining if the variable has a single use. +- `A8-5-3` - `AvoidAutoWithBracedInitialization.ql`: + - Fix regression where `auto x{0}` was no longer detected as a braced initialization with type `auto` with the latest CodeQL versions. + - No longer falsely detect cases where braced initialization was not used, but where the inferred type would be `std::initializer_list`. +- `A7-3-1` - `DefinitionNotConsideredForUnqualifiedLookup.ql` + - The locations reported for names occurring in using-declarations has improved in the latest CodeQL versions. +- Updated the CodeQL version to `2.9.4`. \ No newline at end of file From 1a6a71e4789d7acaa35c3140516b54009a3ede03 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 9 Sep 2022 21:44:22 +0200 Subject: [PATCH 13/37] Bump to 2.9.0-dev --- c/cert/src/qlpack.yml | 2 +- c/cert/test/qlpack.yml | 2 +- c/common/src/qlpack.yml | 2 +- c/common/test/qlpack.yml | 2 +- c/misra/src/qlpack.yml | 2 +- c/misra/test/qlpack.yml | 2 +- cpp/autosar/src/qlpack.yml | 2 +- cpp/autosar/test/qlpack.yml | 2 +- cpp/cert/src/qlpack.yml | 2 +- cpp/cert/test/qlpack.yml | 2 +- cpp/common/src/qlpack.yml | 2 +- cpp/common/test/qlpack.yml | 2 +- cpp/misra/src/qlpack.yml | 2 +- cpp/misra/test/qlpack.yml | 2 +- cpp/report/src/qlpack.yml | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/c/cert/src/qlpack.yml b/c/cert/src/qlpack.yml index b13e62decc..bb55e66e97 100644 --- a/c/cert/src/qlpack.yml +++ b/c/cert/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cert-c-coding-standards -version: 2.8.0-dev +version: 2.9.0-dev suites: codeql-suites dependencies: codeql/common-c-coding-standards: '*' diff --git a/c/cert/test/qlpack.yml b/c/cert/test/qlpack.yml index bd8995b986..cda136263f 100644 --- a/c/cert/test/qlpack.yml +++ b/c/cert/test/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cert-c-coding-standards-tests -version: 2.8.0-dev +version: 2.9.0-dev extractor: cpp dependencies: codeql/cert-c-coding-standards: '*' diff --git a/c/common/src/qlpack.yml b/c/common/src/qlpack.yml index 2d3953f63a..8fc6422c7c 100644 --- a/c/common/src/qlpack.yml +++ b/c/common/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/common-c-coding-standards -version: 2.8.0-dev +version: 2.9.0-dev dependencies: codeql/common-cpp-coding-standards: '*' codeql/cpp-all: 0.2.3 diff --git a/c/common/test/qlpack.yml b/c/common/test/qlpack.yml index ec7387fb99..92b33b0812 100644 --- a/c/common/test/qlpack.yml +++ b/c/common/test/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/common-c-coding-standards-tests -version: 2.8.0-dev +version: 2.9.0-dev extractor: cpp dependencies: codeql/common-c-coding-standards: '*' diff --git a/c/misra/src/qlpack.yml b/c/misra/src/qlpack.yml index 17586fc09f..2e792a47ad 100644 --- a/c/misra/src/qlpack.yml +++ b/c/misra/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/misra-c-coding-standards -version: 2.8.0-dev +version: 2.9.0-dev suites: codeql-suites dependencies: codeql/common-c-coding-standards: '*' diff --git a/c/misra/test/qlpack.yml b/c/misra/test/qlpack.yml index 642e48fdf8..e9ede72bfd 100644 --- a/c/misra/test/qlpack.yml +++ b/c/misra/test/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/misra-c-coding-standards-tests -version: 2.8.0-dev +version: 2.9.0-dev extractor: cpp dependencies: codeql/misra-c-coding-standards: '*' diff --git a/cpp/autosar/src/qlpack.yml b/cpp/autosar/src/qlpack.yml index 56abc6f259..cd6d21d792 100644 --- a/cpp/autosar/src/qlpack.yml +++ b/cpp/autosar/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/autosar-cpp-coding-standards -version: 2.8.0-dev +version: 2.9.0-dev suites: codeql-suites dependencies: codeql/common-cpp-coding-standards: '*' diff --git a/cpp/autosar/test/qlpack.yml b/cpp/autosar/test/qlpack.yml index dcc32f21ff..e0b0d4f32a 100644 --- a/cpp/autosar/test/qlpack.yml +++ b/cpp/autosar/test/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/autosar-cpp-coding-standards-tests -version: 2.8.0-dev +version: 2.9.0-dev extractor: cpp dependencies: codeql/autosar-cpp-coding-standards: '*' diff --git a/cpp/cert/src/qlpack.yml b/cpp/cert/src/qlpack.yml index f2e265f4a3..6841d1a57a 100644 --- a/cpp/cert/src/qlpack.yml +++ b/cpp/cert/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cert-cpp-coding-standards -version: 2.8.0-dev +version: 2.9.0-dev suites: codeql-suites dependencies: codeql/cpp-all: 0.2.3 diff --git a/cpp/cert/test/qlpack.yml b/cpp/cert/test/qlpack.yml index b65f6e59ec..285f088d6e 100644 --- a/cpp/cert/test/qlpack.yml +++ b/cpp/cert/test/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cert-cpp-coding-standards-tests -version: 2.8.0-dev +version: 2.9.0-dev extractor: cpp dependencies: codeql/cert-cpp-coding-standards: '*' diff --git a/cpp/common/src/qlpack.yml b/cpp/common/src/qlpack.yml index 7bcd661037..d057030b9a 100644 --- a/cpp/common/src/qlpack.yml +++ b/cpp/common/src/qlpack.yml @@ -1,4 +1,4 @@ name: codeql/common-cpp-coding-standards -version: 2.8.0-dev +version: 2.9.0-dev dependencies: codeql/cpp-all: 0.2.3 diff --git a/cpp/common/test/qlpack.yml b/cpp/common/test/qlpack.yml index fb42d63ab3..3f34374945 100644 --- a/cpp/common/test/qlpack.yml +++ b/cpp/common/test/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/common-cpp-coding-standards-tests -version: 2.8.0-dev +version: 2.9.0-dev extractor: cpp dependencies: codeql/common-cpp-coding-standards: '*' diff --git a/cpp/misra/src/qlpack.yml b/cpp/misra/src/qlpack.yml index 44d2b10c43..7765092418 100644 --- a/cpp/misra/src/qlpack.yml +++ b/cpp/misra/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/misra-cpp-coding-standards -version: 2.8.0-dev +version: 2.9.0-dev dependencies: codeql/common-cpp-coding-standards: '*' codeql/cpp-all: 0.2.3 diff --git a/cpp/misra/test/qlpack.yml b/cpp/misra/test/qlpack.yml index e8117d58e9..5e8662f3ba 100644 --- a/cpp/misra/test/qlpack.yml +++ b/cpp/misra/test/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/misra-cpp-coding-standards-tests -version: 2.8.0-dev +version: 2.9.0-dev extractor: cpp dependencies: codeql/misra-cpp-coding-standards: '*' \ No newline at end of file diff --git a/cpp/report/src/qlpack.yml b/cpp/report/src/qlpack.yml index f34d8fe4cd..10167f26ce 100644 --- a/cpp/report/src/qlpack.yml +++ b/cpp/report/src/qlpack.yml @@ -1,4 +1,4 @@ name: codeql/report-cpp-coding-standards -version: 2.8.0-dev +version: 2.9.0-dev dependencies: codeql/cpp-all: 0.2.3 From d3f362410688dce2167f974284eab0ea184daaef Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 27 Jan 2023 17:23:30 +0100 Subject: [PATCH 14/37] Update qlpack name in query suites --- c/cert/src/codeql-suites/cert-default.qls | 2 +- c/misra/src/codeql-suites/misra-default.qls | 2 +- cpp/autosar/src/codeql-suites/autosar-advisory.qls | 2 +- cpp/autosar/src/codeql-suites/autosar-audit.qls | 2 +- cpp/autosar/src/codeql-suites/autosar-default.qls | 2 +- cpp/autosar/src/codeql-suites/autosar-required.qls | 2 +- .../src/codeql-suites/autosar-single-translation-unit.qls | 2 +- cpp/cert/src/codeql-suites/cert-default.qls | 2 +- cpp/cert/src/codeql-suites/cert-single-translation-unit.qls | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/c/cert/src/codeql-suites/cert-default.qls b/c/cert/src/codeql-suites/cert-default.qls index c8652930e9..7ad2fc1a99 100644 --- a/c/cert/src/codeql-suites/cert-default.qls +++ b/c/cert/src/codeql-suites/cert-default.qls @@ -1,5 +1,5 @@ - description: CERT C 2016 (Default) -- qlpack: cert-c-coding-standards +- qlpack: codeql/cert-c-coding-standards - include: kind: - problem diff --git a/c/misra/src/codeql-suites/misra-default.qls b/c/misra/src/codeql-suites/misra-default.qls index d32637556d..473232e741 100644 --- a/c/misra/src/codeql-suites/misra-default.qls +++ b/c/misra/src/codeql-suites/misra-default.qls @@ -1,5 +1,5 @@ - description: MISRA C 2012 (Default) -- qlpack: misra-c-coding-standards +- qlpack: codeql/misra-c-coding-standards - include: kind: - problem diff --git a/cpp/autosar/src/codeql-suites/autosar-advisory.qls b/cpp/autosar/src/codeql-suites/autosar-advisory.qls index 163de00856..6460375091 100644 --- a/cpp/autosar/src/codeql-suites/autosar-advisory.qls +++ b/cpp/autosar/src/codeql-suites/autosar-advisory.qls @@ -1,5 +1,5 @@ - description: AUTOSAR C++14 Guidelines 20-11 (Advisory) -- qlpack: autosar-cpp-coding-standards +- qlpack: codeql/autosar-cpp-coding-standards - include: kind: - problem diff --git a/cpp/autosar/src/codeql-suites/autosar-audit.qls b/cpp/autosar/src/codeql-suites/autosar-audit.qls index 6e77e82e77..25f4f1c41c 100644 --- a/cpp/autosar/src/codeql-suites/autosar-audit.qls +++ b/cpp/autosar/src/codeql-suites/autosar-audit.qls @@ -1,5 +1,5 @@ - description: AUTOSAR C++14 Guidelines 20-11 (Audit) -- qlpack: autosar-cpp-coding-standards +- qlpack: codeql/autosar-cpp-coding-standards - include: kind: - problem diff --git a/cpp/autosar/src/codeql-suites/autosar-default.qls b/cpp/autosar/src/codeql-suites/autosar-default.qls index 0296cc54f8..e57d0739d6 100644 --- a/cpp/autosar/src/codeql-suites/autosar-default.qls +++ b/cpp/autosar/src/codeql-suites/autosar-default.qls @@ -1,5 +1,5 @@ - description: AUTOSAR C++14 Guidelines 20-11 (Default) -- qlpack: autosar-cpp-coding-standards +- qlpack: codeql/autosar-cpp-coding-standards - include: kind: - problem diff --git a/cpp/autosar/src/codeql-suites/autosar-required.qls b/cpp/autosar/src/codeql-suites/autosar-required.qls index b79562c66c..aa4173462c 100644 --- a/cpp/autosar/src/codeql-suites/autosar-required.qls +++ b/cpp/autosar/src/codeql-suites/autosar-required.qls @@ -1,5 +1,5 @@ - description: AUTOSAR C++14 Guidelines 20-11 (Required) -- qlpack: autosar-cpp-coding-standards +- qlpack: codeql/autosar-cpp-coding-standards - include: kind: - problem diff --git a/cpp/autosar/src/codeql-suites/autosar-single-translation-unit.qls b/cpp/autosar/src/codeql-suites/autosar-single-translation-unit.qls index 6ba3032fa8..d56a7e306a 100644 --- a/cpp/autosar/src/codeql-suites/autosar-single-translation-unit.qls +++ b/cpp/autosar/src/codeql-suites/autosar-single-translation-unit.qls @@ -1,5 +1,5 @@ - description: AUTOSAR C++14 Guidelines 20-11 (Single Translation Unit) -- qlpack: autosar-cpp-coding-standards +- qlpack: codeql/autosar-cpp-coding-standards - include: kind: - problem diff --git a/cpp/cert/src/codeql-suites/cert-default.qls b/cpp/cert/src/codeql-suites/cert-default.qls index 6ec40af9b9..a003e05ed2 100644 --- a/cpp/cert/src/codeql-suites/cert-default.qls +++ b/cpp/cert/src/codeql-suites/cert-default.qls @@ -1,5 +1,5 @@ - description: CERT C++ 2016 (Default) -- qlpack: cert-cpp-coding-standards +- qlpack: codeql/cert-cpp-coding-standards - include: kind: - problem diff --git a/cpp/cert/src/codeql-suites/cert-single-translation-unit.qls b/cpp/cert/src/codeql-suites/cert-single-translation-unit.qls index ca1315d6ff..0d3f99cbf0 100644 --- a/cpp/cert/src/codeql-suites/cert-single-translation-unit.qls +++ b/cpp/cert/src/codeql-suites/cert-single-translation-unit.qls @@ -1,5 +1,5 @@ - description: CERT C++ 2016 (Single Translation Unit) -- qlpack: cert-cpp-coding-standards +- qlpack: codeql/cert-cpp-coding-standards - include: kind: - problem From 6549297d8b69982c27a92786d25013b000d93a53 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 27 Jan 2023 18:00:16 +0100 Subject: [PATCH 15/37] Merge fix --- c/cert/test/qlpack.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/c/cert/test/qlpack.yml b/c/cert/test/qlpack.yml index d8e1d54d7b..94d0f1f703 100644 --- a/c/cert/test/qlpack.yml +++ b/c/cert/test/qlpack.yml @@ -1,6 +1,5 @@ name: codeql/cert-c-coding-standards-tests version: 2.13.0-dev -libraryPathDependencies: cert-c-coding-standards extractor: cpp dependencies: codeql/cert-c-coding-standards: '*' From 55ff60385e7e0d114894dadfef05aed7acd19c5a Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 27 Jan 2023 18:20:24 +0100 Subject: [PATCH 16/37] Fix deprecated libraries --- cpp/common/src/codingstandards/cpp/Config.qll | 6 +++--- .../src/codingstandards/cpp/deviations/Deviations.qll | 10 +++++----- .../GuidelineRecategorizations.qll | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/common/src/codingstandards/cpp/Config.qll b/cpp/common/src/codingstandards/cpp/Config.qll index 7d12340719..5ce3f7a035 100644 --- a/cpp/common/src/codingstandards/cpp/Config.qll +++ b/cpp/common/src/codingstandards/cpp/Config.qll @@ -8,7 +8,7 @@ import codingstandards.cpp.exclusions.RuleMetadata import codingstandards.cpp.deviations.Deviations /** A `coding-standards.xml` configuration file (usually generated from an YAML configuration file). */ -class CodingStandardsFile extends XMLFile { +class CodingStandardsFile extends XmlFile { CodingStandardsFile() { this.getBaseName() = "coding-standards.xml" and // Must be within the users source code. @@ -16,12 +16,12 @@ class CodingStandardsFile extends XMLFile { } } -class CodingStandardsConfigSection extends XMLElement { +class CodingStandardsConfigSection extends XmlElement { CodingStandardsConfigSection() { getParent() instanceof CodingStandardsConfig } } /** A "Coding Standards" configuration file */ -class CodingStandardsConfig extends XMLElement { +class CodingStandardsConfig extends XmlElement { CodingStandardsConfig() { any(CodingStandardsFile csf).getARootElement() = this and this.getName() = "codingstandards" diff --git a/cpp/common/src/codingstandards/cpp/deviations/Deviations.qll b/cpp/common/src/codingstandards/cpp/deviations/Deviations.qll index 9399607e3d..4dfadd12eb 100644 --- a/cpp/common/src/codingstandards/cpp/deviations/Deviations.qll +++ b/cpp/common/src/codingstandards/cpp/deviations/Deviations.qll @@ -17,7 +17,7 @@ predicate applyDeviationsAtQueryLevel() { } /** An element which tells the analysis whether to report deviated results. */ -class CodingStandardsReportDeviatedAlerts extends XMLElement { +class CodingStandardsReportDeviatedAlerts extends XmlElement { CodingStandardsReportDeviatedAlerts() { getParent() instanceof CodingStandardsConfig and hasName("report-deviated-alerts") @@ -35,7 +35,7 @@ class DeviationPermits extends CodingStandardsConfigSection { } /** A deviation permit record, that is specified by a permit identifier */ -class DeviationPermit extends XMLElement { +class DeviationPermit extends XmlElement { DeviationPermit() { getParent() instanceof DeviationPermits and hasName("deviation-permits-entry") @@ -118,7 +118,7 @@ class DeviationPermit extends XMLElement { } /** A deviation record, that is a specified rule or query */ -class DeviationRecord extends XMLElement { +class DeviationRecord extends XmlElement { DeviationRecord() { getParent() instanceof DeviationRecords and hasName("deviations-entry") @@ -134,13 +134,13 @@ class DeviationRecord extends XMLElement { private string getRawPermitId() { result = getAChild("permit-id").getTextValue() } - private XMLElement getRawRaisedBy() { result = getAChild("raised-by") } + private XmlElement getRawRaisedBy() { result = getAChild("raised-by") } private string getRawRaisedByName() { result = getRawRaisedBy().getAChild("name").getTextValue() } private string getRawRaisedByDate() { result = getRawRaisedBy().getAChild("date").getTextValue() } - private XMLElement getRawApprovedBy() { result = getAChild("approved-by") } + private XmlElement getRawApprovedBy() { result = getAChild("approved-by") } private string getRawApprovedByName() { result = getRawApprovedBy().getAChild("name").getTextValue() diff --git a/cpp/common/src/codingstandards/cpp/guideline_recategorizations/GuidelineRecategorizations.qll b/cpp/common/src/codingstandards/cpp/guideline_recategorizations/GuidelineRecategorizations.qll index ff148a4a72..ec5731f1bf 100644 --- a/cpp/common/src/codingstandards/cpp/guideline_recategorizations/GuidelineRecategorizations.qll +++ b/cpp/common/src/codingstandards/cpp/guideline_recategorizations/GuidelineRecategorizations.qll @@ -12,7 +12,7 @@ class GuidelineRecategorizations extends CodingStandardsConfigSection { GuidelineRecategorizations() { hasName("guideline-recategorizations") } } -class GuidelineRecategorization extends XMLElement { +class GuidelineRecategorization extends XmlElement { GuidelineRecategorization() { getParent() instanceof GuidelineRecategorizations and hasName("guideline-recategorizations-entry") From e40d9336dba0726a590ca02785bb22d940ea2ae2 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 2 Feb 2023 11:03:04 +0100 Subject: [PATCH 17/37] Revert "Fix deprecated libraries" This reverts commit 55ff60385e7e0d114894dadfef05aed7acd19c5a. This only applies to CodeQL 2.10.4 and later, while 2.9.4 is being targeted here. --- cpp/common/src/codingstandards/cpp/Config.qll | 6 +++--- .../src/codingstandards/cpp/deviations/Deviations.qll | 10 +++++----- .../GuidelineRecategorizations.qll | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/common/src/codingstandards/cpp/Config.qll b/cpp/common/src/codingstandards/cpp/Config.qll index 5ce3f7a035..7d12340719 100644 --- a/cpp/common/src/codingstandards/cpp/Config.qll +++ b/cpp/common/src/codingstandards/cpp/Config.qll @@ -8,7 +8,7 @@ import codingstandards.cpp.exclusions.RuleMetadata import codingstandards.cpp.deviations.Deviations /** A `coding-standards.xml` configuration file (usually generated from an YAML configuration file). */ -class CodingStandardsFile extends XmlFile { +class CodingStandardsFile extends XMLFile { CodingStandardsFile() { this.getBaseName() = "coding-standards.xml" and // Must be within the users source code. @@ -16,12 +16,12 @@ class CodingStandardsFile extends XmlFile { } } -class CodingStandardsConfigSection extends XmlElement { +class CodingStandardsConfigSection extends XMLElement { CodingStandardsConfigSection() { getParent() instanceof CodingStandardsConfig } } /** A "Coding Standards" configuration file */ -class CodingStandardsConfig extends XmlElement { +class CodingStandardsConfig extends XMLElement { CodingStandardsConfig() { any(CodingStandardsFile csf).getARootElement() = this and this.getName() = "codingstandards" diff --git a/cpp/common/src/codingstandards/cpp/deviations/Deviations.qll b/cpp/common/src/codingstandards/cpp/deviations/Deviations.qll index 4dfadd12eb..9399607e3d 100644 --- a/cpp/common/src/codingstandards/cpp/deviations/Deviations.qll +++ b/cpp/common/src/codingstandards/cpp/deviations/Deviations.qll @@ -17,7 +17,7 @@ predicate applyDeviationsAtQueryLevel() { } /** An element which tells the analysis whether to report deviated results. */ -class CodingStandardsReportDeviatedAlerts extends XmlElement { +class CodingStandardsReportDeviatedAlerts extends XMLElement { CodingStandardsReportDeviatedAlerts() { getParent() instanceof CodingStandardsConfig and hasName("report-deviated-alerts") @@ -35,7 +35,7 @@ class DeviationPermits extends CodingStandardsConfigSection { } /** A deviation permit record, that is specified by a permit identifier */ -class DeviationPermit extends XmlElement { +class DeviationPermit extends XMLElement { DeviationPermit() { getParent() instanceof DeviationPermits and hasName("deviation-permits-entry") @@ -118,7 +118,7 @@ class DeviationPermit extends XmlElement { } /** A deviation record, that is a specified rule or query */ -class DeviationRecord extends XmlElement { +class DeviationRecord extends XMLElement { DeviationRecord() { getParent() instanceof DeviationRecords and hasName("deviations-entry") @@ -134,13 +134,13 @@ class DeviationRecord extends XmlElement { private string getRawPermitId() { result = getAChild("permit-id").getTextValue() } - private XmlElement getRawRaisedBy() { result = getAChild("raised-by") } + private XMLElement getRawRaisedBy() { result = getAChild("raised-by") } private string getRawRaisedByName() { result = getRawRaisedBy().getAChild("name").getTextValue() } private string getRawRaisedByDate() { result = getRawRaisedBy().getAChild("date").getTextValue() } - private XmlElement getRawApprovedBy() { result = getAChild("approved-by") } + private XMLElement getRawApprovedBy() { result = getAChild("approved-by") } private string getRawApprovedByName() { result = getRawApprovedBy().getAChild("name").getTextValue() diff --git a/cpp/common/src/codingstandards/cpp/guideline_recategorizations/GuidelineRecategorizations.qll b/cpp/common/src/codingstandards/cpp/guideline_recategorizations/GuidelineRecategorizations.qll index ec5731f1bf..ff148a4a72 100644 --- a/cpp/common/src/codingstandards/cpp/guideline_recategorizations/GuidelineRecategorizations.qll +++ b/cpp/common/src/codingstandards/cpp/guideline_recategorizations/GuidelineRecategorizations.qll @@ -12,7 +12,7 @@ class GuidelineRecategorizations extends CodingStandardsConfigSection { GuidelineRecategorizations() { hasName("guideline-recategorizations") } } -class GuidelineRecategorization extends XmlElement { +class GuidelineRecategorization extends XMLElement { GuidelineRecategorization() { getParent() instanceof GuidelineRecategorizations and hasName("guideline-recategorizations-entry") From 104a317cec0007c850c20d06d965a1d4f2b81af3 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 2 Feb 2023 11:04:20 +0100 Subject: [PATCH 18/37] Modernize the `scripts/generate_modules/queries` qlpack --- scripts/generate_modules/queries/codeql-pack.lock.yml | 6 ++++++ scripts/generate_modules/queries/qlpack.yml | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 scripts/generate_modules/queries/codeql-pack.lock.yml diff --git a/scripts/generate_modules/queries/codeql-pack.lock.yml b/scripts/generate_modules/queries/codeql-pack.lock.yml new file mode 100644 index 0000000000..b0f02a9d1f --- /dev/null +++ b/scripts/generate_modules/queries/codeql-pack.lock.yml @@ -0,0 +1,6 @@ +--- +dependencies: + codeql/cpp-all: + version: 0.2.3 +compiled: false +lockVersion: 1.0.0 diff --git a/scripts/generate_modules/queries/qlpack.yml b/scripts/generate_modules/queries/qlpack.yml index d165177f23..61cf146c4d 100644 --- a/scripts/generate_modules/queries/qlpack.yml +++ b/scripts/generate_modules/queries/qlpack.yml @@ -1,3 +1,4 @@ -name: standard-library-extraction-cpp-coding-standards +name: codeql/standard-library-extraction-cpp-coding-standards version: 0.0.0 -libraryPathDependencies: codeql-cpp \ No newline at end of file +dependencies: + codeql/cpp-all: 0.2.3 From 8398907c91c275a2b2d3035e7a1caee59e87f930 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 2 Feb 2023 12:10:28 +0100 Subject: [PATCH 19/37] Fix path filter in verify standard library dependencies workflow --- .github/workflows/verify-standard-library-dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify-standard-library-dependencies.yml b/.github/workflows/verify-standard-library-dependencies.yml index 0b05736ddb..91f4a5075b 100644 --- a/.github/workflows/verify-standard-library-dependencies.yml +++ b/.github/workflows/verify-standard-library-dependencies.yml @@ -9,7 +9,7 @@ on: - next paths: - "supported_codeql_configs.json" - - "qlpack.yml" + - "**/qlpack.yml" workflow_dispatch: jobs: From 9cd9638bcc70ff592e54ff4d4f6e7186b1476c35 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 2 Feb 2023 12:41:32 +0100 Subject: [PATCH 20/37] Install codeql packs in "Run analysis report tests" workflow --- .codeqlmanifest.json | 2 +- .github/workflows/tooling-unit-tests.yml | 5 +++++ scripts/reports/deviations.py | 2 +- scripts/reports/diagnostics.py | 2 +- scripts/reports/guideline_recategorizations.py | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json index d6277f5949..e6aec98053 100644 --- a/.codeqlmanifest.json +++ b/.codeqlmanifest.json @@ -1 +1 @@ -{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml" ] } +{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml", "scripts/generate_modules/queries/qlpack.yml" ] } diff --git a/.github/workflows/tooling-unit-tests.yml b/.github/workflows/tooling-unit-tests.yml index 5531c11d72..d2ed099a2e 100644 --- a/.github/workflows/tooling-unit-tests.yml +++ b/.github/workflows/tooling-unit-tests.yml @@ -64,6 +64,11 @@ jobs: codeql-home: ${{ github.workspace }}/codeql_home add-to-path: false + - name: Install CodeQL packs + uses: ./.github/actions/install-codeql-packs + with: + cli_path: ${{ github.workspace }}/codeql_home/codeql + - name: Run PyTest env: CODEQL_HOME: ${{ github.workspace }}/codeql_home diff --git a/scripts/reports/deviations.py b/scripts/reports/deviations.py index d7baca5665..d511d35dbb 100644 --- a/scripts/reports/deviations.py +++ b/scripts/reports/deviations.py @@ -42,7 +42,7 @@ def __init__(self, database_path, repo_root): # Get a list of deviations print("Running the deviation query...") self.codeql_summary.codeql.run_queries( - database_path, *query_paths, search_path=str(repo_root), no_rerun=True) + database_path, *query_paths, no_rerun=True) print("Decoding deviation query results") diff --git a/scripts/reports/diagnostics.py b/scripts/reports/diagnostics.py index 4cf17ec9a1..78535a8f9a 100644 --- a/scripts/reports/diagnostics.py +++ b/scripts/reports/diagnostics.py @@ -44,7 +44,7 @@ def __init__(self, database_path, repo_root): # Run all the diagnostics over the database print("Running the diagnostic queries...") self.codeql_summary.codeql.run_queries( - database_path, *queries, search_path=str(repo_root), no_rerun=True) + database_path, *queries, no_rerun=True) print("Decoding diagnostic query results") self.extraction_errors = self.codeql_summary.codeql.decode_results( diff --git a/scripts/reports/guideline_recategorizations.py b/scripts/reports/guideline_recategorizations.py index cc30182596..ebd0fa59de 100644 --- a/scripts/reports/guideline_recategorizations.py +++ b/scripts/reports/guideline_recategorizations.py @@ -40,7 +40,7 @@ def __init__(self, database_path, repo_root): # Get a list of guideline recategorizations print("Running the guideline recategorizations queries...") self.codeql_summary.codeql.run_queries( - database_path, *query_paths, search_path=str(repo_root), no_rerun=True) + database_path, *query_paths, no_rerun=True) print("Decoding guideline recategorizations queries results") From 9c20bfc065041af4e27e07ff98f251c60d9aeee1 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Fri, 3 Feb 2023 12:52:50 +0100 Subject: [PATCH 21/37] Update development_handbook.md --- docs/development_handbook.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/development_handbook.md b/docs/development_handbook.md index 22bb18821c..1d1a9a0584 100644 --- a/docs/development_handbook.md +++ b/docs/development_handbook.md @@ -32,9 +32,7 @@ | 0.24.0 | 2022-07-05 | Remco Vermeulen | Update release process to include steps for external help files. | | 0.25.0 | 2022-07-22 | Jeroen Ketema | Document the existence and purpose of the `next` branch. | | 0.26.0 | 2022-08-10 | Remco Vermeulen | Address incorrect package file generation command. This was missing the required language argument. - | | 0.27.0 | 2022-11-08 | Luke Cartey | Update the versions of C we intend to support to exclude C90, which reflects the intended scope at the outset of the project. - | | 0.28.0 | 2023-01-27 | David Bartolomeo | Add section on installing QL dependencies and update CLI commands to account for the migration to CodeQL packs. ## Scope of work From 78acc97edd5b23eec43345fc514ffa6c9f70a85c Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 6 Feb 2023 16:53:55 +0100 Subject: [PATCH 22/37] Add properties to qlpack.yml files --- c/cert/src/qlpack.yml | 3 +++ c/cert/test/qlpack.yml | 1 + c/common/src/qlpack.yml | 1 + c/common/test/qlpack.yml | 1 + c/misra/src/qlpack.yml | 3 +++ c/misra/test/qlpack.yml | 1 + cpp/autosar/src/qlpack.yml | 3 +++ cpp/autosar/test/qlpack.yml | 1 + cpp/cert/src/qlpack.yml | 3 +++ cpp/cert/test/qlpack.yml | 1 + cpp/common/src/qlpack.yml | 1 + cpp/common/test/qlpack.yml | 1 + cpp/misra/src/qlpack.yml | 1 + cpp/misra/test/qlpack.yml | 1 + cpp/report/src/qlpack.yml | 1 + scripts/generate_modules/queries/qlpack.yml | 1 + 16 files changed, 24 insertions(+) diff --git a/c/cert/src/qlpack.yml b/c/cert/src/qlpack.yml index 4e8696753c..51f0b7642b 100644 --- a/c/cert/src/qlpack.yml +++ b/c/cert/src/qlpack.yml @@ -1,6 +1,9 @@ name: codeql/cert-c-coding-standards version: 2.13.0-dev +description: CERT C 2016 suites: codeql-suites +defaultSuiteFile: codeql-suites/cert-default.qls +license: MIT dependencies: codeql/common-c-coding-standards: '*' codeql/cpp-all: 0.2.3 diff --git a/c/cert/test/qlpack.yml b/c/cert/test/qlpack.yml index 94d0f1f703..a5abfa9649 100644 --- a/c/cert/test/qlpack.yml +++ b/c/cert/test/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/cert-c-coding-standards-tests version: 2.13.0-dev extractor: cpp +license: MIT dependencies: codeql/cert-c-coding-standards: '*' diff --git a/c/common/src/qlpack.yml b/c/common/src/qlpack.yml index 32d6970588..507b04d9dd 100644 --- a/c/common/src/qlpack.yml +++ b/c/common/src/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/common-c-coding-standards version: 2.13.0-dev +license: MIT dependencies: codeql/common-cpp-coding-standards: '*' codeql/cpp-all: 0.2.3 diff --git a/c/common/test/qlpack.yml b/c/common/test/qlpack.yml index 1988c80f77..4adc7808cd 100644 --- a/c/common/test/qlpack.yml +++ b/c/common/test/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/common-c-coding-standards-tests version: 2.13.0-dev extractor: cpp +license: MIT dependencies: codeql/common-c-coding-standards: '*' diff --git a/c/misra/src/qlpack.yml b/c/misra/src/qlpack.yml index e885ea299c..2f140abac5 100644 --- a/c/misra/src/qlpack.yml +++ b/c/misra/src/qlpack.yml @@ -1,6 +1,9 @@ name: codeql/misra-c-coding-standards version: 2.13.0-dev +description: MISRA C 2012 suites: codeql-suites +defaultSuiteFile: codeql-suites/misra-default.qls +license: MIT dependencies: codeql/common-c-coding-standards: '*' codeql/cpp-all: 0.2.3 diff --git a/c/misra/test/qlpack.yml b/c/misra/test/qlpack.yml index 80f708e69a..8affcf40b0 100644 --- a/c/misra/test/qlpack.yml +++ b/c/misra/test/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/misra-c-coding-standards-tests version: 2.13.0-dev extractor: cpp +license: MIT dependencies: codeql/misra-c-coding-standards: '*' diff --git a/cpp/autosar/src/qlpack.yml b/cpp/autosar/src/qlpack.yml index 2c9e235ac4..f61a1205d1 100644 --- a/cpp/autosar/src/qlpack.yml +++ b/cpp/autosar/src/qlpack.yml @@ -1,6 +1,9 @@ name: codeql/autosar-cpp-coding-standards version: 2.13.0-dev +description: AUTOSAR C++14 Guidelines 20-11 suites: codeql-suites +defaultSuiteFile: codeql-suites/autosar-default.qls +license: MIT dependencies: codeql/common-cpp-coding-standards: '*' codeql/cpp-all: 0.2.3 diff --git a/cpp/autosar/test/qlpack.yml b/cpp/autosar/test/qlpack.yml index 9757b3dc4e..8b07be43ef 100644 --- a/cpp/autosar/test/qlpack.yml +++ b/cpp/autosar/test/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/autosar-cpp-coding-standards-tests version: 2.13.0-dev extractor: cpp +license: MIT dependencies: codeql/autosar-cpp-coding-standards: '*' diff --git a/cpp/cert/src/qlpack.yml b/cpp/cert/src/qlpack.yml index b2bfe797ba..e539039367 100644 --- a/cpp/cert/src/qlpack.yml +++ b/cpp/cert/src/qlpack.yml @@ -1,6 +1,9 @@ name: codeql/cert-cpp-coding-standards version: 2.13.0-dev +description: CERT C++ 2016 suites: codeql-suites +defaultSuiteFile: codeql-suites/cert-default.qls +license: MIT dependencies: codeql/cpp-all: 0.2.3 codeql/common-cpp-coding-standards: '*' diff --git a/cpp/cert/test/qlpack.yml b/cpp/cert/test/qlpack.yml index f5fc44e682..2badf1e20c 100644 --- a/cpp/cert/test/qlpack.yml +++ b/cpp/cert/test/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/cert-cpp-coding-standards-tests version: 2.13.0-dev extractor: cpp +license: MIT dependencies: codeql/cert-cpp-coding-standards: '*' diff --git a/cpp/common/src/qlpack.yml b/cpp/common/src/qlpack.yml index f52cad36f0..d0c449086f 100644 --- a/cpp/common/src/qlpack.yml +++ b/cpp/common/src/qlpack.yml @@ -1,4 +1,5 @@ name: codeql/common-cpp-coding-standards version: 2.13.0-dev +license: MIT dependencies: codeql/cpp-all: 0.2.3 diff --git a/cpp/common/test/qlpack.yml b/cpp/common/test/qlpack.yml index ab68ae894e..19e27618e2 100644 --- a/cpp/common/test/qlpack.yml +++ b/cpp/common/test/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/common-cpp-coding-standards-tests version: 2.13.0-dev extractor: cpp +license: MIT dependencies: codeql/common-cpp-coding-standards: '*' diff --git a/cpp/misra/src/qlpack.yml b/cpp/misra/src/qlpack.yml index 98e6234d99..ea3f5cbd5f 100644 --- a/cpp/misra/src/qlpack.yml +++ b/cpp/misra/src/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/misra-cpp-coding-standards version: 2.13.0-dev +license: MIT dependencies: codeql/common-cpp-coding-standards: '*' codeql/cpp-all: 0.2.3 diff --git a/cpp/misra/test/qlpack.yml b/cpp/misra/test/qlpack.yml index bc07630c8f..f1f98fb65b 100644 --- a/cpp/misra/test/qlpack.yml +++ b/cpp/misra/test/qlpack.yml @@ -1,5 +1,6 @@ name: codeql/misra-cpp-coding-standards-tests version: 2.13.0-dev extractor: cpp +license: MIT dependencies: codeql/misra-cpp-coding-standards: '*' \ No newline at end of file diff --git a/cpp/report/src/qlpack.yml b/cpp/report/src/qlpack.yml index 974b048917..3587858997 100644 --- a/cpp/report/src/qlpack.yml +++ b/cpp/report/src/qlpack.yml @@ -1,4 +1,5 @@ name: codeql/report-cpp-coding-standards version: 2.13.0-dev +license: MIT dependencies: codeql/cpp-all: 0.2.3 diff --git a/scripts/generate_modules/queries/qlpack.yml b/scripts/generate_modules/queries/qlpack.yml index 61cf146c4d..29906305c6 100644 --- a/scripts/generate_modules/queries/qlpack.yml +++ b/scripts/generate_modules/queries/qlpack.yml @@ -1,4 +1,5 @@ name: codeql/standard-library-extraction-cpp-coding-standards version: 0.0.0 +license: MIT dependencies: codeql/cpp-all: 0.2.3 From 74ff65d4ad33c24a88c91b3d4dc401cff56d0620 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 6 Feb 2023 18:48:27 +0100 Subject: [PATCH 23/37] Replace .codeqlmanifest.json with codeql-workspace.yml --- .github/workflows/code-scanning-pack-gen.yml | 2 +- codeql-workspace.yml | 6 ++++++ scripts/get_workspace_packs.py | 12 ++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 codeql-workspace.yml diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index ca03cb01a5..194dbf3a7f 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -91,7 +91,7 @@ jobs: codeql query compile --threads 0 c cd .. - zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas + zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/codeql-workspace.yml codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas - name: Upload GHAS Query Pack uses: actions/upload-artifact@v2 diff --git a/codeql-workspace.yml b/codeql-workspace.yml new file mode 100644 index 0000000000..d51b909203 --- /dev/null +++ b/codeql-workspace.yml @@ -0,0 +1,6 @@ +provide: + - "cpp/*/src/qlpack.yml" + - "cpp/*/test/qlpack.yml" + - "c/*/src/qlpack.yml" + - "c/*/test/qlpack.yml" + - "scripts/generate_modules/queries/qlpack.yml" diff --git a/scripts/get_workspace_packs.py b/scripts/get_workspace_packs.py index fc9054c641..e348f85d57 100644 --- a/scripts/get_workspace_packs.py +++ b/scripts/get_workspace_packs.py @@ -1,14 +1,14 @@ import glob -import json +import yaml import os def get_workspace_packs(root): - # Find the packs by globbing using the 'provide' patterns in the manifest. + # Find the packs by globbing using the 'provide' patterns in the CodeQL workspace file. os.chdir(root) - with open('.codeqlmanifest.json') as manifest_file: - manifest = json.load(manifest_file) + with open('codeql-workspace.yml') as codeql_workspace_file: + codeql_workspace = yaml.load(codeql_workspace_file) packs = [] - for pattern in manifest['provide']: + for pattern in codeql_workspace['provide']: packs.extend(glob.glob(pattern, recursive=True)) - + return packs From 70c41846fe9ad52ab4a73bb7f2b5d7db8a995da6 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 6 Feb 2023 18:48:57 +0100 Subject: [PATCH 24/37] remove .codeqlmanifest.json --- .codeqlmanifest.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .codeqlmanifest.json diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json deleted file mode 100644 index e6aec98053..0000000000 --- a/.codeqlmanifest.json +++ /dev/null @@ -1 +0,0 @@ -{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml", "scripts/generate_modules/queries/qlpack.yml" ] } From 3cf49f12576c497a0bcbbdf09a9d23cf5854fb69 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Mon, 6 Feb 2023 19:53:36 +0100 Subject: [PATCH 25/37] Update codeql_unit_tests.yml --- .github/workflows/codeql_unit_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeql_unit_tests.yml b/.github/workflows/codeql_unit_tests.yml index aa47483ee4..84a24238c3 100644 --- a/.github/workflows/codeql_unit_tests.yml +++ b/.github/workflows/codeql_unit_tests.yml @@ -47,6 +47,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.9" + + - name: Install Python dependencies + run: pip install -r scripts/requirements.txt - name: Cache CodeQL id: cache-codeql From f7572edb6c2db512300442107fa83bb0972f0275 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 12:18:50 +0100 Subject: [PATCH 26/37] Build codeql pack --- .github/workflows/code-scanning-pack-gen.yml | 4 ++-- cpp/misra/src/qlpack.yml | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index 194dbf3a7f..27653cf22c 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -87,8 +87,8 @@ jobs: run: | PATH=$PATH:$CODEQL_HOME/codeql - codeql query compile --threads 0 cpp - codeql query compile --threads 0 c + find c -name qlpack.yml -exec codeql pack create --threads=0 {} \; + find cpp -name qlpack.yml -exec codeql pack create --threads=0 {} \; cd .. zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/codeql-workspace.yml codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas diff --git a/cpp/misra/src/qlpack.yml b/cpp/misra/src/qlpack.yml index ea3f5cbd5f..9829f925e5 100644 --- a/cpp/misra/src/qlpack.yml +++ b/cpp/misra/src/qlpack.yml @@ -1,5 +1,11 @@ name: codeql/misra-cpp-coding-standards version: 2.13.0-dev +description: MISRA C++ 2008 +suites: codeql-suites +defaultSuite: + qlpack: codeql/autosar-cpp-coding-standards + include: + tags contains: /^external/autosar/id/m.*/ license: MIT dependencies: codeql/common-cpp-coding-standards: '*' From 871a52c04a7c751caa35b98103583fbd42b91fbf Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 14:03:14 +0100 Subject: [PATCH 27/37] Update code-scanning-pack-gen.yml --- .github/workflows/code-scanning-pack-gen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index 27653cf22c..194dbf3a7f 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -87,8 +87,8 @@ jobs: run: | PATH=$PATH:$CODEQL_HOME/codeql - find c -name qlpack.yml -exec codeql pack create --threads=0 {} \; - find cpp -name qlpack.yml -exec codeql pack create --threads=0 {} \; + codeql query compile --threads 0 cpp + codeql query compile --threads 0 c cd .. zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/codeql-workspace.yml codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas From fae4944cb99bdfe847ce381c5dc4cc04839e47f3 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 14:50:36 +0100 Subject: [PATCH 28/37] Revert "remove .codeqlmanifest.json" This reverts commit 70c41846fe9ad52ab4a73bb7f2b5d7db8a995da6. --- .codeqlmanifest.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 .codeqlmanifest.json diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json new file mode 100644 index 0000000000..e6aec98053 --- /dev/null +++ b/.codeqlmanifest.json @@ -0,0 +1 @@ +{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml", "scripts/generate_modules/queries/qlpack.yml" ] } From febe9904f572eaacdb0e2fd2fdfe162de8ff7b9f Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 14:52:03 +0100 Subject: [PATCH 29/37] Revert commit 74ff65d --- .github/workflows/code-scanning-pack-gen.yml | 2 +- codeql-workspace.yml | 6 ------ scripts/get_workspace_packs.py | 12 ++++++------ 3 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 codeql-workspace.yml diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index 194dbf3a7f..ca03cb01a5 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -91,7 +91,7 @@ jobs: codeql query compile --threads 0 c cd .. - zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/codeql-workspace.yml codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas + zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas - name: Upload GHAS Query Pack uses: actions/upload-artifact@v2 diff --git a/codeql-workspace.yml b/codeql-workspace.yml deleted file mode 100644 index d51b909203..0000000000 --- a/codeql-workspace.yml +++ /dev/null @@ -1,6 +0,0 @@ -provide: - - "cpp/*/src/qlpack.yml" - - "cpp/*/test/qlpack.yml" - - "c/*/src/qlpack.yml" - - "c/*/test/qlpack.yml" - - "scripts/generate_modules/queries/qlpack.yml" diff --git a/scripts/get_workspace_packs.py b/scripts/get_workspace_packs.py index e348f85d57..fc9054c641 100644 --- a/scripts/get_workspace_packs.py +++ b/scripts/get_workspace_packs.py @@ -1,14 +1,14 @@ import glob -import yaml +import json import os def get_workspace_packs(root): - # Find the packs by globbing using the 'provide' patterns in the CodeQL workspace file. + # Find the packs by globbing using the 'provide' patterns in the manifest. os.chdir(root) - with open('codeql-workspace.yml') as codeql_workspace_file: - codeql_workspace = yaml.load(codeql_workspace_file) + with open('.codeqlmanifest.json') as manifest_file: + manifest = json.load(manifest_file) packs = [] - for pattern in codeql_workspace['provide']: + for pattern in manifest['provide']: packs.extend(glob.glob(pattern, recursive=True)) - + return packs From f1ccadeae074fdbff50b514e605a7284df53aa37 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 18:34:46 +0100 Subject: [PATCH 30/37] Update code-scanning-pack-gen.yml --- .github/workflows/code-scanning-pack-gen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index ca03cb01a5..ea70c04009 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -87,8 +87,8 @@ jobs: run: | PATH=$PATH:$CODEQL_HOME/codeql - codeql query compile --threads 0 cpp - codeql query compile --threads 0 c + find c -name qlpack.yml -exec codeql pack create --threads=0 {} \; + find cpp -name qlpack.yml -exec codeql pack create --threads=0 {} \; cd .. zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas From 25638c8ddce1448f3aae6ba89fc5642a1c18a0b5 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 18:49:39 +0100 Subject: [PATCH 31/37] Revert "Update code-scanning-pack-gen.yml" This reverts commit f1ccadeae074fdbff50b514e605a7284df53aa37. --- .github/workflows/code-scanning-pack-gen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index ea70c04009..ca03cb01a5 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -87,8 +87,8 @@ jobs: run: | PATH=$PATH:$CODEQL_HOME/codeql - find c -name qlpack.yml -exec codeql pack create --threads=0 {} \; - find cpp -name qlpack.yml -exec codeql pack create --threads=0 {} \; + codeql query compile --threads 0 cpp + codeql query compile --threads 0 c cd .. zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas From 588efba6140610f894ae844d229e0f94cb0d917d Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 20:00:12 +0100 Subject: [PATCH 32/37] Add .codeqlmanifest.json back --- c/.codeqlmanifest.json | 3 +++ cpp/.codeqlmanifest.json | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 c/.codeqlmanifest.json create mode 100644 cpp/.codeqlmanifest.json diff --git a/c/.codeqlmanifest.json b/c/.codeqlmanifest.json new file mode 100644 index 0000000000..c40bc9cfd5 --- /dev/null +++ b/c/.codeqlmanifest.json @@ -0,0 +1,3 @@ +{ "provide": [ + "*/src/qlpack.yml", + "*/test/qlpack.yml" ] } \ No newline at end of file diff --git a/cpp/.codeqlmanifest.json b/cpp/.codeqlmanifest.json new file mode 100644 index 0000000000..c40bc9cfd5 --- /dev/null +++ b/cpp/.codeqlmanifest.json @@ -0,0 +1,3 @@ +{ "provide": [ + "*/src/qlpack.yml", + "*/test/qlpack.yml" ] } \ No newline at end of file From f235db6d29a43482ef51ef8eba8c45391c10c618 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 20:08:26 +0100 Subject: [PATCH 33/37] Delete .codeqlmanifest.json --- cpp/.codeqlmanifest.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 cpp/.codeqlmanifest.json diff --git a/cpp/.codeqlmanifest.json b/cpp/.codeqlmanifest.json deleted file mode 100644 index c40bc9cfd5..0000000000 --- a/cpp/.codeqlmanifest.json +++ /dev/null @@ -1,3 +0,0 @@ -{ "provide": [ - "*/src/qlpack.yml", - "*/test/qlpack.yml" ] } \ No newline at end of file From 1a6d68ecae6381a13f4c221e0234dec4dbac81e3 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Tue, 7 Feb 2023 20:08:36 +0100 Subject: [PATCH 34/37] Delete .codeqlmanifest.json --- c/.codeqlmanifest.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 c/.codeqlmanifest.json diff --git a/c/.codeqlmanifest.json b/c/.codeqlmanifest.json deleted file mode 100644 index c40bc9cfd5..0000000000 --- a/c/.codeqlmanifest.json +++ /dev/null @@ -1,3 +0,0 @@ -{ "provide": [ - "*/src/qlpack.yml", - "*/test/qlpack.yml" ] } \ No newline at end of file From 1c9a91eb402bcb6d3b758db174850b89f88431ed Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 8 Feb 2023 12:33:07 +0100 Subject: [PATCH 35/37] Remove defaultSuite property --- c/cert/src/qlpack.yml | 1 - c/misra/src/qlpack.yml | 1 - cpp/autosar/src/qlpack.yml | 1 - cpp/cert/src/qlpack.yml | 1 - cpp/misra/src/qlpack.yml | 4 ---- 5 files changed, 8 deletions(-) diff --git a/c/cert/src/qlpack.yml b/c/cert/src/qlpack.yml index 51f0b7642b..aa7fe3ba4d 100644 --- a/c/cert/src/qlpack.yml +++ b/c/cert/src/qlpack.yml @@ -2,7 +2,6 @@ name: codeql/cert-c-coding-standards version: 2.13.0-dev description: CERT C 2016 suites: codeql-suites -defaultSuiteFile: codeql-suites/cert-default.qls license: MIT dependencies: codeql/common-c-coding-standards: '*' diff --git a/c/misra/src/qlpack.yml b/c/misra/src/qlpack.yml index 2f140abac5..7eb3a8b873 100644 --- a/c/misra/src/qlpack.yml +++ b/c/misra/src/qlpack.yml @@ -2,7 +2,6 @@ name: codeql/misra-c-coding-standards version: 2.13.0-dev description: MISRA C 2012 suites: codeql-suites -defaultSuiteFile: codeql-suites/misra-default.qls license: MIT dependencies: codeql/common-c-coding-standards: '*' diff --git a/cpp/autosar/src/qlpack.yml b/cpp/autosar/src/qlpack.yml index f61a1205d1..2a74a6bb20 100644 --- a/cpp/autosar/src/qlpack.yml +++ b/cpp/autosar/src/qlpack.yml @@ -2,7 +2,6 @@ name: codeql/autosar-cpp-coding-standards version: 2.13.0-dev description: AUTOSAR C++14 Guidelines 20-11 suites: codeql-suites -defaultSuiteFile: codeql-suites/autosar-default.qls license: MIT dependencies: codeql/common-cpp-coding-standards: '*' diff --git a/cpp/cert/src/qlpack.yml b/cpp/cert/src/qlpack.yml index e539039367..04c216db95 100644 --- a/cpp/cert/src/qlpack.yml +++ b/cpp/cert/src/qlpack.yml @@ -2,7 +2,6 @@ name: codeql/cert-cpp-coding-standards version: 2.13.0-dev description: CERT C++ 2016 suites: codeql-suites -defaultSuiteFile: codeql-suites/cert-default.qls license: MIT dependencies: codeql/cpp-all: 0.2.3 diff --git a/cpp/misra/src/qlpack.yml b/cpp/misra/src/qlpack.yml index 9829f925e5..2afcad806b 100644 --- a/cpp/misra/src/qlpack.yml +++ b/cpp/misra/src/qlpack.yml @@ -2,10 +2,6 @@ name: codeql/misra-cpp-coding-standards version: 2.13.0-dev description: MISRA C++ 2008 suites: codeql-suites -defaultSuite: - qlpack: codeql/autosar-cpp-coding-standards - include: - tags contains: /^external/autosar/id/m.*/ license: MIT dependencies: codeql/common-cpp-coding-standards: '*' From f515ab5bc19a7625492bd07ceaf88ca8df988cf0 Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 8 Feb 2023 16:07:30 +0100 Subject: [PATCH 36/37] Replace .codeqlmanifest.json with codeql-workspace.yml --- .codeqlmanifest.json | 1 - .github/workflows/code-scanning-pack-gen.yml | 6 +++--- codeql-workspace.yml | 7 +++++++ scripts/get_workspace_packs.py | 14 +++++++------- 4 files changed, 17 insertions(+), 11 deletions(-) delete mode 100644 .codeqlmanifest.json create mode 100644 codeql-workspace.yml diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json deleted file mode 100644 index e6aec98053..0000000000 --- a/.codeqlmanifest.json +++ /dev/null @@ -1 +0,0 @@ -{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml", "scripts/generate_modules/queries/qlpack.yml" ] } diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index ca03cb01a5..27653cf22c 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -87,11 +87,11 @@ jobs: run: | PATH=$PATH:$CODEQL_HOME/codeql - codeql query compile --threads 0 cpp - codeql query compile --threads 0 c + find c -name qlpack.yml -exec codeql pack create --threads=0 {} \; + find cpp -name qlpack.yml -exec codeql pack create --threads=0 {} \; cd .. - zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas + zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/codeql-workspace.yml codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas - name: Upload GHAS Query Pack uses: actions/upload-artifact@v2 diff --git a/codeql-workspace.yml b/codeql-workspace.yml new file mode 100644 index 0000000000..6c88c2e088 --- /dev/null +++ b/codeql-workspace.yml @@ -0,0 +1,7 @@ +provide: + - "cpp/*/src/qlpack.yml" + - "cpp/*/test/qlpack.yml" + - "c/*/src/qlpack.yml" + - "c/*/test/qlpack.yml" + - "scripts/generate_modules/queries/qlpack.yml" + \ No newline at end of file diff --git a/scripts/get_workspace_packs.py b/scripts/get_workspace_packs.py index fc9054c641..1833f1685c 100644 --- a/scripts/get_workspace_packs.py +++ b/scripts/get_workspace_packs.py @@ -1,14 +1,14 @@ import glob -import json +import yaml import os def get_workspace_packs(root): - # Find the packs by globbing using the 'provide' patterns in the manifest. + # Find the packs by globbing using the 'provide' patterns in the CodeQL workspace file. os.chdir(root) - with open('.codeqlmanifest.json') as manifest_file: - manifest = json.load(manifest_file) + with open('codeql-workspace.yml') as codeql_workspace_file: + codeql_workspace = yaml.load(codeql_workspace_file) packs = [] - for pattern in manifest['provide']: + for pattern in codeql_workspace['provide']: packs.extend(glob.glob(pattern, recursive=True)) - - return packs + + return packs \ No newline at end of file From d3d4bd368ecb31cebd82f32d64e802869dcfe58a Mon Sep 17 00:00:00 2001 From: Mauro Baluda Date: Wed, 8 Feb 2023 17:31:07 +0100 Subject: [PATCH 37/37] Revert "Replace .codeqlmanifest.json with codeql-workspace.yml" This reverts commit f515ab5bc19a7625492bd07ceaf88ca8df988cf0. --- .codeqlmanifest.json | 1 + .github/workflows/code-scanning-pack-gen.yml | 6 +++--- codeql-workspace.yml | 7 ------- scripts/get_workspace_packs.py | 14 +++++++------- 4 files changed, 11 insertions(+), 17 deletions(-) create mode 100644 .codeqlmanifest.json delete mode 100644 codeql-workspace.yml diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json new file mode 100644 index 0000000000..e6aec98053 --- /dev/null +++ b/.codeqlmanifest.json @@ -0,0 +1 @@ +{ "provide": [ "cpp/*/src/qlpack.yml", "cpp/*/test/qlpack.yml", "c/*/src/qlpack.yml", "c/*/test/qlpack.yml", "scripts/generate_modules/queries/qlpack.yml" ] } diff --git a/.github/workflows/code-scanning-pack-gen.yml b/.github/workflows/code-scanning-pack-gen.yml index 27653cf22c..ca03cb01a5 100644 --- a/.github/workflows/code-scanning-pack-gen.yml +++ b/.github/workflows/code-scanning-pack-gen.yml @@ -87,11 +87,11 @@ jobs: run: | PATH=$PATH:$CODEQL_HOME/codeql - find c -name qlpack.yml -exec codeql pack create --threads=0 {} \; - find cpp -name qlpack.yml -exec codeql pack create --threads=0 {} \; + codeql query compile --threads 0 cpp + codeql query compile --threads 0 c cd .. - zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/codeql-workspace.yml codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas + zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/configuration codeql-coding-standards/scripts/reports codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/guideline_recategorization codeql-coding-standards/scripts/shared codeql-coding-standards/scripts/schemas - name: Upload GHAS Query Pack uses: actions/upload-artifact@v2 diff --git a/codeql-workspace.yml b/codeql-workspace.yml deleted file mode 100644 index 6c88c2e088..0000000000 --- a/codeql-workspace.yml +++ /dev/null @@ -1,7 +0,0 @@ -provide: - - "cpp/*/src/qlpack.yml" - - "cpp/*/test/qlpack.yml" - - "c/*/src/qlpack.yml" - - "c/*/test/qlpack.yml" - - "scripts/generate_modules/queries/qlpack.yml" - \ No newline at end of file diff --git a/scripts/get_workspace_packs.py b/scripts/get_workspace_packs.py index 1833f1685c..fc9054c641 100644 --- a/scripts/get_workspace_packs.py +++ b/scripts/get_workspace_packs.py @@ -1,14 +1,14 @@ import glob -import yaml +import json import os def get_workspace_packs(root): - # Find the packs by globbing using the 'provide' patterns in the CodeQL workspace file. + # Find the packs by globbing using the 'provide' patterns in the manifest. os.chdir(root) - with open('codeql-workspace.yml') as codeql_workspace_file: - codeql_workspace = yaml.load(codeql_workspace_file) + with open('.codeqlmanifest.json') as manifest_file: + manifest = json.load(manifest_file) packs = [] - for pattern in codeql_workspace['provide']: + for pattern in manifest['provide']: packs.extend(glob.glob(pattern, recursive=True)) - - return packs \ No newline at end of file + + return packs