Implement package expressions2

Author: MichaelRFairhurst

c/misra/src/rules/RULE-18-9/ArrayToPointerConversionOfTemporaryObject.ql

@@ -16,6 +16,7 @@
 import cpp
 import codingstandards.c.misra
 import codingstandards.c.Objects
+import codingstandards.cpp.Expr
 
 /**
  * Holds if the value of an expression is used or stored.
@@ -37,32 +38,14 @@ predicate isUsedOrStored(Expr e) {
   e = any(ClassAggregateLiteral l).getAFieldExpr(_)
 }
 
-/**
- * Find expressions that defer their value directly to an inner expression
- * value.
- *
- * When an array is on the rhs of a comma expr, or in the then/else branch of a
- * ternary expr, and the result us used as a pointer, then the ArrayToPointer
- * conversion is marked inside comma expr/ternary expr, on the operands. These
- * conversions are only non-compliant if they flow into an operation or store.
- *
- * Full flow analysis with localFlowStep should not be necessary, and may cast a
- * wider net than needed for some queries, potentially resulting in false
- * positives.
- */
-Expr temporaryObjectFlowStep(Expr e) {
-  e = result.(CommaExpr).getRightOperand()
-  or
-  e = result.(ConditionalExpr).getThen()
-  or
-  e = result.(ConditionalExpr).getElse()
-}
-
 from FieldAccess fa, TemporaryObjectIdentity temporary, ArrayToPointerConversion conversion
 where
   not isExcluded(conversion, InvalidMemory3Package::arrayToPointerConversionOfTemporaryObjectQuery()) and
   fa = temporary.getASubobjectAccess() and
   conversion.getExpr() = fa and
-  isUsedOrStored(temporaryObjectFlowStep*(conversion.getExpr()))
+  exists(Expr useOrStore |
+    temporaryObjectFlowStep*(conversion.getExpr(), useOrStore) and
+    isUsedOrStored(useOrStore)
+  )
 select conversion, "Array to pointer conversion of array $@ from temporary object $@.",
   fa.getTarget(), fa.getTarget().getName(), temporary, temporary.toString()

change_notes/2025-05-20-update-integer-literal-lexing.md

@@ -0,0 +1,2 @@
+ - All queries related to integer suffixes:
+   - No visible changes expected: the regex for parsing integer suffixes, and how they are treated after lexing, has been refactored.

change_notes/2025-06-03-refactor-temporary-object-flow-step.md

@@ -0,0 +1,2 @@
+ - `RULE-18-9` - `ArraytoPointerConversionOfTemporaryObject.ql`
+   - The behavior for finding flow steps of temporary objects (for example, from ternary branches to the ternary expr result) has been extracted for reuse in other rules, no visible changes expected.

cpp/common/src/codingstandards/cpp/Cpp14Literal.qll

@@ -12,11 +12,33 @@ module Cpp14Literal {
   /** Convenience for implementing class `UnrecognizedNumericLiteral` */
   abstract private class RecognizedNumericLiteral extends StandardLibrary::Literal { }
 
+  /** An integer literal suffix (e.g. 'u', 'll', etc, or even an empty string). */
+  class IntegerLiteralSuffix extends string {
+    string uPart;
+    string lPart;
+
+    IntegerLiteralSuffix() {
+      uPart = ["", "u", "U"] and
+      lPart = ["", "l", "L"] + ["", "l", "L"] and
+      this = uPart + lPart
+    }
+
+    predicate isSigned() { uPart = "" }
+
+    predicate isUnsigned() { uPart != "" }
+
+    int getLCount() { result = lPart.length() }
+  }
+
   /** An integer literal. */
   abstract class IntegerLiteral extends NumericLiteral {
     predicate isSigned() { not isUnsigned() }
 
-    predicate isUnsigned() { getValueText().regexpMatch(".*[uU][lL]?$") }
+    predicate isUnsigned() { getSuffix().isUnsigned() }
+
+    IntegerLiteralSuffix getSuffix() {
+      result = getValueText().regexpCapture("[^uUlL]*([uU]?[lL]*)\\s*$", 1)
+    }
   }
 
   /**

cpp/common/src/codingstandards/cpp/Expr.qll

@@ -242,6 +242,28 @@ predicate isCompileTimeEvaluatedExpression(Expr expression) {
   )
 }
 
+/**
+ * Find expressions that defer their value directly to an inner expression
+ * value -- how temporary object can flow without being copied or having their
+ * address taken.
+ *
+ * Full flow analysis with localFlowStep is often not necessary for certain
+ * rules related to temporary objects, and may cast a wider net than needed for
+ * those queries.
+ *
+ * When an array is on the rhs of a comma expr, or in the then/else branch of a
+ * ternary expr, and the result us used as a pointer, then the ArrayToPointer
+ * conversion is marked inside comma expr/ternary expr, on the operands. These
+ * conversions are only non-compliant if they flow into an operation or store.
+ */
+predicate temporaryObjectFlowStep(Expr source, Expr sink) {
+  source = sink.(CommaExpr).getRightOperand()
+  or
+  source = sink.(ConditionalExpr).getThen()
+  or
+  source = sink.(ConditionalExpr).getElse()
+}
+
 predicate isDirectCompileTimeEvaluatedExpression(Expr expression) {
   expression instanceof Literal
   or

cpp/common/src/codingstandards/cpp/Lambda.qll

@@ -0,0 +1,238 @@
+import codingstandards.cpp.alertreporting.CustomPathStateProblem
+import codingstandards.cpp.Expr
+import codingstandards.cpp.Scope
+
+signature class LambdaSig extends LambdaExpression;
+
+class ImplicitThisCapturingLambdaExpr extends LambdaExpression {
+  ImplicitThisCapturingLambdaExpr() {
+    exists(LambdaCapture capture |
+      capture = getACapture() and
+      capture.getField().getName() = "(captured this)" and
+      capture.isImplicit()
+    )
+  }
+}
+
+class ImplicitCaptureLambdaExpr extends LambdaExpression {
+  LambdaCapture capture;
+
+  ImplicitCaptureLambdaExpr() { capture = getCapture(_) and capture.isImplicit() }
+
+  LambdaCapture getImplicitCapture() { result = capture }
+}
+
+/**
+ * A module to find lambda expressions that are eventually copied or moved.
+ *
+ * Unfortunately, CodeQL does not capture all lambda flow or all lambda copies/moves. However,
+ * since lambdas can only be used in an extremely limited number of ways, we can easily roll our
+ * own dataflow-like analysis as a custom path problem, to match lambdas to stores.
+ */
+module TransientLambda<LambdaSig Source> {
+  final class FinalLocatable = Locatable;
+
+  /**
+   * Create a custom path problem, which inherently performs a graph search to find paths from start
+   * nodes (in this case, lambda expressions) to end nodes (in this case, copies and moves of
+   * lambdas).
+   */
+  private module TransientLambdaConfig implements CustomPathStateProblemConfigSig {
+    class Node extends FinalLocatable {
+      Node() {
+        this instanceof Source
+        or
+        this instanceof Variable
+        or
+        this.(VariableAccess).getTarget() instanceof Parameter
+        or
+        this instanceof Expr
+        or
+        this instanceof Function
+        or
+        this instanceof NewExpr
+      }
+    }
+
+    class State = TranslationUnit;
+
+    // Do not search past the first copy or move of a lambda.
+    predicate searchPastEnd() { none() }
+
+    predicate start(Node n, TranslationUnit state) { n instanceof Source and state = n.getFile() }
+
+    bindingset[state]
+    pragma[inline_late]
+    predicate end(Node n, TranslationUnit state) {
+      n instanceof Variable and not n instanceof Parameter
+      or
+      n instanceof Function and
+      not functionDefinedInTranslationUnit(n, state)
+      or
+      exists(NewExpr alloc |
+        alloc = n and
+        alloc.getAllocatedType().stripTopLevelSpecifiers() instanceof Closure
+      )
+    }
+
+    predicate edge(Node a, TranslationUnit s1, Node b, TranslationUnit s2) {
+      s2 = s1 and
+      (
+        a = b.(Variable).getInitializer().getExpr()
+        or
+        exists(Call fc, int i |
+          a = fc.getArgument(i) and
+          (
+            b = fc.getTarget().getParameter(i)
+            or
+            b = fc.getTarget()
+          )
+        )
+        or
+        exists(Call fc, Function f, ReturnStmt ret |
+          f = fc.getTarget() and
+          ret.getEnclosingFunction() = f and
+          a = ret.getExpr() and
+          b = fc
+        )
+        or
+        b = a.(Parameter).getAnAccess()
+        or
+        temporaryObjectFlowStep(a, b)
+        or
+        a = b.(NewExpr).getInitializer()
+      )
+    }
+  }
+
+  import CustomPathStateProblem<TransientLambdaConfig> as TransientFlow
+
+  module PathProblem {
+    import TransientFlow
+  }
+
+  predicate isStored(Source lambda, Element store, string reason) {
+    TransientFlow::problem(lambda, store) and
+    (
+      if store instanceof Function and not functionDefinedInTranslationUnit(store, lambda.getFile())
+      then reason = "passed to a different translation unit"
+      else reason = "copied or moved"
+    )
+  }
+}
+
+/**
+ * An alterate module for detecting transient lambdas which uses the standard CodeQL dataflow
+ * library.
+ *
+ * Ideally, this module eventually replaces `TransientLambda`, however, current CodeQL support for
+ * flow of lambdas is unreliable and incomplete/inconsistent. This implementation does not detect
+ * all cases correctly, but it is a starting place to revisit at a later time.
+ *
+ * In the current dataflow library, there are many missing nodes and edges, making this currently
+ * difficult or impossible to implement correctly.
+ */
+module TransientLambdaDataFlow<LambdaSig Source> {
+  import semmle.code.cpp.dataflow.new.DataFlow as DataFlow
+  import DataFlow::DataFlow as NewDataFlow
+
+  final class FinalLocatable = Locatable;
+
+  /**
+   * Create a custom path problem, which inherently performs a graph search to find paths from start
+   * nodes (in this case, lambda expressions) to end nodes (in this case, copies and moves of
+   * lambdas).
+   */
+  module TransientLambdaConfig implements NewDataFlow::StateConfigSig {
+    class FlowState = TranslationUnit;
+
+    predicate isSource(NewDataFlow::Node n, TranslationUnit state) {
+      n.asExpr() instanceof Source and state = n.asExpr().getFile()
+    }
+
+    predicate isSink(NewDataFlow::Node n) {
+      n.asOperand().getDef().getAst() instanceof VariableDeclarationEntry
+      or
+      exists(n.asVariable()) and not exists(n.asParameter())
+      or
+      exists(NewExpr alloc |
+        alloc.getAllocatedType() instanceof Closure and
+        alloc.getInitializer() = n.asExpr()
+      )
+      or
+      // Detect casting to std::function, which results in a copy of the lambda.
+      exists(Conversion conv | conv.getExpr() = n.asExpr())
+      or
+      // Detect all function calls, in case the definition is in a different translation unit.
+      // We cannot detect this with stateful dataflow, for performance reasons.
+      exists(FunctionCall fc | fc.getAnArgument() = n.asExpr())
+    }
+
+    predicate isSink(NewDataFlow::Node n, TranslationUnit state) {
+      // Ideally, we would be able to check here for calls to functions defined outside of the
+      // translation unit, but in the current stateful dataflow library, this will result in a
+      // cartesian product of all nodes with all translation units. This limitation doesn't exist
+      // in the alternate `TransientLambda` module which uses `CustomPathStateProblem`.
+      //
+      // Since this predicate holds for none(), it may seem that we don't need to use stateful flow.
+      // However, stateful flow is still a good idea so that we can add isBarrier() to prevent flow
+      // out of the translation unit. That should be possible to do without introducing a
+      // cartesian product.
+      //
+      // To work around the cartesian product, this predicate holds for none() and `isSink(n)`
+      // should hold for all function calls. After flow has found lambda/function call pairs, we
+      // can filter out those pairs where the function is defined in a different translation unit.
+      //
+      // This isn't quite implemented yet.
+      none()
+    }
+
+    predicate isBarrierOut(NewDataFlow::Node n, TranslationUnit state) {
+      // TODO: Implement a barrier to prevent flow out of the translation unit.
+      none()
+    }
+
+    predicate isAdditionalFlowStep(
+      NewDataFlow::Node a, TranslationUnit s1, NewDataFlow::Node b, TranslationUnit s2
+    ) {
+      // Add additional flow steps to handle:
+      //
+      // auto x = []() { ... };
+      //
+      // Which isn't represented in the dataflow graph otherwise.
+      pragma[only_bind_out](s2) = s1 and
+      (
+        pragma[only_bind_out](a.asExpr()) =
+          b.asOperand()
+              .getDef()
+              .getAst()
+              .(VariableDeclarationEntry)
+              .getVariable()
+              .getInitializer()
+              .getExpr()
+        or
+        a.asExpr().(Conversion).getExpr() = b.asExpr()
+      )
+    }
+  }
+
+  import NewDataFlow::GlobalWithState<TransientLambdaConfig> as TransientFlow
+
+  module PathProblem {
+    import TransientFlow::PathGraph
+  }
+
+  predicate isStored(Source lambda, Element store) {
+    exists(NewDataFlow::Node sink |
+      TransientFlow::flow(NewDataFlow::exprNode(lambda), sink) and
+      store = sink.asOperand().getDef().getAst() and
+      not exists(FunctionCall fc |
+        fc.getAnArgument() = sink.asExpr() and
+        exists(FunctionDeclarationEntry funcDef |
+          funcDef = fc.getTarget().getDefinition() and
+          funcDef.getFile() = lambda.getFile().(TranslationUnit).getATransitivelyIncludedFile()
+        )
+      )
+    )
+  }
+}

cpp/common/src/codingstandards/cpp/Locations.qll

@@ -28,3 +28,14 @@ bindingset[filepath, lineNumber]
 int getLastColumnNumber(string filepath, int lineNumber) {
   result = max(Location l | l.hasLocationInfo(filepath, _, _, lineNumber, _) | l.getEndColumn())
 }
+
+bindingset[a, b]
+predicate shareEnding(Locatable a, Locatable b) {
+  exists(Location la, Location lb |
+    la = a.getLocation() and
+    lb = b.getLocation() and
+    la.getFile() = lb.getFile() and
+    la.getEndLine() = lb.getEndLine() and
+    la.getEndColumn() = lb.getEndColumn()
+  )
+}

cpp/common/src/codingstandards/cpp/Scope.qll

@@ -307,6 +307,15 @@ predicate inSameTranslationUnitLate(File f1, File f2) {
   )
 }
 
+bindingset[f, tu]
+pragma[inline_late]
+predicate functionDefinedInTranslationUnit(Function f, TranslationUnit tu) {
+  exists(FunctionDeclarationEntry fde | fde = f.getDefinition() |
+    fde = f.getDefinition() and
+    fde.getFile() = tu.getATransitivelyIncludedFile()
+  )
+}
+
 /** A file that is a C/C++ source file */
 class SourceFile extends File {
   SourceFile() {