Fix #7294: Add limit on trees that can be inlined in a compilation unit

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -1214,6 +1214,12 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   /** A key to be used in a context property that tracks enclosing inlined calls */
   private val InlinedCalls = Property.Key[List[Tree]]()
 
+  /** A key to be used in a context property that tracks the number of inlined trees */
+  private val InlinedTrees = Property.Key[Counter]()
+  final class Counter {
+    var count: Int = 0
+  }
+
   /** Record an enclosing inlined call.
    *  EmptyTree calls (for parameters) cancel the next-enclosing call in the list instead of being added to it.
    *  We assume parameters are never nested inside parameters.
@@ -1230,14 +1236,25 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
       else
         call :: oldIC
 
-    ctx.fresh.setProperty(InlinedCalls, newIC)
+    val ctx1 = ctx.fresh.setProperty(InlinedCalls, newIC)
+    if oldIC.isEmpty then ctx1.setProperty(InlinedTrees, new Counter) else ctx1
   }
 
   /** All enclosing calls that are currently inlined, from innermost to outermost.
    */
   def enclosingInlineds(implicit ctx: Context): List[Tree] =
     ctx.property(InlinedCalls).getOrElse(Nil)
 
+  /** Record inlined trees */
+  def addInlinedTrees(n: Int)(implicit ctx: Context): Unit =
+    ctx.property(InlinedTrees).foreach(_.count += n)
+
+  /** Check if the limit on the number of inlined trees has been reached */
+  def reachedInlinedTreesLimit(implicit ctx: Context): Boolean =
+    ctx.property(InlinedTrees) match
+      case Some(c) => c.count > ctx.settings.XmaxInlinedTrees.value
+      case None => false
+
   /** The source file where the symbol of the `inline` method referred to by `call`
    *  is defined
    */

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -72,6 +72,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val Xhelp: Setting[Boolean] = BooleanSetting("-X", "Print a synopsis of advanced options.")
   val XnoForwarders: Setting[Boolean] = BooleanSetting("-Xno-forwarders", "Do not generate static forwarders in mirror classes.")
   val XmaxInlines: Setting[Int] = IntSetting("-Xmax-inlines", "Maximal number of successive inlines.", 32)
+  val XmaxInlinedTrees: Setting[Int] = IntSetting("-Xmax-inlined-trees", "Maximal number of inlined trees.", 2_000_000)
   val Xmigration: Setting[ScalaVersion] = VersionSetting("-Xmigration", "Warn about constructs whose behavior may have changed since version.")
   val Xprint: Setting[List[String]] = PhasesSetting("-Xprint", "Print out program after")
   val XprintTypes: Setting[Boolean] = BooleanSetting("-Xprint-types", "Print tree types (debugging option).")

compiler/src/dotty/tools/dotc/typer/Inliner.scala

@@ -29,6 +29,7 @@ import Nullables.{given _}
 import collection.mutable
 import reporting.trace
 import util.Spans.Span
+import util.NoSourcePosition
 import dotty.tools.dotc.transform.{Splicer, TreeMapWithStages}
 
 object Inliner {
@@ -71,14 +72,17 @@ object Inliner {
    *            and body that replace it.
    */
   def inlineCall(tree: Tree)(using Context): Tree = {
+    val startId = ctx.source.nextId
+
     if tree.symbol.denot != SymDenotations.NoDenotation && tree.symbol.owner.companionModule == defn.CompiletimeTestingPackageObject
       if (tree.symbol == defn.CompiletimeTesting_typeChecks) return Intrinsics.typeChecks(tree)
       if (tree.symbol == defn.CompiletimeTesting_typeCheckErrors) return Intrinsics.typeCheckErrors(tree)
 
-   /** Set the position of all trees logically contained in the expansion of
-    *  inlined call `call` to the position of `call`. This transform is necessary
-    *  when lifting bindings from the expansion to the outside of the call.
-    */
+
+    /** Set the position of all trees logically contained in the expansion of
+     *  inlined call `call` to the position of `call`. This transform is necessary
+     *  when lifting bindings from the expansion to the outside of the call.
+     */
     def liftFromInlined(call: Tree) = new TreeMap:
       override def transform(t: Tree)(using Context) =
         if call.span.exists then
@@ -115,20 +119,28 @@ object Inliner {
 
     // assertAllPositioned(tree)   // debug
     val tree1 = liftBindings(tree, identity)
-    if (bindings.nonEmpty)
-      cpy.Block(tree)(bindings.toList, inlineCall(tree1))
-    else if (enclosingInlineds.length < ctx.settings.XmaxInlines.value) {
-      val body = bodyToInline(tree.symbol) // can typecheck the tree and thereby produce errors
-      new Inliner(tree, body).inlined(tree.sourcePos)
-    }
-    else
-      errorTree(
-        tree,
-        i"""|Maximal number of successive inlines (${ctx.settings.XmaxInlines.value}) exceeded,
-            |Maybe this is caused by a recursive inline method?
-            |You can use -Xmax-inlines to change the limit.""",
-        (tree :: enclosingInlineds).last.sourcePos
-      )
+    val tree2 =
+      if bindings.nonEmpty then
+        cpy.Block(tree)(bindings.toList, inlineCall(tree1))
+      else if enclosingInlineds.length < ctx.settings.XmaxInlines.value && !reachedInlinedTreesLimit then
+        val body = bodyToInline(tree.symbol) // can typecheck the tree and thereby produce errors
+        new Inliner(tree, body).inlined(tree.sourcePos)
+      else
+        val (reason, setting) =
+          if reachedInlinedTreesLimit then ("inlined trees", ctx.settings.XmaxInlinedTrees)
+          else ("successive inlines", ctx.settings.XmaxInlines)
+        errorTree(
+          tree,
+          i"""|Maximal number of $reason (${setting.value}) exceeded,
+              |Maybe this is caused by a recursive inline method?
+              |You can use ${setting.name} to change the limit.""",
+          (tree :: enclosingInlineds).last.sourcePos
+        )
+
+    val endId = ctx.source.nextId
+    addInlinedTrees(endId - startId)
+
+    tree2
   }
 
   /** Try to inline a pattern with an inline unapply method. Fail with error if the maximal

tests/neg/i7294-a.scala

@@ -0,0 +1,9 @@
+package foo
+
+trait Foo { def g(x: Int): Any }
+
+inline given f[T <: Foo] as T = ??? match {
+  case x: T => x.g(10) // error
+}
+
+@main def Test = f // error // error

tests/neg/i7294-b.scala

@@ -0,0 +1,9 @@
+package foo
+
+trait Foo { def g(x: Any): Any }
+
+inline given f[T <: Foo] as T = ??? match {
+  case x: T => x.g(10) // error
+}
+
+@main def Test = f // error // error