Avoid exponential retries on inlining overflow

In some situations Typer tries several alternatives around an inlined call. If overflow is detected
at depth 32 then this could cause 2^32 different retries which leads to OOM.

Prevent this with a switch that stops further inline calls as long as an overflowing call stack
is not completely unwound.

Fixes #12116

Author: odersky

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -922,6 +922,9 @@ object Contexts {
 
     private[core] var denotTransformers: Array[DenotTransformer] = _
 
+    /** Flag to suppress inlining, set after overflow */
+    private[dotc] var stopInlining: Boolean = false
+
     // Reporters state
     private[dotc] var indent: Int = 0
 

compiler/src/dotty/tools/dotc/typer/Inliner.scala

@@ -70,6 +70,7 @@ object Inliner {
         || (ctx.phase == Phases.typerPhase && needsTransparentInlining(tree))
       )
       && !ctx.typer.hasInliningErrors
+      && !ctx.base.stopInlining
   }
 
   private def needsTransparentInlining(tree: Tree)(using Context): Boolean =
@@ -140,6 +141,7 @@ object Inliner {
         val body = bodyToInline(tree.symbol) // can typecheck the tree and thereby produce errors
         new Inliner(tree, body).inlined(tree.srcPos)
       else
+        ctx.base.stopInlining = true
         val (reason, setting) =
           if reachedInlinedTreesLimit then ("inlined trees", ctx.settings.XmaxInlinedTrees)
           else ("successive inlines", ctx.settings.XmaxInlines)
@@ -150,6 +152,10 @@ object Inliner {
               |You can use ${setting.name} to change the limit.""",
           (tree :: enclosingInlineds).last.srcPos
         )
+    if ctx.base.stopInlining && enclosingInlineds.isEmpty then
+      ctx.base.stopInlining = false
+        // we have completely backed out of the call that overflowed;
+        // reset so that further inline calls can be expanded
     tree2
   }
 

tests/neg/i12116.scala

@@ -0,0 +1,15 @@
+import compiletime.erasedValue
+
+object test1:
+
+  transparent inline def length[T]: Int =
+      erasedValue[T] match
+          case _: (h *: t) => 1 + length[t]
+          case _: EmptyTuple => 0
+
+  transparent inline def foo(): Int = 1 + foo()
+
+  val y = length[(1, 2, 3)] // error
+  val x = foo() // error
+
+

tests/neg/i12116a.scala

@@ -0,0 +1,15 @@
+import compiletime.erasedValue
+
+object test1:
+
+  inline def length[T]: Int =
+      erasedValue[T] match
+          case _: (h *: t) => 1 + length[t]
+          case _: EmptyTuple => 0
+
+  inline def foo(): Int = 1 + foo()
+
+  val y = length[(1, 2, 3)] // error
+  val x = foo() // error
+
+