Preventing compilation of a @tailrec method when it does not rewrite, but an inner method does

Adding warnings if there is an annotated def at the top level that is referenced from an inner def

Author: Lucy Martin

compiler/src/dotty/tools/dotc/reporting/ErrorMessageID.scala

@@ -207,6 +207,8 @@ enum ErrorMessageID(val isActive: Boolean = true) extends java.lang.Enum[ErrorMe
   case MatchTypeLegacyPatternID // errorNumber: 191
   case UnstableInlineAccessorID // errorNumber: 192
   case VolatileOnValID // errorNumber: 193
+  
+  case TailrecNestedCallID //errorNumber: 194
 
   def errorNumber = ordinal - 1
 

compiler/src/dotty/tools/dotc/reporting/messages.scala

@@ -1907,6 +1907,20 @@ class TailrecNotApplicable(symbol: Symbol)(using Context)
   def explain(using Context) = ""
 }
 
+class TailrecNestedCall(definition: Symbol, innerDef: Symbol)(using Context)
+  extends SyntaxMsg(TailrecNestedCallID) {
+  def msg(using Context) = {
+    s"The tail recursive def ${definition.name} contains a recursive call inside the non-inlined inner def ${innerDef.name}"
+  }
+
+  def explain(using Context) =
+    """Tail recursion is only validated and optimised directly in the definition
+      |any calls to the recursive method via an inner def cannot be validated as
+      |tail recursive, nor optimised if they are. To enable tail recursion from
+      |inner calls, mark the inner def as inline
+      |""".stripMargin
+}
+
 class FailureToEliminateExistential(tp: Type, tp1: Type, tp2: Type, boundSyms: List[Symbol], classRoot: Symbol)(using Context)
   extends Message(FailureToEliminateExistentialID) {
   def kind = MessageKind.Compatibility

compiler/src/dotty/tools/dotc/transform/TailRec.scala

@@ -4,7 +4,10 @@ package transform
 import ast.{TreeTypeMap, tpd}
 import config.Printers.tailrec
 import core.*
-import Contexts.*, Flags.*, Symbols.*, Decorators.em
+import Contexts.*
+import Flags.*
+import Symbols.*
+import Decorators.em
 import Constants.Constant
 import NameKinds.{TailLabelName, TailLocalName, TailTempName}
 import StdNames.nme
@@ -429,10 +432,20 @@ class TailRec extends MiniPhase {
           assert(false, "We should never have gotten inside a pattern")
           tree
 
-        case tree: ValOrDefDef =>
+        case tree: ValDef =>
           if (isMandatory) noTailTransform(tree.rhs)
           tree
 
+        case tree: DefDef =>
+          if (isMandatory)
+            // We cant tail recurse through nested definitions, so dont want to propagate to child nodes
+            // We dont want to fail if there is a call that would recurse (as this would be a non self recurse), so dont
+            // want to call noTailTransform
+            // We can however warn in this case, as its likely in this situation that someone would expect a tail
+            // recursion optimization and enabling this to optimise would be a simple case of inlining the inner method
+            new NestedTailRecAlerter(method, tree.symbol).traverse(tree)
+          tree
+
         case _: Super | _: This | _: Literal | _: TypeTree | _: TypeDef | EmptyTree =>
           tree
 
@@ -446,14 +459,28 @@ class TailRec extends MiniPhase {
 
         case Return(expr, from) =>
           val fromSym = from.symbol
-          val inTailPosition = !fromSym.is(Label) || tailPositionLabeledSyms.contains(fromSym)
+          val inTailPosition = (!fromSym.is(Label) || tailPositionLabeledSyms.contains(fromSym)) // Label returns are only tail if the label is in tail position
+            && (!fromSym.is(Method) || (fromSym eq method)) // Method returns are only tail if we are looking at the original method
           cpy.Return(tree)(transform(expr, inTailPosition), from)
 
         case _ =>
           super.transform(tree)
       }
     }
   }
+
+  class NestedTailRecAlerter(method: Symbol, inner: Symbol) extends TreeTraverser {
+    override def traverse(tree: tpd.Tree)(using Context): Unit =
+      tree match {
+        case a: Apply =>
+          if (a.fun.symbol eq method) {
+            report.warning(new TailrecNestedCall(method, inner), a.srcPos)
+          }
+          traverseChildren(tree)
+        case _ =>
+          traverseChildren(tree)
+      }
+  }
 }
 
 object TailRec {

tests/neg/i20105.scala

@@ -0,0 +1,8 @@
+@tailrec
+def foo(): Unit =
+  def bar(): Unit =
+    if (???)
+      foo()
+    else
+      bar()
+  bar()

tests/warn/i20105.scala

@@ -0,0 +1,9 @@
+@tailrec
+def foo(): Unit =
+  def bar(): Unit =
+    if (???)
+      foo() // warn
+    else
+      bar()
+  bar()
+  foo()