Fixes for transitive inlining

compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala

@@ -41,6 +41,15 @@ class TreeTypeMap(
   val substTo: List[Symbol] = Nil)(using Context) extends tpd.TreeMap {
   import tpd._
 
+  def copy(
+      typeMap: Type => Type,
+      treeMap: tpd.Tree => tpd.Tree,
+      oldOwners: List[Symbol],
+      newOwners: List[Symbol],
+      substFrom: List[Symbol],
+      substTo: List[Symbol])(using Context): TreeTypeMap =
+    new TreeTypeMap(typeMap, treeMap, oldOwners, newOwners, substFrom, substTo)
+
   /** If `sym` is one of `oldOwners`, replace by corresponding symbol in `newOwners` */
   def mapOwner(sym: Symbol): Symbol = sym.subst(oldOwners, newOwners)
 
@@ -76,8 +85,11 @@ class TreeTypeMap(
       updateDecls(prevStats.tail, newStats.tail)
     }
 
-  /** If true, stop return Inlined(Empty, _, _) nodes unchanged */
-  def stopAtInlinedArgument: Boolean = false
+  def transformInlined(tree: tpd.Inlined)(using Context): tpd.Tree =
+    val Inlined(call, bindings, expanded) = tree
+    val (tmap1, bindings1) = transformDefs(bindings)
+    val expanded1 = tmap1.transform(expanded)
+    cpy.Inlined(tree)(call, bindings1, expanded1)
 
   override def transform(tree: tpd.Tree)(using Context): tpd.Tree = treeMap(tree) match {
     case impl @ Template(constr, parents, self, _) =>
@@ -109,13 +121,8 @@ class TreeTypeMap(
           val (tmap1, stats1) = transformDefs(stats)
           val expr1 = tmap1.transform(expr)
           cpy.Block(blk)(stats1, expr1)
-        case inlined @ Inlined(call, bindings, expanded) =>
-          if stopAtInlinedArgument && call.isEmpty then
-            inlined
-          else
-            val (tmap1, bindings1) = transformDefs(bindings)
-            val expanded1 = tmap1.transform(expanded)
-            cpy.Inlined(inlined)(call, bindings1, expanded1)
+        case inlined: Inlined =>
+          transformInlined(inlined)
         case cdef @ CaseDef(pat, guard, rhs) =>
           val tmap = withMappedSyms(patVars(pat))
           val pat1 = tmap.transform(pat)
@@ -171,7 +178,7 @@ class TreeTypeMap(
       assert(!to.exists(substFrom contains _))
       assert(!from.exists(newOwners contains _))
       assert(!to.exists(oldOwners contains _))
-      new TreeTypeMap(
+      copy(
         typeMap,
         treeMap,
         from ++ oldOwners,

compiler/src/dotty/tools/dotc/typer/Inliner.scala

@@ -706,11 +706,40 @@ class Inliner(call: tpd.Tree, rhsToInline: tpd.Tree)(using Context) {
     def inlinedFromOutside(tree: Tree)(span: Span): Tree =
       Inlined(EmptyTree, Nil, tree)(using ctx.withSource(inlinedMethod.topLevelClass.source)).withSpan(span)
 
+    // InlinerMap is a TreeTypeMap with special treatment for inlined arguments:
+    // They are generally left alone (not mapped further, and if they wrap a type
+    // the type Inlined wrapper gets dropped
+    class InlinerMap(
+        typeMap: Type => Type,
+        treeMap: Tree => Tree,
+        oldOwners: List[Symbol],
+        newOwners: List[Symbol],
+        substFrom: List[Symbol],
+        substTo: List[Symbol])(using Context)
+      extends TreeTypeMap(typeMap, treeMap, oldOwners, newOwners, substFrom, substTo):
+
+      override def copy(
+          typeMap: Type => Type,
+          treeMap: Tree => Tree,
+          oldOwners: List[Symbol],
+          newOwners: List[Symbol],
+          substFrom: List[Symbol],
+          substTo: List[Symbol])(using Context) =
+        new InlinerMap(typeMap, treeMap, oldOwners, newOwners, substFrom, substTo)
+
+      override def transformInlined(tree: Inlined)(using Context) =
+        if tree.call.isEmpty then
+          tree.expansion match
+            case expansion: TypeTree => expansion
+            case _ => tree
+        else super.transformInlined(tree)
+    end InlinerMap
+
     // A tree type map to prepare the inlined body for typechecked.
     // The translation maps references to `this` and parameters to
     // corresponding arguments or proxies on the type and term level. It also changes
     // the owner from the inlined method to the current owner.
-    val inliner = new TreeTypeMap(
+    val inliner = new InlinerMap(
       typeMap =
         new DeepTypeMap {
           def apply(t: Type) = t match {
@@ -746,16 +775,16 @@ class Inliner(call: tpd.Tree, rhsToInline: tpd.Tree)(using Context) {
               val inlinedSingleton = singleton(t).withSpan(argSpan)
               inlinedFromOutside(inlinedSingleton)(tree.span)
             case Some(t) if tree.isType =>
-              TypeTree(t).withSpan(argSpan)
+              inlinedFromOutside(TypeTree(t).withSpan(argSpan))(tree.span)
             case _ => tree
           }
         case tree => tree
       },
       oldOwners = inlinedMethod :: Nil,
-      newOwners = ctx.owner :: Nil
-    )(using inlineCtx) {
-      override def stopAtInlinedArgument: Boolean = true
-    }
+      newOwners = ctx.owner :: Nil,
+      substFrom = Nil,
+      substTo = Nil
+    )(using inlineCtx)
 
     // Apply inliner to `rhsToInline`, split off any implicit bindings from result, and
     // make them part of `bindingsBuf`. The expansion is then the tree that remains.
@@ -995,26 +1024,31 @@ class Inliner(call: tpd.Tree, rhsToInline: tpd.Tree)(using Context) {
       */
     def betaReduce(tree: Tree)(using Context): Tree = tree match {
       case Apply(Select(cl @ closureDef(ddef), nme.apply), args) if defn.isFunctionType(cl.tpe) =>
-        ddef.tpe.widen match {
-          case mt: MethodType if ddef.paramss.head.length == args.length =>
-            val bindingsBuf = new mutable.ListBuffer[ValOrDefDef]
-            val argSyms = mt.paramNames.lazyZip(mt.paramInfos).lazyZip(args).map { (name, paramtp, arg) =>
-              arg.tpe.dealias match {
-                case ref @ TermRef(NoPrefix, _) => ref.symbol
-                case _ =>
-                  paramBindingDef(name, paramtp, arg, bindingsBuf)(
-                    using ctx.withSource(cl.source)
-                  ).symbol
+        // closureDef also returns a result for closures wrapped in Inlined nodes.
+        // These need to be preserved.
+        def recur(cl: Tree): Tree = cl match
+          case Inlined(call, bindings, expr) =>
+            cpy.Inlined(cl)(call, bindings, recur(expr))
+          case _ => ddef.tpe.widen match
+            case mt: MethodType if ddef.paramss.head.length == args.length =>
+              val bindingsBuf = new mutable.ListBuffer[ValOrDefDef]
+              val argSyms = mt.paramNames.lazyZip(mt.paramInfos).lazyZip(args).map { (name, paramtp, arg) =>
+                arg.tpe.dealias match {
+                  case ref @ TermRef(NoPrefix, _) => ref.symbol
+                  case _ =>
+                    paramBindingDef(name, paramtp, arg, bindingsBuf)(
+                      using ctx.withSource(cl.source)
+                    ).symbol
+                }
               }
-            }
-            val expander = new TreeTypeMap(
-              oldOwners = ddef.symbol :: Nil,
-              newOwners = ctx.owner :: Nil,
-              substFrom = ddef.paramss.head.map(_.symbol),
-              substTo = argSyms)
-            Block(bindingsBuf.toList, expander.transform(ddef.rhs))
-          case _ => tree
-        }
+              val expander = new TreeTypeMap(
+                oldOwners = ddef.symbol :: Nil,
+                newOwners = ctx.owner :: Nil,
+                substFrom = ddef.paramss.head.map(_.symbol),
+                substTo = argSyms)
+              Block(bindingsBuf.toList, expander.transform(ddef.rhs)).withSpan(tree.span)
+            case _ => tree
+        recur(cl)
       case _ => tree
     }
 

tests/pos/i11350.scala

@@ -0,0 +1,5 @@
+//case class A[T](action: A[T] ?=> String) // now disallowed
+
+class A1[T](action: A1[T] ?=> String = (_: A1[T]) ?=> "") // works
+//case class A2[T](action: A2[?] ?=> String) // now disallowed
+//case class A3[T](action: A3[T] => String) // now disallowed

tests/pos/i11894.scala

@@ -0,0 +1,7 @@
+class CallbackTo[A](val x: A):
+  inline def runNow(): A = x
+  inline def toScalaFn: () => A = () => runNow()
+  def toOption(): Option[A] =
+    val y: CallbackTo[Option[A]] = ???
+    val f: () => Option[A] = y.toScalaFn // error
+    f()

tests/pos/i11894a.scala

@@ -0,0 +1,24 @@
+object CallbackTo {
+  extension [A](self: CallbackTo[Option[A]]) {
+    transparent inline def asOption: Option[A] =
+      self.toScalaFn()
+  }
+}
+
+final class CallbackTo[A] (val x: List[A]) {
+
+  transparent inline def runNow(): A =
+    x.head
+
+  transparent inline def toScalaFn: () => A =
+    () => runNow()
+
+  def map[B](f: A => B): CallbackTo[B] =
+    ???
+
+  def toOption: Option[A] = {
+    val x = map[Option[A]](Some(_))
+    val y = x: CallbackTo[Option[A]] // ok: type is what we expect
+    y.asOption // error
+  }
+}

tests/pos/i11894b.scala

@@ -0,0 +1,24 @@
+object CallbackTo {
+  extension [A](self: CallbackTo[Option[A]]) {
+    inline def asOption: Option[A] =
+      self.toScalaFn()
+  }
+}
+
+final class CallbackTo[A] (val x: List[A]) {
+
+  inline def runNow(): A =
+    x.head
+
+  inline def toScalaFn: () => A =
+    () => runNow()
+
+  def map[B](f: A => B): CallbackTo[B] =
+    ???
+
+  def toOption: Option[A] = {
+    val x = map[Option[A]](Some(_))
+    val y = x: CallbackTo[Option[A]] // ok: type is what we expect
+    y.asOption // error
+  }
+}