JSR-45: better support for debuging inlined calls

compiler/src/dotty/tools/backend/jvm/BCodeSkelBuilder.scala

@@ -25,6 +25,8 @@ import dotty.tools.dotc.util.Spans._
 import dotty.tools.dotc.report
 import dotty.tools.dotc.transform.SymUtils._
 
+import InlinedSourceMaps._
+
 /*
  *
  *  @author  Miguel Garcia, http://lamp.epfl.ch/~magarcia/ScalaCompilerCornerReloaded/
@@ -80,6 +82,8 @@ trait BCodeSkelBuilder extends BCodeHelpers {
     var isCZParcelable             = false
     var isCZStaticModule           = false
 
+    var sourceMap: InlinedSourceMap = null
+
     /* ---------------- idiomatic way to ask questions to typer ---------------- */
 
     def paramTKs(app: Apply, take: Int = -1): List[BType] = app match {
@@ -100,6 +104,7 @@ trait BCodeSkelBuilder extends BCodeHelpers {
 
     def genPlainClass(cd0: TypeDef) = cd0 match {
       case TypeDef(_, impl: Template) =>
+
       assert(cnode == null, "GenBCode detected nested methods.")
       innerClassBufferASM.clear()
 
@@ -277,7 +282,8 @@ trait BCodeSkelBuilder extends BCodeHelpers {
                   superClass, interfaceNames.toArray)
 
       if (emitSource) {
-        cnode.visitSource(cunit.source.file.name, null /* SourceDebugExtension */)
+        sourceMap = sourceMapFor(cunit)(s => classBTypeFromSymbol(s).internalName)
+        cnode.visitSource(cunit.source.file.name, sourceMap.debugExtension.orNull)
       }
 
       enclosingMethodAttribute(claszSymbol, internalName, asmMethodType(_).descriptor) match {
@@ -371,6 +377,8 @@ trait BCodeSkelBuilder extends BCodeHelpers {
     var shouldEmitCleanup          = false
     // line numbers
     var lastEmittedLineNr          = -1
+    // by real line number we mean line number that is not pointing to virtual lines added by inlined calls
+    var lastRealLineNr             = -1
 
     object bc extends JCodeMethodN {
       override def jmethod = PlainSkelBuilder.this.mnode
@@ -539,19 +547,27 @@ trait BCodeSkelBuilder extends BCodeHelpers {
         case labnode: asm.tree.LabelNode => (labnode.getLabel == lbl);
         case _ => false } )
     }
-    def lineNumber(tree: Tree): Unit = {
-      if (!emitLines || !tree.span.exists) return;
-      val nr = ctx.source.offsetToLine(tree.span.point) + 1
-      if (nr != lastEmittedLineNr) {
+
+    def emitNr(nr: Int): Unit =
+      if nr != lastEmittedLineNr then
         lastEmittedLineNr = nr
-        lastInsn match {
+        lastInsn match
           case lnn: asm.tree.LineNumberNode =>
             // overwrite previous landmark as no instructions have been emitted for it
             lnn.line = nr
           case _ =>
             mnode.visitLineNumber(nr, currProgramPoint())
-        }
-      }
+
+    def lineNumber(tree: Tree): Unit = {
+      if !emitLines || !tree.span.exists then return;
+      if tree.source != cunit.source then
+        sourceMap.lineFor(tree.sourcePos, lastRealLineNr) match
+          case Some(nr) => emitNr(nr)
+          case None => ()
+      else
+        val nr = ctx.source.offsetToLine(tree.span.point) + 1
+        lastRealLineNr = nr
+        emitNr(nr)
     }
 
     // on entering a method

compiler/src/dotty/tools/backend/jvm/InlinedSourceMaps.scala

@@ -0,0 +1,163 @@
+package dotty.tools
+package backend
+package jvm
+
+import dotc.CompilationUnit
+import dotc.ast.tpd._
+import dotc.util.{ SourcePosition, SourceFile }
+import dotc.core.Contexts._
+import dotc.core.Symbols.Symbol
+import dotc.report
+import dotc.typer.Inliner.InliningPosition
+import collection.mutable
+
+/**
+ * Tool for generating virtual lines for inlined calls and keeping track of them.
+
+ * How it works:
+ * - For every inlined call it assumes that empty lines are appended to the source file. These
+ *   lines are not added anywhere in physical form. We only assume that they exist only to be used
+ *   by `LineNumberTable` and `SourceDebugExtension`. The number of these virtual lines is every
+ *   time equal to the size of line range of the expansion of inlined call.
+ * - It generates SMAP (as defined by JSR-45) containing two strata. The first stratum (`Scala`)
+ *   is describing the mapping from the real source files to the real and virtual lines in our
+ *   assumed source. The second stratum (`ScalaDebug`) is mapping from virtual lines to
+ *   corresponding inlined calls.
+ * - Generated SMAP is written to the bytecode in `SourceDebugExtension`
+ * - During the generation of the bytecode backed is asking `InlinedSourceMap` about position of
+ *   all trees that have source different from the main source of given compilation unit.
+ *   The response to that request is number of the virtual line that is corresponding to particular
+ *   line from the other source.
+ * - Debuggers can use information stored in `LineNumberTable` and `SourceDebugExtension` to
+ *   correctly guess which line of inlined method is currently executed. They can also construct
+ *   stack frames for inlined calls.
+ **/
+object InlinedSourceMaps:
+  private case class Request(targetPos: SourcePosition, origPos: SourcePosition, firstFakeLine: Int)
+
+  private class File(id: Int, name: String, path: Option[String]):
+    def write(b: mutable.StringBuilder): Unit =
+      if path.isDefined then b ++= "+ "
+      b append id
+      b += ' '
+      b ++= name
+      b += '\n'
+      path.foreach { p =>
+        b ++= p
+        b += '\n'
+      }
+  end File
+
+  private class Mapping(
+    inputStartLine: Int,
+    fileId: Int,
+    repeatCount: Int,
+    outputStartLine: Int,
+    increment: Int
+  ):
+    extension (b: mutable.StringBuilder) def appendNotDefault(prefix: Char, value: Int): Unit =
+      if value != 1 then
+        b += prefix
+        b append value
+
+    def write(b: mutable.StringBuilder): Unit =
+      b append (inputStartLine + 1)
+      b.appendNotDefault('#', fileId)
+      b.appendNotDefault(',', repeatCount)
+      b += ':'
+      b append (outputStartLine + 1)
+      b.appendNotDefault(',', increment)
+      b += '\n'
+  end Mapping
+
+  private class Stratum(name: String, files: List[File], mappings: List[Mapping]):
+      def write(b: mutable.StringBuilder): Unit =
+        b ++= "*S "
+        b ++= name
+        b ++= "\n*F\n"
+        files.foreach(_.write(b))
+        b ++= "*L\n"
+        mappings.foreach(_.write(b))
+        b ++= "*E\n"
+  end Stratum
+
+  def sourceMapFor(cunit: CompilationUnit)(internalNameProvider: Symbol => String)(using Context): InlinedSourceMap =
+    val requests = mutable.ListBuffer.empty[(SourcePosition, SourcePosition)]
+    var internalNames = Map.empty[SourceFile, String]
+
+    class RequestCollector(enclosingFile: SourceFile) extends TreeTraverser:
+      override def traverse(tree: Tree)(using Context): Unit =
+        if tree.source != enclosingFile && tree.source != cunit.source then
+          tree.getAttachment(InliningPosition) match
+            case Some(InliningPosition(targetPos, cls)) =>
+              requests += (targetPos -> tree.sourcePos)
+
+              cls match
+                case Some(symbol) if !internalNames.isDefinedAt(tree.source) =>
+                  internalNames += (tree.source -> internalNameProvider(symbol))
+                  // We are skipping any internal name info if we already have one stored in our map
+                  // because a debugger will use internal name only to localize matching source.
+                  // Both old and new internal names are associated with the same source file
+                  // so it doesn't matter if internal name is not matching used symbol.
+                case _ => ()
+              RequestCollector(tree.source).traverseChildren(tree)
+            case None =>
+              // Not exactly sure in which cases it is happening. Should we report warning?
+              RequestCollector(tree.source).traverseChildren(tree)
+        else traverseChildren(tree)
+    end RequestCollector
+
+    var lastLine = cunit.tpdTree.sourcePos.endLine
+    def allocate(origPos: SourcePosition): Int =
+      val line = lastLine + 1
+      lastLine += origPos.lines.length
+      line
+
+    RequestCollector(cunit.source).traverse(cunit.tpdTree)
+    val allocated = requests.sortBy(_._1.start).map(r => Request(r._1, r._2, allocate(r._2)))
+    InlinedSourceMap(cunit, allocated.toList, internalNames)
+  end sourceMapFor
+
+  class InlinedSourceMap private[InlinedSourceMaps] (
+    cunit: CompilationUnit,
+    requests: List[Request],
+    internalNames: Map[SourceFile, String])(using Context):
+
+    def debugExtension: Option[String] = Option.when(requests.nonEmpty) {
+      val scalaStratum =
+        val files = cunit.source :: requests.map(_.origPos.source).distinct.filter(_ != cunit.source)
+        val mappings = requests.map { case Request(_, origPos, firstFakeLine) =>
+          Mapping(origPos.startLine, files.indexOf(origPos.source) + 1, origPos.lines.length, firstFakeLine, 1)
+        }
+        Stratum("Scala",
+          files.zipWithIndex.map { case (f, n) => File(n + 1, f.name, internalNames.get(f)) },
+          Mapping(0, 1, cunit.tpdTree.sourcePos.lines.length, 0, 1) +: mappings
+        )
+
+      val debugStratum =
+        val mappings = requests.map { case Request(targetPos, origPos, firstFakeLine) =>
+          Mapping(targetPos.startLine, 1, 1, firstFakeLine, origPos.lines.length)
+        }
+        Stratum("ScalaDebug", File(1, cunit.source.name, None) :: Nil, mappings)
+
+
+      val b = new StringBuilder
+      b ++= "SMAP\n"
+      b ++= cunit.source.name
+      b += '\n'
+      b ++= "Scala\n"
+      scalaStratum.write(b)
+      debugStratum.write(b)
+      b.toString
+    }
+
+    def lineFor(sourcePos: SourcePosition, lastRealNr: Int): Option[Int] =
+      requests.find(r => r.origPos.contains(sourcePos) && r.targetPos.endLine + 1 >= lastRealNr) match
+        case Some(request) =>
+          val offset = sourcePos.startLine - request.origPos.startLine
+          Some(request.firstFakeLine + offset + 1)
+        case None =>
+          report.warning(s"${sourcePos.show} was inlined in ${cunit.source} but its inlining position was not recorded.")
+          None
+
+

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -37,7 +37,6 @@ class Compiler {
   /** Phases dealing with the frontend up to trees ready for TASTY pickling */
   protected def frontendPhases: List[List[Phase]] =
     List(new FrontEnd) ::           // Compiler frontend: scanner, parser, namer, typer
-    List(new YCheckPositions) ::    // YCheck positions
     List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks
     List(new semanticdb.ExtractSemanticDB) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking

compiler/src/dotty/tools/dotc/transform/ExpandPrivate.scala

@@ -65,7 +65,7 @@ class ExpandPrivate extends MiniPhase with IdentityDenotTransformer { thisPhase
   private def ensurePrivateAccessible(d: SymDenotation)(using Context) =
     if (isVCPrivateParamAccessor(d))
       d.ensureNotPrivate.installAfter(thisPhase)
-    else if (d.is(PrivateTerm) && !d.owner.is(Package) && d.owner != ctx.owner.lexicallyEnclosingClass) {
+    else if (d.is(PrivateTerm) && !d.owner.is(Package) && d.owner != ctx.owner.lexicallyEnclosingClass && !d.is(InlineProxy)) {
       // Paths `p1` and `p2` are similar if they have a common suffix that follows
       // possibly different directory paths. That is, their common suffix extends
       // in both cases either to the start of the path or to a file separator character.

compiler/src/dotty/tools/dotc/typer/Inliner.scala

@@ -30,12 +30,16 @@ import transform.{PostTyper, Inlining}
 import collection.mutable
 import reporting.trace
 import util.Spans.Span
+import util.Property
 import dotty.tools.dotc.transform.{Splicer, TreeMapWithStages}
 import quoted.QuoteUtils
 
 object Inliner {
   import tpd._
 
+  object InliningPosition extends Property.StickyKey[InliningPosition]
+  case class InliningPosition(sourcePos: SourcePosition, topLevelSymbol: Option[Symbol])
+
   /** `sym` is an inline method with a known body to inline.
    */
   def hasBodyToInline(sym: SymDenotation)(using Context): Boolean =
@@ -223,58 +227,11 @@ object Inliner {
 
   /** Replace `Inlined` node by a block that contains its bindings and expansion */
   def dropInlined(inlined: Inlined)(using Context): Tree =
-    val tree1 =
-      if inlined.bindings.isEmpty then inlined.expansion
-      else cpy.Block(inlined)(inlined.bindings, inlined.expansion)
-    // Reposition in the outer most inlined call
-    if (enclosingInlineds.nonEmpty) tree1 else reposition(tree1, inlined.span)
-
-  def reposition(tree: Tree, callSpan: Span)(using Context): Tree = {
-    // Reference test tests/run/i4947b
-
-    val curSource = ctx.compilationUnit.source
-
-    // Tree copier that changes the source of all trees to `curSource`
-    val cpyWithNewSource = new TypedTreeCopier {
-      override protected def sourceFile(tree: tpd.Tree): SourceFile = curSource
-      override protected val untpdCpy: untpd.UntypedTreeCopier = new untpd.UntypedTreeCopier {
-        override protected def sourceFile(tree: untpd.Tree): SourceFile = curSource
-      }
-    }
-
-    /** Removes all Inlined trees, replacing them with blocks.
-     *  Repositions all trees directly inside an inlined expansion of a non empty call to the position of the call.
-     *  Any tree directly inside an empty call (inlined in the inlined code) retains their position.
-     *
-     *  Until we implement JSR-45, we cannot represent in output positions in other source files.
-     *  So, reposition inlined code from other files with the call position.
-     */
-    class Reposition extends TreeMap(cpyWithNewSource) {
-
-      override def transform(tree: Tree)(using Context): Tree = {
-        def fixSpan[T <: untpd.Tree](copied: T): T =
-          copied.withSpan(if tree.source == curSource then tree.span else callSpan)
-        def finalize(copied: untpd.Tree) =
-          fixSpan(copied).withAttachmentsFrom(tree).withTypeUnchecked(tree.tpe)
-
-        inContext(ctx.withSource(curSource)) {
-          tree match
-            case tree: Ident => finalize(untpd.Ident(tree.name)(curSource))
-            case tree: Literal => finalize(untpd.Literal(tree.const)(curSource))
-            case tree: This => finalize(untpd.This(tree.qual)(curSource))
-            case tree: JavaSeqLiteral => finalize(untpd.JavaSeqLiteral(transform(tree.elems), transform(tree.elemtpt))(curSource))
-            case tree: SeqLiteral => finalize(untpd.SeqLiteral(transform(tree.elems), transform(tree.elemtpt))(curSource))
-            case tree: Bind => finalize(untpd.Bind(tree.name, transform(tree.body))(curSource))
-            case tree: TypeTree => finalize(tpd.TypeTree(tree.tpe))
-            case tree: DefTree => super.transform(tree).setDefTree
-            case EmptyTree => tree
-            case _ => fixSpan(super.transform(tree))
-        }
-      }
-    }
+    val topLevelClass = Some(inlined.call.symbol.topLevelClass).filter(_.exists)
+    val inliningPosition = InliningPosition(inlined.sourcePos, topLevelClass)
 
-    (new Reposition).transform(tree)
-  }
+    val withPos = inlined.expansion.withAttachment(InliningPosition, inliningPosition)
+    if inlined.bindings.isEmpty then withPos else cpy.Block(inlined)(inlined.bindings, withPos)
 
   /** Leave only a call trace consisting of
    *  - a reference to the top-level class from which the call was inlined,

tests/generic-java-signatures/derivedNames.scala

@@ -8,6 +8,8 @@ object Test {
     val out2 = "Test$Foo$A<Test$Foo<T1>$B$>"               // Linux and sometimes Windows
     if (scala.util.Properties.isWin)
       assert(returnType.toString == out1 || returnType.toString == out2)
+    else if (scala.util.Properties.isMac)
+      assert(returnType.toString == out1)
     else
       assert(returnType.toString == out2)
   }