record end markers in trees and semanticdb

Author: bishabosha

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -182,6 +182,7 @@ object desugar {
         tpt     = TypeTree(defn.UnitType),
         rhs     = setterRhs
       ).withMods((mods | Accessor) &~ (CaseAccessor | GivenOrImplicit | Lazy))
+       .dropEndSpan
       Thicket(vdef1, setter)
     }
     else vdef1
@@ -874,6 +875,7 @@ object desugar {
       val modul = ValDef(moduleName, clsRef, New(clsRef, Nil))
         .withMods(mods.toTermFlags & RetainedModuleValFlags | ModuleValCreationFlags)
         .withSpan(mdef.span.startPos)
+        .withEndSpan(copyFrom=mdef)
       val ValDef(selfName, selfTpt, _) = impl.self
       val selfMods = impl.self.mods
       if (!selfTpt.isEmpty) report.error(ObjectMayNotHaveSelfType(mdef), impl.self.srcPos)

compiler/src/dotty/tools/dotc/ast/Trees.scala

@@ -327,9 +327,43 @@ object Trees {
 
   extension (mdef: untpd.DefTree) def mods: untpd.Modifiers = mdef.rawMods
 
-  abstract class NamedDefTree[-T >: Untyped](implicit @constructorOnly src: SourceFile) extends NameTree[T] with DefTree[T] {
+  /** PackageDef | NamedDefTree */
+  sealed trait WithEndMarker:
+    self: Attachment.Container =>
+
+    import WithEndMarker.*
+
+    final def endToken: SimpleName = endMarker.stripModuleClassSuffix.lastPart
+
+    protected def endMarker: Name
+
+    final def withEndSpan(span: Span): self.type =
+      self.withAttachment(EndMarker, span)
+
+    final def withEndSpan(copyFrom: WithEndMarker): self.type =
+      copyFrom.endSpan.foreach(span => withEndSpan(span=span))
+      this
+
+    final def dropEndSpan: self.type =
+      self.removeAttachment(EndMarker)
+      this
+
+    final def endSpan: Option[Span] = self.getAttachment(EndMarker)
+
+  object WithEndMarker:
+    /** Property key for trees with an `end` marker */
+    private val EndMarker: Property.StickyKey[Span] = Property.StickyKey()
+
+  end WithEndMarker
+
+  abstract class NamedDefTree[-T >: Untyped](implicit @constructorOnly src: SourceFile)
+  extends NameTree[T] with DefTree[T] with WithEndMarker {
     type ThisTree[-T >: Untyped] <: NamedDefTree[T]
 
+    protected def endMarker =
+      if name == nme.CONSTRUCTOR then nme.this_
+      else name
+
     /** The position of the name defined by this definition.
      *  This is a point position if the definition is synthetic, or a range position
      *  if the definition comes from source.
@@ -857,9 +891,10 @@ object Trees {
 
   /** package pid { stats } */
   case class PackageDef[-T >: Untyped] private[ast] (pid: RefTree[T], stats: List[Tree[T]])(implicit @constructorOnly src: SourceFile)
-    extends ProxyTree[T] {
+    extends ProxyTree[T] with WithEndMarker {
     type ThisTree[-T >: Untyped] = PackageDef[T]
     def forwardTo: RefTree[T] = pid
+    protected def endMarker: Name = pid.name
   }
 
   /** arg @annot */

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -1283,8 +1283,8 @@ object Parsers {
           in.token == IDENTIFIER && in.name == nme.extension
         case PackageDef(pid: RefTree, _) =>
           in.isIdent && in.name == pid.name
-        case PatDef(_, IdPattern(id, _) :: Nil, _, _) =>
-         in.isIdent && in.name == id.name
+        case PatDef(_, IdPattern(id, _) :: Nil, _, _) => // TODO: it looks like this case is never reached
+          in.isIdent && in.name == id.name
         case stat: MemberDef if stat.mods.is(Given) => in.token == GIVEN
         case _: PatDef => in.token == VAL
         case _: If => in.token == IF
@@ -1295,9 +1295,21 @@ object Parsers {
         case _: (ForYield | ForDo) => in.token == FOR
         case _ => false
 
+      def matchesWithUpdated(stat: Tree): Boolean = {
+        val didMatch = matches(stat)
+        if didMatch then
+          stat match
+            case stat: WithEndMarker =>
+              val end = in.lastCharOffset
+              stat.withEndSpan(span=Span(end - stat.endToken.length, end))
+            case _ =>
+              ()
+        didMatch
+      }
+
       if in.token == END then
         val start = in.skipToken()
-        if stats.isEmpty || !matches(stats.last) then
+        if stats.isEmpty || !matchesWithUpdated(stats.last) then
           syntaxError("misaligned end marker", Span(start, in.lastCharOffset))
         in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion
         in.nextToken()

compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala

@@ -6,7 +6,7 @@ import core._
 import Phases._
 import ast.tpd._
 import ast.untpd.given
-import ast.Trees.mods
+import ast.Trees.{mods, WithEndMarker}
 import Contexts._
 import Symbols._
 import Flags._
@@ -139,47 +139,46 @@ class ExtractSemanticDB extends Phase:
             case tree => registerDefinition(tree.symbol, tree.span, Set.empty, tree.source)
           tree.stats.foreach(traverse)
         case tree: NamedDefTree =>
-          if tree.symbol.isAllOf(ModuleValCreationFlags) then
-            return
-          if !excludeDef(tree.symbol)
-          && tree.span.hasLength then
-            registerDefinition(tree.symbol, tree.nameSpan, symbolKinds(tree), tree.source)
-            val privateWithin = tree.symbol.privateWithin
-            if privateWithin.exists then
-              registerUseGuarded(None, privateWithin, spanOfSymbol(privateWithin, tree.span, tree.source), tree.source)
-          else if !excludeSymbol(tree.symbol) then
-            registerSymbol(tree.symbol, symbolName(tree.symbol), symbolKinds(tree))
-          tree match
-          case tree: ValDef
-          if tree.symbol.isAllOf(EnumValue) =>
-            tree.rhs match
-            case Block(TypeDef(_, template: Template) :: _, _) => // simple case with specialised extends clause
-              template.parents.filter(!_.span.isZeroExtent).foreach(traverse)
-            case _ => // calls $new
-          case tree: ValDef
-          if tree.symbol.isSelfSym =>
-            if tree.tpt.span.hasLength then
-              traverse(tree.tpt)
-          case tree: DefDef
-          if tree.symbol.isConstructor => // ignore typeparams for secondary ctors
-            tree.trailingParamss.foreach(_.foreach(traverse))
-            traverse(tree.rhs)
-          case tree: (DefDef | ValDef)
-          if tree.symbol.isSyntheticWithIdent =>
+          if !tree.symbol.isAllOf(ModuleValCreationFlags) then
+            if !excludeDef(tree.symbol)
+            && tree.span.hasLength then
+              registerDefinition(tree.symbol, tree.nameSpan, symbolKinds(tree), tree.source)
+              val privateWithin = tree.symbol.privateWithin
+              if privateWithin.exists then
+                registerUseGuarded(None, privateWithin, spanOfSymbol(privateWithin, tree.span, tree.source), tree.source)
+            else if !excludeSymbol(tree.symbol) then
+              registerSymbol(tree.symbol, symbolName(tree.symbol), symbolKinds(tree))
             tree match
-              case tree: DefDef =>
-                tree.paramss.foreach(_.foreach(param => registerSymbolSimple(param.symbol)))
-              case tree: ValDef if tree.symbol.is(Given) => traverse(tree.tpt)
-              case _ =>
-            if !tree.symbol.isGlobal then
-              localBodies(tree.symbol) = tree.rhs
-            // ignore rhs
-          case PatternValDef(pat, rhs) =>
-            traverse(rhs)
-            PatternValDef.collectPats(pat).foreach(traverse)
-          case tree =>
-            if !excludeChildren(tree.symbol) then
-              traverseChildren(tree)
+            case tree: ValDef
+            if tree.symbol.isAllOf(EnumValue) =>
+              tree.rhs match
+              case Block(TypeDef(_, template: Template) :: _, _) => // simple case with specialised extends clause
+                template.parents.filter(!_.span.isZeroExtent).foreach(traverse)
+              case _ => // calls $new
+            case tree: ValDef
+            if tree.symbol.isSelfSym =>
+              if tree.tpt.span.hasLength then
+                traverse(tree.tpt)
+            case tree: DefDef
+            if tree.symbol.isConstructor => // ignore typeparams for secondary ctors
+              tree.trailingParamss.foreach(_.foreach(traverse))
+              traverse(tree.rhs)
+            case tree: (DefDef | ValDef)
+            if tree.symbol.isSyntheticWithIdent =>
+              tree match
+                case tree: DefDef =>
+                  tree.paramss.foreach(_.foreach(param => registerSymbolSimple(param.symbol)))
+                case tree: ValDef if tree.symbol.is(Given) => traverse(tree.tpt)
+                case _ =>
+              if !tree.symbol.isGlobal then
+                localBodies(tree.symbol) = tree.rhs
+              // ignore rhs
+            case PatternValDef(pat, rhs) =>
+              traverse(rhs)
+              PatternValDef.collectPats(pat).foreach(traverse)
+            case tree =>
+              if !excludeChildren(tree.symbol) then
+                traverseChildren(tree)
         case tree: Template =>
           val ctorSym = tree.constr.symbol
           if !excludeDef(ctorSym) then
@@ -240,6 +239,12 @@ class ExtractSemanticDB extends Phase:
         case _ =>
           traverseChildren(tree)
 
+      tree match
+        case tree: WithEndMarker =>
+          for endSpan <- tree.endSpan do
+            registerUseGuarded(None, tree.symbol, endSpan, tree.source)
+        case _ =>
+
     end traverse
 
     private def funParamSymbol(funSym: Symbol)(using Context): Name => String =

tests/semanticdb/expect/EndMarkers.expect.scala

@@ -0,0 +1,59 @@
+package endmarkers:
+
+  class MultiCtor/*<-endmarkers::MultiCtor#*/(val i/*<-endmarkers::MultiCtor#i.*/: Int/*->scala::Int#*/):
+    def this()/*<-endmarkers::MultiCtor#`<init>`(+1).*/ =
+      this(23)
+    end this/*->endmarkers::MultiCtor#`<init>`(+1).*/
+  end MultiCtor/*->endmarkers::MultiCtor#*/
+
+  /*<-endmarkers::EndMarkers$package.*/def topLevelMethod/*<-endmarkers::EndMarkers$package.topLevelMethod().*/: String/*->scala::Predef.String#*/ =
+    "hello"
+  end topLevelMethod/*->endmarkers::EndMarkers$package.topLevelMethod().*/
+
+  val topLevelVal/*<-endmarkers::EndMarkers$package.topLevelVal.*/: Int/*->scala::Int#*/ =
+    23
+  end topLevelVal/*->endmarkers::EndMarkers$package.topLevelVal.*/
+
+  var topLevelVar/*<-endmarkers::EndMarkers$package.topLevelVar().*/: String/*->scala::Predef.String#*/ =
+    ""
+  end topLevelVar/*->endmarkers::EndMarkers$package.topLevelVar().*/
+
+  class Container/*<-endmarkers::Container#*/:
+
+    def foo/*<-endmarkers::Container#foo().*/ =
+      (/*->scala::Tuple3.apply().*/1,2,3)
+    end foo/*->endmarkers::Container#foo().*/
+
+    val bar/*<-endmarkers::Container#bar.*/ =
+      (/*->scala::Tuple3.apply().*/4,5,6)
+    end bar/*->endmarkers::Container#bar.*/
+
+    var baz/*<-endmarkers::Container#baz().*/ =
+      15
+    end baz/*->endmarkers::Container#baz().*/
+
+  end Container/*->endmarkers::Container#*/
+
+  def topLevelWithLocals/*<-endmarkers::EndMarkers$package.topLevelWithLocals().*/: Unit/*->scala::Unit#*/ =
+
+    val localVal/*<-local0*/ =
+      37
+    end localVal/*->local0*/
+
+    var localVar/*<-local1*/ =
+      43
+    end localVar/*->local1*/
+
+    def localDef/*<-local2*/ =
+      97
+    end localDef/*->local2*/
+
+  end topLevelWithLocals/*->endmarkers::EndMarkers$package.topLevelWithLocals().*/
+
+  object TestObj/*<-endmarkers::TestObj.*/:
+
+    def foo/*<-endmarkers::TestObj.foo().*/ = 23
+
+  end TestObj/*->endmarkers::TestObj.*/
+
+end endmarkers

tests/semanticdb/expect/EndMarkers.scala

@@ -0,0 +1,59 @@
+package endmarkers:
+
+  class MultiCtor(val i: Int):
+    def this() =
+      this(23)
+    end this
+  end MultiCtor
+
+  def topLevelMethod: String =
+    "hello"
+  end topLevelMethod
+
+  val topLevelVal: Int =
+    23
+  end topLevelVal
+
+  var topLevelVar: String =
+    ""
+  end topLevelVar
+
+  class Container:
+
+    def foo =
+      (1,2,3)
+    end foo
+
+    val bar =
+      (4,5,6)
+    end bar
+
+    var baz =
+      15
+    end baz
+
+  end Container
+
+  def topLevelWithLocals: Unit =
+
+    val localVal =
+      37
+    end localVal
+
+    var localVar =
+      43
+    end localVar
+
+    def localDef =
+      97
+    end localDef
+
+  end topLevelWithLocals
+
+  object TestObj:
+
+    def foo = 23
+
+  end TestObj
+
+end endmarkers

tests/semanticdb/expect/EndMarkers2.expect.scala

@@ -0,0 +1,7 @@
+package object endmark/*<-endmarkers2::package.*/ers2:
+
+  type Foo/*<-endmarkers2::package.Foo#*/ =
+    Unit/*->scala::Unit#*/
+  end Foo/*->endmarkers2::package.Foo#*/
+
+end endmarkers2/*->endmarkers2::package.*/

tests/semanticdb/expect/EndMarkers2.scala

@@ -0,0 +1,7 @@
+package object endmarkers2:
+
+  type Foo =
+    Unit
+  end Foo
+
+end endmarkers2