Make positions fit for meta

In particular:

 - get rid of envelope, it's too complicated and hides too many errors
 - check that everywhere in parsed trees the position range of a parent
   node contains the position ranges of its children.
 - check that all non-empty trees coming from parser have positions.

The commit contains lots of fixes to make these checks pass.
In particular, it changes the scheme how definitions are positioned.
Previously the position of a definition was the token range of the
name defined by it. That does obviously not work with the parent/children
invariant. Now, the position is the whole definition range, with the
point at the defined name (care is taken to not count backticks).
Namer is changed to still use the token range of defined name as the
position of the symbol.

Author: odersky

src/dotty/tools/dotc/ast/Desugar.scala

@@ -505,7 +505,7 @@ object desugar {
       val clsTmpl = cpy.Template(tmpl)(self = clsSelf, body = tmpl.body)
       val cls = TypeDef(clsName, clsTmpl)
         .withMods(mods.toTypeFlags & RetainedModuleClassFlags | ModuleClassCreationFlags)
-      Thicket(modul, classDef(cls))
+      Thicket(modul, classDef(cls).withPos(mdef.pos))
     }
   }
 
@@ -516,7 +516,7 @@ object desugar {
   def patDef(pdef: PatDef)(implicit ctx: Context): Tree = {
     val PatDef(mods, pats, tpt, rhs) = pdef
     val pats1 = if (tpt.isEmpty) pats else pats map (Typed(_, tpt))
-    flatTree(pats1 map (makePatDef(mods, _, rhs)))
+    flatTree(pats1 map (makePatDef(pdef, mods, _, rhs)))
   }
 
   /** If `pat` is a variable pattern,
@@ -534,9 +534,9 @@ object desugar {
    *  If the original pattern variable carries a type annotation, so does the corresponding
    *  ValDef or DefDef.
    */
-  def makePatDef(mods: Modifiers, pat: Tree, rhs: Tree)(implicit ctx: Context): Tree = pat match {
+  def makePatDef(original: Tree, mods: Modifiers, pat: Tree, rhs: Tree)(implicit ctx: Context): Tree = pat match {
     case VarPattern(named, tpt) =>
-      derivedValDef(named, tpt, rhs, mods)
+      derivedValDef(original, named, tpt, rhs, mods)
     case _ =>
       val rhsUnchecked = makeAnnotated(defn.UncheckedAnnot, rhs)
       val vars = getVariables(pat)
@@ -553,7 +553,7 @@ object desugar {
         case Nil =>
           matchExpr
         case (named, tpt) :: Nil =>
-          derivedValDef(named, tpt, matchExpr, mods)
+          derivedValDef(original, named, tpt, matchExpr, mods)
         case _ =>
           val tmpName = ctx.freshName().toTermName
           val patMods = mods & (AccessFlags | Lazy) | Synthetic
@@ -564,8 +564,8 @@ object desugar {
           val restDefs =
             for (((named, tpt), n) <- vars.zipWithIndex)
             yield
-              if (mods is Lazy) derivedDefDef(named, tpt, selector(n), mods &~ Lazy)
-              else derivedValDef(named, tpt, selector(n), mods)
+              if (mods is Lazy) derivedDefDef(original, named, tpt, selector(n), mods &~ Lazy)
+              else derivedValDef(original, named, tpt, selector(n), mods)
           flatTree(firstDef :: restDefs)
       }
   }
@@ -663,14 +663,18 @@ object desugar {
   def makeAnnotated(cls: Symbol, tree: Tree)(implicit ctx: Context) =
     Annotated(untpd.New(untpd.TypeTree(cls.typeRef), Nil), tree)
 
-  private def derivedValDef(named: NameTree, tpt: Tree, rhs: Tree, mods: Modifiers)(implicit ctx: Context) = {
-    val vdef = ValDef(named.name.asTermName, tpt, rhs).withMods(mods).withPos(named.pos)
+  private def derivedValDef(original: Tree, named: NameTree, tpt: Tree, rhs: Tree, mods: Modifiers)(implicit ctx: Context) = {
+    val vdef = ValDef(named.name.asTermName, tpt, rhs)
+      .withMods(mods)
+      .withPos(original.pos.withPoint(named.pos.start))
     val mayNeedSetter = valDef(vdef)
     mayNeedSetter
    }
 
-  private def derivedDefDef(named: NameTree, tpt: Tree, rhs: Tree, mods: Modifiers) =
-    DefDef(named.name.asTermName, Nil, Nil, tpt, rhs).withMods(mods).withPos(named.pos)
+  private def derivedDefDef(original: Tree, named: NameTree, tpt: Tree, rhs: Tree, mods: Modifiers) =
+    DefDef(named.name.asTermName, Nil, Nil, tpt, rhs)
+      .withMods(mods)
+      .withPos(original.pos.withPoint(named.pos.start))
 
   /** Main desugaring method */
   def apply(tree: Tree)(implicit ctx: Context): Tree = {
@@ -760,7 +764,7 @@ object desugar {
        */
       def makeLambda(pat: Tree, body: Tree): Tree = pat match {
         case VarPattern(named, tpt) =>
-          Function(derivedValDef(named, tpt, EmptyTree, Modifiers(Param)) :: Nil, body)
+          Function(derivedValDef(pat, named, tpt, EmptyTree, Modifiers(Param)) :: Nil, body)
         case _ =>
           makeCaseLambda(CaseDef(pat, EmptyTree, body) :: Nil, unchecked = false)
       }
@@ -863,7 +867,7 @@ object desugar {
           val rhss = valeqs map { case GenAlias(_, rhs) => rhs }
           val (defpat0, id0) = makeIdPat(pat)
           val (defpats, ids) = (pats map makeIdPat).unzip
-          val pdefs = (defpats, rhss).zipped map (makePatDef(Modifiers(), _, _))
+          val pdefs = (valeqs, defpats, rhss).zipped.map(makePatDef(_, Modifiers(), _, _))
           val rhs1 = makeFor(nme.map, nme.flatMap, GenFrom(defpat0, rhs) :: Nil, Block(pdefs, makeTuple(id0 :: ids)))
           val allpats = pat :: pats
           val vfrom1 = new IrrefutableGenFrom(makeTuple(allpats), rhs1)
@@ -946,7 +950,7 @@ object desugar {
         makeFor(nme.map, nme.flatMap, enums, body) orElse tree
       case PatDef(mods, pats, tpt, rhs) =>
         val pats1 = if (tpt.isEmpty) pats else pats map (Typed(_, tpt))
-        flatTree(pats1 map (makePatDef(mods, _, rhs)))
+        flatTree(pats1 map (makePatDef(tree, mods, _, rhs)))
       case ParsedTry(body, handler, finalizer) =>
         handler match {
           case Match(EmptyTree, cases) => Try(body, cases, finalizer)

src/dotty/tools/dotc/ast/NavigateAST.scala

@@ -19,10 +19,9 @@ object NavigateAST {
       case _ =>
         val loosePath = untypedPath(tree, exactMatch = false)
         throw new
-          Error(i"""no untyped tree for $tree, pos = ${tree.pos}, envelope = ${tree.envelope}
+          Error(i"""no untyped tree for $tree, pos = ${tree.pos}
                    |best matching path =\n$loosePath%\n====\n%
-                   |path positions = ${loosePath.map(_.pos)}
-                   |path envelopes = ${loosePath.map(_.envelope)}""")
+                   |path positions = ${loosePath.map(_.pos)}""")
     }
 
   /** The reverse path of untyped trees starting with a tree that closest matches
@@ -40,7 +39,7 @@ object NavigateAST {
   def untypedPath(tree: tpd.Tree, exactMatch: Boolean = false)(implicit ctx: Context): List[Positioned] =
     tree match {
       case tree: MemberDef[_] =>
-        untypedPath(tree.envelope) match {
+        untypedPath(tree.pos) match {
           case path @ (last: DefTree[_]) :: _ => path
           case path if !exactMatch => path
           case _ => Nil
@@ -76,7 +75,7 @@ object NavigateAST {
       path
     }
     def singlePath(p: Positioned, path: List[Positioned]): List[Positioned] =
-      if (p.envelope contains pos) childPath(p.productIterator, p :: path)
+      if (p.pos contains pos) childPath(p.productIterator, p :: path)
       else path
     singlePath(from, Nil)
   }

src/dotty/tools/dotc/ast/Positioned.scala

@@ -3,6 +3,10 @@ package ast
 
 import util.Positions._
 import util.DotClass
+import core.Contexts.Context
+import core.Decorators._
+import core.Flags.JavaDefined
+import core.StdNames.nme
 
 /** A base class for things that have positions (currently: modifiers and trees)
  */
@@ -16,19 +20,14 @@ abstract class Positioned extends DotClass with Product {
    */
   def pos: Position = curPos
 
-  /** Destructively update `curPos` to given position. Also, set any missing
+ /** Destructively update `curPos` to given position. Also, set any missing
    *  positions in children.
    */
   protected def setPos(pos: Position): Unit = {
     curPos = pos
     if (pos.exists) setChildPositions(pos.toSynthetic)
   }
 
-  /** The envelope containing the item in its entirety. Envelope is different from
-   *  `pos` for definitions (instances of MemberDef).
-   */
-  def envelope: Position = pos.toSynthetic
-
   /** A positioned item like this one with the position set to `pos`.
    *  if the positioned item is source-derived, a clone is returned.
    *  If the positioned item is synthetic, the position is updated
@@ -106,16 +105,15 @@ abstract class Positioned extends DotClass with Product {
     }
   }
 
-  /** The initial, synthetic position. This is usually the union of all positioned children's
-   *  envelopes.
+  /** The initial, synthetic position. This is usually the union of all positioned children's positions.
    */
   protected def initialPos: Position = {
     var n = productArity
     var pos = NoPosition
     while (n > 0) {
       n -= 1
       productElement(n) match {
-        case p: Positioned => pos = pos union p.envelope
+        case p: Positioned => pos = pos union p.pos
         case xs: List[_] => pos = unionPos(pos, xs)
         case _ =>
       }
@@ -124,7 +122,7 @@ abstract class Positioned extends DotClass with Product {
   }
 
   private def unionPos(pos: Position, xs: List[_]): Position = xs match {
-    case (p: Positioned) :: xs1 => unionPos(pos union p.envelope, xs1)
+    case (p: Positioned) :: xs1 => unionPos(pos union p.pos, xs1)
     case _ => pos
   }
 
@@ -138,7 +136,7 @@ abstract class Positioned extends DotClass with Product {
         false
     }
     (this eq that) ||
-      (this.envelope contains that.pos) && {
+      (this.pos contains that.pos) && {
         var n = productArity
         var found = false
         while (n > 0 && !found) {
@@ -148,4 +146,48 @@ abstract class Positioned extends DotClass with Product {
         found
       }
   }
+
+  /** Check that all positioned items in this tree satisfy the following conditions:
+   *  - Parent positions contain child positions
+   *  - If item is a non-empty tree, it has a position
+   */
+  def checkPos(complete: Boolean)(implicit ctx: Context): Unit = try {
+    import untpd._
+    def check(p: Any): Unit = p match {
+      case p: Positioned =>
+        assert(pos contains p.pos,
+          s"""position error, parent position does not contain child positon
+                 |parent = $this,
+                 |parent position = $pos,
+                 |child = $p,
+                 |child position = ${p.pos}""".stripMargin)
+        p match {
+          case tree: Tree if !tree.isEmpty =>
+            assert(tree.pos.exists,
+              s"position error: position not set for $tree # ${tree.uniqueId}")
+          case _ =>
+        }
+        p.checkPos(complete)
+      case xs: List[_] =>
+        xs.foreach(check)
+      case _ =>
+    }
+    this match {
+      case tree: DefDef if tree.name == nme.CONSTRUCTOR && tree.mods.is(JavaDefined) =>
+        // Special treatment for constructors coming from Java:
+        // Leave out tparams, they are copied with wrong positions from parent class
+        check(tree.mods)
+        check(tree.vparamss)
+      case _ =>
+        var n = productArity
+        while (n > 0) {
+          n -= 1
+          check(productElement(n))
+        }
+    }
+  } catch {
+    case ex: AssertionError =>
+      println(i"error while checking $this")
+      throw ex
+  }
 }

src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -480,7 +480,7 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
     require(sym.pos.exists)
     object accum extends TreeAccumulator[List[Tree]] {
       def apply(x: List[Tree], tree: Tree)(implicit ctx: Context): List[Tree] = {
-        if (tree.envelope.contains(sym.pos))
+        if (tree.pos.contains(sym.pos))
           if (definedSym(tree) == sym) tree :: x
           else {
             val x1 = foldOver(x, tree)

src/dotty/tools/dotc/ast/Trees.scala

@@ -341,8 +341,6 @@ object Trees {
     }
 
     protected def setMods(mods: Modifiers[T @uncheckedVariance]) = myMods = mods
-
-    override def envelope: Position = rawMods.pos.union(pos).union(initialPos)
   }
 
   /** A ValDef or DefDef tree */
@@ -619,7 +617,6 @@ object Trees {
     type ThisTree[-T >: Untyped] = Bind[T]
     override def isType = name.isTypeName
     override def isTerm = name.isTermName
-    override def envelope: Position = pos union initialPos
   }
 
   /** tree_1 | ... | tree_n */
@@ -740,6 +737,7 @@ object Trees {
       val newTrees = trees.map(_.withPos(pos))
       new Thicket[T](newTrees).asInstanceOf[this.type]
     }
+    override def pos = (NoPosition /: trees) ((pos, t) => pos union t.pos)
     override def foreachInThicket(op: Tree[T] => Unit): Unit =
       trees foreach (_.foreachInThicket(op))
   }

src/dotty/tools/dotc/ast/untpd.scala

@@ -258,7 +258,7 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo {
   implicit class UntypedTreeDecorator(val self: Tree) extends AnyVal {
     def locateEnclosing(base: List[Tree], pos: Position): List[Tree] = {
       def encloses(elem: Any) = elem match {
-        case t: Tree => t.envelope contains pos
+        case t: Tree => t.pos contains pos
         case _ => false
       }
       base.productIterator find encloses match {

src/dotty/tools/dotc/config/Config.scala

@@ -63,6 +63,9 @@ object Config {
     */
   final val checkNoDoubleBindings = true
 
+  /** Check positions for consistency after parsing */
+  final val checkPositions = true
+
   /** Show subtype traces for all deep subtype recursions */
   final val traceDeepSubTypeRecursions = false