Merge pull request #2532 from dotty-staging/ide

IDE support for Dotty via the Language Server Protocol, including a Visual Studio Code extension

Author: smarter

.gitignore

@@ -17,6 +17,9 @@ project/local-plugins.sbt
 .ensime
 .ensime_cache/
 
+# npm
+node_modules
+
 # Scala-IDE specific
 .scala_dependencies
 .cache
@@ -28,6 +31,11 @@ project/local-plugins.sbt
 classes/
 */bin/
 
+# Dotty IDE
+/.dotty-ide-dev-port
+/.dotty-ide-artifact
+/.dotty-ide.json
+
 # idea
 .idea
 .idea_modules

build.sbt

@@ -12,6 +12,7 @@ val `dotty-library-bootstrapped` = Build.`dotty-library-bootstrapped`
 val `dotty-sbt-bridge` = Build.`dotty-sbt-bridge`
 val `dotty-sbt-bridge-bootstrapped` = Build.`dotty-sbt-bridge-bootstrapped`
 val `dotty-sbt-scripted-tests` = Build.`dotty-sbt-scripted-tests`
+val `dotty-language-server` = Build.`dotty-language-server`
 val sjsSandbox = Build.sjsSandbox
 val `dotty-bench` = Build.`dotty-bench`
 val `scala-library` = Build.`scala-library`
@@ -21,5 +22,6 @@ val scalap = Build.scalap
 val dist = Build.dist
 
 val `sbt-dotty` = Build.`sbt-dotty`
+val `vscode-dotty` = Build.`vscode-dotty`
 
 inThisBuild(Build.thisBuildSettings)

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -25,7 +25,7 @@ import Symbols._
 import Denotations._
 import Phases._
 import java.lang.AssertionError
-import java.io.{FileOutputStream, File => JFile}
+import java.io.{DataOutputStream, File => JFile}
 
 import scala.tools.asm
 import scala.tools.asm.tree._
@@ -205,12 +205,16 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
             val dataAttr = new CustomAttr(nme.TASTYATTR.mangledString, binary)
             val store = if (mirrorC ne null) mirrorC else plainC
             store.visitAttribute(dataAttr)
+            val outTastyFile = getFileForClassfile(outF, store.name, ".tasty")
             if (ctx.settings.emitTasty.value) {
-              val outTastyFile = getFileForClassfile(outF, store.name, ".tasty").file
-              val fos = new FileOutputStream(outTastyFile, false)
-              fos.write(binary)
-              fos.close()
-
+              val outstream = new DataOutputStream(outTastyFile.bufferedOutput)
+
+              try outstream.write(binary)
+              finally outstream.close()
+            } else if (!outTastyFile.isVirtual) {
+              // Create an empty file to signal that a tasty section exist in the corresponding .class
+              // This is much cheaper and simpler to check than doing classfile parsing
+              outTastyFile.create()
             }
           }
 

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -19,6 +19,4 @@ class CompilationUnit(val source: SourceFile) {
 
   /** Pickled TASTY binaries, indexed by class. */
   var pickled: Map[ClassSymbol, Array[Byte]] = Map()
-
-  var unpicklers: Map[ClassSymbol, TastyUnpickler] = Map()
 }

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -129,7 +129,7 @@ class Compiler {
     val start = bootstrap.fresh
       .setOwner(defn.RootClass)
       .setTyper(new Typer)
-      .setMode(Mode.ImplicitsEnabled)
+      .addMode(Mode.ImplicitsEnabled)
       .setTyperState(new MutableTyperState(ctx.typerState, ctx.typerState.reporter, isCommittable = true))
       .setFreshNames(new FreshNameCreator.Default)
     ctx.initialize()(start) // re-initialize the base context with start

compiler/src/dotty/tools/dotc/FromTasty.scala

@@ -83,17 +83,15 @@ object FromTasty extends Driver {
           case clsd: ClassDenotation =>
             clsd.infoOrCompleter match {
               case info: ClassfileLoader =>
-                info.load(clsd) match {
-                  case Some(unpickler: DottyUnpickler) =>
-                    val List(unpickled) = unpickler.body(ctx.addMode(Mode.ReadPositions))
+                info.load(clsd)
+                val unpickled = clsd.symbol.asClass.tree
+                if (unpickled != null) {
                     val unit1 = new CompilationUnit(new SourceFile(clsd.symbol.sourceFile, Seq()))
                     unit1.tpdTree = unpickled
-                    unit1.unpicklers += (clsd.classSymbol -> unpickler.unpickler)
                     force.traverse(unit1.tpdTree)
                     unit1
-                  case _ =>
+                } else
                     cannotUnpickle(s"its class file ${info.classfile} does not have a TASTY attribute")
-                }
               case info =>
                 cannotUnpickle(s"its info of type ${info.getClass} is not a ClassfileLoader")
             }

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -555,7 +555,7 @@ object desugar {
       val clsRef = Ident(clsName)
       val modul = ValDef(moduleName, clsRef, New(clsRef, Nil))
         .withMods(mods | ModuleCreationFlags | mods.flags & AccessFlags)
-        .withPos(mdef.pos)
+        .withPos(mdef.pos.startPos)
       val ValDef(selfName, selfTpt, _) = impl.self
       val selfMods = impl.self.mods
       if (!selfTpt.isEmpty) ctx.error(ObjectMayNotHaveSelfType(mdef), impl.self.pos)
@@ -785,7 +785,7 @@ object desugar {
     /**    { label def lname(): Unit = rhs; call }
      */
     def labelDefAndCall(lname: TermName, rhs: Tree, call: Tree) = {
-      val ldef = DefDef(lname, Nil, ListOfNil, TypeTree(defn.UnitType), rhs).withFlags(Label)
+      val ldef = DefDef(lname, Nil, ListOfNil, TypeTree(defn.UnitType), rhs).withFlags(Label | Synthetic)
       Block(ldef, call)
     }
 

compiler/src/dotty/tools/dotc/ast/NavigateAST.scala

@@ -4,7 +4,7 @@ package ast
 import core.Contexts.Context
 import core.Decorators._
 import util.Positions._
-import Trees.{MemberDef, DefTree}
+import Trees.{MemberDef, DefTree, WithLazyField}
 
 /** Utility functions to go from typed to untyped ASTs */
 object NavigateAST {
@@ -61,8 +61,13 @@ object NavigateAST {
   /** The reverse path from node `from` to the node that closest encloses position `pos`,
    *  or `Nil` if no such path exists. If a non-empty path is returned it starts with
    *  the node closest enclosing `pos` and ends with `from`.
+   *
+   *  @param skipZeroExtent  If true, skip over zero-extent nodes in the search. These nodes
+   *                         do not correspond to code the user wrote since their start and
+   *                         end point are the same, so this is useful when trying to reconcile
+   *                         nodes with source code.
    */
-  def pathTo(pos: Position, from: Positioned)(implicit ctx: Context): List[Positioned] = {
+  def pathTo(pos: Position, from: Positioned, skipZeroExtent: Boolean = false)(implicit ctx: Context): List[Positioned] = {
     def childPath(it: Iterator[Any], path: List[Positioned]): List[Positioned] = {
       while (it.hasNext) {
         val path1 = it.next match {
@@ -75,8 +80,17 @@ object NavigateAST {
       path
     }
     def singlePath(p: Positioned, path: List[Positioned]): List[Positioned] =
-      if (p.pos contains pos) childPath(p.productIterator, p :: path)
-      else path
+      if (p.pos.exists && !(skipZeroExtent && p.pos.isZeroExtent) && p.pos.contains(pos)) {
+        // FIXME: We shouldn't be manually forcing trees here, we should replace
+        // our usage of `productIterator` by something in `Positioned` that takes
+        // care of low-level details like this for us.
+        p match {
+          case p: WithLazyField[_] =>
+            p.forceIfLazy
+          case _ =>
+        }
+        childPath(p.productIterator, p :: path)
+      } else path
     singlePath(from, Nil)
   }
-}
+}

compiler/src/dotty/tools/dotc/ast/Positioned.scala

@@ -72,8 +72,16 @@ abstract class Positioned extends DotClass with Product {
                                             // is known, from left to right.
     def fillIn(ps: List[Positioned], start: Int, end: Int): Unit = ps match {
       case p :: ps1 =>
-        p.setPos(Position(start, end))
-        fillIn(ps1, end, end)
+        // If a tree has no position or a zero-extent position, it should be
+        // synthetic. We can preserve this invariant by always setting a
+        // zero-extent position for these trees here.
+        if (!p.pos.exists || p.pos.isZeroExtent) {
+          p.setPos(Position(start, start))
+          fillIn(ps1, start, end)
+        } else {
+          p.setPos(Position(start, end))
+          fillIn(ps1, end, end)
+        }
       case nil =>
     }
     while (true) {

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -98,6 +98,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val YforceSbtPhases = BooleanSetting("-Yforce-sbt-phases", "Run the phases used by sbt for incremental compilation (ExtractDependencies and ExtractAPI) even if the compiler is ran outside of sbt, for debugging.")
   val YdumpSbtInc = BooleanSetting("-Ydump-sbt-inc", "For every compiled foo.scala, output the API representation and dependencies used for sbt incremental compilation in foo.inc, implies -Yforce-sbt-phases.")
   val YcheckAllPatmat = BooleanSetting("-Ycheck-all-patmat", "Check exhaustivity and redundancy of all pattern matching (used for testing the algorithm)")
+  val YretainTrees = BooleanSetting("-Yretain-trees", "Retain trees for top-level classes, accessible from ClassSymbol#tree")
 
   /** Area-specific debug output */
   val Yexplainlowlevel = BooleanSetting("-Yexplain-lowlevel", "When explaining type errors, show types at a lower level.")

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -62,7 +62,8 @@ trait SymDenotations { this: Context =>
       || owner.isRefinementClass
       || owner.is(Scala2x)
       || (owner.unforcedDecls.lookupAll(denot.name) contains denot.symbol)
-      || denot.isSelfSym)
+      || denot.isSelfSym
+      || denot.isLocalDummy)
   } catch {
     case ex: StaleSymbol => false
   }

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -17,6 +17,7 @@ import Decorators.{PreNamedString, StringInterpolators}
 import classfile.ClassfileParser
 import util.Stats
 import scala.util.control.NonFatal
+import ast.Trees._
 
 object SymbolLoaders {
   /** A marker trait for a completer that replaces the original
@@ -320,9 +321,14 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader {
   override def doComplete(root: SymDenotation)(implicit ctx: Context): Unit =
     load(root)
 
-  def load(root: SymDenotation)(implicit ctx: Context): Option[ClassfileParser.Embedded] = {
+  def load(root: SymDenotation)(implicit ctx: Context): Unit = {
     val (classRoot, moduleRoot) = rootDenots(root.asClass)
-    new ClassfileParser(classfile, classRoot, moduleRoot)(ctx).run()
+    (new ClassfileParser(classfile, classRoot, moduleRoot)(ctx)).run() match {
+      case Some(unpickler: tasty.DottyUnpickler) if ctx.settings.YretainTrees.value =>
+        classRoot.symbol.asClass.unpickler = unpickler
+        moduleRoot.symbol.asClass.unpickler = unpickler
+      case _ =>
+    }
   }
 }
 

compiler/src/dotty/tools/dotc/core/Symbols.scala

@@ -20,7 +20,8 @@ import DenotTransformers._
 import StdNames._
 import NameOps._
 import NameKinds.LazyImplicitName
-import ast.tpd.Tree
+import ast.tpd
+import tpd.Tree
 import ast.TreeTypeMap
 import Constants.Constant
 import reporting.diagnostic.Message
@@ -552,6 +553,34 @@ object Symbols {
 
     type ThisName = TypeName
 
+    /** If this is a top-level class, and if `-Yretain-trees` is set, return the TypeDef tree
+     *  for this class, otherwise EmptyTree.
+     */
+    def tree(implicit ctx: Context): tpd.Tree /* tpd.TypeDef | tpd.EmptyTree */ = {
+      // TODO: Consider storing this tree like we store lazy trees for inline functions
+      if (unpickler != null && !denot.isAbsent) {
+        assert(myTree.isEmpty)
+
+        import ast.Trees._
+
+        def findTree(tree: tpd.Tree): Option[tpd.TypeDef] = tree match {
+          case PackageDef(_, stats) =>
+            stats.flatMap(findTree).headOption
+          case tree: tpd.TypeDef if tree.symbol == this =>
+            Some(tree)
+          case _ =>
+              None
+        }
+        val List(unpickledTree) = unpickler.body(ctx.addMode(Mode.ReadPositions))
+        unpickler = null
+
+        myTree = findTree(unpickledTree).get
+      }
+      myTree
+    }
+    private[dotc] var myTree: tpd.Tree = tpd.EmptyTree
+    private[dotc] var unpickler: tasty.DottyUnpickler = _
+
     /** The source or class file from which this class was generated, null if not applicable. */
     override def associatedFile(implicit ctx: Context): AbstractFile =
       if (assocFile != null || (this.owner is PackageClass) || this.isEffectiveRoot) assocFile

compiler/src/dotty/tools/dotc/core/TypeOps.scala

@@ -314,11 +314,14 @@ trait TypeOps { this: Context => // TODO: Make standalone object.
       case _ =>
         tpe
     }
-    tpe.prefix match {
-      case pre: ThisType if pre.cls is Package => tryInsert(pre.cls)
-      case pre: TermRef if pre.symbol is Package => tryInsert(pre.symbol.moduleClass)
-      case _ => tpe
-    }
+    if (tpe.symbol.isRoot)
+      tpe
+    else
+      tpe.prefix match {
+        case pre: ThisType if pre.cls is Package => tryInsert(pre.cls)
+        case pre: TermRef if pre.symbol is Package => tryInsert(pre.symbol.moduleClass)
+        case _ => tpe
+      }
   }
 
   /** Normalize a list of parent types of class `cls` that may contain refinements

compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala

@@ -430,8 +430,9 @@ class ClassfileParser(
     tag match {
       case STRING_TAG =>
         if (skip) None else Some(Literal(Constant(pool.getName(index).toString)))
-      case BOOL_TAG | BYTE_TAG | CHAR_TAG | SHORT_TAG | INT_TAG |
-        LONG_TAG | FLOAT_TAG | DOUBLE_TAG =>
+      case BOOL_TAG | BYTE_TAG | CHAR_TAG | SHORT_TAG =>
+        if (skip) None else Some(Literal(pool.getConstant(index, tag)))
+      case INT_TAG | LONG_TAG | FLOAT_TAG | DOUBLE_TAG =>
         if (skip) None else Some(Literal(pool.getConstant(index)))
       case CLASS_TAG =>
         if (skip) None else Some(Literal(Constant(pool.getType(index))))
@@ -827,7 +828,12 @@ class ClassfileParser(
         def getMember(sym: Symbol, name: Name)(implicit ctx: Context): Symbol =
           if (static)
             if (sym == classRoot.symbol) staticScope.lookup(name)
-            else sym.companionModule.info.member(name).symbol
+            else {
+              var module = sym.companionModule
+              if (!module.exists && sym.isAbsent)
+                module = sym.scalacLinkedClass
+              module.info.member(name).symbol
+            }
           else
             if (sym == classRoot.symbol) instanceScope.lookup(name)
             else sym.info.member(name).symbol
@@ -1040,14 +1046,28 @@ class ClassfileParser(
       getClassSymbol(index)
     }
 
-    def getConstant(index: Int)(implicit ctx: Context): Constant = {
+    def getConstant(index: Int, tag: Int = -1)(implicit ctx: Context): Constant = {
       if (index <= 0 || len <= index) errorBadIndex(index)
       var value = values(index)
       if (value eq null) {
         val start = starts(index)
         value = (in.buf(start).toInt: @switch) match {
           case CONSTANT_STRING =>
             Constant(getName(in.getChar(start + 1).toInt).toString)
+          case CONSTANT_INTEGER if tag != -1 =>
+            val value = in.getInt(start + 1)
+            (tag: @switch) match {
+              case BOOL_TAG =>
+                Constant(value != 0)
+              case BYTE_TAG =>
+                Constant(value.toByte)
+              case CHAR_TAG =>
+                Constant(value.toChar)
+              case SHORT_TAG =>
+                Constant(value.toShort)
+              case _ =>
+                errorBadTag(tag)
+            }
           case CONSTANT_INTEGER =>
             Constant(in.getInt(start + 1))
           case CONSTANT_FLOAT =>

compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

@@ -46,8 +46,15 @@ class DottyUnpickler(bytes: Array[Byte]) extends ClassfileParser.Embedded {
   def enter(roots: Set[SymDenotation])(implicit ctx: Context): Unit =
     treeUnpickler.enterTopLevel(roots)
 
+  /** Only used if `-Yretain-trees` is set. */
+  private[this] var myBody: List[Tree] = _
   /** The unpickled trees, and the source file they come from. */
   def body(implicit ctx: Context): List[Tree] = {
-    treeUnpickler.unpickle()
+    def computeBody() = treeUnpickler.unpickle()
+    if (ctx.settings.YretainTrees.value) {
+      if (myBody == null)
+        myBody = computeBody()
+      myBody
+    } else computeBody()
   }
 }