Add sbt incremental compilation support

To test this with sbt, see
https://github.com/lampepfl/dotty/wiki/Using-Dotty-with-sbt

The following flags are added:
 - -Yforce-sbt-phases: Run the phases used by sbt for incremental compilation
   (ExtractDependencies and ExtractAPI) even if the compiler is ran outside of
   sbt, for debugging.
 - -Ydump-sbt-inc: For every compiled foo.scala, output the API
   representation and dependencies used for sbt incremental compilation
   in foo.inc, implies -Yforce-sbt-phases.

This commit introduces two new phases which do not transform trees:
- `ExtractDependencies` which extracts the dependency information of the current
  compilation unit and sends it to sbt via callbacks
- `ExtractAPI` which creates a representation of the API of the current compilation
  unit and sends it to sbt via callbacks

Briefly, when a file changes sbt will recompile it, if its API has
changed (determined by what `ExtractAPI` sent) then sbt will determine
which reverse-dependencies (determined by what `ExtractDependencies`
sent) of the API have to be recompiled depending on what changed.
See http://www.scala-sbt.org/0.13/docs/Understanding-Recompilation.html for
more information on how sbt incremental compilation works.

This phase was originally based on
https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt
which attempts to integrate the sbt phases into scalac (and is itself based
on https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt),
but it has been heavily refactored and adapted to Dotty. The main
functional differences are:
- ExtractDependencies runs right after Frontend (so that we don't lose
  dependency informations because of the simplifications done by PostTyper),
  but ExtractAPI runs right after PostTyper (so that SuperAccessors are
  part of the API).
- `ExtractAPI` only extract types as they are defined and never "as seen
  from" some some specific prefix, see its documentation for more details.
- `ExtractDependenciesTraverser` and `ExtractUsedNames` have been fused into
  one tree traversal in `ExtractDependenciesCollector`.

TODO: Try to run these phases in parallel with the rest of the compiler
pipeline since they're independent (except for the sbt callbacks in `GenBCode`) ?

Author: smarter

AUTHORS.md

@@ -42,7 +42,7 @@ The majority of the dotty codebase is new code, with the exception of the compon
 > The utilities package is a mix of new and adapted components. The files in  [scala/scala](https://github.com/scala/scala) were originally authored by many people,
 > including Paul Phillips, Martin Odersky, Sean McDirmid, and others.
   
-`dotty.tools.io`   
+`dotty.tools.io`
 
 > The I/O support library was adapted from current Scala compiler. Original authors were Paul Phillips and others.
 
@@ -52,3 +52,12 @@ The majority of the dotty codebase is new code, with the exception of the compon
 > [scala/scala](https://github.com/scala/scala). It has been reworked to fit
 > the needs of dotty. Original authors include: Adrian Moors, Lukas Rytz,
 > Grzegorz Kossakowski, Paul Phillips
+
+`dotty.tools.dotc.sbt`
+
+> The sbt compiler phases are based on
+> https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt
+> which attempts to integrate the sbt phases into scalac and is itself based on
+> the [compiler bridge in sbt 0.13](https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt),
+> but has been heavily adapted and refactored.
+> Original authors were Mark Harrah, Grzegorz Kossakowski, Martin Duhemm, Adriaan Moors and others.

bin/dotc

@@ -23,6 +23,7 @@ SCALA_BINARY_VERSION=2.11
 SCALA_COMPILER_VERSION=$(getLastStringOnLineWith "scala-compiler")
 DOTTY_VERSION=$(getLastStringOnLineWith "version in")
 JLINE_VERSION=$(getLastStringOnLineWith "jline")
+SBT_VERSION=$(grep "sbt.version=" "$DOTTY_ROOT/project/build.properties" | sed 's/sbt.version=//')
 bootcp=true
 bootstrapped=false
 default_java_opts="-Xmx768m -Xms768m"
@@ -100,13 +101,19 @@ then
   JLINE_JAR=$HOME/.ivy2/cache/jline/jline/jars/jline-$JLINE_VERSION.jar
 fi
 
-if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR"  -o ! -f "$SCALA_COMPILER_JAR"  -o ! -f "$JLINE_JAR" ]
+if [ "$SBT_INTERFACE_JAR" == "" ]
+then
+  SBT_INTERFACE_JAR=$HOME/.ivy2/cache/org.scala-sbt/interface/jars/interface-$SBT_VERSION.jar
+fi
+
+if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR"  -o ! -f "$SCALA_COMPILER_JAR"  -o ! -f "$JLINE_JAR" -o ! -f "$SBT_INTERFACE_JAR" ]
 then
   echo To use this script please set
   echo SCALA_LIBRARY_JAR to point to scala-library-$SCALA_VERSION.jar "(currently $SCALA_LIBRARY_JAR)"
   echo SCALA_REFLECT_JAR to point to scala-reflect-$SCALA_VERSION.jar "(currently $SCALA_REFLECT_JAR)"
   echo SCALA_COMPILER_JAR to point to scala-compiler-$SCALA_VERSION.jar with bcode patches "(currently $SCALA_COMPILER_JAR)"
   echo JLINE_JAR to point to jline-$JLINE_VERSION.jar "(currently $JLINE_JAR)"
+  echo SBT_INTERFACE_JAR to point to interface-$SBT_VERSION.jar "(currently $SBT_INTERFACE_JAR)"
 fi
 
 ifdebug () {
@@ -196,9 +203,9 @@ trap onExit INT
 classpathArgs () {
   if [[ "true" == $bootstrapped ]]; then 
     checkjar $DOTTY_JAR "test:runMain dotc.build" src
-    toolchain="$DOTTY_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR"
+    toolchain="$DOTTY_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR:$SBT_INTERFACE_JAR"
   else
-    toolchain="$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR"
+    toolchain="$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR:$SBT_INTERFACE_JAR"
   fi
   bcpJars="$INTERFACES_JAR:$MAIN_JAR"
   cpJars="$INTERFACES_JAR:$MAIN_JAR:$TEST_JAR"

project/Build.scala

@@ -88,7 +88,8 @@ object DottyBuild extends Build {
       libraryDependencies ++= Seq("org.scala-lang.modules" %% "scala-xml" % "1.0.1",
                                   "org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test",
                                   "com.novocode" % "junit-interface" % "0.11" % "test",
-                                  "jline" % "jline" % "2.12"),
+                                  "jline" % "jline" % "2.12",
+                                  "org.scala-sbt" % "interface" % sbtVersion.value),
 
       // enable improved incremental compilation algorithm
       incOptions := incOptions.value.withNameHashing(true),

src/dotty/tools/backend/jvm/GenBCode.scala

@@ -389,6 +389,8 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
             val className = jclassName.replace('/', '.')
             if (ctx.compilerCallback != null)
               ctx.compilerCallback.onClassGenerated(sourceFile, convertAbstractFile(outFile), className)
+            if (ctx.sbtCallback != null)
+              ctx.sbtCallback.generatedClass(sourceFile.jfile.orElse(null), outFile.file, className)
           }
           catch {
             case e: FileConflictException =>

src/dotty/tools/dotc/Compiler.scala

@@ -42,7 +42,9 @@ class Compiler {
   def phases: List[List[Phase]] =
     List(
       List(new FrontEnd),           // Compiler frontend: scanner, parser, namer, typer
+      List(new sbt.ExtractDependencies), // Sends information on classes' dependencies to sbt via callbacks
       List(new PostTyper),          // Additional checks and cleanups after type checking
+      List(new sbt.ExtractAPI),     // Sends a representation of the API of classes to sbt via callbacks
       List(new Pickler),            // Generate TASTY info
       List(new FirstTransform,      // Some transformations to put trees into a canonical form
            new CheckReentrant),     // Internal use only: Check that compiled program has no data races involving global vars

src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -160,6 +160,8 @@ class ScalaSettings extends Settings.SettingGroup {
   val YtestPickler = BooleanSetting("-Ytest-pickler", "self-test for pickling functionality; should be used with -Ystop-after:pickler")
   val YcheckReentrant = BooleanSetting("-Ycheck-reentrant", "check that compiled program does not contain vars that can be accessed from a global root.")
   val YkeepComments = BooleanSetting("-Ykeep-comments", "Keep comments when scanning source files.")
+  val YforceSbtPhases = BooleanSetting("-Yforce-sbt-phases", "Run the phases used by sbt for incremental compilation (ExtractDependencies and ExtractAPI) even if the compiler is ran outside of sbt, for debugging.")
+  val YdumpSbtInc = BooleanSetting("-Ydump-sbt-inc", "For every compiled foo.scala, output the API representation and dependencies used for sbt incremental compilation in foo.inc, implies -Yforce-sbt-phases.")
   def stop = YstopAfter
 
   /** Area-specific debug output.

src/dotty/tools/dotc/core/Contexts.scala

@@ -29,6 +29,7 @@ import printing._
 import config.{Settings, ScalaSettings, Platform, JavaPlatform, SJSPlatform}
 import language.implicitConversions
 import DenotTransformers.DenotTransformer
+import xsbti.AnalysisCallback
 
 object Contexts {
 
@@ -84,6 +85,12 @@ object Contexts {
       _compilerCallback = callback
     def compilerCallback: CompilerCallback = _compilerCallback
 
+    /** The sbt callback implementation if we are run from sbt, null otherwise */
+    private[this] var _sbtCallback: AnalysisCallback = _
+    protected def sbtCallback_=(callback: AnalysisCallback) =
+      _sbtCallback = callback
+    def sbtCallback: AnalysisCallback = _sbtCallback
+
     /** The current context */
     private[this] var _period: Period = _
     protected def period_=(period: Period) = {
@@ -426,6 +433,7 @@ object Contexts {
     def setPeriod(period: Period): this.type = { this.period = period; this }
     def setMode(mode: Mode): this.type = { this.mode = mode; this }
     def setCompilerCallback(callback: CompilerCallback): this.type = { this.compilerCallback = callback; this }
+    def setSbtCallback(callback: AnalysisCallback): this.type = { this.sbtCallback = callback; this }
     def setTyperState(typerState: TyperState): this.type = { this.typerState = typerState; this }
     def setReporter(reporter: Reporter): this.type = setTyperState(typerState.withReporter(reporter))
     def setNewTyperState: this.type = setTyperState(typerState.fresh(isCommittable = true))

src/dotty/tools/dotc/core/Types.scala

@@ -2484,10 +2484,14 @@ object Types {
 
   abstract class ParamType extends BoundType {
     def paramNum: Int
+    def paramName: Name
   }
 
   abstract case class MethodParam(binder: MethodType, paramNum: Int) extends ParamType with SingletonType {
     type BT = MethodType
+
+    def paramName = binder.paramNames(paramNum)
+
     override def underlying(implicit ctx: Context): Type = binder.paramTypes(paramNum)
     def copyBoundType(bt: BT) = new MethodParamImpl(bt, paramNum)
 
@@ -2500,7 +2504,7 @@ object Types {
         false
     }
 
-    override def toString = s"MethodParam(${binder.paramNames(paramNum)})"
+    override def toString = s"MethodParam($paramName)"
   }
 
   class MethodParamImpl(binder: MethodType, paramNum: Int) extends MethodParam(binder, paramNum)
@@ -2530,9 +2534,11 @@ object Types {
       case _ => false
     }
 
+    def paramName = binder.paramNames(paramNum)
+
     override def underlying(implicit ctx: Context): Type = binder.paramBounds(paramNum)
     // no customized hashCode/equals needed because cycle is broken in PolyType
-    override def toString = s"PolyParam(${binder.paramNames(paramNum)})"
+    override def toString = s"PolyParam($paramName)"
 
     override def computeHash = doHash(paramNum, binder.identityHash)
 
@@ -2784,9 +2790,9 @@ object Types {
       if ((prefix eq cls.owner.thisType) || !cls.owner.isClass || ctx.erasedTypes) tp
       else tp.substThis(cls.owner.asClass, prefix)
 
-    private var typeRefCache: Type = null
+    private var typeRefCache: TypeRef = null
 
-    def typeRef(implicit ctx: Context): Type = {
+    def typeRef(implicit ctx: Context): TypeRef = {
       def clsDenot = if (prefix eq cls.owner.thisType) cls.denot else cls.denot.copySymDenotation(info = this)
       if (typeRefCache == null)
         typeRefCache =
@@ -2795,7 +2801,7 @@ object Types {
       typeRefCache
     }
 
-    def symbolicTypeRef(implicit ctx: Context): Type = TypeRef(prefix, cls)
+    def symbolicTypeRef(implicit ctx: Context): TypeRef = TypeRef(prefix, cls)
 
     // cached because baseType needs parents
     private var parentsCache: List[TypeRef] = null