Merge pull request #1 from smarter/incremental

Add support for incremental compilation

Author: smarter

bridge/LICENSE

@@ -1,4 +1,9 @@
-Copyright (c) 2015 The dotty-bridge contributors.
+Copyright (c) 2015-2016 The dotty-bridge contributors.
+All rights reserved.
+
+Some parts of this project are copied or adapted from sbt which is:
+Copyright (c) 2008-2016 Typesafe Inc, Mark Harrah, Grzegorz Kossakowski,
+Josh Suereth, Indrajit Raychaudhuri, Eugene Yokota, and other contributors.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

bridge/README.md

@@ -7,10 +7,9 @@ it allows you to compile your code using Dotty instead of Scala 2.
 
 ### Implementation status
 
-- [X] Minimal `sbt compile` support
-- [ ] Incremental compilation (currently, the full sources will always be recompiled)
+- [X] `sbt compile` support, including [incremental recompilation](http://www.scala-sbt.org/0.13/docs/Understanding-Recompilation.html)
 - [ ] `sbt console` support (may involve some changes to the Dotty REPL)
-- [ ] `sbt doc` support (not possible until we have a Dotty doc tool)
+- [ ] `sbt doc` support
 
 ### Usage
 

bridge/build.sbt

@@ -5,7 +5,9 @@ lazy val root = (project in file(".")).
     description := "sbt compiler bridge for Dotty",
     libraryDependencies := Seq(
       "org.scala-lang" %% "dotty" % "0.1-SNAPSHOT",
-      "org.scala-sbt" % "interface" % sbtVersion.value
+      "org.scala-sbt" % "interface" % sbtVersion.value,
+      "org.scala-sbt" % "api" % sbtVersion.value % "test",
+      "org.specs2" %% "specs2" % "2.3.11" % "test"
     ),
     publishArtifact in packageDoc := false,
     version := "0.1-SNAPSHOT",
@@ -14,5 +16,37 @@ lazy val root = (project in file(".")).
     scalaVersion := "2.11.5",
     // Ideally, the sources should be published with crossPaths := false and the
     // binaries with crossPaths := true, but I haven't figured out how to do this.
-    crossPaths := false
+    crossPaths := false,
+
+    fork in Test := true,
+    parallelExecution in Test := false
   )
+
+// Options for scripted tests
+ScriptedPlugin.scriptedSettings
+scriptedLaunchOpts := Seq("-Xmx1024m")
+scriptedBufferLog := false
+// TODO: Use this instead of manually copying DottyInjectedPlugin.scala
+// everywhere once https://github.com/sbt/sbt/issues/2601 gets fixed.
+/*
+scriptedPrescripted := { f =>
+  IO.write(inj, """
+import sbt._
+import Keys._
+
+object DottyInjectedPlugin extends AutoPlugin {
+  override def requires = plugins.JvmPlugin
+  override def trigger = allRequirements
+
+  override val projectSettings = Seq(
+    scalaVersion := "0.1-SNAPSHOT",
+    scalacOptions += "-language:Scala2",
+    scalaBinaryVersion  := "2.11",
+    autoScalaLibrary := false,
+    libraryDependencies ++= Seq("org.scala-lang" % "scala-library" % "2.11.5"),
+    scalaCompilerBridgeSource := ("ch.epfl.lamp" % "dotty-bridge" % "0.1-SNAPSHOT" % "component").sources()
+  )
+}
+""")
+}
+*/

bridge/project/scripted.sbt

@@ -0,0 +1,3 @@
+libraryDependencies <+= (sbtVersion) { sv =>
+  "org.scala-sbt" % "scripted-plugin" % sv
+}

bridge/src/main/scala/xsbt/CompilerClassLoader.scala

@@ -0,0 +1,90 @@
+package xsbt
+
+import java.net.{URL, URLClassLoader}
+
+/**  A classloader to run the compiler
+ *
+ *   A CompilerClassLoader is constructed from a list of `urls` that need to be on
+ *   the classpath to run the compiler and the classloader used by sbt.
+ *
+ *   To understand why a custom classloader is needed for the compiler, let us
+ *   describe some alternatives that wouldn't work.
+ *   - `new URLClassLoader(urls)`:
+ *     The compiler contains sbt phases that callback to sbt using the `xsbti.*`
+ *     interfaces. If `urls` does not contain the sbt interfaces we'll get a
+ *     `ClassNotFoundException` in the compiler when we try to use them, if
+ *     `urls` does contain the interfaces we'll get a `ClassCastException` or a
+ *     `LinkageError` because if the same class is loaded by two different
+ *     classloaders, they are considered distinct by the JVM.
+ *    - `new URLClassLoader(urls, sbtLoader)`:
+ *      Because of the JVM delegation model, this means that we will only load
+ *      a class from `urls` if it's not present in the parent `sbtLoader`, but
+ *      sbt uses its own version of the scala compiler and scala library which
+ *      is not the one we need to run the compiler.
+ *
+ *  Our solution is to implement a subclass of URLClassLoader with no parent, instead
+ *  we override `loadClass` to load the `xsbti.*` interfaces from `sbtLoader`.
+ */
+class CompilerClassLoader(urls: Array[URL], sbtLoader: ClassLoader)
+    extends URLClassLoader(urls, null) {
+  override def loadClass(className: String, resolve: Boolean): Class[_] =
+    if (className.startsWith("xsbti.")) {
+      // We can't use the loadClass overload with two arguments because it's
+      // protected, but we can do the same by hand (the classloader instance
+      // from which we call resolveClass does not matter).
+      val c = sbtLoader.loadClass(className)
+      if (resolve)
+        resolveClass(c)
+      c
+    } else {
+      super.loadClass(className, resolve)
+    }
+}
+
+object CompilerClassLoader {
+  /** Fix the compiler bridge ClassLoader
+   *
+   *  Soundtrack: https://www.youtube.com/watch?v=imamcajBEJs
+   *
+   *  The classloader that we get from sbt looks like:
+   *
+   *    URLClassLoader(bridgeURLs,
+   *      DualLoader(scalaLoader, notXsbtiFilter, sbtLoader, xsbtiFilter))
+   *
+   *  DualLoader will load the `xsbti.*` interfaces using `sbtLoader` and
+   *  everything else with `scalaLoader`. Once we have loaded the dotty Main
+   *  class using `scalaLoader`, subsequent classes in the dotty compiler will
+   *  also be loaded by `scalaLoader` and _not_ by the DualLoader. But the sbt
+   *  compiler phases are part of dotty and still need access to the `xsbti.*`
+   *  interfaces in `sbtLoader`, therefore DualLoader does not work for us
+   *  (this issue is not present with scalac because the sbt phases are
+   *  currently defined in the compiler bridge itself, not in scalac).
+   *
+   *  CompilerClassLoader is a replacement for DualLoader. Until we can fix
+   *  this in sbt proper, we need to use reflection to construct our own
+   *  fixed classloader:
+   *
+   *    URLClassLoader(bridgeURLs,
+   *      CompilerClassLoader(scalaLoader.getURLs, sbtLoader))
+   *
+   *  @param bridgeLoader  The classloader that sbt uses to load the compiler bridge
+   *  @return A fixed classloader that works with dotty
+   */
+  def fixBridgeLoader(bridgeLoader: ClassLoader) = bridgeLoader match {
+    case bridgeLoader: URLClassLoader =>
+      val dualLoader = bridgeLoader.getParent
+      val dualLoaderClass = dualLoader.getClass
+
+      // DualLoader#parentA and DualLoader#parentB are private
+      val parentAField = dualLoaderClass.getDeclaredField("parentA")
+      parentAField.setAccessible(true)
+      val parentBField = dualLoaderClass.getDeclaredField("parentB")
+      parentBField.setAccessible(true)
+      val scalaLoader = parentAField.get(dualLoader).asInstanceOf[URLClassLoader]
+      val sbtLoader = parentBField.get(dualLoader).asInstanceOf[URLClassLoader]
+
+      val bridgeURLs = bridgeLoader.getURLs
+      new URLClassLoader(bridgeURLs,
+        new CompilerClassLoader(scalaLoader.getURLs, sbtLoader))
+  }
+}

bridge/src/main/scala/xsbt/CompilerInterface.scala

@@ -3,22 +3,41 @@
  */
 package xsbt
 
-import xsbti.{ AnalysisCallback, Logger, Problem, Reporter, Severity }
+import xsbti.{ AnalysisCallback, Logger, Problem, Reporter, Severity, DependencyContext }
+import xsbti.api.SourceAPI
 import xsbti.compile._
 import Log.debug
 import java.io.File
 
+import dotty.tools.dotc.core.Contexts.ContextBase
 import dotty.tools.dotc.{ Main => DottyMain }
+import dotty.tools.dotc.interfaces._
+
+import java.net.URLClassLoader
 
 final class CompilerInterface {
-  def newCompiler(options: Array[String], output: Output, initialLog: Logger, initialDelegate: Reporter, resident: Boolean): CachedCompiler =
-    new CachedCompiler0(options, output, resident)
+  def newCompiler(options: Array[String], output: Output, initialLog: Logger,
+                  initialDelegate: Reporter, resident: Boolean): CachedCompiler = {
+    // The classloader that sbt uses to load the compiler bridge is broken
+    // (see CompilerClassLoader#fixBridgeLoader for details). To workaround
+    // this we construct our own ClassLoader and then run the following code
+    // with it:
+    //   new CachedCompilerImpl(options, output, resident)
+
+    val bridgeLoader = getClass.getClassLoader
+    val fixedLoader = CompilerClassLoader.fixBridgeLoader(bridgeLoader)
+    val cciClass = fixedLoader.loadClass("xsbt.CachedCompilerImpl")
+    cciClass.getConstructors.head
+      .newInstance(options, output, resident: java.lang.Boolean)
+      .asInstanceOf[CachedCompiler]
+  }
 
-  def run(sources: Array[File], changes: DependencyChanges, callback: AnalysisCallback, log: Logger, delegate: Reporter, progress: CompileProgress, cached: CachedCompiler): Unit =
+  def run(sources: Array[File], changes: DependencyChanges, callback: AnalysisCallback, log: Logger,
+          delegate: Reporter, progress: CompileProgress, cached: CachedCompiler): Unit =
     cached.run(sources, changes, callback, log, delegate, progress)
 }
 
-private final class CachedCompiler0(args: Array[String], output: Output, resident: Boolean) extends CachedCompiler {
+class CachedCompilerImpl(args: Array[String], output: Output, resident: Boolean) extends CachedCompiler {
   val outputArgs =
     output match {
       case multi: MultipleOutput =>
@@ -35,7 +54,11 @@ private final class CachedCompiler0(args: Array[String], output: Output, residen
   }
   private[this] def run(sources: List[File], changes: DependencyChanges, callback: AnalysisCallback, log: Logger, compileProgress: CompileProgress): Unit = {
     debug(log, args.mkString("Calling Dotty compiler with arguments  (CompilerInterface):\n\t", "\n\t", ""))
-    val reporter = DottyMain.process(commandArguments(sources.toArray))
+    val ctx = (new ContextBase).initialCtx.fresh
+      .setSbtCallback(callback)
+    val cl = getClass.getClassLoader.asInstanceOf[URLClassLoader]
+
+    val reporter = DottyMain.process(commandArguments(sources.toArray), ctx)
     if (reporter.hasErrors) {
       throw new InterfaceCompileFailed(args, Array())
     }

bridge/src/sbt-test/source-dependencies/abstract-override/A.scala

@@ -0,0 +1,3 @@
+trait A {
+	def x: Int
+}

bridge/src/sbt-test/source-dependencies/abstract-override/B.scala

@@ -0,0 +1,3 @@
+trait B extends A {
+	override def x = 2
+}

bridge/src/sbt-test/source-dependencies/abstract-override/C.scala

@@ -0,0 +1,3 @@
+trait C extends A {
+	def x = 5
+}

bridge/src/sbt-test/source-dependencies/abstract-override/D.scala

@@ -0,0 +1 @@
+trait D extends C with B

bridge/src/sbt-test/source-dependencies/abstract-override/changes/C2.scala

@@ -0,0 +1,3 @@
+trait C extends A {
+	abstract override def x = super.x + 5
+}

bridge/src/sbt-test/source-dependencies/abstract-override/project/DottyInjectedPlugin.scala

@@ -0,0 +1,16 @@
+import sbt._
+import Keys._
+
+object DottyInjectedPlugin extends AutoPlugin {
+  override def requires = plugins.JvmPlugin
+  override def trigger = allRequirements
+
+  override val projectSettings = Seq(
+    scalaVersion := "0.1-SNAPSHOT",
+    scalacOptions += "-language:Scala2",
+    scalaBinaryVersion  := "2.11",
+    autoScalaLibrary := false,
+    libraryDependencies ++= Seq("org.scala-lang" % "scala-library" % "2.11.5"),
+    scalaCompilerBridgeSource := ("ch.epfl.lamp" % "dotty-bridge" % "0.1-SNAPSHOT" % "component").sources()
+  )
+}

bridge/src/sbt-test/source-dependencies/abstract-override/test

@@ -0,0 +1,3 @@
+> compile
+$ copy-file changes/C2.scala C.scala
+-> compile

bridge/src/sbt-test/source-dependencies/abstract-type-override/build.sbt

@@ -0,0 +1,7 @@
+InputKey[Unit]("check-number-of-compiler-iterations") <<= inputTask { (argTask: TaskKey[Seq[String]]) =>
+  (argTask, compile in Compile) map { (args: Seq[String], a: sbt.inc.Analysis) =>
+    assert(args.size == 1)
+    val expectedIterationsNumber = args(0).toInt
+    assert(a.compilations.allCompilations.size == expectedIterationsNumber, "a.compilations.allCompilations.size = %d (expected %d)".format(a.compilations.allCompilations.size, expectedIterationsNumber))
+  }
+}

bridge/src/sbt-test/source-dependencies/abstract-type-override/changes/Bar1.scala

@@ -0,0 +1,4 @@
+object Bar {
+  def bar: Outer.TypeInner = null
+  // comment to trigger recompilation
+}

bridge/src/sbt-test/source-dependencies/abstract-type-override/project/DottyInjectedPlugin.scala

@@ -0,0 +1,16 @@
+import sbt._
+import Keys._
+
+object DottyInjectedPlugin extends AutoPlugin {
+  override def requires = plugins.JvmPlugin
+  override def trigger = allRequirements
+
+  override val projectSettings = Seq(
+    scalaVersion := "0.1-SNAPSHOT",
+    scalacOptions += "-language:Scala2",
+    scalaBinaryVersion  := "2.11",
+    autoScalaLibrary := false,
+    libraryDependencies ++= Seq("org.scala-lang" % "scala-library" % "2.11.5"),
+    scalaCompilerBridgeSource := ("ch.epfl.lamp" % "dotty-bridge" % "0.1-SNAPSHOT" % "component").sources()
+  )
+}

bridge/src/sbt-test/source-dependencies/abstract-type-override/src/main/scala/Bar.scala

@@ -0,0 +1,3 @@
+object Bar {
+  def bar: Outer.TypeInner = null
+}

bridge/src/sbt-test/source-dependencies/abstract-type-override/src/main/scala/Foo.scala

@@ -0,0 +1,5 @@
+object Outer {
+  class Inner { type Xyz }
+
+  type TypeInner = Inner { type Xyz = Int }
+}

bridge/src/sbt-test/source-dependencies/abstract-type-override/src/main/scala/Impl.scala

@@ -0,0 +1,3 @@
+class Impl {
+  def bleep = Bar.bar
+}

bridge/src/sbt-test/source-dependencies/abstract-type-override/test

@@ -0,0 +1,14 @@
+# Test for separate compilation and proper value of
+# the OVERRIDE flag when abstract types, type alias
+# and structural type are involved
+# See https://github.com/sbt/sbt/issues/726 for details
+
+# introduces first compile iteration
+> compile
+# this change adds a comment and does not change api so introduces
+# only one additional compile iteration
+$ copy-file changes/Bar1.scala src/main/scala/Bar.scala
+# second iteration
+#> compile
+# check if there are only two compile iterations performed
+> check-number-of-compiler-iterations 2

bridge/src/sbt-test/source-dependencies/abstract-type/A.scala

@@ -0,0 +1,3 @@
+trait A {
+	type S[_]
+}

bridge/src/sbt-test/source-dependencies/abstract-type/B.scala

@@ -0,0 +1,3 @@
+trait B extends A {
+	type F = S[Int]
+}

bridge/src/sbt-test/source-dependencies/abstract-type/changes/A.scala

@@ -0,0 +1,3 @@
+trait A {
+	type S
+}

bridge/src/sbt-test/source-dependencies/abstract-type/project/DottyInjectedPlugin.scala

@@ -0,0 +1,16 @@
+import sbt._
+import Keys._
+
+object DottyInjectedPlugin extends AutoPlugin {
+  override def requires = plugins.JvmPlugin
+  override def trigger = allRequirements
+
+  override val projectSettings = Seq(
+    scalaVersion := "0.1-SNAPSHOT",
+    scalacOptions += "-language:Scala2",
+    scalaBinaryVersion  := "2.11",
+    autoScalaLibrary := false,
+    libraryDependencies ++= Seq("org.scala-lang" % "scala-library" % "2.11.5"),
+    scalaCompilerBridgeSource := ("ch.epfl.lamp" % "dotty-bridge" % "0.1-SNAPSHOT" % "component").sources()
+  )
+}

bridge/src/sbt-test/source-dependencies/abstract-type/test

@@ -0,0 +1,7 @@
+> compile
+
+# remove type arguments from S
+$ copy-file changes/A.scala A.scala
+
+# Both A.scala and B.scala should be recompiled, producing a compile error
+-> compile

bridge/src/sbt-test/source-dependencies/added/changes/A1.scala

@@ -0,0 +1,6 @@
+package example
+
+object A
+{
+	val x: Int = 3
+}

bridge/src/sbt-test/source-dependencies/added/changes/A2.scala

@@ -0,0 +1,6 @@
+package example
+
+object A
+{
+	val x: Int = B.y
+}

bridge/src/sbt-test/source-dependencies/added/changes/A3.scala

@@ -0,0 +1,6 @@
+package example
+
+object A
+{
+	val x: String = B.y
+}