Skip to content

Commit 401ca9e

Browse files
smartersmarter
committed
Add sbt incremental compilation support
To test this with sbt, see https://github.com/lampepfl/dotty/wiki/Using-Dotty-with-sbt The following flags are added: - -Yforce-sbt-phases: Run the phases used by sbt for incremental compilation (ExtractDependencies and ExtractAPI) even if the compiler is ran outside of sbt, for debugging. - -Ydump-sbt-inc: For every compiled foo.scala, output the sbt API representation and dependencies in foo.inc, implies -Yforce-sbt-phases. This commit introduces two new phases which do not transform trees: - `ExtractDependencies` which extracts the dependency information of the current compilation unit and sends it to sbt via callbacks - `ExtractAPI` which creates a representation of the API of the current compilation unit and sends it to sbt via callbacks Briefly, when a file changes sbt will recompile it, if its API has changed (determined by what `ExtractAPI` sent) then sbt will determine which reverse-dependencies (determined by what `ExtractDependencies` sent) of the API have to be recompiled depending on what changed. This phase was originally based on https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt which attempts to integrate the sbt phases into scalac (and is itself based on https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt), but it has been heavily refactored and adapted to Dotty. The main functional differences are: - ExtractDependencies runs right after Frontend (so that we don't lose dependency informations because of the simplifications done by PostTyper), but ExtractAPI runs right after PostTyper (so that SuperAccessors are part of the API). - `ExtractAPI` only extract types as they are defined and never "as seen from" some some specific prefix, see its documentation for more details. - `ExtractDependenciesTraverser` and `ExtractUsedNames` have been fused into one tree traversal in `ExtractDependenciesCollector`. TODO: Try to run these phases in parallel with the rest of the compiler pipeline since they're independent (except for the sbt callbacks in `GenBCode`) ?
1 parent aca6a9c commit 401ca9e

File tree

12 files changed

+792
-61
lines changed

12 files changed

+792
-61
lines changed

‎AUTHORS.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,15 @@ The majority of the dotty codebase is new code, with the exception of the compon
4242
> The utilities package is a mix of new and adapted components. The files in [scala/scala](https://github.com/scala/scala) were originally authored by many people,
4343
> including Paul Phillips, Martin Odersky, Sean McDirmid, and others.
4444
45-
`dotty.tools.io`
45+
`dotty.tools.io`
4646

4747
> The I/O support library was adapted from current Scala compiler. Original authors were Paul Phillips and others.
48+
49+
`dotty.tools.dotc.sbt`
50+
51+
> The sbt compiler phase is based on
52+
> https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt
53+
> which attempts to integrate the sbt phases into scalac and is itself based on
54+
> the [compiler bridge in sbt 0.13](https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt),
55+
> but has been heavily adapted and refactored.
56+
> Original authors were Mark Harrah, Grzegorz Kossakowski, Martin Duhemm, Adriaan Moors and others.

‎bin/dotc

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ SCALA_BINARY_VERSION=2.11
2323
SCALA_COMPILER_VERSION=$(getLastStringOnLineWith "scala-compiler")
2424
DOTTY_VERSION=$(getLastStringOnLineWith "version in")
2525
JLINE_VERSION=$(getLastStringOnLineWith "jline")
26+
SBT_VERSION=$(grep "sbt.version=" "$DOTTY_ROOT/project/build.properties" | sed 's/sbt.version=//')
2627
bootcp=true
2728
bootstrapped=false
2829
default_java_opts="-Xmx768m -Xms768m"
@@ -100,13 +101,19 @@ then
100101
JLINE_JAR=$HOME/.ivy2/cache/jline/jline/jars/jline-$JLINE_VERSION.jar
101102
fi
102103

103-
if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR" -o ! -f "$SCALA_COMPILER_JAR" -o ! -f "$JLINE_JAR" ]
104+
if [ "$SBT_INTERFACE_JAR" == "" ]
105+
then
106+
SBT_INTERFACE_JAR=$HOME/.ivy2/cache/org.scala-sbt/interface/jars/interface-$SBT_VERSION.jar
107+
fi
108+
109+
if [ ! -f "$SCALA_LIBRARY_JAR" -o ! -f "$SCALA_REFLECT_JAR" -o ! -f "$SCALA_COMPILER_JAR" -o ! -f "$JLINE_JAR" -o ! -f "$SBT_INTERFACE_JAR" ]
104110
then
105111
echo To use this script please set
106112
echo SCALA_LIBRARY_JAR to point to scala-library-$SCALA_VERSION.jar "(currently $SCALA_LIBRARY_JAR)"
107113
echo SCALA_REFLECT_JAR to point to scala-reflect-$SCALA_VERSION.jar "(currently $SCALA_REFLECT_JAR)"
108114
echo SCALA_COMPILER_JAR to point to scala-compiler-$SCALA_VERSION.jar with bcode patches "(currently $SCALA_COMPILER_JAR)"
109115
echo JLINE_JAR to point to jline-$JLINE_VERSION.jar "(currently $JLINE_JAR)"
116+
echo SBT_INTERFACE_JAR to point to interface-$SBT_VERSION.jar "(currently $SBT_INTERFACE_JAR)"
110117
fi
111118

112119
ifdebug () {
@@ -196,9 +203,9 @@ trap onExit INT
196203
classpathArgs () {
197204
if [[ "true" == $bootstrapped ]]; then
198205
checkjar $DOTTY_JAR "test:runMain dotc.build" src
199-
toolchain="$DOTTY_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR"
206+
toolchain="$DOTTY_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR:$SBT_INTERFACE_JAR"
200207
else
201-
toolchain="$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR"
208+
toolchain="$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$JLINE_JAR:$SBT_INTERFACE_JAR"
202209
fi
203210
bcpJars="$INTERFACES_JAR:$MAIN_JAR"
204211
cpJars="$INTERFACES_JAR:$MAIN_JAR:$TEST_JAR"

‎project/Build.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ object DottyBuild extends Build {
8484
libraryDependencies ++= Seq("org.scala-lang.modules" %% "scala-xml" % "1.0.1",
8585
"org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test",
8686
"com.novocode" % "junit-interface" % "0.11" % "test",
87-
"jline" % "jline" % "2.12"),
87+
"jline" % "jline" % "2.12",
88+
"org.scala-sbt" % "interface" % sbtVersion.value),
8889

8990
// enable improved incremental compilation algorithm
9091
incOptions := incOptions.value.withNameHashing(true),

‎src/dotty/tools/backend/jvm/GenBCode.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,8 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
389389
val className = jclassName.replace('/', '.')
390390
if (ctx.compilerCallback != null)
391391
ctx.compilerCallback.onClassGenerated(sourceFile, convertAbstractFile(outFile), className)
392+
if (ctx.sbtCallback != null)
393+
ctx.sbtCallback.generatedClass(sourceFile.jfile.orElse(null), outFile.file, className)
392394
}
393395
catch {
394396
case e: FileConflictException =>

‎src/dotty/tools/dotc/Compiler.scala

Lines changed: 52 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -41,56 +41,58 @@ class Compiler {
4141
*/
4242
def phases: List[List[Phase]] =
4343
List(
44-
List(new FrontEnd), // Compiler frontend: scanner, parser, namer, typer
45-
List(new PostTyper), // Additional checks and cleanups after type checking
46-
List(new Pickler), // Generate TASTY info
47-
List(new FirstTransform, // Some transformations to put trees into a canonical form
48-
new CheckReentrant), // Internal use only: Check that compiled program has no data races involving global vars
49-
List(new RefChecks, // Various checks mostly related to abstract members and overriding
50-
new CheckStatic, // Check restrictions that apply to @static members
51-
new ElimRepeated, // Rewrite vararg parameters and arguments
52-
new NormalizeFlags, // Rewrite some definition flags
53-
new ExtensionMethods, // Expand methods of value classes with extension methods
54-
new ExpandSAMs, // Expand single abstract method closures to anonymous classes
55-
new TailRec, // Rewrite tail recursion to loops
56-
new LiftTry, // Put try expressions that might execute on non-empty stacks into their own methods
57-
new ClassOf), // Expand `Predef.classOf` calls.
58-
List(new PatternMatcher, // Compile pattern matches
59-
new ExplicitOuter, // Add accessors to outer classes from nested ones.
60-
new ExplicitSelf, // Make references to non-trivial self types explicit as casts
61-
new CrossCastAnd, // Normalize selections involving intersection types.
62-
new Splitter), // Expand selections involving union types into conditionals
63-
List(new VCInlineMethods, // Inlines calls to value class methods
64-
new SeqLiterals, // Express vararg arguments as arrays
65-
new InterceptedMethods, // Special handling of `==`, `|=`, `getClass` methods
66-
new Getters, // Replace non-private vals and vars with getter defs (fields are added later)
67-
new ElimByName, // Expand by-name parameters and arguments
68-
new AugmentScala2Traits, // Expand traits defined in Scala 2.11 to simulate old-style rewritings
69-
new ResolveSuper, // Implement super accessors and add forwarders to trait methods
70-
new ArrayConstructors), // Intercept creation of (non-generic) arrays and intrinsify.
71-
List(new Erasure), // Rewrite types to JVM model, erasing all type parameters, abstract types and refinements.
72-
List(new ElimErasedValueType, // Expand erased value types to their underlying implmementation types
73-
new VCElideAllocations, // Peep-hole optimization to eliminate unnecessary value class allocations
74-
new Mixin, // Expand trait fields and trait initializers
75-
new LazyVals, // Expand lazy vals
76-
new Memoize, // Add private fields to getters and setters
77-
new LinkScala2ImplClasses, // Forward calls to the implementation classes of traits defined by Scala 2.11
78-
new NonLocalReturns, // Expand non-local returns
79-
new CapturedVars, // Represent vars captured by closures as heap objects
80-
new Constructors, // Collect initialization code in primary constructors
81-
// Note: constructors changes decls in transformTemplate, no InfoTransformers should be added after it
82-
new FunctionalInterfaces,// Rewrites closures to implement @specialized types of Functions.
83-
new GetClass), // Rewrites getClass calls on primitive types.
84-
List(new LambdaLift, // Lifts out nested functions to class scope, storing free variables in environments
85-
// Note: in this mini-phase block scopes are incorrect. No phases that rely on scopes should be here
86-
new ElimStaticThis, // Replace `this` references to static objects by global identifiers
87-
new Flatten, // Lift all inner classes to package scope
88-
new RestoreScopes), // Repair scopes rendered invalid by moving definitions in prior phases of the group
89-
List(new ExpandPrivate, // Widen private definitions accessed from nested classes
90-
new CollectEntryPoints, // Find classes with main methods
91-
new LabelDefs), // Converts calls to labels to jumps
92-
List(new GenSJSIR), // Generate .js code
93-
List(new GenBCode) // Generate JVM bytecode
44+
List(new FrontEnd), // Compiler frontend: scanner, parser, namer, typer
45+
List(new sbt.ExtractDependencies), // Sends information on classes' dependencies to sbt via callbacks
46+
List(new PostTyper), // Additional checks and cleanups after type checking
47+
List(new sbt.ExtractAPI), // Sends a representation of the API of classes to sbt via callbacks
48+
List(new Pickler), // Generate TASTY info
49+
List(new FirstTransform, // Some transformations to put trees into a canonical form
50+
new CheckReentrant), // Internal use only: Check that compiled program has no data races involving global vars
51+
List(new RefChecks, // Various checks mostly related to abstract members and overriding
52+
new CheckStatic, // Check restrictions that apply to @static members
53+
new ElimRepeated, // Rewrite vararg parameters and arguments
54+
new NormalizeFlags, // Rewrite some definition flags
55+
new ExtensionMethods, // Expand methods of value classes with extension methods
56+
new ExpandSAMs, // Expand single abstract method closures to anonymous classes
57+
new TailRec, // Rewrite tail recursion to loops
58+
new LiftTry, // Put try expressions that might execute on non-empty stacks into their own methods
59+
new ClassOf), // Expand `Predef.classOf` calls.
60+
List(new PatternMatcher, // Compile pattern matches
61+
new ExplicitOuter, // Add accessors to outer classes from nested ones.
62+
new ExplicitSelf, // Make references to non-trivial self types explicit as casts
63+
new CrossCastAnd, // Normalize selections involving intersection types.
64+
new Splitter), // Expand selections involving union types into conditionals
65+
List(new VCInlineMethods, // Inlines calls to value class methods
66+
new SeqLiterals, // Express vararg arguments as arrays
67+
new InterceptedMethods, // Special handling of `==`, `|=`, `getClass` methods
68+
new Getters, // Replace non-private vals and vars with getter defs (fields are added later)
69+
new ElimByName, // Expand by-name parameters and arguments
70+
new AugmentScala2Traits, // Expand traits defined in Scala 2.11 to simulate old-style rewritings
71+
new ResolveSuper, // Implement super accessors and add forwarders to trait methods
72+
new ArrayConstructors), // Intercept creation of (non-generic) arrays and intrinsify.
73+
List(new Erasure), // Rewrite types to JVM model, erasing all type parameters, abstract types and refinements.
74+
List(new ElimErasedValueType, // Expand erased value types to their underlying implmementation types
75+
new VCElideAllocations, // Peep-hole optimization to eliminate unnecessary value class allocations
76+
new Mixin, // Expand trait fields and trait initializers
77+
new LazyVals, // Expand lazy vals
78+
new Memoize, // Add private fields to getters and setters
79+
new LinkScala2ImplClasses, // Forward calls to the implementation classes of traits defined by Scala 2.11
80+
new NonLocalReturns, // Expand non-local returns
81+
new CapturedVars, // Represent vars captured by closures as heap objects
82+
new Constructors, // Collect initialization code in primary constructors
83+
// Note: constructors changes decls in transformTemplate, no InfoTransformers should be added after it
84+
new FunctionalInterfaces, // Rewrites closures to implement @specialized types of Functions.
85+
new GetClass), // Rewrites getClass calls on primitive types.
86+
List(new LambdaLift, // Lifts out nested functions to class scope, storing free variables in environments
87+
// Note: in this mini-phase block scopes are incorrect. No phases that rely on scopes should be here
88+
new ElimStaticThis, // Replace `this` references to static objects by global identifiers
89+
new Flatten, // Lift all inner classes to package scope
90+
new RestoreScopes), // Repair scopes rendered invalid by moving definitions in prior phases of the group
91+
List(new ExpandPrivate, // Widen private definitions accessed from nested classes
92+
new CollectEntryPoints, // Find classes with main methods
93+
new LabelDefs), // Converts calls to labels to jumps
94+
List(new GenSJSIR), // Generate .js code
95+
List(new GenBCode) // Generate JVM bytecode
9496
)
9597

9698
var runId = 1

‎src/dotty/tools/dotc/config/ScalaSettings.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,8 @@ class ScalaSettings extends Settings.SettingGroup {
160160
val YtestPickler = BooleanSetting("-Ytest-pickler", "self-test for pickling functionality; should be used with -Ystop-after:pickler")
161161
val YcheckReentrant = BooleanSetting("-Ycheck-reentrant", "check that compiled program does not contain vars that can be accessed from a global root.")
162162
val YkeepComments = BooleanSetting("-Ykeep-comments", "Keep comments when scanning source files.")
163+
val YforceSbtPhases = BooleanSetting("-Yforce-sbt-phases", "Run the phases used by sbt for incremental compilation (ExtractDependencies and ExtractAPI) even if the compiler is ran outside of sbt, for debugging.")
164+
val YdumpSbtInc = BooleanSetting("-Ydump-sbt-inc", "For every compiled foo.scala, output the sbt API representation and dependencies in foo.inc, implies -Yforce-sbt-phases.")
163165
def stop = YstopAfter
164166

165167
/** Area-specific debug output.

‎src/dotty/tools/dotc/core/Contexts.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import printing._
2929
import config.{Settings, ScalaSettings, Platform, JavaPlatform, SJSPlatform}
3030
import language.implicitConversions
3131
import DenotTransformers.DenotTransformer
32+
import xsbti.AnalysisCallback
3233

3334
object Contexts {
3435

@@ -84,6 +85,12 @@ object Contexts {
8485
_compilerCallback = callback
8586
def compilerCallback: CompilerCallback = _compilerCallback
8687

88+
/** The sbt callback implementation if we are run from sbt, null otherwise */
89+
private[this] var _sbtCallback: AnalysisCallback = _
90+
protected def sbtCallback_=(callback: AnalysisCallback) =
91+
_sbtCallback = callback
92+
def sbtCallback: AnalysisCallback = _sbtCallback
93+
8794
/** The current context */
8895
private[this] var _period: Period = _
8996
protected def period_=(period: Period) = {
@@ -426,6 +433,7 @@ object Contexts {
426433
def setPeriod(period: Period): this.type = { this.period = period; this }
427434
def setMode(mode: Mode): this.type = { this.mode = mode; this }
428435
def setCompilerCallback(callback: CompilerCallback): this.type = { this.compilerCallback = callback; this }
436+
def setSbtCallback(callback: AnalysisCallback): this.type = { this.sbtCallback = callback; this }
429437
def setTyperState(typerState: TyperState): this.type = { this.typerState = typerState; this }
430438
def setReporter(reporter: Reporter): this.type = setTyperState(typerState.withReporter(reporter))
431439
def setNewTyperState: this.type = setTyperState(typerState.fresh(isCommittable = true))

‎src/dotty/tools/dotc/core/Types.scala

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2478,10 +2478,14 @@ object Types {
24782478

24792479
abstract class ParamType extends BoundType {
24802480
def paramNum: Int
2481+
def paramName: Name
24812482
}
24822483

24832484
abstract case class MethodParam(binder: MethodType, paramNum: Int) extends ParamType with SingletonType {
24842485
type BT = MethodType
2486+
2487+
def paramName = binder.paramNames(paramNum)
2488+
24852489
override def underlying(implicit ctx: Context): Type = binder.paramTypes(paramNum)
24862490
def copyBoundType(bt: BT) = new MethodParamImpl(bt, paramNum)
24872491

@@ -2494,7 +2498,7 @@ object Types {
24942498
false
24952499
}
24962500

2497-
override def toString = s"MethodParam(${binder.paramNames(paramNum)})"
2501+
override def toString = s"MethodParam($paramName)"
24982502
}
24992503

25002504
class MethodParamImpl(binder: MethodType, paramNum: Int) extends MethodParam(binder, paramNum)
@@ -2524,9 +2528,11 @@ object Types {
25242528
case _ => false
25252529
}
25262530

2531+
def paramName = binder.paramNames(paramNum)
2532+
25272533
override def underlying(implicit ctx: Context): Type = binder.paramBounds(paramNum)
25282534
// no customized hashCode/equals needed because cycle is broken in PolyType
2529-
override def toString = s"PolyParam(${binder.paramNames(paramNum)})"
2535+
override def toString = s"PolyParam($paramName)"
25302536

25312537
override def computeHash = doHash(paramNum, binder.identityHash)
25322538

@@ -2778,9 +2784,9 @@ object Types {
27782784
if ((prefix eq cls.owner.thisType) || !cls.owner.isClass || ctx.erasedTypes) tp
27792785
else tp.substThis(cls.owner.asClass, prefix)
27802786

2781-
private var typeRefCache: Type = null
2787+
private var typeRefCache: TypeRef = null
27822788

2783-
def typeRef(implicit ctx: Context): Type = {
2789+
def typeRef(implicit ctx: Context): TypeRef = {
27842790
def clsDenot = if (prefix eq cls.owner.thisType) cls.denot else cls.denot.copySymDenotation(info = this)
27852791
if (typeRefCache == null)
27862792
typeRefCache =
@@ -2789,7 +2795,7 @@ object Types {
27892795
typeRefCache
27902796
}
27912797

2792-
def symbolicTypeRef(implicit ctx: Context): Type = TypeRef(prefix, cls)
2798+
def symbolicTypeRef(implicit ctx: Context): TypeRef = TypeRef(prefix, cls)
27932799

27942800
// cached because baseType needs parents
27952801
private var parentsCache: List[TypeRef] = null

0 commit comments

Comments
 (0)