Start building Dotty against the 2.13 stdlib

This required some surgery in Build.scala: when compiling the
non-bootstrapped compiler, we need the 2.12 stdlib on the JVM and
compiler classpaths, because the current reference compiler cannot be
run using the 2.13 stdlib, on the other hand when _running_ the
non-bootstrapped compiler we need to the 2.13 stdlib on the compiler
classpath to produce a bootstrapped compiler that links against 2.13.

Upgrade the Scala 2 PickleFormat version to 5.2 (which is in fact
identical to 5.0).

Remove no longer needed -Ynew-collections flag and update the
related test files.

Author: smarter

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -610,8 +610,6 @@ object desugar {
         }
       }
 
-      // TODO When the Scala library is updated to 2.13.x add the override keyword to this generated method.
-      // (because Product.scala was updated)
       def productElemNameMeth = {
         val methodParam = makeSyntheticParameter(tpt = scalaDot(tpnme.Int))
         val paramRef = Ident(methodParam.name)
@@ -625,7 +623,7 @@ object desugar {
         } :+ defaultCase
         val body = Match(paramRef, patternMatchCases)
         DefDef(nme.productElementName, Nil, List(List(methodParam)), javaDotLangDot(tpnme.String), body)
-          .withFlags(if (defn.isNewCollections) Override | Synthetic else Synthetic)
+          .withFlags(Override | Synthetic)
       }
 
       if (isCaseClass)

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -112,8 +112,6 @@ class ScalaSettings extends Settings.SettingGroup {
   val YdisableFlatCpCaching: Setting[Boolean] = BooleanSetting("-YdisableFlatCpCaching", "Do not cache flat classpath representation of classpath elements from jars across compiler instances.")
 
   val Yscala2Unpickler: Setting[String] = StringSetting("-Yscala2-unpickler", "", "Control where we may get Scala 2 symbols from. This is either \"always\", \"never\", or a classpath.", "always")
-  // TODO: Remove once we drop support for 2.12 standard library
-  val YnewCollections: Setting[Boolean] = BooleanSetting("-Ynew-collections", "Inform the compiler that we are using the 2.13 collection library (even if the 2.12 library is on the classpath).")
 
   val YnoImports: Setting[Boolean] = BooleanSetting("-Yno-imports", "Compile without importing scala.*, java.lang.*, or Predef.")
   val YnoInline: Setting[Boolean] = BooleanSetting("-Yno-inline", "Suppress inlining.")

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -349,17 +349,9 @@ class Definitions {
     @tu lazy val Predef_classOf  : Symbol = ScalaPredefModule.requiredMethod(nme.classOf)
     @tu lazy val Predef_undefined: Symbol = ScalaPredefModule.requiredMethod(nme.???)
 
-  def SubTypeClass(implicit ctx: Context): ClassSymbol =
-    if (isNewCollections)
-      ctx.requiredClass("scala.<:<")
-    else
-      ScalaPredefModule.requiredClass("<:<")
+  def SubTypeClass(implicit ctx: Context): ClassSymbol = ctx.requiredClass("scala.<:<")
 
-  def DummyImplicitClass(implicit ctx: Context): ClassSymbol =
-    if (isNewCollections)
-      ctx.requiredClass("scala.DummyImplicit")
-    else
-      ScalaPredefModule.requiredClass("DummyImplicit")
+  def DummyImplicitClass(implicit ctx: Context): ClassSymbol = ctx.requiredClass("scala.DummyImplicit")
 
   @tu lazy val ScalaRuntimeModule: Symbol = ctx.requiredModule("scala.runtime.ScalaRunTime")
     def runtimeMethodRef(name: PreName): TermRef = ScalaRuntimeModule.requiredMethodRef(name)
@@ -385,10 +377,7 @@ class Definitions {
     def newGenericArrayMethod(implicit ctx: Context): TermSymbol = DottyArraysModule.requiredMethod("newGenericArray")
     def newArrayMethod(implicit ctx: Context): TermSymbol = DottyArraysModule.requiredMethod("newArray")
 
-  // TODO: Remove once we drop support for 2.12 standard library
-  @tu lazy val isNewCollections: Boolean = ctx.settings.YnewCollections.value
-
-  def getWrapVarargsArrayModule: Symbol = if (isNewCollections) ScalaRuntimeModule else ScalaPredefModule
+  def getWrapVarargsArrayModule: Symbol = ScalaRuntimeModule
 
   // The set of all wrap{X, Ref}Array methods, where X is a value type
   val WrapArrayMethods: PerRun[collection.Set[Symbol]] = new PerRun({ implicit ctx =>
@@ -406,9 +395,7 @@ class Definitions {
       List(AnyClass.typeRef), EmptyScope)
   @tu lazy val SingletonType: TypeRef = SingletonClass.typeRef
 
-  @tu lazy val SeqType: TypeRef =
-    if (isNewCollections) ctx.requiredClassRef("scala.collection.immutable.Seq")
-    else ctx.requiredClassRef("scala.collection.Seq")
+  @tu lazy val SeqType: TypeRef = ctx.requiredClassRef("scala.collection.immutable.Seq")
   def SeqClass given Context: ClassSymbol = SeqType.symbol.asClass
     @tu lazy val Seq_apply        : Symbol = SeqClass.requiredMethod(nme.apply)
     @tu lazy val Seq_head         : Symbol = SeqClass.requiredMethod(nme.head)
@@ -540,11 +527,7 @@ class Definitions {
 
   @tu lazy val ThrowableType: TypeRef          = ctx.requiredClassRef("java.lang.Throwable")
   def ThrowableClass given Context: ClassSymbol = ThrowableType.symbol.asClass
-  @tu lazy val SerializableType: TypeRef       =
-    if (isNewCollections)
-      JavaSerializableClass.typeRef
-    else
-      ctx.requiredClassRef("scala.Serializable")
+  @tu lazy val SerializableType: TypeRef       = JavaSerializableClass.typeRef
   def SerializableClass given Context: ClassSymbol = SerializableType.symbol.asClass
 
    @tu lazy val JavaEnumClass: ClassSymbol = {

compiler/src/dotty/tools/dotc/core/unpickleScala2/PickleFormat.scala

@@ -117,7 +117,7 @@ object PickleFormat {
  *   len is remaining length after `len`.
  */
   val MajorVersion: Int = 5
-  val MinorVersion: Int = 0
+  val MinorVersion: Int = 2
 
   final val TERMname = 1
   final val TYPEname = 2

compiler/src/dotty/tools/dotc/typer/Implicits.scala

@@ -145,10 +145,8 @@ object Implicits {
               val isFunctionInS2 =
                 ctx.scala2Mode && tpw.derivesFrom(defn.FunctionClass(1)) && ref.symbol != defn.Predef_conforms
               val isImplicitConversion = tpw.derivesFrom(defn.ConversionClass)
-              val isConforms = // An implementation of <:< counts as a view, except that $conforms is always omitted
-                  tpw.derivesFrom(defn.SubTypeClass) &&
-                    (defn.isNewCollections || // In 2.13, the type of `$conforms` changed from `A <:< A` to `A => A`
-                     ref.symbol != defn.Predef_conforms)
+              // An implementation of <:< counts as a view
+              val isConforms = tpw.derivesFrom(defn.SubTypeClass)
               val hasExtensions = resType match {
                 case SelectionProto(name, _, _, _) =>
                   tpw.memberBasedOnFlags(name, required = ExtensionMethod).exists

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -50,7 +50,6 @@ class CompilationTests extends ParallelTesting {
       compileFile("tests/pos-special/completeFromSource/Test2.scala", defaultOptions.and("-sourcepath", "tests/pos-special")),
       compileFile("tests/pos-special/completeFromSource/Test3.scala", defaultOptions.and("-sourcepath", "tests/pos-special", "-scansource")),
       compileFile("tests/pos-special/completeFromSource/nested/Test4.scala", defaultOptions.and("-sourcepath", "tests/pos-special", "-scansource")),
-      compileFile("tests/pos-special/repeatedArgs213.scala", defaultOptions.and("-Ynew-collections")),
       compileFilesInDir("tests/pos-special/fatal-warnings", defaultOptions.and("-Xfatal-warnings", "-feature")),
       compileList(
         "compileMixed",
@@ -174,7 +173,6 @@ class CompilationTests extends ParallelTesting {
       compileFile("tests/neg-custom-args/i3627.scala", allowDeepSubtypes),
       compileFile("tests/neg-custom-args/matchtype-loop.scala", allowDeepSubtypes),
       compileFile("tests/neg-custom-args/completeFromSource/nested/Test1.scala", defaultOptions.and("-sourcepath", "tests/neg-custom-args", "-scansource")),
-      compileFile("tests/neg-custom-args/repeatedArgs213.scala", defaultOptions.and("-Ynew-collections")),
       compileList("duplicate source", List(
         "tests/neg-custom-args/toplevel-samesource/S.scala",
         "tests/neg-custom-args/toplevel-samesource/nested/S.scala"),

project/Build.scala

@@ -58,7 +58,6 @@ object MyScalaJSPlugin extends AutoPlugin {
 }
 
 object Build {
-  val scalacVersion = "2.12.8"
   val referenceVersion = "0.17.0-RC1"
 
   val baseVersion = "0.18.0"
@@ -71,6 +70,16 @@ object Build {
   val publishedDottyVersion = referenceVersion
   val publishedSbtDottyVersion = "0.3.3"
 
+  /** scala-library version required to compile Dotty.
+   *
+   *  Both the non-bootstrapped and bootstrapped version should match, unless
+   *  we're in the process of upgrading to a new major version of
+   *  scala-library.
+   */
+  def stdlibVersion(implicit mode: Mode): String = mode match {
+    case NonBootstrapped => "2.12.8"
+    case Bootstrapped => "2.13.0"
+  }
 
   val dottyOrganization = "ch.epfl.lamp"
   val dottyGithubUrl = "https://github.com/lampepfl/dotty"
@@ -144,7 +153,8 @@ object Build {
       "-feature",
       "-deprecation",
       "-unchecked",
-      "-Xfatal-warnings",
+      // TODO: reenable once we've fixed all the deprecation warnings.
+      // "-Xfatal-warnings",
       "-encoding", "UTF8",
       "-language:existentials,higherKinds,implicitConversions"
     ),
@@ -256,8 +266,8 @@ object Build {
     // Enforce that the only Scala 2 classfiles we unpickle come from scala-library
     /*
     scalacOptions ++= {
-      val attList = (dependencyClasspath in `dotty-library` in Compile).value
-      val scalaLib = findLib(attList, "scala-library")
+      val cp = (dependencyClasspath in `dotty-library` in Compile).value
+      val scalaLib = findLib(cp, "scala-library")
       Seq("-Yscala2-unpickler", scalaLib)
     },
     */
@@ -268,10 +278,10 @@ object Build {
     // Compile using the non-bootstrapped and non-published dotty
     managedScalaInstance := false,
     scalaInstance := {
-      val externalDeps = externalDependencyClasspath.in(`dotty-doc`, Compile).value
+      val externalNonBootstrappedDeps = externalDependencyClasspath.in(`dotty-doc`, Compile).value
       def getExternalDep(name: String): File =
-        externalDeps.find(_.get(artifact.key).exists(_.name == name))
-          .getOrElse(throw new MessageOnlyException(s"Artifact for $name not found in $externalDeps"))
+        externalNonBootstrappedDeps.find(_.get(artifact.key).exists(_.name == name))
+          .getOrElse(throw new MessageOnlyException(s"Artifact for $name not found in $externalNonBootstrappedDeps"))
           .data
 
       val scalaLibrary = getExternalDep("scala-library")
@@ -286,7 +296,7 @@ object Build {
       val dottyCompiler = packageBin.in(`dotty-compiler`, Compile).value
       val dottyDoc = packageBin.in(`dotty-doc`, Compile).value
 
-      val allJars = Seq(dottyLibrary, dottyInterfaces, dottyCompiler, dottyDoc) ++ externalDeps.map(_.data)
+      val allJars = Seq(dottyLibrary, dottyInterfaces, dottyCompiler, dottyDoc) ++ externalNonBootstrappedDeps.map(_.data)
 
       makeScalaInstance(
         state.value,
@@ -322,6 +332,15 @@ object Build {
 
   /** Projects -------------------------------------------------------------- */
 
+  val dottyCompilerBootstrappedRef = LocalProject("dotty-compiler-bootstrapped")
+
+  /** External dependencies we may want to put on the compiler classpath. */
+  def externalCompilerClasspathTask: Def.Initialize[Task[Def.Classpath]] =
+    // Even if we're running the non-bootstrapped compiler, we want the
+    // dependencies of the bootstrapped compiler since we want to put them on
+    // the compiler classpath, not the JVM classpath.
+    externalDependencyClasspath.in(dottyCompilerBootstrappedRef, Runtime)
+
   // The root project:
   // - aggregates other projects so that "compile", "test", etc are run on all projects at once.
   // - publishes its own empty artifact "dotty" that depends on "dotty-library" and "dotty-compiler",
@@ -337,8 +356,8 @@ object Build {
     val dottyLib = jars("dotty-library")
     val dottyInterfaces = jars("dotty-interfaces")
     val otherDeps = (dependencyClasspath in Compile).value.map(_.data).mkString(File.pathSeparator)
-    val attList = (dependencyClasspath in Runtime).value
-    dottyLib + File.pathSeparator + findLib(attList, "scala-library-")
+    val externalDeps = externalCompilerClasspathTask.value
+    dottyLib + File.pathSeparator + findLib(externalDeps, "scala-library-")
   }
 
   def dottyDocSettings(implicit mode: Mode) = Seq(
@@ -407,7 +426,7 @@ object Build {
     case Bootstrapped => `dotty-doc-bootstrapped`
   }
 
-  def findLib(attList: Seq[Attributed[File]], name: String) = attList
+  def findLib(classpath: Def.Classpath, name: String) = classpath
     .map(_.data.getAbsolutePath)
     .find(_.contains(name))
     .toList.mkString(File.pathSeparator)
@@ -442,7 +461,6 @@ object Build {
       // get libraries onboard
       libraryDependencies ++= Seq(
         "org.scala-lang.modules" % "scala-asm" % "6.0.0-scala-1", // used by the backend
-        "org.scala-lang" % "scala-library" % scalacVersion % "test",
         Dependencies.`compiler-interface`,
         "org.jline" % "jline-reader" % "3.9.0",   // used by the REPL
         "org.jline" % "jline-terminal" % "3.9.0",
@@ -472,7 +490,7 @@ object Build {
       // http://grokbase.com/t/gg/simple-build-tool/135ke5y90p/sbt-setting-jvm-boot-paramaters-for-scala
       // packageAll should always be run before tests
       javaOptions ++= {
-        val attList = (dependencyClasspath in Runtime).value
+        val externalDeps = externalCompilerClasspathTask.value
         val jars = packageAll.value
 
         val ci_build = // propagate if this is a ci build
@@ -499,11 +517,11 @@ object Build {
           "-Ddotty.tests.classes.dottyInterfaces=" + jars("dotty-interfaces"),
           "-Ddotty.tests.classes.dottyLibrary=" + jars("dotty-library"),
           "-Ddotty.tests.classes.dottyCompiler=" + jars("dotty-compiler"),
-          "-Ddotty.tests.classes.compilerInterface=" + findLib(attList, "compiler-interface"),
-          "-Ddotty.tests.classes.scalaLibrary=" + findLib(attList, "scala-library-"),
-          "-Ddotty.tests.classes.scalaAsm=" + findLib(attList, "scala-asm"),
-          "-Ddotty.tests.classes.jlineTerminal=" + findLib(attList, "jline-terminal"),
-          "-Ddotty.tests.classes.jlineReader=" + findLib(attList, "jline-reader")
+          "-Ddotty.tests.classes.compilerInterface=" + findLib(externalDeps, "compiler-interface"),
+          "-Ddotty.tests.classes.scalaLibrary=" + findLib(externalDeps, "scala-library-"),
+          "-Ddotty.tests.classes.scalaAsm=" + findLib(externalDeps, "scala-asm"),
+          "-Ddotty.tests.classes.jlineTerminal=" + findLib(externalDeps, "jline-terminal"),
+          "-Ddotty.tests.classes.jlineReader=" + findLib(externalDeps, "jline-reader")
         )
 
         jarOpts ::: tuning ::: agentOptions ::: ci_build
@@ -541,10 +559,10 @@ object Build {
 
       dotr := {
         val args: List[String] = spaceDelimited("<arg>").parsed.toList
-        val attList = (dependencyClasspath in Runtime).value
+        val externalDeps = externalCompilerClasspathTask.value
         val jars = packageAll.value
 
-        val scalaLib = findLib(attList, "scala-library")
+        val scalaLib = findLib(externalDeps, "scala-library")
         val dottyLib = jars("dotty-library")
 
         def run(args: List[String]): Unit = {
@@ -558,7 +576,7 @@ object Build {
           println("Couldn't find scala-library on classpath, please run using script in bin dir instead")
         } else if (args.contains("-with-compiler")) {
           val args1 = args.filter(_ != "-with-compiler")
-          val asm = findLib(attList, "scala-asm")
+          val asm = findLib(externalDeps, "scala-asm")
           val dottyCompiler = jars("dotty-compiler")
           val dottyInterfaces = jars("dotty-interfaces")
           run(insertClasspathInArgs(args1, List(dottyCompiler, dottyInterfaces, asm).mkString(File.pathSeparator)))
@@ -577,7 +595,7 @@ object Build {
       ivyConfigurations += SourceDeps.hide,
       transitiveClassifiers := Seq("sources"),
       libraryDependencies +=
-        ("org.scala-js" %% "scalajs-ir" % scalaJSVersion % "sourcedeps").withDottyCompat(scalaVersion.value),
+        ("org.scala-js" % "scalajs-ir_2.13.0-RC2" % scalaJSVersion % "sourcedeps").withDottyCompat(scalaVersion.value),
       sourceGenerators in Compile += Def.task {
         val s = streams.value
         val cacheDir = s.cacheDirectory
@@ -606,9 +624,9 @@ object Build {
 
   def runCompilerMain(repl: Boolean = false) = Def.inputTaskDyn {
     val log = streams.value.log
-    val attList = (dependencyClasspath in Runtime).value
+    val externalDeps = externalCompilerClasspathTask.value
     val jars = packageAll.value
-    val scalaLib = findLib(attList, "scala-library-")
+    val scalaLib = findLib(externalDeps, "scala-library-")
     val dottyLib = jars("dotty-library")
     val dottyCompiler = jars("dotty-compiler")
     val args0: List[String] = spaceDelimited("<arg>").parsed.toList
@@ -634,7 +652,7 @@ object Build {
         log.error("-with-compiler should only be used with a bootstrapped compiler")
       }
       val dottyInterfaces = jars("dotty-interfaces")
-      val asm = findLib(attList, "scala-asm")
+      val asm = findLib(externalDeps, "scala-asm")
       extraClasspath ++= Seq(dottyCompiler, dottyInterfaces, asm)
     }
 
@@ -696,8 +714,6 @@ object Build {
 
   // Settings shared between dotty-library and dotty-library-bootstrapped
   lazy val dottyLibrarySettings = Seq(
-    libraryDependencies += "org.scala-lang" % "scala-library" % scalacVersion,
-
     // Needed so that the library sources are visible when `dotty.tools.dotc.core.Definitions#init` is called
     scalacOptions in Compile ++= Seq("-sourcepath", (scalaSource in Compile).value.getAbsolutePath),
 
@@ -767,7 +783,10 @@ object Build {
       scalaSource in Test := baseDirectory.value,
       javaSource  in Test := baseDirectory.value,
 
-      libraryDependencies += (Dependencies.`zinc-api-info` % Test).withDottyCompat(scalaVersion.value)
+      // Tests disabled until zinc-api-info cross-compiles with 2.13,
+      // alternatively we could just copy in sources the part of zinc-api-info we need.
+      sources in Test := Seq(),
+      // libraryDependencies += (Dependencies.`zinc-api-info` % Test).withDottyCompat(scalaVersion.value)
     )
 
   lazy val `dotty-language-server` = project.in(file("language-server")).
@@ -949,20 +968,20 @@ object Build {
       // depend on it via dotty-library, because sbt may rewrite dependencies
       // (see https://github.com/sbt/sbt/pull/2634), but won't rewrite the direct
       // dependencies of scala-library (see https://github.com/sbt/sbt/pull/2897)
-      libraryDependencies += "org.scala-lang" % "scala-library" % scalacVersion
+      libraryDependencies += "org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped)
     )
 
   lazy val `scala-compiler` = project.
     settings(commonDummySettings)
   lazy val `scala-reflect` = project.
     settings(commonDummySettings).
     settings(
-      libraryDependencies := Seq("org.scala-lang" % "scala-reflect" % scalacVersion)
+      libraryDependencies := Seq("org.scala-lang" % "scala-reflect" % stdlibVersion(Bootstrapped))
     )
   lazy val scalap = project.
     settings(commonDummySettings).
     settings(
-      libraryDependencies := Seq("org.scala-lang" % "scalap" % scalacVersion)
+      libraryDependencies := Seq("org.scala-lang" % "scalap" % stdlibVersion(Bootstrapped))
     )
 
 
@@ -1210,6 +1229,9 @@ object Build {
       settings(dottyCompilerSettings)
 
     def asDottyLibrary(implicit mode: Mode): Project = project.withCommonSettings.
+      settings(
+        libraryDependencies += "org.scala-lang" % "scala-library" % stdlibVersion
+      ).
       settings(dottyLibrarySettings)
 
     def asDottyDoc(implicit mode: Mode): Project = project.withCommonSettings.

project/plugins.sbt

@@ -5,7 +5,7 @@
 // Scala IDE project file generator
 addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.4")
 
-addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.0.0-M7")
+addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.0.0-M8")
 
 addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.1")
 

sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala

@@ -94,7 +94,7 @@ object DottyPlugin extends AutoPlugin {
         if (name != "dotty" && name != "dotty-library" && name != "dotty-compiler")
           moduleID.crossVersion match {
             case _: librarymanagement.Binary if scalaVersion.startsWith("0.") =>
-              moduleID.cross(CrossVersion.constant("2.12"))
+              moduleID.cross(CrossVersion.constant("2.13"))
             case _ =>
               moduleID
           }

tests/neg-custom-args/repeatedArgs213.scala renamed to tests/neg/repeatedArgs213.scala

@@ -1,7 +1,6 @@
 import scala.collection.{immutable, mutable}
 import java.nio.file.Paths
 
-// Start of Test
 class repeatedArgs {
   def bar(xs: String*): Int = xs.length