Merge pull request #1636 from dotty-staging/topic/restructure-attempt2

Restructuring repo and issues

Author: smarter

.gitignore

@@ -3,6 +3,7 @@
 *.log
 *.swp
 *~
+tags
 
 # sbt specific
 dist/*
@@ -31,6 +32,7 @@ classes/
 
 # Partest
 dotty.jar
+dotty-lib.jar
 tests/partest-generated/
 tests/locks/
 /test-classes/

bench/src/test/scala/Benchmarks.scala renamed to bench/test/dotty/tools/benchmarks/Benchmarks.scala

@@ -1,12 +1,11 @@
 package dotty.tools.benchmarks
 
-
 import org.scalameter.Key.reports._
 import org.scalameter.PerformanceTest.OnlineRegressionReport
 import org.scalameter.api._
 import org.scalameter.{Context, History, currentContext, persistence}
 import org.scalameter.reporting.RegressionReporter.Tester
-import test.CompilerTest
+import dotty.tools.dotc.CompilerTest
 
 import scala.io.Source
 
@@ -46,13 +45,14 @@ object BenchTests extends OnlineRegressionReport {
   implicit val defaultOptions = List("-d", outputDir)
   val scala2mode = List("-language:Scala2")
 
-  val dottyDir  = "./src/dotty/"
+  val dottyDir  = "../compiler/src/dotty/"
 
-  val stdlibFiles = Source.fromFile("./test/dotc/scala-collections.whitelist", "UTF8").getLines()
+  val stdlibFiles = Source.fromFile("../compiler/test/dotc/scala-collections.whitelist", "UTF8").getLines()
     .map(_.trim) // allow identation
     .filter(!_.startsWith("#")) // allow comment lines prefixed by #
     .map(_.takeWhile(_ != '#').trim) // allow comments in the end of line
     .filter(_.nonEmpty)
+    .map("." + _)
     .toList
 
   def stdLib = compiler.compileList("compileStdLib", stdlibFiles, "-migration" :: scala2mode)

bin/common

@@ -58,7 +58,7 @@ function build_jar {
 
     local sedjar="$3"
     if [ "$sedjar" == "" ]; then
-        sedjar="/.*\.jar/p"
+        sedjar="/tests/d; /javadoc/d; /.*\.jar/p"
     fi
 
     if [ "$jar" == "" ]; then
@@ -74,6 +74,7 @@ function build_jar {
 function update_packages {
     echo "$INTERFACES_JAR" >  $DOTTY_ROOT/.packages
     echo "$MAIN_JAR"       >> $DOTTY_ROOT/.packages
+    echo "$DOTTY_LIB_JAR"  >> $DOTTY_ROOT/.packages
     echo "$TEST_JAR"       >> $DOTTY_ROOT/.packages
 }
 
@@ -84,12 +85,16 @@ function build_all {
     INTERFACES_JAR=$(build_jar dotty-interfaces/package interfaces/target)
     printf "done\n"
 
-    printf "Building dotty..."
-    MAIN_JAR=$(build_jar package "target/scala-$SCALA_BINARY_VERSION")
+    printf "Building dotty-compiler..."
+    MAIN_JAR=$(build_jar dotty-compiler/package "compiler/target/scala-$SCALA_BINARY_VERSION")
+    printf "done\n"
+
+    printf "Building dotty library..."
+    DOTTY_LIB_JAR=$(build_jar dotty-library/package "library/target/scala-$SCALA_BINARY_VERSION")
     printf "done\n"
 
     printf "Building tests..."
-    TEST_JAR=$(build_jar test:package "target/scala-$SCALA_BINARY_VERSION" '/dotty.*-tests\.jar/p')
+    TEST_JAR=$(build_jar test:package "compiler/target/scala-$SCALA_BINARY_VERSION" '/dotty.*-tests\.jar/p')
     printf "done\n"
 
     update_packages
@@ -101,10 +106,11 @@ if  [ ! -f "$DOTTY_ROOT/.packages" ]; then
 else
     IFS=$'\r\n' GLOBIGNORE='*' command eval 'JARS=($(cat $DOTTY_ROOT/.packages))'
 
-    if [ "${#JARS[@]}" == "3" ]; then
+    if [ "${#JARS[@]}" == "4" ]; then
         INTERFACES_JAR="${JARS[0]}"
         MAIN_JAR="${JARS[1]}"
-        TEST_JAR="${JARS[2]}"
+        DOTTY_LIB_JAR="${JARS[2]}"
+        TEST_JAR="${JARS[3]}"
     else
         echo "Failed to parse .packages file"
         build_all
@@ -118,15 +124,17 @@ function check_jar {
     local new_files="$(find "$DOTTY_ROOT/$3" \( -iname "*.scala" -o -iname "*.java" \) -newer "$2")"
     if [ ! -z "$new_files" ]; then
         printf "New files detected in $1, rebuilding..."
+        rm $2
         eval "$4"
         printf "done\n"
         update_packages
     fi
 }
 
-check_jar "dotty-interfaces" $INTERFACES_JAR "interfaces" 'INTERFACES_JAR=$(build_jar dotty-interfaces/package interfaces/target)'
-check_jar "dotty"            $MAIN_JAR       "src"        'MAIN_JAR=$(build_jar package target/scala-$SCALA_BINARY_VERSION)'
-check_jar "dotty-tests"      $TEST_JAR       "test"       'TEST_JAR=$(build_jar test:package target/scala-$SCALA_BINARY_VERSION /dotty.*-tests\.jar/p)'
+check_jar "dotty-interfaces" $INTERFACES_JAR "interfaces/src" 'INTERFACES_JAR=$(build_jar dotty-interfaces/package interfaces/target)'
+check_jar "dotty-compiler"   $MAIN_JAR       "compiler/src"   'MAIN_JAR=$(build_jar dotty-compiler/package compiler/target/scala-$SCALA_BINARY_VERSION)'
+check_jar "dotty-library"    $DOTTY_LIB_JAR  "library/src"    'DOTTY_LIB_JAR=$(build_jar dotty-library/package library/target/scala-$SCALA_BINARY_VERSION)'
+check_jar "dotty-tests"      $TEST_JAR       "compiler/test"  'TEST_JAR=$(build_jar dotty-compiler/test:package compiler/target/scala-$SCALA_BINARY_VERSION /dotty.*-tests\.jar/p)'
 
 # Autodetecting the scala-library location, in case it wasn't provided by an environment variable
 if [ "$SCALA_LIBRARY_JAR" == "" ]; then

bin/dotc

@@ -115,12 +115,12 @@ trap onExit INT
 classpathArgs () {
   if [[ "true" == $bootstrapped ]]; then
     check_jar "dotty-bootstrapped" $DOTTY_JAR "target" 'build_jar "test:runMain dotc.build" target' &> /dev/null
-    toolchain="$DOTTY_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR"
+    toolchain="$DOTTY_JAR:$DOTTY_LIB_JAR:$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR"
   else
-    toolchain="$SCALA_LIBRARY_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR"
+    toolchain="$SCALA_LIBRARY_JAR:$DOTTY_LIB_JAR:$SCALA_REFLECT_JAR:$SCALA_COMPILER_JAR:$SBT_INTERFACE_JAR"
   fi
-  bcpJars="$INTERFACES_JAR:$MAIN_JAR"
-  cpJars="$INTERFACES_JAR:$MAIN_JAR:$TEST_JAR"
+  bcpJars="$INTERFACES_JAR:$MAIN_JAR:$DOTTY_LIB_JAR"
+  cpJars="$INTERFACES_JAR:$MAIN_JAR:$DOTTY_LIB_JAR:$TEST_JAR"
 
   if [[ -n "$cygwin" ]]; then
     if [[ "$OS" = "Windows_NT" ]] && cygpath -m .>/dev/null 2>/dev/null ; then

bin/dotr

@@ -11,8 +11,7 @@ DOTTY_ROOT="$( cd "$DOTTY_ROOT" >& /dev/null && pwd )/.."  # absolute
 # Load common functions and variables
 source $DOTTY_ROOT/bin/common
 
-# CLASS_PATH is derived from the DOTTY_ROOT and SCALA_LIBRARY_JAR
-CLASS_PATH="-Xbootclasspath/a:.:$DOTTY_ROOT/target/scala-$SCALA_BINARY_VERSION/classes/:.:$SCALA_LIBRARY_JAR"
+CLASS_PATH="-classpath .:$DOTTY_LIB_JAR:.:$SCALA_LIBRARY_JAR"
 
 function runMain {
     local jbin=$(which "java")

test/scripts/TestDotc.scala renamed to bin/test/TestScripts.scala

@@ -7,7 +7,7 @@ import scala.io.Source
 import scala.sys.process.{Process, ProcessLogger}
 import java.io.{File => JFile, FileNotFoundException}
 
-class TestDotc {
+class TestScripts {
   private val lineSep = util.Properties.lineSeparator
   private def doUnlessWindows(op: => Unit) =
     if (!System.getProperty("os.name").toLowerCase.contains("windows"))
@@ -28,11 +28,13 @@ class TestDotc {
     }
 
     try {
-      for (jar <- Source.fromFile(".packages").getLines())
+      for (jar <- Source.fromFile("./.packages").getLines())
         delete(jar)
 
-      delete(".packages")
-      delete("src/dotty/tools/dotc/Dummy.scala")
+      delete("./.packages")
+      delete("./compiler/src/dotty/tools/dotc/Dummy.scala")
+      delete("./HelloWorld.class")
+      delete("./HelloWorld$.class")
     } catch {
       case _: FileNotFoundException => ()
     }
@@ -45,15 +47,15 @@ class TestDotc {
    *  execute it using dotr
    */
   @Test def buildAndRunHelloWorld = doUnlessWindows {
-    val (retDotc, dotcOutput) = executeScript("bin/dotc tests/pos/HelloWorld.scala")
+    val (retDotc, dotcOutput) = executeScript("./bin/dotc ./tests/pos/HelloWorld.scala")
 
     // Check correct output of building and running dotc
     assert(
       retDotc == 0,
       s"bin/dotc script did not run properly. Output:$lineSep$dotcOutput"
     )
 
-    val (retDotr, dotrOutput) = executeScript("bin/dotr HelloWorld")
+    val (retDotr, dotrOutput) = executeScript("./bin/dotr HelloWorld")
     assert(
       retDotr == 0 && dotrOutput == "hello world",
       s"Running hello world exited with status: $retDotr and output: $dotrOutput"
@@ -64,24 +66,24 @@ class TestDotc {
    *  rebuild dotty if needed
    */
   @Test def rebuildIfNecessary = doUnlessWindows {
-    val (retFirstBuild, _) = executeScript("bin/dotc tests/pos/HelloWorld.scala")
-    assert(retFirstBuild == 0, "building dotc failed")
+    val (retFirstBuild, out1) = executeScript("./bin/dotc ./tests/pos/HelloWorld.scala")
+    assert(retFirstBuild == 0, s"building dotc failed: $out1")
 
-    // Create a new file
-    new JFile("src/dotty/tools/dotc/Dummy.scala").createNewFile()
+    // Create a new file to force rebuild
+    new JFile("./compiler/src/dotty/tools/dotc/Dummy.scala").createNewFile()
 
-    val (retSecondBuild, output) = executeScript("bin/dotc tests/pos/HelloWorld.scala")
+    val (retSecondBuild, output) = executeScript("./bin/dotc ./tests/pos/HelloWorld.scala")
     assert(
       retSecondBuild == 0 && output.contains("rebuilding"),
       s"Rebuilding the tool should result in jar files being rebuilt. Status: $retSecondBuild, output:$lineSep$output")
   }
 
   /** if no changes to dotty, dotc script should be fast */
   @Test def beFastOnNoChanges = doUnlessWindows {
-    val (retFirstBuild, _) = executeScript("bin/dotc tests/pos/HelloWorld.scala")
+    val (retFirstBuild, _) = executeScript("./bin/dotc ./tests/pos/HelloWorld.scala")
     assert(retFirstBuild == 0, "building dotc failed")
 
-    val (ret, output) = executeScript("bin/dotc tests/pos/HelloWorld.scala")
+    val (ret, output) = executeScript("./bin/dotc ./tests/pos/HelloWorld.scala")
     assert(
       ret == 0 && !output.contains("rebuilding"),
       s"Project recompiled when it didn't need to be. Status $ret, output:$lineSep$output")

src/dotty/tools/backend/sjs/GenSJSIR.scala renamed to compiler/sjs/backend/sjs/GenSJSIR.scala

@@ -17,7 +17,6 @@ import core.DenotTransformers.DenotTransformer
 import core.Denotations.SingleDenotation
 
 import dotty.tools.backend.jvm.{LabelDefs, GenBCode, CollectSuperCalls}
-import dotty.tools.backend.sjs.GenSJSIR
 
 /** The central class of the dotc compiler. The job of a compiler is to create
  *  runs, which process given `phases` in a given `rootContext`.
@@ -98,7 +97,6 @@ class Compiler {
            new DropInlined,         // Drop Inlined nodes, since backend has no use for them
            new MoveStatics,         // Move static methods to companion classes
            new LabelDefs),          // Converts calls to labels to jumps
-      List(new GenSJSIR),           // Generate .js code
       List(new GenBCode)            // Generate JVM bytecode
     )
 
@@ -117,20 +115,7 @@ class Compiler {
    */
   def rootContext(implicit ctx: Context): Context = {
     ctx.initialize()(ctx)
-    val actualPhases = if (ctx.settings.scalajs.value) {
-      // Remove phases that Scala.js does not want
-      phases.mapConserve(_.filter {
-        case _: FunctionalInterfaces => false
-        case _ => true
-      }).filter(_.nonEmpty)
-    } else {
-      // Remove Scala.js-related phases
-      phases.mapConserve(_.filter {
-        case _: GenSJSIR => false
-        case _ => true
-      }).filter(_.nonEmpty)
-    }
-    ctx.setPhasePlan(actualPhases)
+    ctx.setPhasePlan(phases)
     val rootScope = new MutableScope
     val bootstrap = ctx.fresh
       .setPeriod(Period(nextRunId, FirstPhaseId))

src/dotty/tools/backend/sjs/JSCodeGen.scala renamed to compiler/sjs/backend/sjs/JSCodeGen.scala

@@ -179,7 +179,7 @@ object desugar {
           DefDef(
             name = meth.name.defaultGetterName(n),
             tparams = meth.tparams.map(tparam => dropContextBound(toDefParam(tparam))),
-            vparamss = takeUpTo(normalizedVparamss, n),
+            vparamss = takeUpTo(normalizedVparamss.nestedMap(toDefParam), n),
             tpt = TypeTree(),
             rhs = vparam.rhs
           ).withMods(Modifiers(mods.flags & AccessFlags, mods.privateWithin))

src/dotty/tools/backend/sjs/JSDefinitions.scala renamed to compiler/sjs/backend/sjs/JSDefinitions.scala

@@ -464,7 +464,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
         case tree: Select if qualifier.tpe eq tree.qualifier.tpe =>
           tree1.withTypeUnchecked(tree.tpe)
         case _ => tree.tpe match {
-          case tpe: NamedType => tree1.withType(tpe.derivedSelect(qualifier.tpe))
+          case tpe: NamedType => tree1.withType(tpe.derivedSelect(qualifier.tpe.widenIfUnstable))
           case _ => tree1.withTypeUnchecked(tree.tpe)
         }
       }

src/dotty/tools/backend/sjs/JSEncoding.scala renamed to compiler/sjs/backend/sjs/JSEncoding.scala

@@ -16,7 +16,9 @@ class JavaPlatform extends Platform {
   def classPath(implicit ctx: Context): ClassPath = {
     if (currentClassPath.isEmpty)
       currentClassPath = Some(new PathResolver().result)
-    currentClassPath.get
+    val cp = currentClassPath.get
+    //println(cp)
+    cp
   }
 
   // The given symbol is a method with the right name and signature to be a runnable java program.

src/dotty/tools/backend/sjs/JSInterop.scala renamed to compiler/sjs/backend/sjs/JSInterop.scala

@@ -20,7 +20,7 @@ object PathResolver {
   // security exceptions.
   import AccessControl._
 
-  def firstNonEmpty(xs: String*)            = xs find (_ != "") getOrElse ""
+  def firstNonEmpty(xs: String*) = xs find (_ != "") getOrElse ""
 
   /** Map all classpath elements to absolute paths and reconstruct the classpath.
     */
@@ -47,6 +47,7 @@ object PathResolver {
     def sourcePathEnv       =  envOrElse("SOURCEPATH", "")
 
     def javaBootClassPath   = propOrElse("sun.boot.class.path", searchForBootClasspath)
+
     def javaExtDirs         = propOrEmpty("java.ext.dirs")
     def scalaHome           = propOrEmpty("scala.home")
     def scalaExtDirs        = propOrEmpty("scala.ext.dirs")
@@ -55,20 +56,14 @@ object PathResolver {
     def javaUserClassPath   = propOrElse("java.class.path", "")
     def useJavaClassPath    = propOrFalse("scala.usejavacp")
 
-    override def toString = """
+    override def toString = s"""
       |object Environment {
-      |  scalaHome          = %s (useJavaClassPath = %s)
-      |  javaBootClassPath  = <%d chars>
-      |  javaExtDirs        = %s
-      |  javaUserClassPath  = %s
-      |  scalaExtDirs       = %s
-      |}""".trim.stripMargin.format(
-        scalaHome, useJavaClassPath,
-        javaBootClassPath.length,
-        ppcp(javaExtDirs),
-        ppcp(javaUserClassPath),
-        ppcp(scalaExtDirs)
-      )
+      |  scalaHome          = $scalaHome (useJavaClassPath = $useJavaClassPath)
+      |  javaBootClassPath  = <${javaBootClassPath.length} chars>
+      |  javaExtDirs        = ${ppcp(javaExtDirs)}
+      |  javaUserClassPath  = ${ppcp(javaUserClassPath)}
+      |  scalaExtDirs       = ${ppcp(scalaExtDirs)}
+      |}""".trim.stripMargin
   }
 
   /** Default values based on those in Environment as interpreted according
@@ -194,7 +189,7 @@ class PathResolver(implicit ctx: Context) {
     def javaUserClassPath   = if (useJavaClassPath) Defaults.javaUserClassPath else ""
     def scalaBootClassPath  = cmdLineOrElse("bootclasspath", Defaults.scalaBootClassPath)
     def scalaExtDirs        = cmdLineOrElse("extdirs", Defaults.scalaExtDirs)
-    def priorityClassPath   = cmdLineOrElse("prioritypath", "")
+    def priorityClassPath   = cmdLineOrElse("priorityclasspath", "")
     /** Scaladoc doesn't need any bootstrapping, otherwise will create errors such as:
      * [scaladoc] ../scala-trunk/src/reflect/scala/reflect/macros/Reifiers.scala:89: error: object api is not a member of package reflect
      * [scaladoc] case class ReificationException(val pos: reflect.api.PositionApi, val msg: String) extends Throwable(msg)
@@ -213,11 +208,11 @@ class PathResolver(implicit ctx: Context) {
      *  - Otherwise, if CLASSPATH is set, it is that
      *  - If neither of those, then "." is used.
      */
-    def userClassPath = (
+    def userClassPath = {
       if (!settings.classpath.isDefault)
         settings.classpath.value
       else sys.env.getOrElse("CLASSPATH", ".")
-    )
+    }
 
     import context._
 
@@ -243,7 +238,7 @@ class PathResolver(implicit ctx: Context) {
       |  javaBootClassPath    = %s
       |  javaExtDirs          = %s
       |  javaUserClassPath    = %s
-      |    useJavaClassPath   = %s
+      |  useJavaClassPath     = %s
       |  scalaBootClassPath   = %s
       |  scalaExtDirs         = %s
       |  userClassPath        = %s
@@ -260,7 +255,15 @@ class PathResolver(implicit ctx: Context) {
   def containers = Calculated.containers
 
   lazy val result: JavaClassPath = {
-    val cp = new JavaClassPath(containers.toIndexedSeq, context)
+    // Prioritize `dotty.jar` and `dotty-lib.jar` to shadow others
+    val (dottyJars, others) =
+      containers.partition(x => x.name.contains("dotty-lib.jar") || x.name.contains("dotty.jar"))
+    // Then any jars with `dotty` in the name - putting them before scala-library
+    val (dottyCp, remaining) =
+      others.partition(_.name.contains("dotty-"))
+
+    val cp = new JavaClassPath((dottyJars ++ dottyCp ++ remaining).toIndexedSeq, context)
+
     if (settings.Ylogcp.value) {
       Console.println("Classpath built from " + settings.toConciseString(ctx.sstate))
       Console.println("Defaults: " + PathResolver.Defaults)

src/dotty/tools/backend/sjs/JSPositions.scala renamed to compiler/sjs/backend/sjs/JSPositions.scala

@@ -42,7 +42,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val usejavacp = BooleanSetting("-usejavacp", "Utilize the java.class.path in classpath resolution.")
   val verbose = BooleanSetting("-verbose", "Output messages about what the compiler is doing.")
   val version = BooleanSetting("-version", "Print product version and exit.")
-  val pageWidth = IntSetting("-pagewidth", "Set page width", 120)
+  val pageWidth = IntSetting("-pagewidth", "Set page width", 80)
 
   val jvmargs = PrefixSetting("-J<flag>", "-J", "Pass <flag> directly to the runtime system.")
   val defines = PrefixSetting("-Dproperty=value", "-D", "Pass -Dproperty=value directly to the runtime system.")