Make tests depend on environment variables for classpath jars

compiler/test/dotc/tests.scala

@@ -1,5 +1,6 @@
 package dotc
 
+import dotty.Jars
 import dotty.tools.dotc.CompilerTest
 import org.junit.{Before, Test}
 
@@ -32,18 +33,28 @@ class tests extends CompilerTest {
   )
 
   val classPath = {
-    val paths = List(
-      "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar",
-      "./target/scala-2.11/dotty-compiler_2.11-0.1-SNAPSHOT.jar",
-      "../interfaces/target/dotty-interfaces-0.1-SNAPSHOT.jar"
-    ).map { p =>
+    val paths = Jars.dottyTestDeps map { p =>
       val file = new JFile(p)
       assert(
         file.exists,
-        s"""File "$p" couldn't be found. Run `packageAll` from build tool before testing"""
+        s"""|File "$p" couldn't be found. Run `packageAll` from build tool before
+            |testing.
+            |
+            |If running without sbt, test paths need to be setup environment variables:
+            |
+            | - DOTTY_LIBRARY
+            | - DOTTY_COMPILER
+            | - DOTTY_INTERFACES
+            | - DOTTY_EXTRAS
+            |
+            |Where these all contain locations, except extras which is a comma
+            |separated list of jars.
+            |
+            |When compiling with eclipse, you need the sbt-interfaces jar, but
+            |it in extras."""
       )
       file.getAbsolutePath
-    }.mkString(":")
+    } mkString (":")
 
     List("-classpath", paths)
   }
@@ -338,10 +349,16 @@ class tests extends CompilerTest {
   @Test def tasty_tests = compileDir(testsDir, "tasty", testPickling)
 
   @Test def tasty_bootstrap = {
+    val f = new JFile(getClass.getProtectionDomain.getCodeSource.getLocation.getPath)
+    println(f)
+    println(System.getProperty("java.class.path"))
+
+
     val opt = List("-priorityclasspath", defaultOutputDir, "-Ylog-classpath")
     // first compile dotty
     compileDir(dottyDir, ".", List("-deep", "-Ycheck-reentrant", "-strict"))(allowDeepSubtypes)
 
+
     compileDir(dottyDir, "tools", opt)
     compileDir(toolsDir, "dotc", opt)
     compileDir(dotcDir, "ast", opt)

compiler/test/dotty/Jars.scala

@@ -0,0 +1,22 @@
+package dotty
+
+/** Jars used when compiling test, defaults to sbt locations */
+object Jars {
+  val dottyLib: String = sys.env.get("DOTTY_LIB") getOrElse {
+    "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar"
+  }
+
+  val dottyCompiler: String = sys.env.get("DOTTY_COMPILER") getOrElse {
+    "./target/scala-2.11/dotty-compiler_2.11-0.1-SNAPSHOT.jar"
+  }
+
+  val dottyInterfaces: String = sys.env.get("DOTTY_INTERFACE") getOrElse {
+    "../interfaces/target/dotty-interfaces-0.1-SNAPSHOT.jar"
+  }
+
+  val dottyExtras: List[String] = sys.env.get("DOTTY_EXTRAS")
+    .map(_.split(",").toList).getOrElse(Nil)
+
+  val dottyTestDeps: List[String] =
+    dottyLib :: dottyCompiler :: dottyInterfaces :: dottyExtras
+}

compiler/test/dotty/partest/DPConsoleRunner.scala

@@ -88,27 +88,43 @@ extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPat
     """.stripMargin
   }
 
-  /** Tests which are compiled with one or more of the flags in this list will be run
-   *  one by one, without any other test running at the same time.
-   *  This is necessary because some test flags require a lot of memory when running
-   *  the compiler and may exhaust the available memory when run in parallel with other tests.
-   */
-  def sequentialFlags = List("-Ytest-pickler")
+  /** Some tests require a limitation of resources, tests which are compiled
+    * with one or more of the flags in this list will be run with
+    * `limitedThreads`. This is necessary because some test flags require a lot
+    * of memory when running the compiler and may exhaust the available memory
+    * when run in parallel with too many other tests.
+    *
+    * This number could be increased on the CI, but might fail locally if
+    * scaled too extreme - override with:
+    *
+    * ```
+    * -Ddotty.tests.limitedThreads=X
+    * ```
+    */
+  def limitResourceFlags = List("-Ytest-pickler")
+  private val limitedThreads = sys.props.get("dotty.tests.limitedThreads").getOrElse("2")
 
   override def runTestsForFiles(kindFiles: Array[File], kind: String): Array[TestState] = {
-    val (sequentialTests, parallelTests) =
+    val (limitResourceTests, parallelTests) =
       kindFiles partition { kindFile =>
         val flags = kindFile.changeExtension("flags").fileContents
-        sequentialFlags.exists(seqFlag => flags.contains(seqFlag))
+        limitResourceFlags.exists(seqFlag => flags.contains(seqFlag))
       }
 
     val seqResults =
-      if (!sequentialTests.isEmpty) {
+      if (!limitResourceTests.isEmpty) {
         val savedThreads = sys.props("partest.threads")
-        sys.props("partest.threads") = "2"
+        sys.props("partest.threads") = {
+          assert(
+            savedThreads == null || limitedThreads.toInt <= savedThreads.toInt,
+            """|Should not use more threads than the default, when the point
+               |is to limit the amount of resources""".stripMargin
+          )
+          limitedThreads
+        }
 
-        NestUI.echo(s"## we will run ${sequentialTests.length} tests using ${PartestDefaults.numThreads} thread(s)")
-        val res = super.runTestsForFiles(sequentialTests, kind)
+        NestUI.echo(s"## we will run ${limitResourceTests.length} tests using ${PartestDefaults.numThreads} thread(s) in parallel")
+        val res = super.runTestsForFiles(limitResourceTests, kind)
 
         if (savedThreads != null)
           sys.props("partest.threads") = savedThreads
@@ -383,13 +399,9 @@ class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runn
     suiteRunner.fileManager.asInstanceOf[DottyFileManager].extraJarList ::: super.extraClasspath
 
   // override to keep class files if failed and delete clog if ok
-  override def cleanup = if (lastState.isOk) try {
+  override def cleanup = if (lastState.isOk) {
     logFile.delete
     cLogFile.delete
     Directory(outDir).deleteRecursively
-  } catch {
-    case t: Throwable =>
-      println("whhhhhhhhhhhhhhhhhhhhhhhhhhhaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaat")
-      throw t
   }
 }

compiler/test/dotty/tools/DottyTest.scala

@@ -23,10 +23,7 @@ class DottyTest extends ContextEscapeDetection{
     import base.settings._
     val ctx = base.initialCtx.fresh
     ctx.setSetting(ctx.settings.encoding, "UTF8")
-    ctx.setSetting(
-      ctx.settings.classpath,
-      "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar"
-    )
+    ctx.setSetting(ctx.settings.classpath, Jars.dottyLib)
     // when classpath is changed in ctx, we need to re-initialize to get the
     // correct classpath from PathResolver
     base.initialize()(ctx)

compiler/test/dotty/tools/ShowClassTests.scala

@@ -1,4 +1,5 @@
-package dotty.tools
+package dotty
+package tools
 
 import dotc.core._
 import dotc.core.Contexts._
@@ -18,8 +19,7 @@ class ShowClassTests extends DottyTest {
     ctx.setSetting(ctx.settings.encoding, "UTF8")
     ctx.setSetting(
       ctx.settings.classpath,
-      "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar" +
-      ":../interfaces/target/dotty-interfaces-0.1-SNAPSHOT.jar"
+      Jars.dottyLib + ":" + Jars.dottyInterfaces
     )
     base.initialize()(ctx)
     ctx

compiler/test/dotty/tools/dotc/EntryPointsTest.scala

@@ -1,4 +1,5 @@
-package dotty.tools
+package dotty
+package tools
 package dotc
 
 import org.junit.Test
@@ -19,14 +20,7 @@ import scala.collection.mutable.ListBuffer
 class EntryPointsTest {
   private val sources =
     List("../tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
-  private val dottyInterfaces =
-    new java.io.File("../interfaces/dotty-interfaces-0.1-SNAPSHOT.jar").getPath
-  private val dottyLibrary =
-    new java.io.File("../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar").getPath
-  private val args =
-    sources ++
-    List("-d", "../out/") ++
-    List("-classpath", dottyInterfaces + ":" + dottyLibrary)
+  private val args = sources ++ List("-d", "../out/", "-usejavacp")
 
   @Test def runCompiler = {
     val reporter = new CustomReporter

compiler/test/dotty/tools/dotc/InterfaceEntryPointTest.scala

@@ -1,4 +1,5 @@
-package dotty.tools.dotc
+package dotty
+package tools.dotc
 
 import org.junit.Test
 import org.junit.Assert._
@@ -20,15 +21,7 @@ class InterfaceEntryPointTest {
   @Test def runCompilerFromInterface = {
     val sources =
       List("../tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
-    val dottyInterfaces =
-      new java.io.File("../interfaces/dotty-interfaces-0.1-SNAPSHOT.jar").getPath
-    val dottyLibrary =
-      new java.io.File("../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar").getPath
-
-    val args =
-      sources ++
-      List("-d", "../out/") ++
-      List("-classpath", dottyInterfaces + ":" + dottyLibrary)
+    val args = sources ++ List("-d", "../out/", "-usejavacp")
 
     val mainClass = Class.forName("dotty.tools.dotc.Main")
     val process = mainClass.getMethod("process",

compiler/test/dotty/tools/dotc/repl/TestREPL.scala

@@ -1,4 +1,5 @@
-package dotty.tools.dotc
+package dotty
+package tools.dotc
 package repl
 
 import core.Contexts.Context
@@ -23,10 +24,7 @@ class TestREPL(script: String) extends REPL {
     override def context(ctx: Context) = {
       val fresh = ctx.fresh
       fresh.setSetting(ctx.settings.color, "never")
-      fresh.setSetting(
-        ctx.settings.classpath,
-        "../library/target/scala-2.11/dotty-library_2.11-0.1-SNAPSHOT.jar"
-      )
+      fresh.setSetting(ctx.settings.classpath, Jars.dottyLib)
       fresh.initialize()(fresh)
       fresh
     }