From ca3fc43bb3f9eb0068669acf6e64b3c6bd310511 Mon Sep 17 00:00:00 2001 From: vsalvis Date: Sat, 9 May 2015 18:35:04 +0200 Subject: [PATCH 1/4] Run tests for partest --- project/Build.scala | 44 +++++++++---- test/dotc/tests.scala | 8 ++- test/dotty/partest/DPConsoleRunner.scala | 39 ++++++++++-- test/partest | 12 ++++ test/test/CompilerTest.scala | 81 +++++++++++++++++------- tests/run/hello.check | 1 + tests/run/hello.scala | 3 + tests/run/lazyVals.check | 1 + tests/run/lazyVals.scala | 8 +++ 9 files changed, 156 insertions(+), 41 deletions(-) create mode 100755 test/partest create mode 100644 tests/run/hello.check create mode 100644 tests/run/hello.scala create mode 100644 tests/run/lazyVals.check create mode 100644 tests/run/lazyVals.scala diff --git a/project/Build.scala b/project/Build.scala index 3f8a9b5225ec..af8c51211e71 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -2,6 +2,7 @@ import sbt.Keys._ import sbt._ import java.io.{ RandomAccessFile, File } import java.nio.channels.FileLock +import scala.reflect.io.Path object DottyBuild extends Build { @@ -42,16 +43,16 @@ object DottyBuild extends Build { // to get Scala 2.11 resolvers += Resolver.sonatypeRepo("releases"), - // get reflect and xml onboard - libraryDependencies ++= Seq("org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.scala-lang.modules" %% "scala-xml" % "1.0.1", - "me.d-d" % "scala-compiler" % "2.11.5-20150506-175515-8fc7635b56", + // get libraries onboard + partestDeps := Seq("me.d-d" % "scala-compiler" % "2.11.5-20150506-175515-8fc7635b56", + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "org.scala-lang" % "scala-library" % scalaVersion.value % "test"), + libraryDependencies ++= partestDeps.value, + libraryDependencies ++= Seq("org.scala-lang.modules" %% "scala-xml" % "1.0.1", "org.scala-lang.modules" %% "scala-partest" % "1.0.5" % "test", + "com.novocode" % "junit-interface" % "0.11" % "test", "jline" % "jline" % "2.12"), - // get junit onboard - libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test", - // scalac options scalacOptions in Global ++= Seq("-feature", "-deprecation", "-language:_"), @@ -67,9 +68,18 @@ object DottyBuild extends Build { // otherwise it just executes the tests directly lockPartestFile := { val partestLockFile = "." + File.separator + "tests" + File.separator + "partest.lock" - partestLock = new RandomAccessFile(partestLockFile, "rw").getChannel.tryLock + try { + partestLock = new RandomAccessFile(partestLockFile, "rw").getChannel.tryLock + } catch { + case ex: java.nio.channels.OverlappingFileLockException => // locked already + } + }, + runPartestRunner <<= Def.taskDyn { + val jars = Seq((packageBin in Compile).value.getAbsolutePath) ++ + getJarPaths(partestDeps.value, ivyPaths.value.ivyHome) + val dottyJars = "-dottyJars " + jars.length + " " + jars.mkString(" ") + runTask(Test, "dotty.partest.DPConsoleRunner", dottyJars) }, - runPartestRunner <<= runTask(Test, "dotty.partest.DPConsoleRunner", "") dependsOn (test in Test), // Adjust classpath for running dotty mainClass in (Compile, run) := Some("dotty.tools.dotc.Main"), @@ -103,7 +113,7 @@ object DottyBuild extends Build { tuning ::: agentOptions ::: travis_build ::: fullpath } - ) ++ addCommandAlias("partest", ";lockPartestFile;runPartestRunner") + ) ++ addCommandAlias("partest", ";test:compile;lockPartestFile;test:test;runPartestRunner") lazy val dotty = Project(id = "dotty", base = file("."), settings = defaults) @@ -156,5 +166,17 @@ object DottyBuild extends Build { lazy val lockPartestFile = TaskKey[Unit]("lockPartestFile", "Creates the file lock on ./tests/partest.lock") lazy val runPartestRunner = TaskKey[Unit]("runPartestRunner", "Runs partests") - + lazy val partestDeps = SettingKey[Seq[ModuleID]]("partestDeps", "Finds jars for partest dependencies") + + def getJarPaths(modules: Seq[ModuleID], ivyHome: Option[File]): Seq[String] = ivyHome match { + case Some(home) => + modules.map({ module => + val file = Path(home) / Path("cache") / + Path(module.organization) / Path(module.name) / Path("jars") / + Path(module.name + "-" + module.revision + ".jar") + if (!file.isFile) throw new RuntimeException("ERROR: sbt getJarPaths: dependency jar not found: " + file) + else file.jfile.getAbsolutePath + }) + case None => throw new RuntimeException("ERROR: sbt getJarPaths: ivyHome not defined") + } } diff --git a/test/dotc/tests.scala b/test/dotc/tests.scala index 61ebae609c44..04ea1c3efc15 100644 --- a/test/dotc/tests.scala +++ b/test/dotc/tests.scala @@ -35,6 +35,7 @@ class tests extends CompilerTest { val posDir = testsDir + "pos/" val posSpecialDir = testsDir + "pos-special/" val negDir = testsDir + "neg/" + val runDir = testsDir + "run/" val newDir = testsDir + "new/" val sourceDir = "./src/" @@ -136,7 +137,12 @@ class tests extends CompilerTest { @Test def neg_instantiateAbstract = compileFile(negDir, "instantiateAbstract", xerrors = 8) @Test def neg_selfInheritance = compileFile(negDir, "selfInheritance", xerrors = 5) - @Test def dotty = compileDir(toolsDir, "", "-deep" :: allowDeepSubtypes ++ twice) // note the -deep argument + @Test def run_hello = runFile(runDir, "hello") + @Test def run_lazyVals = runFile(runDir, "lazyVals") + + + @Test def dotty = compileDir(dottyDir, "tools", "-deep" :: allowDeepSubtypes ++ twice) // note the -deep argument + @Test def dotc_ast = compileDir(dotcDir, "ast") @Test def dotc_config = compileDir(dotcDir, "config") diff --git a/test/dotty/partest/DPConsoleRunner.scala b/test/dotty/partest/DPConsoleRunner.scala index d21b5e98353a..c17bf7e8cc47 100644 --- a/test/dotty/partest/DPConsoleRunner.scala +++ b/test/dotty/partest/DPConsoleRunner.scala @@ -4,26 +4,41 @@ package dotty.partest +import scala.reflect.io.AbstractFile import scala.tools.partest._ import scala.tools.partest.nest._ +import scala.util.matching.Regex +import tools.nsc.io.{ File => NSCFile } import java.io.File import java.net.URLClassLoader /** Runs dotty partest from the Console, discovering test sources in * DPConfig.testRoot that have been generated automatically by - * DPPrepJUnitRunner. Use `sbt test` to run. + * DPPrepJUnitRunner. Use `sbt partest` to run. */ object DPConsoleRunner { def main(args: Array[String]): Unit = { - new DPConsoleRunner(args mkString (" ")).runPartest + // unfortunately sbt runTask passes args as single string + val jarFinder = """-dottyJars (\d*) (.*)""".r + val (jarList, otherArgs) = args.toList.partition(jarFinder.findFirstIn(_).isDefined) + val extraJars = jarList match { + case Nil => sys.error("Error: DPConsoleRunner needs \"-dottyJars *\".") + case jarFinder(nr, jarString) :: Nil => + val jars = jarString.split(" ").toList + if (jars.length.toString != nr) + sys.error("Error: DPConsoleRunner found wrong number of dottyJars: " + jars + ", expected: " + nr + ". Make sure the path doesn't contain any spaces.") + else jars + case list => sys.error("Error: DPConsoleRunner found several -dottyJars options: " + list) + } + new DPConsoleRunner(otherArgs mkString (" "), extraJars).runPartest } } // console runner has a suite runner which creates a test runner for each test -class DPConsoleRunner(args: String) extends ConsoleRunner(args) { +class DPConsoleRunner(args: String, extraJars: List[String]) extends ConsoleRunner(args) { override val suiteRunner = new DPSuiteRunner ( testSourcePath = optSourcePath getOrElse DPConfig.testRoot, - fileManager = null, // new FileManager(ClassPath split PathResolver.Environment.javaUserClassPath map (Path(_))), // the script sets up our classpath for us via ant + fileManager = new DottyFileManager(extraJars), updateCheck = optUpdateCheck, failed = optFailed) @@ -31,8 +46,15 @@ class DPConsoleRunner(args: String) extends ConsoleRunner(args) { def runPartest = super.run } +class DottyFileManager(extraJars: List[String]) extends FileManager(Nil) { + lazy val extraJarList = extraJars.map(NSCFile(_)) + override lazy val libraryUnderTest = Path(extraJars.find(_.contains("scala-library")).getOrElse("")) + override lazy val reflectUnderTest = Path(extraJars.find(_.contains("scala-reflect")).getOrElse("")) + override lazy val compilerUnderTest = Path(extraJars.find(_.contains("dotty")).getOrElse("")) +} + class DPSuiteRunner(testSourcePath: String, // relative path, like "files", or "pending" - fileManager: FileManager, + fileManager: DottyFileManager, updateCheck: Boolean, failed: Boolean, javaCmdPath: String = PartestDefaults.javaCmd, @@ -83,7 +105,7 @@ extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPat // but it doesn't seem to be used anywhere } -class DPTestRunner(testFile: File, suiteRunner: SuiteRunner) extends nest.Runner(testFile, suiteRunner) { +class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runner(testFile, suiteRunner) { // override to provide DottyCompiler override def newCompiler = new dotty.partest.DPDirectCompiler(this) @@ -117,7 +139,6 @@ class DPTestRunner(testFile: File, suiteRunner: SuiteRunner) extends nest.Runner case class CompSucceeded() extends NegTestState def nerrIsOk(reason: String) = { - import scala.util.matching.Regex val nerrFinder = """compilation failed with (\d+) errors""".r reason match { case nerrFinder(found) => @@ -187,4 +208,8 @@ class DPTestRunner(testFile: File, suiteRunner: SuiteRunner) extends nest.Runner def description = s"dotc $fsString" lazy val result = { pushTranscript(description) ; attemptCompile(fs) } } + + // override to add dotty and scala jars to classpath + override def extraClasspath = suiteRunner.fileManager.asInstanceOf[DottyFileManager].extraJarList ::: super.extraClasspath + } diff --git a/test/partest b/test/partest new file mode 100755 index 000000000000..5794e2e5793e --- /dev/null +++ b/test/partest @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# partest error message references partest script to update check files, but +# doesn't work for dotty because we don't know where tests came from. + +if [ $1='--update-check' ]; +then + echo """ERROR: Since dotty partest runs on generated files, please update the check +files in the original location (run tests) or update the expected error count +(neg tests) in the test file." +else + echo "This script doesn't launch partest, please use sbt partest instead." +fi \ No newline at end of file diff --git a/test/test/CompilerTest.scala b/test/test/CompilerTest.scala index f404c79ba872..be88e8c8e662 100644 --- a/test/test/CompilerTest.scala +++ b/test/test/CompilerTest.scala @@ -5,7 +5,7 @@ import dotty.tools.dotc.{Main, Bench, Driver} import dotty.tools.dotc.reporting.Reporter import scala.collection.mutable.ListBuffer import scala.reflect.io.{ Path, Directory, File => SFile } -import scala.tools.partest.nest.FileManager +import scala.tools.partest.nest.{ FileManager, NestUI } import java.io.{ RandomAccessFile, File => JFile } import org.junit.Test @@ -19,15 +19,19 @@ import org.junit.Test * generated sources. * * Through overriding the partestableXX methods, tests can always be run as - * JUnit compiler tests. + * JUnit compiler tests. Run tests cannot be run by JUnit, only by partest. * - * A test can either be a file or a directory. The test is in a parent - * directory that determines the kind of test: + * A test can either be a file or a directory. Partest will generate a + * -.log file with output of failed tests. Partest reads compiler + * flags and the number of errors expected from a neg test from .flags + * and .nerr files (also generated). The test is in a parent directory + * that determines the kind of test: * - pos: checks that compilation succeeds * - neg: checks that compilation fails with the given number of errors - * (- run: compilation succeeds and running generates the given output.) - * For partest, compiler flags and the number of errors expected from a neg - * test are read from test.flags and test.nerr files (also generated). + * - run: compilation succeeds, partest: test run generates the output in + * .check. Run tests always need to be: + * object Test { def main(args: Array[String]): Unit = ... } + * */ abstract class CompilerTest extends DottyTest { @@ -60,8 +64,11 @@ abstract class CompilerTest extends DottyTest { CompilerTest.init /** Always run with JUnit. */ - def compileLine(cmdLine: String, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = + def compileLine(cmdLine: String, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = { + if (generatePartestFiles) + NestUI.echoWarning("WARNING: compileLine will always run with JUnit, no partest files generated.") compileArgs(cmdLine.split("\n"), xerrors) + } /** Compiles the given code file. * @@ -73,12 +80,15 @@ abstract class CompilerTest extends DottyTest { * @param extension the file extension, .scala by default * @param defaultOptions more arguments to the compiler */ - def compileFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0, extension: String = ".scala") + def compileFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0, + extension: String = ".scala", runTest: Boolean = false) (implicit defaultOptions: List[String]): Unit = { if (!generatePartestFiles || !partestableFile(prefix, fileName, extension, args ++ defaultOptions, xerrors)) { + if (runTest) + NestUI.echoWarning(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$fileName$extension") compileArgs((s"$prefix$fileName$extension" :: args).toArray, xerrors) } else { - val kind = testKind(xerrors) + val kind = testKind(prefix, xerrors, runTest) println(s"generating partest files for test file: $prefix$fileName$extension of kind $kind") val sourceFile = new JFile(prefix + fileName + extension) @@ -90,11 +100,17 @@ abstract class CompilerTest extends DottyTest { } } } + def runFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0, + extension: String = ".scala")(implicit defaultOptions: List[String]): Unit = + compileFile(prefix, fileName, args, xerrors, extension, true) /** Compiles the code files in the given directory together. If args starts * with "-deep", all files in subdirectories (and so on) are included. */ - def compileDir(prefix: String, dirName: String, args: List[String] = Nil, xerrors: Int = 0)(implicit defaultOptions: List[String]): Unit = { + def compileDir(prefix: String, dirName: String, args: List[String] = Nil, xerrors: Int = 0, runTest: Boolean = false) + (implicit defaultOptions: List[String]): Unit = { if (!generatePartestFiles || !partestableDir(prefix, dirName, args ++ defaultOptions, xerrors)) { + if (runTest) + NestUI.echoWarning(s"WARNING: run tests can only be run by partest, JUnit just verifies compilation: $prefix$dirName") val dir = Directory(prefix + dirName) val (files, normArgs) = args match { case "-deep" :: args1 => (dir.deepFiles, args1) @@ -107,18 +123,24 @@ abstract class CompilerTest extends DottyTest { case "-deep" :: args1 => (flattenDir(prefix, dirName), args1 ++ defaultOptions, "deep") case _ => (new JFile(prefix + dirName), args ++ defaultOptions, "shallow") } - val kind = testKind(xerrors) + val kind = testKind(prefix, xerrors, runTest) println(s"generating partest files for test directory ($deep): $prefix$dirName of kind $kind") if (sourceDir.exists) { val firstDest = Directory(DPConfig.testRoot + JFile.separator + kind + JFile.separator + dirName) - computeDestAndCopyFiles(sourceDir, firstDest, kind, args ++ defaultOptions, xerrors.toString) - if (deep == "deep") sourceDir.delete + computeDestAndCopyFiles(sourceDir, firstDest, kind, flags, xerrors.toString) + if (deep == "deep") { + sourceDir.listFiles.foreach(_.delete) + sourceDir.delete + } } else { throw new java.io.FileNotFoundException(s"Unable to locate test dir $prefix$dirName") } } } + def runDir(prefix: String, dirName: String, args: List[String] = Nil, xerrors: Int = 0) + (implicit defaultOptions: List[String]): Unit = + compileDir(prefix, dirName, args, xerrors, true) /** Compiles each source in the directory path separately by calling * compileFile resp. compileDir. */ @@ -161,10 +183,18 @@ abstract class CompilerTest extends DottyTest { } // In particular, don't copy flags from scalac tests - private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java") + private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java", "check") /** Determines what kind of test to run. */ - private def testKind(xerrors: Int) = if (xerrors > 0) "neg" else "pos" + private def testKind(prefixDir: String, xerrors: Int, runTest: Boolean) = { + if (runTest) "run" + else if (xerrors > 0) "neg" + else if (prefixDir.endsWith("run" + JFile.separator)) { + NestUI.echoWarning("WARNING: test is being run as pos test despite being in a run directory. " + + "Use runFile/runDir instead of compileFile/compileDir to do a run test") + "pos" + } else "pos" + } /** The three possibilities: no generated sources exist yet, the same sources * exist already, different sources exist. */ @@ -184,7 +214,7 @@ abstract class CompilerTest extends DottyTest { val flags = oldFlags.map(f => if (f == oldOutput) partestOutput else f) getExisting(dest).isDifferent(source, flags, nerr) match { - case NotExists => copyFiles(source, dest, partestOutput, flags, nerr) + case NotExists => copyFiles(source, dest, partestOutput, flags, nerr, kind) case ExistsSame => // nothing else to do case ExistsDifferent => val nextDest = dest.parent / (dest match { @@ -195,8 +225,8 @@ abstract class CompilerTest extends DottyTest { } } - /** Copies the test sources and creates flags, nerr and output files. */ - private def copyFiles(sourceFile: Path, dest: Path, partestOutput: String, flags: List[String], nerr: String) = { + /** Copies the test sources. Creates flags, nerr, check and output files. */ + private def copyFiles(sourceFile: Path, dest: Path, partestOutput: String, flags: List[String], nerr: String, kind: String) = { recCopyFiles(sourceFile, dest) new JFile(partestOutput).mkdirs @@ -205,17 +235,24 @@ abstract class CompilerTest extends DottyTest { dest.changeExtension("flags").createFile(true).writeAll(flags.mkString(" ")) if (nerr != "0") dest.changeExtension("nerr").createFile(true).writeAll(nerr) + sourceFile.changeExtension("check").ifFile({ check => + if (kind == "run") + FileManager.copyFile(check.jfile, dest.changeExtension("check").jfile) + else + NestUI.echoWarning(s"WARNING: ignoring $check for test kind $kind") + }) + } - /** Recursively copy over files and directories, excluding extensions that - * aren't in extensionsToCopy. */ + /** Recursively copy over source files and directories, excluding extensions + * that aren't in extensionsToCopy. */ private def recCopyFiles(sourceFile: Path, dest: Path): Unit = { processFileDir(sourceFile, { sf => if (extensionsToCopy.contains(sf.extension)) { dest.parent.jfile.mkdirs FileManager.copyFile(sourceFile.jfile, dest.jfile) } else { - println(s"warning: ignoring $sf") + NestUI.echoWarning(s"WARNING: ignoring $sf") } }, { sdir => dest.jfile.mkdirs diff --git a/tests/run/hello.check b/tests/run/hello.check new file mode 100644 index 000000000000..6c70cd9e8165 --- /dev/null +++ b/tests/run/hello.check @@ -0,0 +1 @@ +hello dotty! diff --git a/tests/run/hello.scala b/tests/run/hello.scala new file mode 100644 index 000000000000..9224cec054d3 --- /dev/null +++ b/tests/run/hello.scala @@ -0,0 +1,3 @@ +object Test { + def main(args: Array[String]): Unit = println("hello dotty!") +} diff --git a/tests/run/lazyVals.check b/tests/run/lazyVals.check new file mode 100644 index 000000000000..f70d7bba4ae1 --- /dev/null +++ b/tests/run/lazyVals.check @@ -0,0 +1 @@ +42 \ No newline at end of file diff --git a/tests/run/lazyVals.scala b/tests/run/lazyVals.scala new file mode 100644 index 000000000000..db91dbcd6ef2 --- /dev/null +++ b/tests/run/lazyVals.scala @@ -0,0 +1,8 @@ +object Test { + def foo = { + lazy val s = 42 // needs LazyIntHolder + s + } + + def main(args: Array[String]): Unit = println(foo) +} From 7e3595d1f7efa34ef3ea2409529643cd8864e926 Mon Sep 17 00:00:00 2001 From: vsalvis Date: Mon, 11 May 2015 13:45:49 +0200 Subject: [PATCH 2/4] Better documentation for partest dottyJar option --- project/Build.scala | 16 +++++++++++++--- test/dotty/partest/DPConsoleRunner.scala | 4 +++- test/test/CompilerTest.scala | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index af8c51211e71..7d5d59ec5f96 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -64,20 +64,30 @@ object DottyBuild extends Build { // enable verbose exception messages for JUnit testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector"), testOptions in Test += Tests.Cleanup({ () => if (partestLock != null) partestLock.release }), - // when this file is locked, running test generates the files for partest - // otherwise it just executes the tests directly + // When this file is locked, running test generates the files for partest. + // Otherwise it just executes the tests directly. So running two separate + // sbt instances might result in unexpected behavior if one does partest + // and the other test, since the second still sees the locked file and thus + // generates partest files instead of running JUnit tests, but doesn't + // partest them. lockPartestFile := { val partestLockFile = "." + File.separator + "tests" + File.separator + "partest.lock" try { partestLock = new RandomAccessFile(partestLockFile, "rw").getChannel.tryLock + if (partestLock == null) + throw new RuntimeException("ERROR: sbt partest: file is locked already. Bad things happen when trying to mix test/partest in two concurrent sbt instances.") } catch { - case ex: java.nio.channels.OverlappingFileLockException => // locked already + case ex: java.nio.channels.OverlappingFileLockException => // locked already, Tests.Cleanup didn't run + if (partestLock != null) + partestLock.release + throw new RuntimeException("ERROR: sbt partest: file was still locked, please try again or restart sbt.") } }, runPartestRunner <<= Def.taskDyn { val jars = Seq((packageBin in Compile).value.getAbsolutePath) ++ getJarPaths(partestDeps.value, ivyPaths.value.ivyHome) val dottyJars = "-dottyJars " + jars.length + " " + jars.mkString(" ") + // Provide the jars required on the classpath of run tests runTask(Test, "dotty.partest.DPConsoleRunner", dottyJars) }, diff --git a/test/dotty/partest/DPConsoleRunner.scala b/test/dotty/partest/DPConsoleRunner.scala index c17bf7e8cc47..31b8467d347c 100644 --- a/test/dotty/partest/DPConsoleRunner.scala +++ b/test/dotty/partest/DPConsoleRunner.scala @@ -14,11 +14,13 @@ import java.net.URLClassLoader /** Runs dotty partest from the Console, discovering test sources in * DPConfig.testRoot that have been generated automatically by - * DPPrepJUnitRunner. Use `sbt partest` to run. + * DPPrepJUnitRunner. Use `sbt partest` to run. If additional jars are + * required by some run tests, add them to partestDeps in the sbt Build.scala. */ object DPConsoleRunner { def main(args: Array[String]): Unit = { // unfortunately sbt runTask passes args as single string + // extra jars for run tests are passed with -dottyJars ... val jarFinder = """-dottyJars (\d*) (.*)""".r val (jarList, otherArgs) = args.toList.partition(jarFinder.findFirstIn(_).isDefined) val extraJars = jarList match { diff --git a/test/test/CompilerTest.scala b/test/test/CompilerTest.scala index be88e8c8e662..d710a26d858c 100644 --- a/test/test/CompilerTest.scala +++ b/test/test/CompilerTest.scala @@ -31,7 +31,7 @@ import org.junit.Test * - run: compilation succeeds, partest: test run generates the output in * .check. Run tests always need to be: * object Test { def main(args: Array[String]): Unit = ... } - * + * Classpath jars can be added to partestDeps in the sbt Build.scala. */ abstract class CompilerTest extends DottyTest { From ef22b1d0f878e97a7602dabd1a90ba871bf1fccb Mon Sep 17 00:00:00 2001 From: vsalvis Date: Tue, 12 May 2015 11:15:29 +0200 Subject: [PATCH 3/4] More robust partest/test switching for concurrent sbt instances --- .gitignore | 1 + project/Build.scala | 81 +++++++++++++++++------------------- test/test/CompilerTest.scala | 32 +++++++------- tests/partest.lock | 0 4 files changed, 56 insertions(+), 58 deletions(-) delete mode 100644 tests/partest.lock diff --git a/.gitignore b/.gitignore index 69d86527a270..95d95ca1158a 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ classes/ # Partest tests/partest-generated/ +tests/locks/ /test-classes/ diff --git a/project/Build.scala b/project/Build.scala index 7d5d59ec5f96..d21b4c807154 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -17,8 +17,6 @@ object DottyBuild extends Build { // "-XX:+HeapDumpOnOutOfMemoryError", "-Xmx1g", "-Xss2m" ) - var partestLock: FileLock = null - val defaults = Defaults.defaultSettings ++ Seq( scalaVersion in Global := "2.11.5", version in Global := "0.1-SNAPSHOT", @@ -63,25 +61,18 @@ object DottyBuild extends Build { // enable verbose exception messages for JUnit testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector"), - testOptions in Test += Tests.Cleanup({ () => if (partestLock != null) partestLock.release }), - // When this file is locked, running test generates the files for partest. - // Otherwise it just executes the tests directly. So running two separate - // sbt instances might result in unexpected behavior if one does partest - // and the other test, since the second still sees the locked file and thus - // generates partest files instead of running JUnit tests, but doesn't - // partest them. + testOptions in Test += Tests.Cleanup({ () => partestLockFile.delete }), + lockPartestFile := { - val partestLockFile = "." + File.separator + "tests" + File.separator + "partest.lock" - try { - partestLock = new RandomAccessFile(partestLockFile, "rw").getChannel.tryLock - if (partestLock == null) - throw new RuntimeException("ERROR: sbt partest: file is locked already. Bad things happen when trying to mix test/partest in two concurrent sbt instances.") - } catch { - case ex: java.nio.channels.OverlappingFileLockException => // locked already, Tests.Cleanup didn't run - if (partestLock != null) - partestLock.release - throw new RuntimeException("ERROR: sbt partest: file was still locked, please try again or restart sbt.") - } + // When this file is present, running `test` generates the files for + // partest. Otherwise it just executes the tests directly. + val lockDir = partestLockFile.getParentFile + lockDir.mkdirs + // Cannot have concurrent partests as they write to the same directory. + if (lockDir.list.size > 0) + throw new RuntimeException("ERROR: sbt partest: another partest is already running, pid in lock file: " + lockDir.list.toList.mkString(" ")) + partestLockFile.createNewFile + partestLockFile.deleteOnExit }, runPartestRunner <<= Def.taskDyn { val jars = Seq((packageBin in Compile).value.getAbsolutePath) ++ @@ -99,29 +90,29 @@ object DottyBuild extends Build { // http://grokbase.com/t/gg/simple-build-tool/135ke5y90p/sbt-setting-jvm-boot-paramaters-for-scala javaOptions <++= (managedClasspath in Runtime, packageBin in Compile) map { (attList, bin) => - // put the Scala {library, reflect} in the classpath - val path = for { - file <- attList.map(_.data) - path = file.getAbsolutePath - } yield "-Xbootclasspath/p:" + path - // dotty itself needs to be in the bootclasspath - val fullpath = ("-Xbootclasspath/a:" + bin) :: path.toList - // System.err.println("BOOTPATH: " + fullpath) - - val travis_build = // propagate if this is a travis build - if (sys.props.isDefinedAt(TRAVIS_BUILD)) - List(s"-D$TRAVIS_BUILD=${sys.props(TRAVIS_BUILD)}") ::: travisMemLimit - else - List() - - val tuning = - if (sys.props.isDefinedAt("Oshort")) - // Optimize for short-running applications, see https://github.com/lampepfl/dotty/issues/222 - List("-XX:+TieredCompilation", "-XX:TieredStopAtLevel=1") + // put the Scala {library, reflect} in the classpath + val path = for { + file <- attList.map(_.data) + path = file.getAbsolutePath + } yield "-Xbootclasspath/p:" + path + // dotty itself needs to be in the bootclasspath + val fullpath = ("-Xbootclasspath/a:" + bin) :: path.toList + // System.err.println("BOOTPATH: " + fullpath) + + val travis_build = // propagate if this is a travis build + if (sys.props.isDefinedAt(TRAVIS_BUILD)) + List(s"-D$TRAVIS_BUILD=${sys.props(TRAVIS_BUILD)}") ::: travisMemLimit + else + List() + + val tuning = + if (sys.props.isDefinedAt("Oshort")) + // Optimize for short-running applications, see https://github.com/lampepfl/dotty/issues/222 + List("-XX:+TieredCompilation", "-XX:TieredStopAtLevel=1") else List() - tuning ::: agentOptions ::: travis_build ::: fullpath + ("-DpartestParentID=" + pid) :: tuning ::: agentOptions ::: travis_build ::: fullpath } ) ++ addCommandAlias("partest", ";test:compile;lockPartestFile;test:test;runPartestRunner") @@ -174,10 +165,14 @@ object DottyBuild extends Build { lazy val benchmarks = Project(id = "dotty-bench", settings = benchmarkSettings, base = file("bench")) dependsOn(dotty % "compile->test") - lazy val lockPartestFile = TaskKey[Unit]("lockPartestFile", "Creates the file lock on ./tests/partest.lock") - lazy val runPartestRunner = TaskKey[Unit]("runPartestRunner", "Runs partests") - lazy val partestDeps = SettingKey[Seq[ModuleID]]("partestDeps", "Finds jars for partest dependencies") + // Partest tasks + lazy val lockPartestFile = TaskKey[Unit]("lockPartestFile", "Creates the lock file at ./tests/locks/partest-.lock") + val partestLockFile = new File("." + File.separator + "tests" + File.separator + "locks" + File.separator + s"partest-$pid.lock") + val pid = java.lang.Long.parseLong(java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split("@")(0)) + lazy val runPartestRunner = TaskKey[Unit]("runPartestRunner", "Runs partest") + + lazy val partestDeps = SettingKey[Seq[ModuleID]]("partestDeps", "Finds jars for partest dependencies") def getJarPaths(modules: Seq[ModuleID], ivyHome: Option[File]): Seq[String] = ivyHome match { case Some(home) => modules.map({ module => diff --git a/test/test/CompilerTest.scala b/test/test/CompilerTest.scala index d710a26d858c..0e5c9a6b47fe 100644 --- a/test/test/CompilerTest.scala +++ b/test/test/CompilerTest.scala @@ -14,9 +14,9 @@ import org.junit.Test /** This class has two modes: it can directly run compiler tests, or it can * generate the necessary file structure for partest in the directory * DPConfig.testRoot. Both modes are regular JUnit tests. Which mode is used - * depends on the existence of the tests/runPartest.flag file which is created - * by sbt to trigger partest generation. Sbt can then run partest on the - * generated sources. + * depends on the existence of the tests/locks/partest-ppid.lock file which is + * created by sbt to trigger partest generation. Sbt will then run partest on + * the generated sources. * * Through overriding the partestableXX methods, tests can always be run as * JUnit compiler tests. Run tests cannot be run by JUnit, only by partest. @@ -45,18 +45,20 @@ abstract class CompilerTest extends DottyTest { def partestableList(testName: String, files: List[String], args: List[String], xerrors: Int) = true val generatePartestFiles = { - val partestLockFile = "." + JFile.separator + "tests" + JFile.separator + "partest.lock" - try { - val partestLock = new RandomAccessFile(partestLockFile, "rw").getChannel.tryLock - if (partestLock != null) { // file not locked by sbt -> don't generate partest - partestLock.release - false - } else true - } catch { - // if sbt doesn't fork in Test, the tryLock request will throw instead of - // returning null, because locks are per JVM, not per thread - case ex: java.nio.channels.OverlappingFileLockException => true - } + /* Because we fork in test, the JVM in which this JUnit test runs has a + * different pid from the one that started the partest. But the forked VM + * receives the pid of the parent as system property. If the lock file + * exists, the parent is requesting partest generation. This mechanism + * allows one sbt instance to run test (JUnit only) and another partest. + * We cannot run two instances of partest at the same time, because they're + * writing to the same directories. The sbt lock file generation prevents + * this. + */ + val pid = System.getProperty("partestParentID") + if (pid == null) + false + else + new JFile("." + JFile.separator + "tests" + JFile.separator + "locks" + JFile.separator + s"partest-$pid.lock").exists } // Delete generated files from previous run diff --git a/tests/partest.lock b/tests/partest.lock deleted file mode 100644 index e69de29bb2d1..000000000000 From f64762b150f7d3510ae783ab632b0aa849fae3a1 Mon Sep 17 00:00:00 2001 From: vsalvis Date: Tue, 12 May 2015 11:48:08 +0200 Subject: [PATCH 4/4] Removing pickle tests because directory doesn't exist anymore. --- project/Build.scala | 4 ++-- test/dotc/tests.scala | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index d21b4c807154..504185e65f6c 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -167,8 +167,8 @@ object DottyBuild extends Build { // Partest tasks lazy val lockPartestFile = TaskKey[Unit]("lockPartestFile", "Creates the lock file at ./tests/locks/partest-.lock") - val partestLockFile = new File("." + File.separator + "tests" + File.separator + "locks" + File.separator + s"partest-$pid.lock") - val pid = java.lang.Long.parseLong(java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split("@")(0)) + lazy val partestLockFile = new File("." + File.separator + "tests" + File.separator + "locks" + File.separator + s"partest-$pid.lock") + def pid = java.lang.Long.parseLong(java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split("@")(0)) lazy val runPartestRunner = TaskKey[Unit]("runPartestRunner", "Runs partest") diff --git a/test/dotc/tests.scala b/test/dotc/tests.scala index 04ea1c3efc15..88d076fd3b61 100644 --- a/test/dotc/tests.scala +++ b/test/dotc/tests.scala @@ -45,7 +45,8 @@ class tests extends CompilerTest { val coreDir = dotcDir + "core/" @Test def pickle_pickleOK = compileDir(testsDir, "pickling", testPickling) - @Test def pickle_pickling = compileDir(coreDir, "pickling", testPickling) +// This directory doesn't exist anymore +// @Test def pickle_pickling = compileDir(coreDir, "pickling", testPickling) @Test def pickle_ast = compileDir(dotcDir, "ast", testPickling) //@Test def pickle_core = compileDir(dotcDir, "core", testPickling, xerrors = 2) // two spurious comparison errors in Types and TypeOps @@ -148,8 +149,8 @@ class tests extends CompilerTest { @Test def dotc_config = compileDir(dotcDir, "config") @Test def dotc_core = compileDir(dotcDir, "core")("-Yno-double-bindings" :: allowDeepSubtypes)// twice omitted to make tests run faster - - @Test def dotc_core_pickling = compileDir(coreDir, "pickling")(allowDeepSubtypes)// twice omitted to make tests run faster +// This directory doesn't exist anymore +// @Test def dotc_core_pickling = compileDir(coreDir, "pickling")(allowDeepSubtypes)// twice omitted to make tests run faster @Test def dotc_transform = compileDir(dotcDir, "transform")// twice omitted to make tests run faster