diff --git a/.gitmodules b/.gitmodules index 4982e6d0ba8a..fde8547ad49b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -59,3 +59,6 @@ [submodule "community-build/community-projects/utest"] path = community-build/community-projects/utest url = https://github.com/dotty-staging/utest.git +[submodule "community-build/community-projects/os-lib"] + path = community-build/community-projects/os-lib + url = https://github.com/dotty-staging/os-lib.git diff --git a/community-build/community-projects/os-lib b/community-build/community-projects/os-lib new file mode 160000 index 000000000000..b29d68ffee5a --- /dev/null +++ b/community-build/community-projects/os-lib @@ -0,0 +1 @@ +Subproject commit b29d68ffee5a6e0aedbfdd048d10059f250134e4 diff --git a/community-build/community-projects/sourcecode b/community-build/community-projects/sourcecode index 9d8a4b1b6a96..49c20bd80cb6 160000 --- a/community-build/community-projects/sourcecode +++ b/community-build/community-projects/sourcecode @@ -1 +1 @@ -Subproject commit 9d8a4b1b6a96d76bf287ce43aa2446d691b115e0 +Subproject commit 49c20bd80cb61a53c0b11e80b9946bbe513a4c0f diff --git a/community-build/community-projects/utest b/community-build/community-projects/utest index 0fe508a5be98..4953683a23fe 160000 --- a/community-build/community-projects/utest +++ b/community-build/community-projects/utest @@ -1 +1 @@ -Subproject commit 0fe508a5be98854c9ac1fd317b62434cb24dfbe5 +Subproject commit 4953683a23fe8f60f8f25f061a7b3095f8ddbeac diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index 31ea2d291017..2881b6ef65f1 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -7,207 +7,223 @@ import org.junit.{Ignore, Test} import org.junit.Assert.{assertEquals, fail} import org.junit.experimental.categories.Category -@Category(Array(classOf[TestCategory])) -class CommunityBuildTest { - lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir")) - lazy val compilerVersion: String = { - val file = communitybuildDir.resolve("dotty-bootstrapped.version") - new String(Files.readAllBytes(file), UTF_8) - } +lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir")) - def testSbt(project: String, testCommand: String, updateCommand: String, extraSbtArgs: Seq[String] = Nil) = { - // Workaround for https://github.com/sbt/sbt/issues/4395 - new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs() - val pluginFilePath = communitybuildDir.resolve("sbt-dotty-sbt").toAbsolutePath().toString() +lazy val compilerVersion: String = + val file = communitybuildDir.resolve("dotty-bootstrapped.version") + new String(Files.readAllBytes(file), UTF_8) - // Run the sbt command with the compiler version and sbt plugin set in the build - val arguments = { - val sbtProps = Option(System.getProperty("sbt.ivy.home")) match { - case Some(ivyHome) => - Seq(s"-Dsbt.ivy.home=$ivyHome") - case _ => - Seq() - } - extraSbtArgs ++ sbtProps ++ Seq( - "-sbt-version", "1.2.7", - s"--addPluginSbtFile=$pluginFilePath", - s";clean ;set updateOptions in Global ~= (_.withLatestSnapshots(false)) ;++$compilerVersion! $testCommand" - ) - } +lazy val sbtPluginFilePath: String = + // Workaround for https://github.com/sbt/sbt/issues/4395 + new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs() + communitybuildDir.resolve("sbt-dotty-sbt").toAbsolutePath().toString() - test(project, "sbt", arguments) - } +def log(msg: String) = println(Console.GREEN + msg + Console.RESET) - def testMill(project: String, testCommand: String, extraMillArgs: Seq[String] = Nil) = - test(project, "./mill", extraMillArgs :+ testCommand) +/** Executes shell command, returns false in case of error. */ +def exec(projectDir: Path, binary: String, arguments: String*): Int = + val command = binary +: arguments + log(command.mkString(" ")) + val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO() + val process = builder.start() + val exitCode = process.waitFor() + exitCode - /** Build the given project with the published local compiler and sbt plugin. - * - * This test reads the compiler version from community-build/dotty-bootstrapped.version - * and expects community-build/sbt-dotty-sbt to set the compiler plugin. - * - * @param project The project name, should be a git submodule in community-build/ - * @param command The sbt command used to test the project - * @param updateCommand The sbt command used to update the project - * @param extraSbtArgs Extra arguments to pass to sbt - */ - def test(project: String, command: String, arguments: Seq[String]): Unit = { - def log(msg: String) = println(Console.GREEN + msg + Console.RESET) - log(s"Building $project with dotty-bootstrapped $compilerVersion...") +sealed trait CommunityProject + private var published = false - val projectDir = communitybuildDir.resolve("community-projects").resolve(project) + val project: String + val updateCommand: String + val testCommand: String + val publishCommand: String + val dependencies: List[CommunityProject] + val binaryName: String + val runCommandsArgs: List[String] = Nil - if (!Files.exists(projectDir.resolve(".git"))) { - fail(s""" - | - |Missing $project submodule. You can initialize this module using - | - | git submodule update --init community-build/community-projects/$project - | - |""".stripMargin) - } + final val projectDir = communitybuildDir.resolve("community-projects").resolve(project) - /** Executes shell command, returns false in case of error. */ - def exec(binary: String, arguments: String*): Int = { - val command = binary +: arguments - log(command.mkString(" ")) - val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO() - val process = builder.start() - val exitCode = process.waitFor() - exitCode - } + /** Is this project running in the test or update mode in the + * context of the given suite? @see `run` for more details. + */ + final def isUpdateMode(given suite: CommunityBuildTest) = + suite.isInstanceOf[CommunityBuildUpdate] + + /** Depending on the mode of operation, either + * runs the test or updates the project. Updating + * means that all the dependencies are fetched but + * minimal other extra other work is done. Updating + * is necessary since we run tests each time on a fresh + * Docker container. We run the update on Docker container + * creation time to create the cache of the dependencies + * and avoid network overhead. See https://github.com/lampepfl/dotty-drone + * for more infrastructural details. + */ + final def run()(given suite: CommunityBuildTest) = + val runCmd = if isUpdateMode then updateCommand else testCommand + if !isUpdateMode then dependencies.foreach(_.publish()) + suite.test(project, binaryName, runCommandsArgs :+ runCmd) + + /** Publish this project to the local Maven repository */ + final def publish(): Unit = + if !published + log(s"Publishing $project") + if publishCommand eq null + throw RuntimeException(s"Publish command is not specified for $project. Project details:\n$this") + val exitCode = exec(projectDir, binaryName, (runCommandsArgs :+ publishCommand): _*) + if exitCode != 0 + throw RuntimeException(s"Publish command exited with code $exitCode for project $project. Project details:\n$this") + published = true +end CommunityProject + +final case class MillCommunityProject(project: String, baseCommand: String, + dependencies: List[CommunityProject] = Nil) extends CommunityProject + override val binaryName: String = "./mill" + override val updateCommand = s"$baseCommand.compileClasspath" + override val testCommand = s"$baseCommand.test" + override val publishCommand = s"$baseCommand.publishLocal" + override val runCommandsArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + +final case class SbtCommunityProject(project: String, sbtTestCommand: String, + sbtUpdateCommand: String, extraSbtArgs: List[String] = Nil, + dependencies: List[CommunityProject] = Nil) extends CommunityProject + override val binaryName: String = "sbt" + private val baseCommand = s";clean ;set updateOptions in Global ~= (_.withLatestSnapshots(false)) ;++$compilerVersion! " + override val publishCommand = null + override val testCommand = s"$baseCommand$sbtTestCommand" + override val updateCommand = s"$baseCommand$sbtUpdateCommand" + + override val runCommandsArgs: List[String] = + // Run the sbt command with the compiler version and sbt plugin set in the build + val sbtProps = Option(System.getProperty("sbt.ivy.home")) match + case Some(ivyHome) => List(s"-Dsbt.ivy.home=$ivyHome") + case _ => Nil + extraSbtArgs ++ sbtProps ++ List( + "-sbt-version", "1.2.7", + s"--addPluginSbtFile=$sbtPluginFilePath") + +object projects + val utest = MillCommunityProject( + project = "utest", + baseCommand = s"utest.jvm[$compilerVersion]", + ) - val exitCode = exec(command, arguments: _*) + val sourcecode = MillCommunityProject( + project = "sourcecode", + baseCommand = s"sourcecode.jvm[$compilerVersion]", + ) - if (exitCode != 0) { - fail(s""" - | - |$command exited with an error code. To reproduce without JUnit, use: - | - | sbt community-build/prepareCommunityBuild - | cd community-build/community-projects/$project - | $command ${arguments.init.mkString(" ")} "${arguments.last}" - | - |For a faster feedback loop on SBT projects, one can try to extract a direct call to dotc - |using the sbt export command. For instance, for scalacheck, use - | sbt export jvm/test:compileIncremental - | - |""".stripMargin) - } - } + val oslib = MillCommunityProject( + project = "os-lib", + baseCommand = s"os[$compilerVersion]", + dependencies = List(utest, sourcecode) + ) - @Test def utest = testMill( - project = "utest", - testCommand = s"utest.jvm[$compilerVersion].test", - extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + val oslibWatch = MillCommunityProject( + project = "os-lib", + baseCommand = s"os.watch[$compilerVersion]", + dependencies = List(utest, sourcecode) ) - @Test def intent = testSbt( + val intent = SbtCommunityProject( project = "intent", - testCommand = "test", - updateCommand = "update" + sbtTestCommand = "test", + sbtUpdateCommand = "update" ) - @Test def algebra = testSbt( + val algebra = SbtCommunityProject( project = "algebra", - testCommand = "coreJVM/compile", - updateCommand = "coreJVM/update" + sbtTestCommand = "coreJVM/compile", + sbtUpdateCommand = "coreJVM/update" ) - @Test def scalacheck = testSbt( + val scalacheck = SbtCommunityProject( project = "scalacheck", - testCommand = "jvm/test:compile", - updateCommand = "jvm/test:update" + sbtTestCommand = "jvm/test:compile", + sbtUpdateCommand = "jvm/test:update" ) - @Test def scalatest = testSbt( + val scalatest = SbtCommunityProject( project = "scalatest", - testCommand = ";scalacticDotty/clean;scalacticTestDotty/test;scalatestTestDotty/test", - updateCommand = "scalatest/update" + sbtTestCommand = ";scalacticDotty/clean;scalacticTestDotty/test;scalatestTestDotty/test", + sbtUpdateCommand = "scalatest/update" ) - @Test def scalaXml = testSbt( + val scalaXml = SbtCommunityProject( project = "scala-xml", - testCommand = "xml/test", - updateCommand = "xml/update" + sbtTestCommand = "xml/test", + sbtUpdateCommand = "xml/update" ) - @Test def scopt = testSbt( + val scopt = SbtCommunityProject( project = "scopt", - testCommand = "scoptJVM/compile", - updateCommand = "scoptJVM/update" + sbtTestCommand = "scoptJVM/compile", + sbtUpdateCommand = "scoptJVM/update" ) - @Test def scalap = testSbt( + val scalap = SbtCommunityProject( project = "scalap", - testCommand = "scalap/compile", - updateCommand = "scalap/update" + sbtTestCommand = "scalap/compile", + sbtUpdateCommand = "scalap/update" ) - @Test def squants = testSbt( + val squants = SbtCommunityProject( project = "squants", - testCommand = "squantsJVM/compile", - updateCommand = "squantsJVM/update" + sbtTestCommand = "squantsJVM/compile", + sbtUpdateCommand = "squantsJVM/update" ) - @Test def betterfiles = testSbt( + val betterfiles = SbtCommunityProject( project = "betterfiles", - testCommand = "dotty-community-build/compile", - updateCommand = "dotty-community-build/update" + sbtTestCommand = "dotty-community-build/compile", + sbtUpdateCommand = "dotty-community-build/update" ) - @Test def ScalaPB = testSbt( + val ScalaPB = SbtCommunityProject( project = "ScalaPB", - testCommand = "dotty-community-build/compile", - updateCommand = "dotty-community-build/update" + sbtTestCommand = "dotty-community-build/compile", + sbtUpdateCommand = "dotty-community-build/update" ) - @Test def minitest = testSbt( + val minitest = SbtCommunityProject( project = "minitest", - testCommand = "dotty-community-build/compile", - updateCommand = "dotty-community-build/update" + sbtTestCommand = "dotty-community-build/compile", + sbtUpdateCommand = "dotty-community-build/update" ) - @Test def fastparse = testSbt( + val fastparse = SbtCommunityProject( project = "fastparse", - testCommand = "dotty-community-build/compile;dotty-community-build/test:compile", - updateCommand = "dotty-community-build/update" - ) - - @Test def sourcecode = testMill( - project = "sourcecode", - testCommand = s"sourcecode.jvm[$compilerVersion].test", - extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + sbtTestCommand = "dotty-community-build/compile;dotty-community-build/test:compile", + sbtUpdateCommand = "dotty-community-build/update" ) - @Test def stdLib213 = testSbt( + val stdLib213 = SbtCommunityProject( project = "stdLib213", - testCommand = "library/compile", - updateCommand = "library/update", - extraSbtArgs = Seq("-Dscala.build.compileWithDotty=true") + sbtTestCommand = "library/compile", + sbtUpdateCommand = "library/update", + extraSbtArgs = List("-Dscala.build.compileWithDotty=true") ) - @Test def shapeless = testSbt( + val shapeless = SbtCommunityProject( project = "shapeless", - testCommand = "test", - updateCommand = "update" + sbtTestCommand = "test", + sbtUpdateCommand = "update" ) - @Test def xmlInterpolator = testSbt( + val xmlInterpolator = SbtCommunityProject( project = "xml-interpolator", - testCommand = "test", - updateCommand = "update" + sbtTestCommand = "test", + sbtUpdateCommand = "update" ) - @Test def semanticdb = testSbt( + val semanticdb = SbtCommunityProject( project = "semanticdb", - testCommand = "test:compile", - updateCommand = "update" + sbtTestCommand = "test:compile", + sbtUpdateCommand = "update" ) - @Test def effpi = testSbt( + val effpi = SbtCommunityProject( project = "effpi", // We set `useEffpiPlugin := false` because we don't want to run their // compiler plugin since it relies on external binaries (from the model @@ -217,26 +233,94 @@ class CommunityBuildTest { // has not been updated since 2018, so no 2.13 compat. Some akka tests are dropped due to MutableBehaviour being // dropped in the 2.13 compatible release - // testCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:compile; plugin/test:compile; benchmarks/test:compile; examples/test:compile; pluginBenchmarks/test:compile", - // updateCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:update; plugin/test:update; benchmarks/test:update; examples/test:update; pluginBenchmarks/test:update" + // sbtTestCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:compile; plugin/test:compile; benchmarks/test:compile; examples/test:compile; pluginBenchmarks/test:compile", + // sbtUpdateCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:update; plugin/test:update; benchmarks/test:update; examples/test:update; pluginBenchmarks/test:update" - testCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:compile; benchmarks/test:compile; examples/test:compile; pluginBenchmarks/test:compile", - updateCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:update; benchmarks/test:update; examples/test:update; pluginBenchmarks/test:update" + sbtTestCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:compile; benchmarks/test:compile; examples/test:compile; pluginBenchmarks/test:compile", + sbtUpdateCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:update; benchmarks/test:update; examples/test:update; pluginBenchmarks/test:update" ) - // TODO @oderky? It got broken by #5458 - // @Test def pdbp = test( + // TODO @odersky? It got broken by #5458 + // val pdbp = test( // project = "pdbp", - // testCommand = "compile", - // updateCommand = "update" + // sbtTestCommand = "compile", + // sbtUpdateCommand = "update" // ) +end projects + +@Category(Array(classOf[TestCategory])) +class CommunityBuildTest { + given CommunityBuildTest = this + + /** Build the given project with the published local compiler and sbt plugin. + * + * This test reads the compiler version from community-build/dotty-bootstrapped.version + * and expects community-build/sbt-dotty-sbt to set the compiler plugin. + * + * @param project The project name, should be a git submodule in community-build/ + * @param command The binary file of the program used to test the project – usually + * a build tool like SBT or Mill + * @param arguments Arguments to pass to the testing program + */ + def test(project: String, command: String, arguments: Seq[String]): Unit = { + log(s"Building $project with dotty-bootstrapped $compilerVersion...") + + val projectDir = communitybuildDir.resolve("community-projects").resolve(project) + + if (!Files.exists(projectDir.resolve(".git"))) { + fail(s""" + | + |Missing $project submodule. You can initialize this module using + | + | git submodule update --init community-build/community-projects/$project + | + |""".stripMargin) + } + + val exitCode = exec(projectDir, command, arguments: _*) + + if (exitCode != 0) { + fail(s""" + | + |$command exited with an error code. To reproduce without JUnit, use: + | + | sbt community-build/prepareCommunityBuild + | cd community-build/community-projects/$project + | $command ${arguments.init.mkString(" ")} "${arguments.last}" + | + |For a faster feedback loop on SBT projects, one can try to extract a direct call to dotc + |using the sbt export command. For instance, for scalacheck, use + | sbt export jvm/test:compileIncremental + | + |""".stripMargin) + } + } + + @Test def intent = projects.intent.run() + @Test def algebra = projects.algebra.run() + @Test def scalacheck = projects.scalacheck.run() + @Test def scalatest = projects.scalatest.run() + @Test def scalaXml = projects.scalaXml.run() + @Test def scopt = projects.scopt.run() + @Test def scalap = projects.scalap.run() + @Test def squants = projects.squants.run() + @Test def betterfiles = projects.betterfiles.run() + @Test def ScalaPB = projects.ScalaPB.run() + @Test def minitest = projects.minitest.run() + @Test def fastparse = projects.fastparse.run() + @Test def utest = projects.utest.run() + @Test def sourcecode = projects.sourcecode.run() + @Test def oslib = projects.oslib.run() + @Test def oslibWatch = projects.oslibWatch.run() + @Test def stdLib213 = projects.stdLib213.run() + @Test def shapeless = projects.shapeless.run() + @Test def xmlInterpolator = projects.xmlInterpolator.run() + @Test def semanticdb = projects.semanticdb.run() + @Test def effpi = projects.effpi.run() } class TestCategory class UpdateCategory @Category(Array(classOf[UpdateCategory])) -class CommunityBuildUpdate extends CommunityBuildTest { - override def testSbt(project: String, testCommand: String, updateCommand: String, extraSbtArgs: Seq[String]): Unit = - super.testSbt(project, updateCommand, null, extraSbtArgs) -} +class CommunityBuildUpdate extends CommunityBuildTest