From b9f09089eea4464f0950111aec315410b1f36abe Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 20 Nov 2019 14:27:22 +0100 Subject: [PATCH 1/7] Add os-lib to the community build --- .gitmodules | 3 +++ community-build/community-projects/os-lib | 1 + .../dotty/communitybuild/CommunityBuildTest.scala | 12 ++++++++++++ 3 files changed, 16 insertions(+) create mode 160000 community-build/community-projects/os-lib diff --git a/.gitmodules b/.gitmodules index 4982e6d0ba8a..fde8547ad49b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -59,3 +59,6 @@ [submodule "community-build/community-projects/utest"] path = community-build/community-projects/utest url = https://github.com/dotty-staging/utest.git +[submodule "community-build/community-projects/os-lib"] + path = community-build/community-projects/os-lib + url = https://github.com/dotty-staging/os-lib.git diff --git a/community-build/community-projects/os-lib b/community-build/community-projects/os-lib new file mode 160000 index 000000000000..9eb7e55d2c95 --- /dev/null +++ b/community-build/community-projects/os-lib @@ -0,0 +1 @@ +Subproject commit 9eb7e55d2c9583786741fc919a22720b314fd93a diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index 31ea2d291017..f78319495059 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -104,6 +104,18 @@ class CommunityBuildTest { extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") ) + @Test def oslib = testMill( + project = "os-lib", + testCommand = s"os[$compilerVersion].test", + extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + ) + + @Test def oslibWatch = testMill( + project = "os-lib", + testCommand = s"os.watch[$compilerVersion].test", + extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + ) + @Test def intent = testSbt( project = "intent", testCommand = "test", From 3099a9834cd1ecee13692295f2a85c886279b711 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 20 Nov 2019 15:21:13 +0100 Subject: [PATCH 2/7] Update sourcecode and utest to enable local publishing --- community-build/community-projects/sourcecode | 2 +- community-build/community-projects/utest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/community-build/community-projects/sourcecode b/community-build/community-projects/sourcecode index 9d8a4b1b6a96..49c20bd80cb6 160000 --- a/community-build/community-projects/sourcecode +++ b/community-build/community-projects/sourcecode @@ -1 +1 @@ -Subproject commit 9d8a4b1b6a96d76bf287ce43aa2446d691b115e0 +Subproject commit 49c20bd80cb61a53c0b11e80b9946bbe513a4c0f diff --git a/community-build/community-projects/utest b/community-build/community-projects/utest index 0fe508a5be98..4953683a23fe 160000 --- a/community-build/community-projects/utest +++ b/community-build/community-projects/utest @@ -1 +1 @@ -Subproject commit 0fe508a5be98854c9ac1fd317b62434cb24dfbe5 +Subproject commit 4953683a23fe8f60f8f25f061a7b3095f8ddbeac From bc3b8139980cf6c23bda4ba9883147fe7804e8b6 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 20 Nov 2019 16:13:37 +0100 Subject: [PATCH 3/7] Make Mill projects publishable locally --- .../communitybuild/CommunityBuildTest.scala | 103 +++++++++++------- 1 file changed, 64 insertions(+), 39 deletions(-) diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index f78319495059..43a2eb877fd6 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -8,7 +8,7 @@ import org.junit.Assert.{assertEquals, fail} import org.junit.experimental.categories.Category @Category(Array(classOf[TestCategory])) -class CommunityBuildTest { +class CommunityBuildTest { suite => lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir")) lazy val compilerVersion: String = { @@ -16,6 +16,18 @@ class CommunityBuildTest { new String(Files.readAllBytes(file), UTF_8) } + def log(msg: String) = println(Console.GREEN + msg + Console.RESET) + + /** Executes shell command, returns false in case of error. */ + def exec(projectDir: Path, binary: String, arguments: String*): Int = { + val command = binary +: arguments + log(command.mkString(" ")) + val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO() + val process = builder.start() + val exitCode = process.waitFor() + exitCode + } + def testSbt(project: String, testCommand: String, updateCommand: String, extraSbtArgs: Seq[String] = Nil) = { // Workaround for https://github.com/sbt/sbt/issues/4395 new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs() @@ -39,9 +51,6 @@ class CommunityBuildTest { test(project, "sbt", arguments) } - def testMill(project: String, testCommand: String, extraMillArgs: Seq[String] = Nil) = - test(project, "./mill", extraMillArgs :+ testCommand) - /** Build the given project with the published local compiler and sbt plugin. * * This test reads the compiler version from community-build/dotty-bootstrapped.version @@ -53,8 +62,6 @@ class CommunityBuildTest { * @param extraSbtArgs Extra arguments to pass to sbt */ def test(project: String, command: String, arguments: Seq[String]): Unit = { - def log(msg: String) = println(Console.GREEN + msg + Console.RESET) - log(s"Building $project with dotty-bootstrapped $compilerVersion...") val projectDir = communitybuildDir.resolve("community-projects").resolve(project) @@ -69,17 +76,7 @@ class CommunityBuildTest { |""".stripMargin) } - /** Executes shell command, returns false in case of error. */ - def exec(binary: String, arguments: String*): Int = { - val command = binary +: arguments - log(command.mkString(" ")) - val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO() - val process = builder.start() - val exitCode = process.waitFor() - exitCode - } - - val exitCode = exec(command, arguments: _*) + val exitCode = exec(projectDir, command, arguments: _*) if (exitCode != 0) { fail(s""" @@ -98,23 +95,49 @@ class CommunityBuildTest { } } - @Test def utest = testMill( - project = "utest", - testCommand = s"utest.jvm[$compilerVersion].test", - extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") - ) - - @Test def oslib = testMill( - project = "os-lib", - testCommand = s"os[$compilerVersion].test", - extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") - ) - - @Test def oslibWatch = testMill( - project = "os-lib", - testCommand = s"os.watch[$compilerVersion].test", - extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") - ) + case class MillCommunityProject(project: String, testCommand: String, + publishCommand: String = "", extraArgs: List[String] = Nil, + dependencies: List[MillCommunityProject] = Nil) + final def test() = + dependencies.foreach(_.publish()) + suite.test(project, "./mill", extraArgs :+ testCommand) + + final def publish() = + log(s"Publishing $project") + val projectDir = communitybuildDir.resolve("community-projects").resolve(project) + if publishCommand.isEmpty + throw RuntimeException(s"Missing publish command for project $this") + exec(projectDir, "./mill", (extraArgs :+ publishCommand): _*) + + object projects { + val utest = MillCommunityProject( + project = "utest", + testCommand = s"utest.jvm[$compilerVersion].test", + publishCommand = s"utest.jvm[$compilerVersion].publishLocal", + extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + ) + + val sourcecode = MillCommunityProject( + project = "sourcecode", + testCommand = s"sourcecode.jvm[$compilerVersion].test", + publishCommand = s"sourcecode.jvm[$compilerVersion].publishLocal", + extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + ) + + val oslib = MillCommunityProject( + project = "os-lib", + testCommand = s"os[$compilerVersion].test", + extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + dependencies = List(utest, sourcecode) + ) + + val oslibWatch = MillCommunityProject( + project = "os-lib", + testCommand = s"os.watch[$compilerVersion].test", + extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + dependencies = List(utest, sourcecode) + ) + } @Test def intent = testSbt( project = "intent", @@ -188,11 +211,13 @@ class CommunityBuildTest { updateCommand = "dotty-community-build/update" ) - @Test def sourcecode = testMill( - project = "sourcecode", - testCommand = s"sourcecode.jvm[$compilerVersion].test", - extraMillArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") - ) + @Test def utest = projects.utest.test() + + @Test def sourcecode = projects.sourcecode.test() + + @Test def oslib = projects.oslib.test() + + @Test def oslibWatch = projects.oslibWatch.test() @Test def stdLib213 = testSbt( project = "stdLib213", From 7a77cea06133a0f4080d7687bda8c2f16bbdff7f Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 20 Nov 2019 16:18:05 +0100 Subject: [PATCH 4/7] Avoid publishing the same project twice --- .../communitybuild/CommunityBuildTest.scala | 109 ++++++++++-------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index 43a2eb877fd6..a3e71643e33c 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -7,8 +7,30 @@ import org.junit.{Ignore, Test} import org.junit.Assert.{assertEquals, fail} import org.junit.experimental.categories.Category -@Category(Array(classOf[TestCategory])) -class CommunityBuildTest { suite => +case class MillCommunityProject(project: String, testCommand: String, + publishCommand: String = "", extraArgs: List[String] = Nil, + dependencies: List[MillCommunityProject] = Nil) + import communityBuildCommons.{ log, exec, communitybuildDir } + + private var published = false + + final def test()(given suite: CommunityBuildTest) = + dependencies.foreach(_.publish()) + suite.test(project, "./mill", extraArgs :+ testCommand) + + final def publish() = + if !published + log(s"Publishing $project") + val projectDir = communitybuildDir.resolve("community-projects").resolve(project) + if publishCommand.isEmpty + throw RuntimeException(s"Missing publish command for $project, project details:\n$this") + val exitCode = exec(projectDir, "./mill", (extraArgs :+ publishCommand): _*) + if exitCode != 0 + throw RuntimeException(s"Publish command exited with code $exitCode for project $project. Project details:\n$this") + published = true +end MillCommunityProject + +object communityBuildCommons lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir")) lazy val compilerVersion: String = { @@ -27,6 +49,45 @@ class CommunityBuildTest { suite => val exitCode = process.waitFor() exitCode } +end communityBuildCommons + +object projects + import communityBuildCommons._ + + val utest = MillCommunityProject( + project = "utest", + testCommand = s"utest.jvm[$compilerVersion].test", + publishCommand = s"utest.jvm[$compilerVersion].publishLocal", + extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + ) + + val sourcecode = MillCommunityProject( + project = "sourcecode", + testCommand = s"sourcecode.jvm[$compilerVersion].test", + publishCommand = s"sourcecode.jvm[$compilerVersion].publishLocal", + extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + ) + + val oslib = MillCommunityProject( + project = "os-lib", + testCommand = s"os[$compilerVersion].test", + extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + dependencies = List(utest, sourcecode) + ) + + val oslibWatch = MillCommunityProject( + project = "os-lib", + testCommand = s"os.watch[$compilerVersion].test", + extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + dependencies = List(utest, sourcecode) + ) +end projects + +@Category(Array(classOf[TestCategory])) +class CommunityBuildTest { + import communityBuildCommons._ + + given CommunityBuildTest = this def testSbt(project: String, testCommand: String, updateCommand: String, extraSbtArgs: Seq[String] = Nil) = { // Workaround for https://github.com/sbt/sbt/issues/4395 @@ -95,50 +156,6 @@ class CommunityBuildTest { suite => } } - case class MillCommunityProject(project: String, testCommand: String, - publishCommand: String = "", extraArgs: List[String] = Nil, - dependencies: List[MillCommunityProject] = Nil) - final def test() = - dependencies.foreach(_.publish()) - suite.test(project, "./mill", extraArgs :+ testCommand) - - final def publish() = - log(s"Publishing $project") - val projectDir = communitybuildDir.resolve("community-projects").resolve(project) - if publishCommand.isEmpty - throw RuntimeException(s"Missing publish command for project $this") - exec(projectDir, "./mill", (extraArgs :+ publishCommand): _*) - - object projects { - val utest = MillCommunityProject( - project = "utest", - testCommand = s"utest.jvm[$compilerVersion].test", - publishCommand = s"utest.jvm[$compilerVersion].publishLocal", - extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") - ) - - val sourcecode = MillCommunityProject( - project = "sourcecode", - testCommand = s"sourcecode.jvm[$compilerVersion].test", - publishCommand = s"sourcecode.jvm[$compilerVersion].publishLocal", - extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), - ) - - val oslib = MillCommunityProject( - project = "os-lib", - testCommand = s"os[$compilerVersion].test", - extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), - dependencies = List(utest, sourcecode) - ) - - val oslibWatch = MillCommunityProject( - project = "os-lib", - testCommand = s"os.watch[$compilerVersion].test", - extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), - dependencies = List(utest, sourcecode) - ) - } - @Test def intent = testSbt( project = "intent", testCommand = "test", From 707fca67ec751cc1883caf810411fdd8e48bf59c Mon Sep 17 00:00:00 2001 From: Anatolii Date: Wed, 20 Nov 2019 17:54:18 +0100 Subject: [PATCH 5/7] Disable flaky os-lib tests; make mill use cache --- community-build/community-projects/os-lib | 2 +- .../communitybuild/CommunityBuildTest.scala | 30 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/community-build/community-projects/os-lib b/community-build/community-projects/os-lib index 9eb7e55d2c95..b29d68ffee5a 160000 --- a/community-build/community-projects/os-lib +++ b/community-build/community-projects/os-lib @@ -1 +1 @@ -Subproject commit 9eb7e55d2c9583786741fc919a22720b314fd93a +Subproject commit b29d68ffee5a6e0aedbfdd048d10059f250134e4 diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index a3e71643e33c..e866bd95eacd 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -7,18 +7,19 @@ import org.junit.{Ignore, Test} import org.junit.Assert.{assertEquals, fail} import org.junit.experimental.categories.Category -case class MillCommunityProject(project: String, testCommand: String, - publishCommand: String = "", extraArgs: List[String] = Nil, - dependencies: List[MillCommunityProject] = Nil) +final case class MillCommunityProject(project: String, testCommand: String, + updateCommand: String, publishCommand: String = "", + extraArgs: List[String] = Nil, dependencies: List[MillCommunityProject] = Nil) import communityBuildCommons.{ log, exec, communitybuildDir } private var published = false - final def test()(given suite: CommunityBuildTest) = - dependencies.foreach(_.publish()) - suite.test(project, "./mill", extraArgs :+ testCommand) + def test(update: Boolean)(given suite: CommunityBuildTest) = + val millCmd = if update then updateCommand else testCommand + if !update then dependencies.foreach(_.publish()) + suite.test(project, "./mill", extraArgs :+ millCmd) - final def publish() = + def publish() = if !published log(s"Publishing $project") val projectDir = communitybuildDir.resolve("community-projects").resolve(project) @@ -58,12 +59,14 @@ object projects project = "utest", testCommand = s"utest.jvm[$compilerVersion].test", publishCommand = s"utest.jvm[$compilerVersion].publishLocal", + updateCommand = "utest.jvm[$compilerVersion].compileClasspath", extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") ) val sourcecode = MillCommunityProject( project = "sourcecode", testCommand = s"sourcecode.jvm[$compilerVersion].test", + updateCommand = s"sourcecode.jvm[$compilerVersion].compileClasspath", publishCommand = s"sourcecode.jvm[$compilerVersion].publishLocal", extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), ) @@ -71,6 +74,7 @@ object projects val oslib = MillCommunityProject( project = "os-lib", testCommand = s"os[$compilerVersion].test", + updateCommand = s"os[$compilerVersion].compileClasspath", extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), dependencies = List(utest, sourcecode) ) @@ -78,6 +82,7 @@ object projects val oslibWatch = MillCommunityProject( project = "os-lib", testCommand = s"os.watch[$compilerVersion].test", + updateCommand = s"os.watch[$compilerVersion].compileClasspath", extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), dependencies = List(utest, sourcecode) ) @@ -88,6 +93,7 @@ class CommunityBuildTest { import communityBuildCommons._ given CommunityBuildTest = this + val updateCacheRun = false def testSbt(project: String, testCommand: String, updateCommand: String, extraSbtArgs: Seq[String] = Nil) = { // Workaround for https://github.com/sbt/sbt/issues/4395 @@ -228,13 +234,13 @@ class CommunityBuildTest { updateCommand = "dotty-community-build/update" ) - @Test def utest = projects.utest.test() + @Test def utest = projects.utest.test(updateCacheRun) - @Test def sourcecode = projects.sourcecode.test() + @Test def sourcecode = projects.sourcecode.test(updateCacheRun) - @Test def oslib = projects.oslib.test() + @Test def oslib = projects.oslib.test(updateCacheRun) - @Test def oslibWatch = projects.oslibWatch.test() + @Test def oslibWatch = projects.oslibWatch.test(updateCacheRun) @Test def stdLib213 = testSbt( project = "stdLib213", @@ -291,6 +297,8 @@ class UpdateCategory @Category(Array(classOf[UpdateCategory])) class CommunityBuildUpdate extends CommunityBuildTest { + override val updateCacheRun = true + override def testSbt(project: String, testCommand: String, updateCommand: String, extraSbtArgs: Seq[String]): Unit = super.testSbt(project, updateCommand, null, extraSbtArgs) } From 6e1bc2292c8aa0aed26fefc56ca770e3458aeae2 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Thu, 21 Nov 2019 12:14:28 +0100 Subject: [PATCH 6/7] Improve community build testing framework --- .../communitybuild/CommunityBuildTest.scala | 418 +++++++++--------- 1 file changed, 220 insertions(+), 198 deletions(-) diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index e866bd95eacd..96a62a113b4d 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -7,267 +7,223 @@ import org.junit.{Ignore, Test} import org.junit.Assert.{assertEquals, fail} import org.junit.experimental.categories.Category -final case class MillCommunityProject(project: String, testCommand: String, - updateCommand: String, publishCommand: String = "", - extraArgs: List[String] = Nil, dependencies: List[MillCommunityProject] = Nil) - import communityBuildCommons.{ log, exec, communitybuildDir } +lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir")) + +lazy val compilerVersion: String = + val file = communitybuildDir.resolve("dotty-bootstrapped.version") + new String(Files.readAllBytes(file), UTF_8) + +lazy val sbtPluginFilePath: String = + // Workaround for https://github.com/sbt/sbt/issues/4395 + new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs() + communitybuildDir.resolve("sbt-dotty-sbt").toAbsolutePath().toString() + +def log(msg: String) = println(Console.GREEN + msg + Console.RESET) + +/** Executes shell command, returns false in case of error. */ +def exec(projectDir: Path, binary: String, arguments: String*): Int = + val command = binary +: arguments + log(command.mkString(" ")) + val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO() + val process = builder.start() + val exitCode = process.waitFor() + exitCode + + +sealed trait CommunityProject private var published = false - def test(update: Boolean)(given suite: CommunityBuildTest) = - val millCmd = if update then updateCommand else testCommand - if !update then dependencies.foreach(_.publish()) - suite.test(project, "./mill", extraArgs :+ millCmd) + val project: String + val updateCommand: String + val testCommand: String + val publishCommand: String + val dependencies: List[CommunityProject] + val binaryName: String + val runCommandsArgs: List[String] = Nil + + final val projectDir = communitybuildDir.resolve("community-projects").resolve(project) - def publish() = + /** Is this project running in the test or update mode in the + * context of the given suite? @see `run` for more details. + */ + final def isUpdateMode(given suite: CommunityBuildTest) = + suite.isInstanceOf[CommunityBuildUpdate] + + /** Depending on the mode of operation, either + * runs the test or updates the project. Updating + * means that all the dependencies are fetched but + * minimal other extra other work is done. Updating + * is necessary since we run tests each time on a fresh + * Docker container. We run the update on Docker container + * creation time to create the cache of the dependencies + * and avoid network overhead. See https://github.com/lampepfl/dotty-drone + * for more infrastructural details. + */ + final def run()(given suite: CommunityBuildTest) = + val runCmd = if isUpdateMode then updateCommand else testCommand + if !isUpdateMode then dependencies.foreach(_.publish()) + suite.test(project, binaryName, runCommandsArgs :+ runCmd) + + /** Publish this project to the local Maven repository */ + final def publish(): Unit = if !published log(s"Publishing $project") - val projectDir = communitybuildDir.resolve("community-projects").resolve(project) - if publishCommand.isEmpty - throw RuntimeException(s"Missing publish command for $project, project details:\n$this") - val exitCode = exec(projectDir, "./mill", (extraArgs :+ publishCommand): _*) + if publishCommand eq null + throw RuntimeException(s"Publish command is not specified for $project. Project details:\n$this") + val exitCode = exec(projectDir, binaryName, (runCommandsArgs :+ publishCommand): _*) if exitCode != 0 throw RuntimeException(s"Publish command exited with code $exitCode for project $project. Project details:\n$this") published = true -end MillCommunityProject - -object communityBuildCommons - lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir")) - - lazy val compilerVersion: String = { - val file = communitybuildDir.resolve("dotty-bootstrapped.version") - new String(Files.readAllBytes(file), UTF_8) - } - - def log(msg: String) = println(Console.GREEN + msg + Console.RESET) - - /** Executes shell command, returns false in case of error. */ - def exec(projectDir: Path, binary: String, arguments: String*): Int = { - val command = binary +: arguments - log(command.mkString(" ")) - val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO() - val process = builder.start() - val exitCode = process.waitFor() - exitCode - } -end communityBuildCommons +end CommunityProject + +final case class MillCommunityProject(project: String, baseCommand: String, + dependencies: List[CommunityProject] = Nil) extends CommunityProject + override val binaryName: String = "./mill" + override val updateCommand = s"$baseCommand.compileClasspath" + override val testCommand = s"$baseCommand.test" + override val publishCommand = s"$baseCommand.publishLocal" + override val runCommandsArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + +final case class SbtCommunityProject(project: String, sbtTestCommand: String, + sbtUpdateCommand: String, extraSbtArgs: List[String] = Nil, + dependencies: List[CommunityProject] = Nil) extends CommunityProject + override val binaryName: String = "sbt" + private val baseCommand = s";clean ;set updateOptions in Global ~= (_.withLatestSnapshots(false)) ;++$compilerVersion! " + override val publishCommand = null + override val testCommand = s"$baseCommand$sbtTestCommand" + override val updateCommand = s"$baseCommand$sbtUpdateCommand" + + override val runCommandsArgs: List[String] = + // Run the sbt command with the compiler version and sbt plugin set in the build + val sbtProps = Option(System.getProperty("sbt.ivy.home")) match + case Some(ivyHome) => List(s"-Dsbt.ivy.home=$ivyHome") + case _ => Nil + extraSbtArgs ++ sbtProps ++ List( + "-sbt-version", "1.2.7", + s"--addPluginSbtFile=$sbtPluginFilePath") object projects - import communityBuildCommons._ - val utest = MillCommunityProject( project = "utest", - testCommand = s"utest.jvm[$compilerVersion].test", - publishCommand = s"utest.jvm[$compilerVersion].publishLocal", - updateCommand = "utest.jvm[$compilerVersion].compileClasspath", - extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion") + baseCommand = s"utest.jvm[$compilerVersion]", ) val sourcecode = MillCommunityProject( project = "sourcecode", - testCommand = s"sourcecode.jvm[$compilerVersion].test", - updateCommand = s"sourcecode.jvm[$compilerVersion].compileClasspath", - publishCommand = s"sourcecode.jvm[$compilerVersion].publishLocal", - extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + baseCommand = s"sourcecode.jvm[$compilerVersion]", ) val oslib = MillCommunityProject( project = "os-lib", - testCommand = s"os[$compilerVersion].test", - updateCommand = s"os[$compilerVersion].compileClasspath", - extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + baseCommand = s"os[$compilerVersion]", dependencies = List(utest, sourcecode) ) val oslibWatch = MillCommunityProject( project = "os-lib", - testCommand = s"os.watch[$compilerVersion].test", - updateCommand = s"os.watch[$compilerVersion].compileClasspath", - extraArgs = List("-i", "-D", s"dottyVersion=$compilerVersion"), + baseCommand = s"os.watch[$compilerVersion]", dependencies = List(utest, sourcecode) ) -end projects - -@Category(Array(classOf[TestCategory])) -class CommunityBuildTest { - import communityBuildCommons._ - - given CommunityBuildTest = this - val updateCacheRun = false - - def testSbt(project: String, testCommand: String, updateCommand: String, extraSbtArgs: Seq[String] = Nil) = { - // Workaround for https://github.com/sbt/sbt/issues/4395 - new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs() - val pluginFilePath = communitybuildDir.resolve("sbt-dotty-sbt").toAbsolutePath().toString() - - // Run the sbt command with the compiler version and sbt plugin set in the build - val arguments = { - val sbtProps = Option(System.getProperty("sbt.ivy.home")) match { - case Some(ivyHome) => - Seq(s"-Dsbt.ivy.home=$ivyHome") - case _ => - Seq() - } - extraSbtArgs ++ sbtProps ++ Seq( - "-sbt-version", "1.2.7", - s"--addPluginSbtFile=$pluginFilePath", - s";clean ;set updateOptions in Global ~= (_.withLatestSnapshots(false)) ;++$compilerVersion! $testCommand" - ) - } - - test(project, "sbt", arguments) - } - - /** Build the given project with the published local compiler and sbt plugin. - * - * This test reads the compiler version from community-build/dotty-bootstrapped.version - * and expects community-build/sbt-dotty-sbt to set the compiler plugin. - * - * @param project The project name, should be a git submodule in community-build/ - * @param command The sbt command used to test the project - * @param updateCommand The sbt command used to update the project - * @param extraSbtArgs Extra arguments to pass to sbt - */ - def test(project: String, command: String, arguments: Seq[String]): Unit = { - log(s"Building $project with dotty-bootstrapped $compilerVersion...") - - val projectDir = communitybuildDir.resolve("community-projects").resolve(project) - - if (!Files.exists(projectDir.resolve(".git"))) { - fail(s""" - | - |Missing $project submodule. You can initialize this module using - | - | git submodule update --init community-build/community-projects/$project - | - |""".stripMargin) - } - - val exitCode = exec(projectDir, command, arguments: _*) - - if (exitCode != 0) { - fail(s""" - | - |$command exited with an error code. To reproduce without JUnit, use: - | - | sbt community-build/prepareCommunityBuild - | cd community-build/community-projects/$project - | $command ${arguments.init.mkString(" ")} "${arguments.last}" - | - |For a faster feedback loop on SBT projects, one can try to extract a direct call to dotc - |using the sbt export command. For instance, for scalacheck, use - | sbt export jvm/test:compileIncremental - | - |""".stripMargin) - } - } - @Test def intent = testSbt( + val intent = SbtCommunityProject( project = "intent", - testCommand = "test", - updateCommand = "update" + sbtTestCommand = "test", + sbtUpdateCommand = "update" ) - @Test def algebra = testSbt( + val algebra = SbtCommunityProject( project = "algebra", - testCommand = "coreJVM/compile", - updateCommand = "coreJVM/update" + sbtTestCommand = "coreJVM/compile", + sbtUpdateCommand = "coreJVM/update" ) - @Test def scalacheck = testSbt( + val scalacheck = SbtCommunityProject( project = "scalacheck", - testCommand = "jvm/test:compile", - updateCommand = "jvm/test:update" + sbtTestCommand = "jvm/test:compile", + sbtUpdateCommand = "jvm/test:update" ) - @Test def scalatest = testSbt( + val scalatest = SbtCommunityProject( project = "scalatest", - testCommand = ";scalacticDotty/clean;scalacticTestDotty/test;scalatestTestDotty/test", - updateCommand = "scalatest/update" + sbtTestCommand = ";scalacticDotty/clean;scalacticTestDotty/test;scalatestTestDotty/test", + sbtUpdateCommand = "scalatest/update" ) - @Test def scalaXml = testSbt( + val scalaXml = SbtCommunityProject( project = "scala-xml", - testCommand = "xml/test", - updateCommand = "xml/update" + sbtTestCommand = "xml/test", + sbtUpdateCommand = "xml/update" ) - @Test def scopt = testSbt( + val scopt = SbtCommunityProject( project = "scopt", - testCommand = "scoptJVM/compile", - updateCommand = "scoptJVM/update" + sbtTestCommand = "scoptJVM/compile", + sbtUpdateCommand = "scoptJVM/update" ) - @Test def scalap = testSbt( + val scalap = SbtCommunityProject( project = "scalap", - testCommand = "scalap/compile", - updateCommand = "scalap/update" + sbtTestCommand = "scalap/compile", + sbtUpdateCommand = "scalap/update" ) - @Test def squants = testSbt( + val squants = SbtCommunityProject( project = "squants", - testCommand = "squantsJVM/compile", - updateCommand = "squantsJVM/update" + sbtTestCommand = "squantsJVM/compile", + sbtUpdateCommand = "squantsJVM/update" ) - @Test def betterfiles = testSbt( + val betterfiles = SbtCommunityProject( project = "betterfiles", - testCommand = "dotty-community-build/compile", - updateCommand = "dotty-community-build/update" + sbtTestCommand = "dotty-community-build/compile", + sbtUpdateCommand = "dotty-community-build/update" ) - @Test def ScalaPB = testSbt( + val ScalaPB = SbtCommunityProject( project = "ScalaPB", - testCommand = "dotty-community-build/compile", - updateCommand = "dotty-community-build/update" + sbtTestCommand = "dotty-community-build/compile", + sbtUpdateCommand = "dotty-community-build/update" ) - @Test def minitest = testSbt( + val minitest = SbtCommunityProject( project = "minitest", - testCommand = "dotty-community-build/compile", - updateCommand = "dotty-community-build/update" + sbtTestCommand = "dotty-community-build/compile", + sbtUpdateCommand = "dotty-community-build/update" ) - @Test def fastparse = testSbt( + val fastparse = SbtCommunityProject( project = "fastparse", - testCommand = "dotty-community-build/compile;dotty-community-build/test:compile", - updateCommand = "dotty-community-build/update" + sbtTestCommand = "dotty-community-build/compile;dotty-community-build/test:compile", + sbtUpdateCommand = "dotty-community-build/update" ) - @Test def utest = projects.utest.test(updateCacheRun) - - @Test def sourcecode = projects.sourcecode.test(updateCacheRun) - - @Test def oslib = projects.oslib.test(updateCacheRun) - - @Test def oslibWatch = projects.oslibWatch.test(updateCacheRun) - - @Test def stdLib213 = testSbt( + val stdLib213 = SbtCommunityProject( project = "stdLib213", - testCommand = "library/compile", - updateCommand = "library/update", - extraSbtArgs = Seq("-Dscala.build.compileWithDotty=true") + sbtTestCommand = "library/compile", + sbtUpdateCommand = "library/update", + extraSbtArgs = List("-Dscala.build.compileWithDotty=true") ) - @Test def shapeless = testSbt( + val shapeless = SbtCommunityProject( project = "shapeless", - testCommand = "test", - updateCommand = "update" + sbtTestCommand = "test", + sbtUpdateCommand = "update" ) - @Test def xmlInterpolator = testSbt( + val xmlInterpolator = SbtCommunityProject( project = "xml-interpolator", - testCommand = "test", - updateCommand = "update" + sbtTestCommand = "test", + sbtUpdateCommand = "update" ) - @Test def semanticdb = testSbt( + val semanticdb = SbtCommunityProject( project = "semanticdb", - testCommand = "test:compile", - updateCommand = "update" + sbtTestCommand = "test:compile", + sbtUpdateCommand = "update" ) - @Test def effpi = testSbt( + val effpi = SbtCommunityProject( project = "effpi", // We set `useEffpiPlugin := false` because we don't want to run their // compiler plugin since it relies on external binaries (from the model @@ -277,28 +233,94 @@ class CommunityBuildTest { // has not been updated since 2018, so no 2.13 compat. Some akka tests are dropped due to MutableBehaviour being // dropped in the 2.13 compatible release - // testCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:compile; plugin/test:compile; benchmarks/test:compile; examples/test:compile; pluginBenchmarks/test:compile", - // updateCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:update; plugin/test:update; benchmarks/test:update; examples/test:update; pluginBenchmarks/test:update" + // sbtTestCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:compile; plugin/test:compile; benchmarks/test:compile; examples/test:compile; pluginBenchmarks/test:compile", + // sbtUpdateCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:update; plugin/test:update; benchmarks/test:update; examples/test:update; pluginBenchmarks/test:update" - testCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:compile; benchmarks/test:compile; examples/test:compile; pluginBenchmarks/test:compile", - updateCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:update; benchmarks/test:update; examples/test:update; pluginBenchmarks/test:update" + sbtTestCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:compile; benchmarks/test:compile; examples/test:compile; pluginBenchmarks/test:compile", + sbtUpdateCommand = ";set ThisBuild / useEffpiPlugin := false; effpi/test:update; benchmarks/test:update; examples/test:update; pluginBenchmarks/test:update" ) - // TODO @oderky? It got broken by #5458 - // @Test def pdbp = test( + // TODO @odersky? It got broken by #5458 + // val pdbp = test( // project = "pdbp", - // testCommand = "compile", - // updateCommand = "update" + // sbtTestCommand = "compile", + // sbtUpdateCommand = "update" // ) +end projects + +@Category(Array(classOf[TestCategory])) +class CommunityBuildTest { + given CommunityBuildTest = this + + /** Build the given project with the published local compiler and sbt plugin. + * + * This test reads the compiler version from community-build/dotty-bootstrapped.version + * and expects community-build/sbt-dotty-sbt to set the compiler plugin. + * + * @param project The project name, should be a git submodule in community-build/ + * @param command The sbt command used to test the project + * @param updateCommand The sbt command used to update the project + * @param extraSbtArgs Extra arguments to pass to sbt + */ + def test(project: String, command: String, arguments: Seq[String]): Unit = { + log(s"Building $project with dotty-bootstrapped $compilerVersion...") + + val projectDir = communitybuildDir.resolve("community-projects").resolve(project) + + if (!Files.exists(projectDir.resolve(".git"))) { + fail(s""" + | + |Missing $project submodule. You can initialize this module using + | + | git submodule update --init community-build/community-projects/$project + | + |""".stripMargin) + } + + val exitCode = exec(projectDir, command, arguments: _*) + + if (exitCode != 0) { + fail(s""" + | + |$command exited with an error code. To reproduce without JUnit, use: + | + | sbt community-build/prepareCommunityBuild + | cd community-build/community-projects/$project + | $command ${arguments.init.mkString(" ")} "${arguments.last}" + | + |For a faster feedback loop on SBT projects, one can try to extract a direct call to dotc + |using the sbt export command. For instance, for scalacheck, use + | sbt export jvm/test:compileIncremental + | + |""".stripMargin) + } + } + + @Test def intent = projects.intent.run() + @Test def algebra = projects.algebra.run() + @Test def scalacheck = projects.scalacheck.run() + @Test def scalatest = projects.scalatest.run() + @Test def scalaXml = projects.scalaXml.run() + @Test def scopt = projects.scopt.run() + @Test def scalap = projects.scalap.run() + @Test def squants = projects.squants.run() + @Test def betterfiles = projects.betterfiles.run() + @Test def ScalaPB = projects.ScalaPB.run() + @Test def minitest = projects.minitest.run() + @Test def fastparse = projects.fastparse.run() + @Test def utest = projects.utest.run() + @Test def sourcecode = projects.sourcecode.run() + @Test def oslib = projects.oslib.run() + @Test def oslibWatch = projects.oslibWatch.run() + @Test def stdLib213 = projects.stdLib213.run() + @Test def shapeless = projects.shapeless.run() + @Test def xmlInterpolator = projects.xmlInterpolator.run() + @Test def semanticdb = projects.semanticdb.run() + @Test def effpi = projects.effpi.run() } class TestCategory class UpdateCategory @Category(Array(classOf[UpdateCategory])) -class CommunityBuildUpdate extends CommunityBuildTest { - override val updateCacheRun = true - - override def testSbt(project: String, testCommand: String, updateCommand: String, extraSbtArgs: Seq[String]): Unit = - super.testSbt(project, updateCommand, null, extraSbtArgs) -} +class CommunityBuildUpdate extends CommunityBuildTest From c1a6f61dc1442ea4fcf49eca4df0b7cf808752b7 Mon Sep 17 00:00:00 2001 From: Anatolii Date: Fri, 22 Nov 2019 13:23:54 +0100 Subject: [PATCH 7/7] Fix CommunityBuildTest docs --- .../scala/dotty/communitybuild/CommunityBuildTest.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index 96a62a113b4d..2881b6ef65f1 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -257,10 +257,10 @@ class CommunityBuildTest { * This test reads the compiler version from community-build/dotty-bootstrapped.version * and expects community-build/sbt-dotty-sbt to set the compiler plugin. * - * @param project The project name, should be a git submodule in community-build/ - * @param command The sbt command used to test the project - * @param updateCommand The sbt command used to update the project - * @param extraSbtArgs Extra arguments to pass to sbt + * @param project The project name, should be a git submodule in community-build/ + * @param command The binary file of the program used to test the project – usually + * a build tool like SBT or Mill + * @param arguments Arguments to pass to the testing program */ def test(project: String, command: String, arguments: Seq[String]): Unit = { log(s"Building $project with dotty-bootstrapped $compilerVersion...")