From 4aaffe49e528c48e7aaa7cf24cdadb30b612714d Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Thu, 9 Jan 2020 16:50:24 +0100 Subject: [PATCH 01/11] Upgrade stdlib to support sbt 1.3 This corresponds to https://github.com/scala/scala/pull/8525. --- community-build/community-projects/stdLib213 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community-build/community-projects/stdLib213 b/community-build/community-projects/stdLib213 index ec4624b48262..1886950613d8 160000 --- a/community-build/community-projects/stdLib213 +++ b/community-build/community-projects/stdLib213 @@ -1 +1 @@ -Subproject commit ec4624b482626020e154d2e6161ef5f5f38f166c +Subproject commit 1886950613d85af824435b2c072e9eedb12d6851 From 19db2f20275d851ae90ed34e65cd2f9aa2e20761 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Thu, 9 Jan 2020 17:28:35 +0100 Subject: [PATCH 02/11] Upgrade scalap to support sbt 1.3 --- community-build/community-projects/scalap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community-build/community-projects/scalap b/community-build/community-projects/scalap index fe6239e99c07..a20c669b8f26 160000 --- a/community-build/community-projects/scalap +++ b/community-build/community-projects/scalap @@ -1 +1 @@ -Subproject commit fe6239e99c07bf838e57fe0dc9930208a5486ad3 +Subproject commit a20c669b8f264a7b08cfccd875049420007deada From bcdd787e5f85238849602e0a3b10947d73a78f2d Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Wed, 8 Jan 2020 20:54:47 +0100 Subject: [PATCH 03/11] Upgrade dotty-semanticdb to work with sbt 1.3 --- community-build/community-projects/semanticdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community-build/community-projects/semanticdb b/community-build/community-projects/semanticdb index 05b8001beade..168b1f6881dd 160000 --- a/community-build/community-projects/semanticdb +++ b/community-build/community-projects/semanticdb @@ -1 +1 @@ -Subproject commit 05b8001beade3492f7eb8ab9425dd7454f66fe4b +Subproject commit 168b1f6881dd2f79025ae3db61bf7f56120c369b From 2b810e97d85e587870561d34d4f7fcac5441e471 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Thu, 9 Jan 2020 22:41:27 +0100 Subject: [PATCH 04/11] Remove obsolete scripted test For the same reason that it was removed from sbt: https://github.com/sbt/sbt/commit/ffa69ea5d690b5d8d548e0eefd98e83587a30de2 --- .../relative-source-error/changes/absolute.sbt | 1 - .../relative-source-error/changes/relative.sbt | 1 - .../project/DottyInjectedPlugin.scala | 12 ------------ .../relative-source-error/project/plugins.sbt | 1 - .../relative-source-error/src/A.scala | 1 - .../source-dependencies/relative-source-error/test | 7 ------- 6 files changed, 23 deletions(-) delete mode 100644 sbt-dotty/sbt-test/source-dependencies/relative-source-error/changes/absolute.sbt delete mode 100644 sbt-dotty/sbt-test/source-dependencies/relative-source-error/changes/relative.sbt delete mode 100644 sbt-dotty/sbt-test/source-dependencies/relative-source-error/project/DottyInjectedPlugin.scala delete mode 100644 sbt-dotty/sbt-test/source-dependencies/relative-source-error/project/plugins.sbt delete mode 100644 sbt-dotty/sbt-test/source-dependencies/relative-source-error/src/A.scala delete mode 100644 sbt-dotty/sbt-test/source-dependencies/relative-source-error/test diff --git a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/changes/absolute.sbt b/sbt-dotty/sbt-test/source-dependencies/relative-source-error/changes/absolute.sbt deleted file mode 100644 index a5f2d007a58c..000000000000 --- a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/changes/absolute.sbt +++ /dev/null @@ -1 +0,0 @@ -scalaSource in Compile := baseDirectory.value / "src" \ No newline at end of file diff --git a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/changes/relative.sbt b/sbt-dotty/sbt-test/source-dependencies/relative-source-error/changes/relative.sbt deleted file mode 100644 index 8e898fe02829..000000000000 --- a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/changes/relative.sbt +++ /dev/null @@ -1 +0,0 @@ -scalaSource in Compile := file("src") \ No newline at end of file diff --git a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/project/DottyInjectedPlugin.scala b/sbt-dotty/sbt-test/source-dependencies/relative-source-error/project/DottyInjectedPlugin.scala deleted file mode 100644 index 6a77cb33121e..000000000000 --- a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/project/DottyInjectedPlugin.scala +++ /dev/null @@ -1,12 +0,0 @@ -import sbt._ -import Keys._ - -object DottyInjectedPlugin extends AutoPlugin { - override def requires = plugins.JvmPlugin - override def trigger = allRequirements - - override val projectSettings = Seq( - scalaVersion := sys.props("plugin.scalaVersion"), - scalacOptions += "-language:Scala2Compat" - ) -} diff --git a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/project/plugins.sbt b/sbt-dotty/sbt-test/source-dependencies/relative-source-error/project/plugins.sbt deleted file mode 100644 index c17caab2d98c..000000000000 --- a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/project/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % sys.props("plugin.version")) diff --git a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/src/A.scala b/sbt-dotty/sbt-test/source-dependencies/relative-source-error/src/A.scala deleted file mode 100644 index 528ffce71c5f..000000000000 --- a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/src/A.scala +++ /dev/null @@ -1 +0,0 @@ -object A \ No newline at end of file diff --git a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/test b/sbt-dotty/sbt-test/source-dependencies/relative-source-error/test deleted file mode 100644 index fb8e9dce835f..000000000000 --- a/sbt-dotty/sbt-test/source-dependencies/relative-source-error/test +++ /dev/null @@ -1,7 +0,0 @@ -$ copy-file changes/relative.sbt build.sbt -> reload --> compile - -$ copy-file changes/absolute.sbt build.sbt -> reload -> compile From 7c07bf795664ac25ab20e56657c3f96c08fddba3 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Thu, 9 Jan 2020 22:03:58 +0100 Subject: [PATCH 05/11] Test that IDE can be started in CI, upgrade ref compiler dotty-library-bootstrappedJS scalacOptions handling was broken, I fixed it by upgrading the reference compiler to simplify the handling of -Yerased-terms, but I also removed all scalajs projects from IDE startup as explained in the comment. --- .drone.yml | 2 +- project/Build.scala | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.drone.yml b/.drone.yml index 839cbb9a1aae..9cb33a3e0bfe 100644 --- a/.drone.yml +++ b/.drone.yml @@ -40,7 +40,7 @@ steps: depends_on: [ clone ] commands: - cp -R . /tmp/2/ && cd /tmp/2/ - - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;dotty-staging/test ;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test" + - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;dotty-staging/test ;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test ;configureIDE" - ./project/scripts/bootstrapCmdTests - name: community_build diff --git a/project/Build.scala b/project/Build.scala index 43c8ea9b47c1..21fd9e09f2c7 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -54,11 +54,15 @@ object MyScalaJSPlugin extends AutoPlugin { // Typecheck the Scala.js IR found on the classpath scalaJSLinkerConfig ~= (_.withCheckIR(true)), + + // Exclude all these projects from `configureIDE/launchIDE` since they + // take time to compile, print a bunch of warnings, and are rarely edited. + excludeFromIDE := true ) } object Build { - val referenceVersion = "0.21.0-RC1" + val referenceVersion = "0.22.0-bin-20200114-193f7de-NIGHTLY" val baseVersion = "0.22.0" val baseSbtDottyVersion = "0.3.5" @@ -730,16 +734,18 @@ object Build { case Bootstrapped => `dotty-compiler-bootstrapped` } - // Settings shared between dotty-library and dotty-library-bootstrapped + // Settings shared between dotty-library, dotty-library-bootstrapped and dotty-library-bootstrappedJS lazy val dottyLibrarySettings = Seq( - // Needed so that the library sources are visible when `dotty.tools.dotc.core.Definitions#init` is called - scalacOptions in Compile ++= Seq("-sourcepath", (sourceDirectories in Compile).value.map(_.getAbsolutePath).distinct.mkString(File.pathSeparator)), + scalacOptions in Compile ++= Seq( + // Needed so that the library sources are visible when `dotty.tools.dotc.core.Definitions#init` is called + "-sourcepath", (sourceDirectories in Compile).value.map(_.getAbsolutePath).distinct.mkString(File.pathSeparator), + // support declaration of scala.compiletime.erasedValue + "-Yerased-terms" + ), ) lazy val `dotty-library` = project.in(file("library")).asDottyLibrary(NonBootstrapped) lazy val `dotty-library-bootstrapped`: Project = project.in(file("library")).asDottyLibrary(Bootstrapped) - // TODO: move -Yerased-terms to dottyLibrarySettings on reference compiler update - .settings(scalacOptions in Compile += "-Yerased-terms") // support declaration of scala.compiletime.erasedValue def dottyLibrary(implicit mode: Mode): Project = mode match { case NonBootstrapped => `dotty-library` @@ -761,7 +767,6 @@ object Build { settings( unmanagedSourceDirectories in Compile := (unmanagedSourceDirectories in (`dotty-library-bootstrapped`, Compile)).value, - scalacOptions += "-Yerased-terms", // support declaration of scala.compiletime.erasedValue ) lazy val tastyCoreSettings = Seq( From 46a120ca5aecc28788bb0cc804f30789cfc49234 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Fri, 10 Jan 2020 15:45:45 +0100 Subject: [PATCH 06/11] Fix tastyBootstrap test to also compile tasty-core tasty-core is a dotty project, so when bootstrapping it needs to be recompiled and cannot be reused as is, the test did not fail so far since we didn't break anything, but it could be made to break by bumping the tasty version in TastyFormat. Also use change the names of the output directories of lib, dotty1 and dotty2 to match the group name. --- .../dotty/tools/dotc/CompilationTests.scala | 43 +++++++++++-------- tests/idempotency/BootstrapChecker.scala | 2 +- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 8e5cf657a624..246a7331106b 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -7,6 +7,7 @@ import org.junit.Assert._ import org.junit.Assume._ import org.junit.experimental.categories.Category +import java.io.File import java.nio.file._ import java.util.stream.{ Stream => JStream } import scala.collection.JavaConverters._ @@ -197,24 +198,26 @@ class CompilationTests extends ParallelTesting { */ @Test def tastyBootstrap: Unit = { implicit val testGroup: TestGroup = TestGroup("tastyBootstrap/tests") + val libGroup = TestGroup("tastyBootstrap/lib") + val tastyCoreGroup = TestGroup("tastyBootstrap/tastyCore") val dotty1Group = TestGroup("tastyBootstrap/dotty1") val dotty2Group = TestGroup("tastyBootstrap/dotty2") - val libGroup = TestGroup("tastyBootstrap/lib") // Make sure that the directory is clean dotty.tools.io.Directory(defaultOutputDir + "tastyBootstrap").deleteRecursively() - val sep = java.io.File.pathSeparator - val opt = TestFlags( - // compile with bootstrapped library on cp: - defaultOutputDir + libGroup + "/src/" + sep + - // as well as bootstrapped compiler: - defaultOutputDir + dotty1Group + "/dotty/" + sep + - // and the other compiler dependenies: - Properties.compilerInterface + sep + Properties.scalaLibrary + sep + Properties.scalaAsm + sep + - Properties.dottyInterfaces + sep + Properties.tastyCore + sep + Properties.jlineTerminal + sep + - Properties.jlineReader, + List( + // compile with bootstrapped library on cp: + defaultOutputDir + libGroup + "/lib/", + // and bootstrapped tasty-core: + defaultOutputDir + tastyCoreGroup + "/tastyCore/", + // as well as bootstrapped compiler: + defaultOutputDir + dotty1Group + "/dotty1/", + // and the other compiler dependencies: + Properties.compilerInterface, Properties.scalaLibrary, Properties.scalaAsm, + Properties.dottyInterfaces, Properties.jlineTerminal, Properties.jlineReader, + ).mkString(File.pathSeparator), Array("-Ycheck-reentrant", "-Yemit-tasty-in-class") ) @@ -222,20 +225,23 @@ class CompilationTests extends ParallelTesting { val librarySources = libraryDirs.flatMap(sources(_)) val lib = - compileList("src", librarySources, + compileList("lib", librarySources, defaultOptions.and("-Ycheck-reentrant", "-Yerased-terms", // support declaration of scala.compiletime.erasedValue // "-strict", // TODO: re-enable once we allow : @unchecked in pattern definitions. Right now, lots of narrowing pattern definitions fail. "-priorityclasspath", defaultOutputDir))(libGroup) + val tastyCoreSources = sources(Paths.get("tasty/src")) + val tastyCore = compileList("tastyCore", tastyCoreSources, opt)(tastyCoreGroup) + val compilerSources = sources(Paths.get("compiler/src")) val compilerManagedSources = sources(Properties.dottyCompilerManagedSources) - val dotty1 = compileList("dotty", compilerSources ++ compilerManagedSources, opt)(dotty1Group) - val dotty2 = compileList("dotty", compilerSources ++ compilerManagedSources, opt)(dotty2Group) + val dotty1 = compileList("dotty1", compilerSources ++ compilerManagedSources, opt)(dotty1Group) + val dotty2 = compileList("dotty2", compilerSources ++ compilerManagedSources, opt)(dotty2Group) val tests = { - lib.keepOutput :: dotty1.keepOutput :: aggregateTests( + lib.keepOutput :: tastyCore.keepOutput :: dotty1.keepOutput :: aggregateTests( dotty2, compileShallowFilesInDir("compiler/src/dotty/tools", opt), compileShallowFilesInDir("compiler/src/dotty/tools/dotc", opt), @@ -255,9 +261,10 @@ class CompilationTests extends ParallelTesting { }.map(_.checkCompile()) def assertExists(path: String) = assertTrue(Files.exists(Paths.get(path))) - assertExists(s"out/$dotty1Group/dotty/") - assertExists(s"out/$dotty2Group/dotty/") - assertExists(s"out/$libGroup/src/") + assertExists(s"out/$libGroup/lib/") + assertExists(s"out/$tastyCoreGroup/tastyCore/") + assertExists(s"out/$dotty1Group/dotty1/") + assertExists(s"out/$dotty2Group/dotty2/") compileList("idempotency", List("tests/idempotency/BootstrapChecker.scala", "tests/idempotency/IdempotencyCheck.scala"), defaultOptions).checkRuns() tests.foreach(_.delete()) diff --git a/tests/idempotency/BootstrapChecker.scala b/tests/idempotency/BootstrapChecker.scala index 7c88ed597298..cfd868060409 100644 --- a/tests/idempotency/BootstrapChecker.scala +++ b/tests/idempotency/BootstrapChecker.scala @@ -1,5 +1,5 @@ object Test { def main(args: Array[String]): Unit = - IdempotencyCheck.checkIdempotency("out/tastyBootstrap/dotty1", "out/tastyBootstrap/dotty2") + IdempotencyCheck.checkIdempotency("out/tastyBootstrap/dotty1/dotty1", "out/tastyBootstrap/dotty2/dotty2") } From 86f90d3d8270a0cd8392eb9b9310d78bea8c5d85 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Thu, 16 May 2019 18:22:32 +0200 Subject: [PATCH 07/11] Upgrade to sbt 1.3.6, release sbt-dotty 0.4.0 Switching to sbt 1.3 allows us to pass both the scala-library and dotty-library jars as standard library jars to `ScalaInstance` whereas before we could only pass one of them which could have weird consequences. --- .drone.yml | 18 ++--- community-build/project/build.properties | 1 - .../communitybuild/CommunityBuildTest.scala | 2 +- project/Build.scala | 9 ++- project/build.properties | 2 +- .../dotty/tools/sbtplugin/DottyPlugin.scala | 76 ++++++++----------- 6 files changed, 48 insertions(+), 60 deletions(-) delete mode 100644 community-build/project/build.properties diff --git a/.drone.yml b/.drone.yml index 9cb33a3e0bfe..35a4b6177b49 100644 --- a/.drone.yml +++ b/.drone.yml @@ -27,7 +27,7 @@ steps: - name: test pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: [ clone ] commands: - cp -R . /tmp/1/ && cd /tmp/1/ @@ -36,7 +36,7 @@ steps: - name: test_bootstrapped pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: [ clone ] commands: - cp -R . /tmp/2/ && cd /tmp/2/ @@ -45,7 +45,7 @@ steps: - name: community_build pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: [ clone ] commands: - cp -R . /tmp/3/ && cd /tmp/3/ @@ -55,7 +55,7 @@ steps: - name: test_sbt pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: [ clone ] commands: - cp -R . /tmp/4/ && cd /tmp/4/ @@ -67,7 +67,7 @@ steps: - name: test_java11 pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: [ clone ] commands: - export PATH="/usr/lib/jvm/java-11-openjdk-amd64/bin:$PATH" @@ -80,7 +80,7 @@ steps: - name: documentation pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: - test - test_bootstrapped @@ -99,7 +99,7 @@ steps: - name: publish_nightly pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: - test - test_bootstrapped @@ -126,7 +126,7 @@ steps: - name: publish_release pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: - test - test_bootstrapped @@ -169,7 +169,7 @@ steps: - name: publish_sbt_release pull: default - image: lampepfl/dotty:2019-10-17 + image: lampepfl/dotty:2020-01-09 depends_on: - test - test_bootstrapped diff --git a/community-build/project/build.properties b/community-build/project/build.properties deleted file mode 100644 index 72f902892a13..000000000000 --- a/community-build/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.2.7 diff --git a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala index 1c2c8aba7321..d03ebc905e15 100644 --- a/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala +++ b/community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala @@ -100,7 +100,7 @@ final case class SbtCommunityProject(project: String, sbtTestCommand: String, case Some(ivyHome) => List(s"-Dsbt.ivy.home=$ivyHome") case _ => Nil extraSbtArgs ++ sbtProps ++ List( - "-sbt-version", "1.2.7", + "-sbt-version", "1.3.6", s"--addPluginSbtFile=$sbtPluginFilePath") object projects diff --git a/project/Build.scala b/project/Build.scala index 21fd9e09f2c7..f73866848728 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -65,7 +65,7 @@ object Build { val referenceVersion = "0.22.0-bin-20200114-193f7de-NIGHTLY" val baseVersion = "0.22.0" - val baseSbtDottyVersion = "0.3.5" + val baseSbtDottyVersion = "0.4.0" // Versions used by the vscode extension to create a new project // This should be the latest published releases. @@ -196,6 +196,11 @@ object Build { state }, + // Turn off the sbt supershell because it can mangle the output of some tasks + // (see https://github.com/sbt/sbt/issues/5122, https://github.com/sbt/sbt/issues/5352) + // and in general I find it more distracting than helpful anyway. + useSuperShell := false, + // Credentials to release to Sonatype credentials ++= ( for { @@ -1114,8 +1119,6 @@ object Build { version := "0.1.17-snapshot", // Keep in sync with package.json autoScalaLibrary := false, publishArtifact := false, - includeFilter in unmanagedSources := NothingFilter | "*.ts" | "**.json", - watchSources in Global ++= (unmanagedSources in Compile).value, resourceGenerators in Compile += Def.task { // Resources that will be copied when bootstrapping a new project val buildSbtFile = baseDirectory.value / "out" / "build.sbt" diff --git a/project/build.properties b/project/build.properties index 72f902892a13..00b48d978b25 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.7 +sbt.version=1.3.6 diff --git a/sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala b/sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala index 2b12150520a2..3af93b4d6bd2 100644 --- a/sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala +++ b/sbt-dotty/src/dotty/tools/sbtplugin/DottyPlugin.scala @@ -154,7 +154,7 @@ object DottyPlugin extends AutoPlugin { override val globalSettings: Seq[Def.Setting[_]] = Seq( onLoad in Global := onLoad.in(Global).value.andThen { state => - val requiredVersion = ">=1.2.7" + val requiredVersion = ">=1.3.6" val sbtV = sbtVersion.value if (!VersionNumber(sbtV).matchesSemVer(SemanticSelector(requiredVersion))) @@ -210,41 +210,42 @@ object DottyPlugin extends AutoPlugin { scalaBinaryVersion.value }, - // Ideally, we should have: + // We want: // // 1. Nothing but the Java standard library on the _JVM_ bootclasspath // (starting with Java 9 we cannot inspect it so we don't have a choice) // - // 2. scala-library, dotty-library, dotty-compiler, dotty-doc on the _JVM_ - // classpath, because we need all of those to actually run the compiler - // and the doc tool. + // 2. scala-library, dotty-library, dotty-compiler and its dependencies on the _JVM_ + // classpath, because we need all of those to actually run the compiler. // NOTE: All of those should have the *same version* (equal to scalaVersion // for everything but scala-library). + // (Complication: because dottydoc is a separate artifact with its own dependencies, + // running it requires putting extra dependencies on the _JVM_ classpath) // - // 3. scala-library, dotty-library on the _compiler_ bootclasspath because - // user code should always have access to the symbols from these jars but - // should not be able to shadow them (the compiler bootclasspath has - // higher priority than the compiler classpath). + // 3. scala-library, dotty-library on the _compiler_ bootclasspath or + // classpath (the only difference between them is that the compiler + // bootclasspath has higher priority, but that should never + // make a difference in a sane environment). // NOTE: the versions of {scala,dotty}-library used here do not necessarily // match the one used in 2. because a dependency of the current project might - // require more recent versions, this is OK. + // require a more recent standard library version, this is OK + // TODO: ... but if macros are used we might be forced to use the same + // versions in the JVM and compiler classpaths to avoid problems, this + // needs to be investigated. // // 4. every other dependency of the user project on the _compiler_ // classpath. // - // Unfortunately, zinc assumes that the compiler bootclasspath is only - // made of one jar (scala-library), so to make this work we'll need to - // either change sbt's bootclasspath handling or wait until the - // dotty-library jar and scala-library jars are merged into one jar. - // Furthermore, zinc will put on the compiler bootclasspath the + // By default, zinc will put on the compiler bootclasspath the // scala-library used on the JVM classpath, even if the current project // transitively depends on a newer scala-library (this works because Scala // 2 guarantees forward- and backward- binary compatibility, but we don't // necessarily want to keep doing that in Scala 3). // So for the moment, let's just put nothing at all on the compiler - // bootclasspath, and instead have scala-library and dotty-library on the - // compiler classpath. This means that user code could shadow symbols - // from these jars but we can live with that for now. + // bootclasspath, and instead let sbt dependency management choose which + // scala-library and dotty-library to put on the compiler classpath. + // Maybe eventually we should just remove the compiler bootclasspath since + // it's a source of complication with only dubious benefits. // sbt crazy scoping rules mean that when we override `classpathOptions` // below we also override `classpathOptions in console` which is normally @@ -404,7 +405,7 @@ object DottyPlugin extends AutoPlugin { /** Create a scalaInstance task that uses Dotty based on `moduleName`. */ def dottyScalaInstanceTask(moduleName: String): Initialize[Task[ScalaInstance]] = Def.task { - val ivyConfig0 = mkIvyConfiguration.value + val ivyConfig0 = Classpaths.mkIvyConfiguration.value // When compiling non-bootstrapped projects in the build of Dotty itself, // dependency resolution might pick a local project which is not what we // want. We avoid this by dropping the inter-project resolver. @@ -436,37 +437,22 @@ object DottyPlugin extends AutoPlugin { ) } - // Copy-pasted from sbt until we upgrade to a version of sbt - // with https://github.com/sbt/sbt/pull/5271 in. - def mkIvyConfiguration: Initialize[Task[InlineIvyConfiguration]] = - Def.task { - val (rs, other) = (fullResolvers.value.toVector, otherResolvers.value.toVector) - val s = streams.value - Classpaths.warnResolversConflict(rs ++: other, s.log) - InlineIvyConfiguration() - .withPaths(ivyPaths.value) - .withResolvers(rs) - .withOtherResolvers(other) - .withModuleConfigurations(moduleConfigurations.value.toVector) - .withLock(Defaults.lock(appConfiguration.value)) - .withChecksums((checksums in update).value.toVector) - .withResolutionCacheDir(crossTarget.value / "resolution-cache") - .withUpdateOptions(updateOptions.value) - .withLog(s.log) - } - + // Adapted from private mkScalaInstance in sbt def makeScalaInstance( state: State, dottyVersion: String, scalaLibrary: File, dottyLibrary: File, compiler: File, all: Seq[File] ): ScalaInstance = { - val loader = state.classLoaderCache(all.toList) - val loaderLibraryOnly = state.classLoaderCache(List(dottyLibrary, scalaLibrary)) + val libraryLoader = state.classLoaderCache(List(dottyLibrary, scalaLibrary)) + class DottyLoader + extends URLClassLoader(all.map(_.toURI.toURL).toArray, libraryLoader) + val fullLoader = state.classLoaderCache.cachedCustomClassloader( + all.toList, + () => new DottyLoader + ) new ScalaInstance( dottyVersion, - loader, - loaderLibraryOnly, - scalaLibrary, // Should be a Seq also containing dottyLibrary but zinc - // doesn't support this, see comment above our redefinition - // of `classpathOption` + fullLoader, + libraryLoader, + Array(dottyLibrary, scalaLibrary), compiler, all.toArray, None) From cd8f1b80f98792bfcbd28fe0f8e0e22e64db12dc Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Fri, 10 Jan 2020 15:02:59 +0100 Subject: [PATCH 08/11] staging: fix classpath discovery with sbt >= 1.3 This fixes the sbt-dotty/quoted-example-project scripted test after the previous commit. Getting a classpath from a classloader is impossible in general, so we really shouldn't be relying on it, but I fixed the thing we currently use to at least work with sbt >= 1.3, sbt now uses multiple layers of classloaders so we have to recurse on the parent of the classloader to find all the URLs. I also made it more correct by not appending System.getProperty("java.class.path") to the classpath, this is incorrect in general and can lead to classpath pollution (e.g., when launched from sbt where java.class.path will contain the jars used by sbt itself, which might include a different version of scala-library than the one we use). --- .../scala/quoted/staging/QuoteDriver.scala | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/staging/src/scala/quoted/staging/QuoteDriver.scala b/staging/src/scala/quoted/staging/QuoteDriver.scala index 68441e61cf84..06168b257af2 100644 --- a/staging/src/scala/quoted/staging/QuoteDriver.scala +++ b/staging/src/scala/quoted/staging/QuoteDriver.scala @@ -11,6 +11,9 @@ import dotty.tools.dotc.reporting._ import scala.quoted._ import scala.quoted.staging.Toolbox import java.net.URLClassLoader +import java.nio.file.Paths +import java.io.File +import scala.annotation.tailrec /** Driver to compile quoted code * @@ -53,7 +56,7 @@ private class QuoteDriver(appClassloader: ClassLoader) extends Driver { override def initCtx: Context = { val ictx = contextBase.initialCtx - ictx.settings.classpath.update(getCurrentClasspath(appClassloader))(ictx) + ictx.settings.classpath.update(classpathFromClassloader(appClassloader))(ictx) ictx } @@ -64,17 +67,29 @@ private class QuoteDriver(appClassloader: ClassLoader) extends Driver { ctx.setReporter(new ThrowingReporter(ctx.reporter)) } - private def getCurrentClasspath(cl: ClassLoader): String = { - val classpath0 = System.getProperty("java.class.path") - cl match { - case cl: URLClassLoader => - // Loads the classes loaded by this class loader - // When executing `run` or `test` in sbt the classpath is not in the property java.class.path - import java.nio.file.Paths - val newClasspath = cl.getURLs.map(url => Paths.get(url.toURI).toString) - newClasspath.mkString("", java.io.File.pathSeparator, if (classpath0 == "") "" else java.io.File.pathSeparator + classpath0) - case _ => classpath0 - } + /** Attempt to recreate a classpath from a classloader. + * + * BEWARE: with exotic enough classloaders, this may not work at all or do + * the wrong thing. + */ + private def classpathFromClassloader(cl: ClassLoader): String = { + @tailrec + def loop(cl: ClassLoader, suffixClasspath: String): String = + cl match { + case cl: URLClassLoader => + val updatedClasspath = cl.getURLs + .map(url => Paths.get(url.toURI).toAbsolutePath.toString) + .mkString( + "", + File.pathSeparator, + if (suffixClasspath.isEmpty) "" else File.pathSeparator + suffixClasspath + ) + loop(cl.getParent, updatedClasspath) + case _ => + suffixClasspath + } + + loop(cl, "") } } From 2cc8671165743ce7751648480149d78191afbde9 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 10 Jan 2020 16:15:29 +0100 Subject: [PATCH 09/11] Make classpathFromClassloader non quadratic, add tests The tests ensure that we've fixed #7897. Co-Authored-By: Guillaume Martres --- .../sbt-dotty/i7897/build-no-fork.sbt | 5 ++++ sbt-dotty/sbt-test/sbt-dotty/i7897/build.sbt | 5 ++++ .../sbt-dotty/i7897/project/build.properties | 1 + .../sbt-dotty/i7897/project/plugins.sbt | 1 + .../i7897/src/main/scala/hello/i7897.scala | 14 +++++++++ sbt-dotty/sbt-test/sbt-dotty/i7897/test | 3 ++ .../scala/quoted/staging/QuoteDriver.scala | 30 ++++++++----------- 7 files changed, 42 insertions(+), 17 deletions(-) create mode 100644 sbt-dotty/sbt-test/sbt-dotty/i7897/build-no-fork.sbt create mode 100644 sbt-dotty/sbt-test/sbt-dotty/i7897/build.sbt create mode 100644 sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties create mode 100644 sbt-dotty/sbt-test/sbt-dotty/i7897/project/plugins.sbt create mode 100644 sbt-dotty/sbt-test/sbt-dotty/i7897/src/main/scala/hello/i7897.scala create mode 100644 sbt-dotty/sbt-test/sbt-dotty/i7897/test diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/build-no-fork.sbt b/sbt-dotty/sbt-test/sbt-dotty/i7897/build-no-fork.sbt new file mode 100644 index 000000000000..802dff0748fe --- /dev/null +++ b/sbt-dotty/sbt-test/sbt-dotty/i7897/build-no-fork.sbt @@ -0,0 +1,5 @@ +scalaVersion := sys.props("plugin.scalaVersion") + +libraryDependencies += "ch.epfl.lamp" %% "dotty-staging" % scalaVersion.value + +fork := false diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/build.sbt b/sbt-dotty/sbt-test/sbt-dotty/i7897/build.sbt new file mode 100644 index 000000000000..0f1833c230ab --- /dev/null +++ b/sbt-dotty/sbt-test/sbt-dotty/i7897/build.sbt @@ -0,0 +1,5 @@ +scalaVersion := sys.props("plugin.scalaVersion") + +libraryDependencies += "ch.epfl.lamp" %% "dotty-staging" % scalaVersion.value + +fork := true diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties b/sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties new file mode 100644 index 000000000000..00b48d978b25 --- /dev/null +++ b/sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.3.6 diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/project/plugins.sbt b/sbt-dotty/sbt-test/sbt-dotty/i7897/project/plugins.sbt new file mode 100644 index 000000000000..c17caab2d98c --- /dev/null +++ b/sbt-dotty/sbt-test/sbt-dotty/i7897/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % sys.props("plugin.version")) diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/src/main/scala/hello/i7897.scala b/sbt-dotty/sbt-test/sbt-dotty/i7897/src/main/scala/hello/i7897.scala new file mode 100644 index 000000000000..eb4b5ac08be4 --- /dev/null +++ b/sbt-dotty/sbt-test/sbt-dotty/i7897/src/main/scala/hello/i7897.scala @@ -0,0 +1,14 @@ +import scala.quoted._, staging._ + +given Toolbox = Toolbox.make(getClass.getClassLoader) + +val f: Array[Int] => Int = run { + val stagedSum: Expr[Array[Int] => Int] = '{ (arr: Array[Int]) => 6 } + println(stagedSum.show) + stagedSum +} + +object Main { + def main(args: Array[String]): Unit = + f.apply(Array(1, 2, 3)) // Returns 6 +} diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/test b/sbt-dotty/sbt-test/sbt-dotty/i7897/test new file mode 100644 index 000000000000..bd20c79fdc9e --- /dev/null +++ b/sbt-dotty/sbt-test/sbt-dotty/i7897/test @@ -0,0 +1,3 @@ +> run +$ copy-file build-no-fork.sbt build.sbt +> run diff --git a/staging/src/scala/quoted/staging/QuoteDriver.scala b/staging/src/scala/quoted/staging/QuoteDriver.scala index 06168b257af2..dd23554b455f 100644 --- a/staging/src/scala/quoted/staging/QuoteDriver.scala +++ b/staging/src/scala/quoted/staging/QuoteDriver.scala @@ -73,23 +73,19 @@ private class QuoteDriver(appClassloader: ClassLoader) extends Driver { * the wrong thing. */ private def classpathFromClassloader(cl: ClassLoader): String = { - @tailrec - def loop(cl: ClassLoader, suffixClasspath: String): String = - cl match { - case cl: URLClassLoader => - val updatedClasspath = cl.getURLs - .map(url => Paths.get(url.toURI).toAbsolutePath.toString) - .mkString( - "", - File.pathSeparator, - if (suffixClasspath.isEmpty) "" else File.pathSeparator + suffixClasspath - ) - loop(cl.getParent, updatedClasspath) - case _ => - suffixClasspath - } - - loop(cl, "") + val classpathBuff = List.newBuilder[String] + def collectClassLoaderPaths(cl: ClassLoader): Unit = cl match { + case cl: URLClassLoader => + collectClassLoaderPaths(cl.getParent) + // Parent classloaders are searched before their child, so the part of + // the classpath coming from the child is added at the _end_ of the + // classpath. + classpathBuff ++= + cl.getURLs.iterator.map(url => Paths.get(url.toURI).toAbsolutePath.toString) + case _ => + } + collectClassLoaderPaths(cl) + classpathBuff.result().mkString(java.io.File.pathSeparator) } } From 3677971b5d81fbf100f7d181298bd48e4878f233 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Mon, 13 Jan 2020 21:23:32 +0100 Subject: [PATCH 10/11] Fix scripted test i7897 - Move build-no-fork.sbt in a subdirectory, otherwise it would be loaded by sbt. - Call reload after changing build.sbt, otherwise nothing would actually change. - Remove build.properties, no longer needed since we now use sbt 1.3.6 by default. --- .../sbt-test/sbt-dotty/i7897/{ => changes}/build-no-fork.sbt | 0 sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties | 1 - sbt-dotty/sbt-test/sbt-dotty/i7897/test | 3 ++- 3 files changed, 2 insertions(+), 2 deletions(-) rename sbt-dotty/sbt-test/sbt-dotty/i7897/{ => changes}/build-no-fork.sbt (100%) delete mode 100644 sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/build-no-fork.sbt b/sbt-dotty/sbt-test/sbt-dotty/i7897/changes/build-no-fork.sbt similarity index 100% rename from sbt-dotty/sbt-test/sbt-dotty/i7897/build-no-fork.sbt rename to sbt-dotty/sbt-test/sbt-dotty/i7897/changes/build-no-fork.sbt diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties b/sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties deleted file mode 100644 index 00b48d978b25..000000000000 --- a/sbt-dotty/sbt-test/sbt-dotty/i7897/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.3.6 diff --git a/sbt-dotty/sbt-test/sbt-dotty/i7897/test b/sbt-dotty/sbt-test/sbt-dotty/i7897/test index bd20c79fdc9e..42d5b04677a8 100644 --- a/sbt-dotty/sbt-test/sbt-dotty/i7897/test +++ b/sbt-dotty/sbt-test/sbt-dotty/i7897/test @@ -1,3 +1,4 @@ > run -$ copy-file build-no-fork.sbt build.sbt +$ copy-file changes/build-no-fork.sbt build.sbt +> reload > run From 63339549c6b36087ae259176685ef062dfd01ece Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Mon, 13 Jan 2020 21:14:19 +0100 Subject: [PATCH 11/11] Fix classpath detection when using the REPL This was broken after I stopped appending `System.getProperty("java.class.path")` to the constructed classpath. Fixed with a hack, but this is still better than looking at java.class.path. --- .../scala/quoted/staging/QuoteDriver.scala | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/staging/src/scala/quoted/staging/QuoteDriver.scala b/staging/src/scala/quoted/staging/QuoteDriver.scala index dd23554b455f..fe4410bafeb6 100644 --- a/staging/src/scala/quoted/staging/QuoteDriver.scala +++ b/staging/src/scala/quoted/staging/QuoteDriver.scala @@ -74,15 +74,32 @@ private class QuoteDriver(appClassloader: ClassLoader) extends Driver { */ private def classpathFromClassloader(cl: ClassLoader): String = { val classpathBuff = List.newBuilder[String] - def collectClassLoaderPaths(cl: ClassLoader): Unit = cl match { - case cl: URLClassLoader => - collectClassLoaderPaths(cl.getParent) - // Parent classloaders are searched before their child, so the part of - // the classpath coming from the child is added at the _end_ of the - // classpath. - classpathBuff ++= - cl.getURLs.iterator.map(url => Paths.get(url.toURI).toAbsolutePath.toString) - case _ => + def collectClassLoaderPaths(cl: ClassLoader): Unit = { + if (cl != null) { + cl match { + case cl: URLClassLoader => + // This is wrong if we're in a subclass of URLClassLoader + // that filters loading classes from its parent ¯\_(ツ)_/¯ + collectClassLoaderPaths(cl.getParent) + // Parent classloaders are searched before their child, so the part of + // the classpath coming from the child is added at the _end_ of the + // classpath. + classpathBuff ++= + cl.getURLs.iterator.map(url => Paths.get(url.toURI).toAbsolutePath.toString) + case _ => + // HACK: We can't just collect the classpath from arbitrary parent + // classloaders since the current classloader might intentionally + // filter loading classes from its parent (for example + // BootFilteredLoader in the sbt launcher does this and we really + // don't want to include the scala-library that sbt depends on + // here), but we do need to look at the parent of the REPL + // classloader, so we special case it. We can't do this using a type + // test since the REPL classloader class itself is normally loaded + // with a different classloader. + if (cl.getClass.getName == classOf[AbstractFileClassLoader].getName) + collectClassLoaderPaths(cl.getParent) + } + } } collectClassLoaderPaths(cl) classpathBuff.result().mkString(java.io.File.pathSeparator)