Refactor stdlib-bootstrapped

Author: nicolasstucki

.github/workflows/ci.yaml

@@ -134,7 +134,7 @@ jobs:
 
       - name: Cmd Tests
         run: |
-          ./project/scripts/sbt ";dist/pack; scala3-bootstrapped/compile; scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;stdlib-bootstrapped/test:run ;stdlib-full-bootstrapped/test:run ;stdlib-bootstrapped-tasty-tests/test; scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test"
+          ./project/scripts/sbt ";dist/pack; scala3-bootstrapped/compile; scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;stdlib-bootstrapped/test:run ;stdlib-bootstrapped-compat/test:run ;stdlib-bootstrapped-tasty-tests/test; scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test"
           ./project/scripts/cmdTests
           ./project/scripts/bootstrappedOnlyCmdTests
 
@@ -253,7 +253,7 @@ jobs:
 
       - name: MiMa
         run: |
-          ./project/scripts/sbt ";scala3-interfaces/mimaReportBinaryIssues ;scala3-library-bootstrapped/mimaReportBinaryIssues ;scala3-library-bootstrappedJS/mimaReportBinaryIssues; tasty-core-bootstrapped/mimaReportBinaryIssues; stdlib-bootstrapped/mimaReportBinaryIssues; stdlib-full-bootstrapped/mimaReportBinaryIssues"
+          ./project/scripts/sbt ";scala3-interfaces/mimaReportBinaryIssues ;scala3-library-bootstrapped/mimaReportBinaryIssues ;scala3-library-bootstrappedJS/mimaReportBinaryIssues; tasty-core-bootstrapped/mimaReportBinaryIssues; stdlib-bootstrapped/mimaReportBinaryIssues; stdlib-bootstrapped-compat/mimaReportBinaryIssues; stdlib-with-tasty/mimaReportBinaryIssues"
 
   community_build_a:
     runs-on: [self-hosted, Linux]

build.sbt

@@ -14,8 +14,9 @@ val `scala3-language-server` = Build.`scala3-language-server`
 val `scala3-bench` = Build.`scala3-bench`
 val `scala3-bench-bootstrapped` = Build.`scala3-bench-bootstrapped`
 val `scala3-bench-micro` = Build.`scala3-bench-micro`
+val `stdlib-bootstrapped-compat` = Build.`stdlib-bootstrapped-compat`
+val `stdlib-with-tasty` = Build.`stdlib-with-tasty`
 val `stdlib-bootstrapped` = Build.`stdlib-bootstrapped`
-val `stdlib-full-bootstrapped` = Build.`stdlib-full-bootstrapped`
 val `stdlib-bootstrapped-tasty-tests` = Build.`stdlib-bootstrapped-tasty-tests`
 val `tasty-core` = Build.`tasty-core`
 val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped`

project/Build.scala

@@ -922,69 +922,13 @@ object Build {
       javaOptions := (`scala3-compiler-bootstrapped` / javaOptions).value
     )
 
-  /** Scala library compiled by dotty using the latest published sources of the library */
-  lazy val `stdlib-bootstrapped` = project.in(file("stdlib-bootstrapped")).
-    withCommonSettings(Bootstrapped).
-    dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
-    settings(stdliBootstrappedCommonSettings).
-    settings(
-      Compile/scalacOptions ++= {
-        Seq(
-          "-sourcepath",
-          Seq(
-            baseDirectory.value / "stdlib-bootstrapped" / "src",
-            (Compile/sourceManaged).value / "scala-library-src",
-          ).mkString(File.pathSeparator),
-        )
-      },
-      mimaCheckDirection := "both",
-      mimaBinaryIssueFilters ++= MiMaFilters.LibraryBootstrappedForward,
-      libraryDependencies += // For jar packaging
-        ("org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped)),
-      exportJars := true,
-      Compile / packageBin / artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
-        artifact.name + "-" + stdlibVersion(Bootstrapped) + "-" + module.revision + "." + artifact.extension
-      },
-      Compile / packageBin / mappings := {
-        val s = streams.value
-        val report = updateClassifiers.value
-        val scalaLibraryJar =
-          // FIXME
-          report.select(
-              configuration = configurationFilter("sourcedeps"),
-              module = (_: ModuleID).name == "scala-library",
-              artifact = artifactFilter(`type` = "src")).headOption.getOrElse {
-            sys.error(s"Could not fetch scala-library sources")
-          }.getParentFile / "scala-library-2.13.10.jar"
-        val cacheDir = s.cacheDirectory
-        val trgDir = (Compile / sourceManaged).value / "scala-library-bin"
-        FileFunction.cached(cacheDir / "fetchScalaLibraryBin",
-            FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
-          s.log.info(s"Unpacking scala-library classfiles to $trgDir...")
-          if (trgDir.exists)
-            IO.delete(trgDir)
-          IO.createDirectory(trgDir)
-          IO.unzip(scalaLibraryJar, trgDir)
-
-          (trgDir ** "*.class").get.toSet
-        } (Set(scalaLibraryJar)).toSeq
-        val allScalaLibraryJarFiles =
-          trgDir.allPaths.get().filterNot(_.getName.endsWith(".MF")) // TODO use parts of the old manifest?
-        val allScalaLibraryJarMappings: Seq[(java.io.File, String)] =
-          allScalaLibraryJarFiles.zip(allScalaLibraryJarFiles.map(_.relativeTo(trgDir).get.getPath))
-
-        val tastyFileMappings: Seq[(java.io.File, String)] =
-          (Compile / packageBin / mappings).value.filter(_._1.getName.endsWith(".tasty"))
-
-        allScalaLibraryJarMappings ++ tastyFileMappings
-      }
-
-    )
-
   /** Scala library compiled by dotty using the latest published sources of the library
-   *  and the current Scala 3 livrary sources
+   *  and the current Scala 3 library sources.
+   *
+   *  This version of the library is not binary compatible with the Scala 2 compiled library.
+   *  It compiles every file as Scala 3 sources.
    */
-  lazy val `stdlib-full-bootstrapped` = project.in(file("stdlib-full-bootstrapped")).
+  lazy val `stdlib-bootstrapped` = project.in(file("stdlib-bootstrapped")).
     withCommonSettings(Bootstrapped).
     dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
     settings(stdliBootstrappedCommonSettings).
@@ -1031,6 +975,81 @@ object Build {
       }.taskValue,
     )
 
+  /** Scala 2 library compiled by dotty using the latest published sources of the library.
+   *
+   *  This version of the library is compiled by Scala 3 while being binary compatible with Scala 2.
+   */
+  lazy val `stdlib-bootstrapped-compat` = project.in(file("stdlib-bootstrapped-compat")).
+    withCommonSettings(Bootstrapped).
+    dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
+    settings(stdliBootstrappedCommonSettings).
+    settings(
+      Compile/scalacOptions ++= {
+        Seq(
+          "-sourcepath",
+          Seq(
+            baseDirectory.value / "stdlib-bootstrapped" / "src",
+            (Compile/sourceManaged).value / "scala-library-src",
+          ).mkString(File.pathSeparator),
+        )
+      },
+      mimaCheckDirection := "both",
+      mimaBinaryIssueFilters ++= MiMaFilters.LibraryBootstrappedForward,
+    )
+
+  /** Re-packages stdlib compiled be Scala 2 adding TASTy generated by `stdlib-bootstrapped-compat` */
+  lazy val `stdlib-with-tasty` = project.in(file("stdlib-with-tasty")).
+    withCommonSettings(Bootstrapped).
+    dependsOn(`stdlib-bootstrapped-compat`).
+    settings(commonBootstrappedSettings).
+    settings(
+      moduleName := "scala-library",
+      exportJars := true,
+      Compile / packageBin / artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
+        artifact.name + "-" + stdlibVersion(Bootstrapped) + "-" + module.revision + "." + artifact.extension
+      },
+      (Compile / packageBin / mappings) := {
+        val allScalaLibraryJarMappings: Seq[(java.io.File, String)] = {
+          val s = streams.value
+          val report = updateClassifiers.value
+          val scalaLibraryJar =
+            // FIXME
+            report.select(
+                configuration = configurationFilter("compile"),
+                module = (_: ModuleID).name == "scala-library",
+                artifact = artifactFilter(`type` = "src"))
+                .headOption.getOrElse {
+              sys.error(s"Could not fetch scala-library sources")
+            }.getParentFile / "scala-library-2.13.10.jar"
+          val cacheDir = s.cacheDirectory
+          val trgDir = (Compile / sourceManaged).value / "scala-library-bin" // FIXME not really managed sources
+          FileFunction.cached(cacheDir / "fetchScalaLibraryBin",
+              FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
+            s.log.info(s"Unpacking scala-library classfiles to $trgDir...")
+            if (trgDir.exists)
+              IO.delete(trgDir)
+            IO.createDirectory(trgDir)
+            IO.unzip(scalaLibraryJar, trgDir)
+            (trgDir ** "*").get.toSet
+          } (Set(scalaLibraryJar)).toSeq
+          val allScalaLibraryJarFiles =
+            trgDir.allPaths.get().filterNot(_.getName.endsWith(".MF")) // TODO use parts of the old manifest?
+          allScalaLibraryJarFiles.zip(allScalaLibraryJarFiles.map(_.relativeTo(trgDir).get.getPath))
+        }
+        val tastyFileMappings =
+          (`stdlib-bootstrapped-compat` / Compile / packageBin / mappings).value
+            .filter(_._1.getName.endsWith(".tasty"))
+        allScalaLibraryJarMappings ++ tastyFileMappings
+      },
+      mimaCurrentClassfiles :=
+        mimaCurrentClassfiles.value.getParentFile() /
+          (moduleName.value + "-" + stdlibVersion(Bootstrapped) + "-" + scalaVersion.value.replace("-nonbootstrapped", "") + ".jar"),
+      mimaPreviousArtifacts +=
+        ("org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped)),
+      mimaCheckDirection := "both",
+      mimaBinaryIssueFilters := Seq(),
+    )
+
   lazy val stdliBootstrappedCommonSettings =
     commonBootstrappedSettings ++
     Seq(
@@ -1094,7 +1113,7 @@ object Build {
         "scala.annotation.unspecialized",
       ),
       mimaCheckDirection := "backward",
-      mimaBinaryIssueFilters ++= MiMaFilters.LibraryBootstrapped,
+      mimaBinaryIssueFilters ++= MiMaFilters.LibraryBootstrappedBackwards,
       run := {
         val args: Seq[String] = spaceDelimited("<arg>").parsed
         val reference = (Compile/sourceManaged).value / "scala-library-src"
@@ -1154,7 +1173,7 @@ object Build {
     settings(commonBootstrappedSettings).
     settings(
       javaOptions := (`scala3-compiler-bootstrapped` / javaOptions).value,
-      javaOptions += "-Ddotty.scala.library=" + (`stdlib-full-bootstrapped` / Compile / packageBin).value.getAbsolutePath
+      javaOptions += "-Ddotty.scala.library=" + (`stdlib-bootstrapped` / Compile / packageBin).value.getAbsolutePath
     )
 
   lazy val `scala3-sbt-bridge` = project.in(file("sbt-bridge/src")).

project/MiMaFilters.scala

@@ -50,7 +50,7 @@ object MiMaFilters {
     ProblemFilters.exclude[MissingClassProblem]("dotty.tools.dotc.interfaces.DiagnosticRelatedInformation")
   )
 
-  val LibraryBootstrapped: Seq[ProblemFilter] = Seq(
+  val LibraryBootstrappedBackwards: Seq[ProblemFilter] = Seq(
     // Files that are not compiled in the bootstrapped library
     ProblemFilters.exclude[MissingClassProblem]("scala.AnyVal"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("scala.Unit.this"),

stdlib-bootstrapped-compat/test/Main.scala

@@ -0,0 +1,12 @@
+package hello
+
+enum Color:
+  case Red, Green, Blue
+
+object HelloWorld:
+  def main(args: Array[String]): Unit = {
+    println("hello dotty.superbootstrapped!")
+    println(Color.Red)
+    println(Color.Green)
+    println(Color.Blue)
+  }