diff --git a/compiler/src/dotty/tools/dotc/Driver.scala b/compiler/src/dotty/tools/dotc/Driver.scala index 5640ae16a1d1..c0a6b9176454 100644 --- a/compiler/src/dotty/tools/dotc/Driver.scala +++ b/compiler/src/dotty/tools/dotc/Driver.scala @@ -86,6 +86,7 @@ class Driver { protected def fromTastySetup(fileNames0: List[String], ctx0: Context): (List[String], Context) = given Context = ctx0 if (ctx0.settings.fromTasty.value) { + val fromTastyIgnoreList = ctx0.settings.YfromTastyIgnoreList.value.toSet // Resolve classpath and class names of tasty files val (classPaths, classNames) = fileNames0.flatMap { name => val path = Paths.get(name) @@ -96,7 +97,7 @@ class Driver { Nil else if name.endsWith(".jar") then new dotty.tools.io.Jar(File(name)).toList.collect { - case e if e.getName.endsWith(".tasty") => + case e if e.getName.endsWith(".tasty") && !fromTastyIgnoreList(e.getName) => (name, e.getName.stripSuffix(".tasty").replace("/", ".")) } else diff --git a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala index b40a4c4c2b4b..a57b2ee30ac9 100644 --- a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala +++ b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala @@ -45,7 +45,7 @@ class ScalaSettings extends Settings.SettingGroup { val language: Setting[List[String]] = MultiStringSetting("-language", "feature", "Enable one or more language features.") withAbbreviation "--language" val rewrite: Setting[Option[Rewrites]] = OptionSetting[Rewrites]("-rewrite", "When used in conjunction with a `...-migration` source version, rewrites sources to migrate to new version.") withAbbreviation "--rewrite" val silentWarnings: Setting[Boolean] = BooleanSetting("-nowarn", "Silence all warnings.") withAbbreviation "--no-warnings" - val fromTasty: Setting[Boolean] = BooleanSetting("-from-tasty", "Compile classes from tasty in classpath. The arguments are used as class names.") withAbbreviation "--from-tasty" + val fromTasty: Setting[Boolean] = BooleanSetting("-from-tasty", "Compile classes from tasty files. The arguments are .tasty or .jar files.") withAbbreviation "--from-tasty" val newSyntax: Setting[Boolean] = BooleanSetting("-new-syntax", "Require `then` and `do` in control expressions.") val oldSyntax: Setting[Boolean] = BooleanSetting("-old-syntax", "Require `(...)` around conditions.") @@ -158,6 +158,7 @@ class ScalaSettings extends Settings.SettingGroup { val YretainTrees: Setting[Boolean] = BooleanSetting("-Yretain-trees", "Retain trees for top-level classes, accessible from ClassSymbol#tree") val Ysemanticdb: Setting[Boolean] = BooleanSetting("-Ysemanticdb", "Store information in SemanticDB.") val YshowTreeIds: Setting[Boolean] = BooleanSetting("-Yshow-tree-ids", "Uniquely tag all tree nodes in debugging output.") + val YfromTastyIgnoreList: Setting[List[String]] = MultiStringSetting("-Yfrom-tasty-ignore-list", "file", "List of `tasty` files in jar files that will not be loaded when using -from-tasty") val YprofileEnabled: Setting[Boolean] = BooleanSetting("-Yprofile-enabled", "Enable profiling.") val YprofileDestination: Setting[String] = StringSetting("-Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "") diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala index 12e8a60c5a28..c4283bb43473 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala @@ -25,10 +25,6 @@ class TastyClassName(bytes: Array[Byte]) { import dotty.tools.tasty.TastyFormat._ def unpickle(reader: TastyReader, tastyName: NameTable): (TermName, TermName) = { import reader._ - def readName() = { - val idx = readNat() - nameAtRef(NameRef(idx)) - } def readNames(packageName: TermName): (TermName, TermName) = { val tag = readByte() if (tag >= firstLengthTreeTag) { @@ -36,7 +32,7 @@ class TastyClassName(bytes: Array[Byte]) { val end = currentAddr + len tag match { case TYPEDEF => - val className = readName() + val className = reader.readName() goto(end) (packageName, className) case IMPORT | VALDEF => @@ -48,7 +44,14 @@ class TastyClassName(bytes: Array[Byte]) { } else tag match { case TERMREFpkg | TYPEREFpkg => - val subPackageName = readName() + val subPackageName = reader.readName() + readNames(subPackageName) + case SHAREDtype => + val addr = reader.readAddr() + val reader2 = reader.subReader(addr, reader.endAddr) + val tag2 = reader2.readByte() + assert(tag2 == TERMREFpkg || tag2 == TYPEREFpkg) + val subPackageName = reader2.readName() readNames(subPackageName) case _ => readNames(packageName) @@ -56,5 +59,11 @@ class TastyClassName(bytes: Array[Byte]) { } readNames(nme.EMPTY_PACKAGE) } + + extension (reader: TastyReader) def readName() = { + val idx = reader.readNat() + nameAtRef(NameRef(idx)) + } } + } diff --git a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala index 56cdb6a7bf5c..1ea2e67a7432 100644 --- a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala +++ b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala @@ -10,6 +10,7 @@ import dotty.tools.dotc.util.ClasspathFromClassloader import scala.quoted._ import java.io.File.pathSeparator +import java.io.File.separator class BootstrappedStdLibTASYyTest: @@ -19,17 +20,14 @@ class BootstrappedStdLibTASYyTest: @Test def testTastyInspector: Unit = loadWithTastyInspector(loadBlacklisted) + /** Test that we can load and compile trees from TASTy in a Jar */ + @Test def testFromTastyInJar: Unit = + compileFromTastyInJar(loadBlacklisted.union(compileBlacklisted)) + /** Test that we can load and compile trees from TASTy */ @Test def testFromTasty: Unit = compileFromTasty(loadBlacklisted.union(compileBlacklisted)) - @Ignore - @Test def testWhiteListFromTasty: Unit = - val whitelist = Set( - "scala.collection.mutable.StringBuilder" - ) - compileFromTasty(x => !whitelist(x)) - @Test def blacklistNoDuplicates = def testDup(name: String, list: List[String], set: Set[String]) = assert(list.size == set.size, @@ -44,14 +42,14 @@ class BootstrappedStdLibTASYyTest: "`compileBlacklist` contains names that are already in `loadBlacklist`: \n ", "\n ", "\n\n")) @Test def blacklistsOnlyContainsClassesThatExist = - val scalaLibJarTastyClassNamesSet = scalaLibJarTastyClassNames.toSet + val scalaLibTastyPathsSet = scalaLibTastyPaths.toSet val intersection = loadBlacklisted & compileBlacklisted - assert(loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty, - loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString( - "`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n")) - assert(compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty, - compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString( - "`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n")) + assert(loadBlacklisted.diff(scalaLibTastyPathsSet).isEmpty, + loadBlacklisted.diff(scalaLibTastyPathsSet).mkString( + "`loadBlacklisted` contains names that are not in `scalaLibTastyPaths`: \n ", "\n ", "\n\n")) + assert(compileBlacklisted.diff(scalaLibTastyPathsSet).isEmpty, + compileBlacklisted.diff(scalaLibTastyPathsSet).mkString( + "`loadBlacklisted` contains names that are not in `scalaLibTastyPaths`: \n ", "\n ", "\n\n")) @Ignore @Test def testLoadBacklistIsMinimal = @@ -79,7 +77,7 @@ class BootstrappedStdLibTASYyTest: val blacklist = blacklist0 - notBlacklisted println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)") try { - compileFromTasty(blacklist) + compileFromTastyInJar(blacklist) shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted } catch { @@ -92,44 +90,59 @@ end BootstrappedStdLibTASYyTest object BootstrappedStdLibTASYyTest: - val scalaLibJarPath = System.getProperty("dotty.scala.library") + def scalaLibJarPath = System.getProperty("dotty.scala.library") + def scalaLibClassesPath = + java.nio.file.Paths.get(scalaLibJarPath).getParent.resolve("classes").normalize - val scalaLibJarTastyClassNames = { - val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath))) - scalaLibJar.toList.map(_.toString).filter(_.endsWith(".tasty")) - .map(_.stripSuffix(".tasty").replace("/", ".")) - .sorted - } + val scalaLibTastyPaths = + new Directory(scalaLibClassesPath).deepFiles + .filter(_.`extension` == "tasty") + .map(_.normalize.path.stripPrefix(scalaLibClassesPath.toString + "/")) + .toList - def loadWithTastyInspector(blacklisted: String => Boolean): Unit = + def loadWithTastyInspector(blacklisted: Set[String]): Unit = val inspector = new scala.tasty.inspector.TastyInspector { def processCompilationUnit(using QuoteContext)(root: qctx.reflect.Tree): Unit = root.showExtractors // Check that we can traverse the full tree () } - val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted) - val hasErrors = inspector.inspectTastyFilesInJar(scalaLibJarPath) + val tastyFiles = scalaLibTastyPaths.filterNot(blacklisted) + val hasErrors = inspector.inspectTastyFiles(tastyFiles.map(x => scalaLibClassesPath.resolve(x).toString)) assert(!hasErrors, "Errors reported while loading from TASTy") - def compileFromTasty(blacklisted: String => Boolean): Unit = { + def compileFromTastyInJar(blacklisted: Set[String]): Unit = { val driver = new dotty.tools.dotc.Driver - val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader) - val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted) + val yFromTastyBlacklist = + blacklisted.mkString("-Yfrom-tasty-ignore-list:", ",", "") val args = Array( - "-classpath", s"$scalaLibJarPath$pathSeparator$currentClasspath", + "-classpath", ClasspathFromClassloader(getClass.getClassLoader), "-from-tasty", - "-nowarn" - ) ++ classNames + "-nowarn", + yFromTastyBlacklist, + scalaLibJarPath, + ) val reporter = driver.process(args) assert(reporter.errorCount == 0, "Errors while re-compiling") } - /** List of classes that cannot be loaded from TASTy */ + def compileFromTasty(blacklisted: Set[String]): Unit = { + val driver = new dotty.tools.dotc.Driver + val tastyFiles = scalaLibTastyPaths.filterNot(blacklisted) + val args = Array( + "-classpath", ClasspathFromClassloader(getClass.getClassLoader), + "-from-tasty", + "-nowarn", + ) ++ tastyFiles.map(x => scalaLibClassesPath.resolve(x).toString) + val reporter = driver.process(args) + assert(reporter.errorCount == 0, "Errors while re-compiling") + } + + /** List of tasty files that cannot be loaded from TASTy */ def loadBlacklist = List[String]( // No issues :) ) - /** List of classes that cannot be recompilied from TASTy */ + /** List of tasty files that cannot be recompilied from TASTy */ def compileBlacklist = List[String]( // See #10048 // failed: java.lang.AssertionError: assertion failed: class Boolean @@ -139,22 +152,22 @@ object BootstrappedStdLibTASYyTest: // at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass$(BCodeHelpers.scala:210) // at dotty.tools.backend.jvm.BCodeSkelBuilder$PlainSkelBuilder.getClassBTypeAndRegisterInnerClass(BCodeSkelBuilder.scala:62) // at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.internalName(BCodeHelpers.scala:237) - "scala.Array", - "scala.Boolean", - "scala.Byte", - "scala.Char", - "scala.Double", - "scala.Float", - "scala.Int", - "scala.Long", - "scala.Short", - "scala.Unit", - ) - - /** Set of classes that cannot be loaded from TASTy */ + "scala/Array.tasty", + "scala/Boolean.tasty", + "scala/Byte.tasty", + "scala/Char.tasty", + "scala/Double.tasty", + "scala/Float.tasty", + "scala/Int.tasty", + "scala/Long.tasty", + "scala/Short.tasty", + "scala/Unit.tasty", + ).map(_.replace("/", separator)) + + /** Set of tasty files that cannot be loaded from TASTy */ def loadBlacklisted = loadBlacklist.toSet - /** Set of classes that cannot be recompilied from TASTy */ + /** Set of tasty files that cannot be recompilied from TASTy */ def compileBlacklisted = compileBlacklist.toSet end BootstrappedStdLibTASYyTest