From 6837f23742c3013340c9642a32dedf684915a3d2 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Tue, 3 Nov 2020 10:52:05 +0100 Subject: [PATCH 1/5] Test stdlib from TASTy directly from Jar Passing the class names will no be supported in the future. --- compiler/src/dotty/tools/dotc/Driver.scala | 3 ++- .../tools/dotc/config/ScalaSettings.scala | 3 ++- .../test/BootstrappedStdLibTASYyTest.scala | 22 ++++++++----------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/Driver.scala b/compiler/src/dotty/tools/dotc/Driver.scala index 5640ae16a1d1..8fd98a7f763b 100644 --- a/compiler/src/dotty/tools/dotc/Driver.scala +++ b/compiler/src/dotty/tools/dotc/Driver.scala @@ -86,6 +86,7 @@ class Driver { protected def fromTastySetup(fileNames0: List[String], ctx0: Context): (List[String], Context) = given Context = ctx0 if (ctx0.settings.fromTasty.value) { + val fromTastyBlacklist = ctx0.settings.YfromTastyBlacklist.value.toSet // Resolve classpath and class names of tasty files val (classPaths, classNames) = fileNames0.flatMap { name => val path = Paths.get(name) @@ -96,7 +97,7 @@ class Driver { Nil else if name.endsWith(".jar") then new dotty.tools.io.Jar(File(name)).toList.collect { - case e if e.getName.endsWith(".tasty") => + case e if e.getName.endsWith(".tasty") && !fromTastyBlacklist(e.getName) => (name, e.getName.stripSuffix(".tasty").replace("/", ".")) } else diff --git a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala index b40a4c4c2b4b..4bdcedea81f4 100644 --- a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala +++ b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala @@ -45,7 +45,7 @@ class ScalaSettings extends Settings.SettingGroup { val language: Setting[List[String]] = MultiStringSetting("-language", "feature", "Enable one or more language features.") withAbbreviation "--language" val rewrite: Setting[Option[Rewrites]] = OptionSetting[Rewrites]("-rewrite", "When used in conjunction with a `...-migration` source version, rewrites sources to migrate to new version.") withAbbreviation "--rewrite" val silentWarnings: Setting[Boolean] = BooleanSetting("-nowarn", "Silence all warnings.") withAbbreviation "--no-warnings" - val fromTasty: Setting[Boolean] = BooleanSetting("-from-tasty", "Compile classes from tasty in classpath. The arguments are used as class names.") withAbbreviation "--from-tasty" + val fromTasty: Setting[Boolean] = BooleanSetting("-from-tasty", "Compile classes from tasty files. The arguments are .tasty or .jar files.") withAbbreviation "--from-tasty" val newSyntax: Setting[Boolean] = BooleanSetting("-new-syntax", "Require `then` and `do` in control expressions.") val oldSyntax: Setting[Boolean] = BooleanSetting("-old-syntax", "Require `(...)` around conditions.") @@ -158,6 +158,7 @@ class ScalaSettings extends Settings.SettingGroup { val YretainTrees: Setting[Boolean] = BooleanSetting("-Yretain-trees", "Retain trees for top-level classes, accessible from ClassSymbol#tree") val Ysemanticdb: Setting[Boolean] = BooleanSetting("-Ysemanticdb", "Store information in SemanticDB.") val YshowTreeIds: Setting[Boolean] = BooleanSetting("-Yshow-tree-ids", "Uniquely tag all tree nodes in debugging output.") + val YfromTastyBlacklist: Setting[List[String]] = MultiStringSetting("-Yfrom-tasty-blacklist", "file", "List of `tasty` files in jar files that will not be loaded when using -from-tasty") val YprofileEnabled: Setting[Boolean] = BooleanSetting("-Yprofile-enabled", "Enable profiling.") val YprofileDestination: Setting[String] = StringSetting("-Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "") diff --git a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala index 56cdb6a7bf5c..94ba6b9f0efb 100644 --- a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala +++ b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala @@ -10,6 +10,7 @@ import dotty.tools.dotc.util.ClasspathFromClassloader import scala.quoted._ import java.io.File.pathSeparator +import java.io.File.separator class BootstrappedStdLibTASYyTest: @@ -23,13 +24,6 @@ class BootstrappedStdLibTASYyTest: @Test def testFromTasty: Unit = compileFromTasty(loadBlacklisted.union(compileBlacklisted)) - @Ignore - @Test def testWhiteListFromTasty: Unit = - val whitelist = Set( - "scala.collection.mutable.StringBuilder" - ) - compileFromTasty(x => !whitelist(x)) - @Test def blacklistNoDuplicates = def testDup(name: String, list: List[String], set: Set[String]) = assert(list.size == set.size, @@ -111,15 +105,17 @@ object BootstrappedStdLibTASYyTest: val hasErrors = inspector.inspectTastyFilesInJar(scalaLibJarPath) assert(!hasErrors, "Errors reported while loading from TASTy") - def compileFromTasty(blacklisted: String => Boolean): Unit = { + def compileFromTasty(blacklisted: Iterable[String]): Unit = { val driver = new dotty.tools.dotc.Driver - val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader) - val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted) + val yFromTastyBlacklist = + blacklisted.map(x => x.replace(".", separator) + ".tasty").mkString("-Yfrom-tasty-blacklist:", ",", "") val args = Array( - "-classpath", s"$scalaLibJarPath$pathSeparator$currentClasspath", + "-classpath", ClasspathFromClassloader(getClass.getClassLoader), "-from-tasty", - "-nowarn" - ) ++ classNames + "-nowarn", + yFromTastyBlacklist, + scalaLibJarPath, + ) val reporter = driver.process(args) assert(reporter.errorCount == 0, "Errors while re-compiling") } From 14e3ac582258b9bb9f382206f14556b24fc10c3e Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Tue, 3 Nov 2020 12:02:28 +0100 Subject: [PATCH 2/5] Test stdlib from tasty directly from tasty files --- .../test/BootstrappedStdLibTASYyTest.scala | 82 ++++++++++++------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala index 94ba6b9f0efb..4324b52bb66d 100644 --- a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala +++ b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala @@ -20,6 +20,10 @@ class BootstrappedStdLibTASYyTest: @Test def testTastyInspector: Unit = loadWithTastyInspector(loadBlacklisted) + /** Test that we can load and compile trees from TASTy in a Jar */ + @Test def testFromTastyInJar: Unit = + compileFromTastyInJar(loadBlacklisted.union(compileBlacklisted)) + /** Test that we can load and compile trees from TASTy */ @Test def testFromTasty: Unit = compileFromTasty(loadBlacklisted.union(compileBlacklisted)) @@ -38,14 +42,14 @@ class BootstrappedStdLibTASYyTest: "`compileBlacklist` contains names that are already in `loadBlacklist`: \n ", "\n ", "\n\n")) @Test def blacklistsOnlyContainsClassesThatExist = - val scalaLibJarTastyClassNamesSet = scalaLibJarTastyClassNames.toSet + val scalaLibTastyPathsSet = scalaLibTastyPaths.toSet val intersection = loadBlacklisted & compileBlacklisted - assert(loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty, - loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString( - "`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n")) - assert(compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty, - compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString( - "`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n")) + assert(loadBlacklisted.diff(scalaLibTastyPathsSet).isEmpty, + loadBlacklisted.diff(scalaLibTastyPathsSet).mkString( + "`loadBlacklisted` contains names that are not in `scalaLibTastyPaths`: \n ", "\n ", "\n\n")) + assert(compileBlacklisted.diff(scalaLibTastyPathsSet).isEmpty, + compileBlacklisted.diff(scalaLibTastyPathsSet).mkString( + "`loadBlacklisted` contains names that are not in `scalaLibTastyPaths`: \n ", "\n ", "\n\n")) @Ignore @Test def testLoadBacklistIsMinimal = @@ -73,7 +77,7 @@ class BootstrappedStdLibTASYyTest: val blacklist = blacklist0 - notBlacklisted println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)") try { - compileFromTasty(blacklist) + compileFromTastyInJar(blacklist) shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted } catch { @@ -86,7 +90,9 @@ end BootstrappedStdLibTASYyTest object BootstrappedStdLibTASYyTest: - val scalaLibJarPath = System.getProperty("dotty.scala.library") + def scalaLibJarPath = System.getProperty("dotty.scala.library") + def scalaLibClassesPath = + java.nio.file.Paths.get(scalaLibJarPath).getParent.resolve("classes").normalize val scalaLibJarTastyClassNames = { val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath))) @@ -95,20 +101,26 @@ object BootstrappedStdLibTASYyTest: .sorted } - def loadWithTastyInspector(blacklisted: String => Boolean): Unit = + val scalaLibTastyPaths = + new Directory(scalaLibClassesPath).deepFiles + .filter(_.`extension` == "tasty") + .map(_.normalize.path.stripPrefix(scalaLibClassesPath.toString + "/")) + .toList + + def loadWithTastyInspector(blacklisted: Set[String]): Unit = val inspector = new scala.tasty.inspector.TastyInspector { def processCompilationUnit(using QuoteContext)(root: qctx.reflect.Tree): Unit = root.showExtractors // Check that we can traverse the full tree () } - val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted) + val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted.map(_.stripSuffix(".tasty").replace("/", "."))) val hasErrors = inspector.inspectTastyFilesInJar(scalaLibJarPath) assert(!hasErrors, "Errors reported while loading from TASTy") - def compileFromTasty(blacklisted: Iterable[String]): Unit = { + def compileFromTastyInJar(blacklisted: Set[String]): Unit = { val driver = new dotty.tools.dotc.Driver val yFromTastyBlacklist = - blacklisted.map(x => x.replace(".", separator) + ".tasty").mkString("-Yfrom-tasty-blacklist:", ",", "") + blacklisted.mkString("-Yfrom-tasty-blacklist:", ",", "") val args = Array( "-classpath", ClasspathFromClassloader(getClass.getClassLoader), "-from-tasty", @@ -120,12 +132,24 @@ object BootstrappedStdLibTASYyTest: assert(reporter.errorCount == 0, "Errors while re-compiling") } - /** List of classes that cannot be loaded from TASTy */ + def compileFromTasty(blacklisted: Set[String]): Unit = { + val driver = new dotty.tools.dotc.Driver + val tastyFiles = scalaLibTastyPaths.filterNot(blacklisted) + val args = Array( + "-classpath", ClasspathFromClassloader(getClass.getClassLoader), + "-from-tasty", + "-nowarn", + ) ++ tastyFiles.map(x => scalaLibClassesPath.resolve(x).toString) + val reporter = driver.process(args) + assert(reporter.errorCount == 0, "Errors while re-compiling") + } + + /** List of tasty files that cannot be loaded from TASTy */ def loadBlacklist = List[String]( // No issues :) ) - /** List of classes that cannot be recompilied from TASTy */ + /** List of tasty files that cannot be recompilied from TASTy */ def compileBlacklist = List[String]( // See #10048 // failed: java.lang.AssertionError: assertion failed: class Boolean @@ -135,22 +159,22 @@ object BootstrappedStdLibTASYyTest: // at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass$(BCodeHelpers.scala:210) // at dotty.tools.backend.jvm.BCodeSkelBuilder$PlainSkelBuilder.getClassBTypeAndRegisterInnerClass(BCodeSkelBuilder.scala:62) // at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.internalName(BCodeHelpers.scala:237) - "scala.Array", - "scala.Boolean", - "scala.Byte", - "scala.Char", - "scala.Double", - "scala.Float", - "scala.Int", - "scala.Long", - "scala.Short", - "scala.Unit", - ) - - /** Set of classes that cannot be loaded from TASTy */ + "scala/Array.tasty", + "scala/Boolean.tasty", + "scala/Byte.tasty", + "scala/Char.tasty", + "scala/Double.tasty", + "scala/Float.tasty", + "scala/Int.tasty", + "scala/Long.tasty", + "scala/Short.tasty", + "scala/Unit.tasty", + ).map(_.replace("/", separator)) + + /** Set of tasty files that cannot be loaded from TASTy */ def loadBlacklisted = loadBlacklist.toSet - /** Set of classes that cannot be recompilied from TASTy */ + /** Set of tasty files that cannot be recompilied from TASTy */ def compileBlacklisted = compileBlacklist.toSet end BootstrappedStdLibTASYyTest From fc9e4d5697618dc10167c425e392977aa19d99a6 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Tue, 3 Nov 2020 10:24:18 +0100 Subject: [PATCH 3/5] Fix #10143: Handle shared package references --- .../dotc/core/tasty/TastyClassName.scala | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala index 12e8a60c5a28..c4283bb43473 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala @@ -25,10 +25,6 @@ class TastyClassName(bytes: Array[Byte]) { import dotty.tools.tasty.TastyFormat._ def unpickle(reader: TastyReader, tastyName: NameTable): (TermName, TermName) = { import reader._ - def readName() = { - val idx = readNat() - nameAtRef(NameRef(idx)) - } def readNames(packageName: TermName): (TermName, TermName) = { val tag = readByte() if (tag >= firstLengthTreeTag) { @@ -36,7 +32,7 @@ class TastyClassName(bytes: Array[Byte]) { val end = currentAddr + len tag match { case TYPEDEF => - val className = readName() + val className = reader.readName() goto(end) (packageName, className) case IMPORT | VALDEF => @@ -48,7 +44,14 @@ class TastyClassName(bytes: Array[Byte]) { } else tag match { case TERMREFpkg | TYPEREFpkg => - val subPackageName = readName() + val subPackageName = reader.readName() + readNames(subPackageName) + case SHAREDtype => + val addr = reader.readAddr() + val reader2 = reader.subReader(addr, reader.endAddr) + val tag2 = reader2.readByte() + assert(tag2 == TERMREFpkg || tag2 == TYPEREFpkg) + val subPackageName = reader2.readName() readNames(subPackageName) case _ => readNames(packageName) @@ -56,5 +59,11 @@ class TastyClassName(bytes: Array[Byte]) { } readNames(nme.EMPTY_PACKAGE) } + + extension (reader: TastyReader) def readName() = { + val idx = reader.readNat() + nameAtRef(NameRef(idx)) + } } + } From 5fe33698c1e1519ef9f9bf524841e4a31b32ad18 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Tue, 3 Nov 2020 12:06:35 +0100 Subject: [PATCH 4/5] Rename -Yfrom-tasty-blacklist to -Yfrom-tasty-ignore-list --- compiler/src/dotty/tools/dotc/Driver.scala | 4 ++-- compiler/src/dotty/tools/dotc/config/ScalaSettings.scala | 2 +- .../test/BootstrappedStdLibTASYyTest.scala | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/Driver.scala b/compiler/src/dotty/tools/dotc/Driver.scala index 8fd98a7f763b..c0a6b9176454 100644 --- a/compiler/src/dotty/tools/dotc/Driver.scala +++ b/compiler/src/dotty/tools/dotc/Driver.scala @@ -86,7 +86,7 @@ class Driver { protected def fromTastySetup(fileNames0: List[String], ctx0: Context): (List[String], Context) = given Context = ctx0 if (ctx0.settings.fromTasty.value) { - val fromTastyBlacklist = ctx0.settings.YfromTastyBlacklist.value.toSet + val fromTastyIgnoreList = ctx0.settings.YfromTastyIgnoreList.value.toSet // Resolve classpath and class names of tasty files val (classPaths, classNames) = fileNames0.flatMap { name => val path = Paths.get(name) @@ -97,7 +97,7 @@ class Driver { Nil else if name.endsWith(".jar") then new dotty.tools.io.Jar(File(name)).toList.collect { - case e if e.getName.endsWith(".tasty") && !fromTastyBlacklist(e.getName) => + case e if e.getName.endsWith(".tasty") && !fromTastyIgnoreList(e.getName) => (name, e.getName.stripSuffix(".tasty").replace("/", ".")) } else diff --git a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala index 4bdcedea81f4..a57b2ee30ac9 100644 --- a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala +++ b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala @@ -158,7 +158,7 @@ class ScalaSettings extends Settings.SettingGroup { val YretainTrees: Setting[Boolean] = BooleanSetting("-Yretain-trees", "Retain trees for top-level classes, accessible from ClassSymbol#tree") val Ysemanticdb: Setting[Boolean] = BooleanSetting("-Ysemanticdb", "Store information in SemanticDB.") val YshowTreeIds: Setting[Boolean] = BooleanSetting("-Yshow-tree-ids", "Uniquely tag all tree nodes in debugging output.") - val YfromTastyBlacklist: Setting[List[String]] = MultiStringSetting("-Yfrom-tasty-blacklist", "file", "List of `tasty` files in jar files that will not be loaded when using -from-tasty") + val YfromTastyIgnoreList: Setting[List[String]] = MultiStringSetting("-Yfrom-tasty-ignore-list", "file", "List of `tasty` files in jar files that will not be loaded when using -from-tasty") val YprofileEnabled: Setting[Boolean] = BooleanSetting("-Yprofile-enabled", "Enable profiling.") val YprofileDestination: Setting[String] = StringSetting("-Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "") diff --git a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala index 4324b52bb66d..8ac567e42734 100644 --- a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala +++ b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala @@ -120,7 +120,7 @@ object BootstrappedStdLibTASYyTest: def compileFromTastyInJar(blacklisted: Set[String]): Unit = { val driver = new dotty.tools.dotc.Driver val yFromTastyBlacklist = - blacklisted.mkString("-Yfrom-tasty-blacklist:", ",", "") + blacklisted.mkString("-Yfrom-tasty-ignore-list:", ",", "") val args = Array( "-classpath", ClasspathFromClassloader(getClass.getClassLoader), "-from-tasty", From 7fab830c2353e77b4e26918e18488416172a0d80 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Tue, 3 Nov 2020 15:13:43 +0100 Subject: [PATCH 5/5] Use tasty files as input for inspector tests --- .../test/BootstrappedStdLibTASYyTest.scala | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala index 8ac567e42734..1ea2e67a7432 100644 --- a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala +++ b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala @@ -94,13 +94,6 @@ object BootstrappedStdLibTASYyTest: def scalaLibClassesPath = java.nio.file.Paths.get(scalaLibJarPath).getParent.resolve("classes").normalize - val scalaLibJarTastyClassNames = { - val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath))) - scalaLibJar.toList.map(_.toString).filter(_.endsWith(".tasty")) - .map(_.stripSuffix(".tasty").replace("/", ".")) - .sorted - } - val scalaLibTastyPaths = new Directory(scalaLibClassesPath).deepFiles .filter(_.`extension` == "tasty") @@ -113,8 +106,8 @@ object BootstrappedStdLibTASYyTest: root.showExtractors // Check that we can traverse the full tree () } - val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted.map(_.stripSuffix(".tasty").replace("/", "."))) - val hasErrors = inspector.inspectTastyFilesInJar(scalaLibJarPath) + val tastyFiles = scalaLibTastyPaths.filterNot(blacklisted) + val hasErrors = inspector.inspectTastyFiles(tastyFiles.map(x => scalaLibClassesPath.resolve(x).toString)) assert(!hasErrors, "Errors reported while loading from TASTy") def compileFromTastyInJar(blacklisted: Set[String]): Unit = {