Loading symbols from TASTy files directly (#17594)

Before this PR we used to parse the classfiles first and when we found a
classfile that has a `TASTY` attribute we switched and loaded the tasty.
Now we find the `.tasty` files directly and load the symbols directly
from them. We still load the class files to check that the UUID in that
classfile matches the UUID in the TASTy file.

When looking for classes in the classpath, we prioritize the TASTy files
over classfiles. This implies that the symbol loader will receive the
`.tasty` files for Scala 3 code and `.class` for Scala 2 and Java code.

A variant of the `ClassfileParser` called `ClassfileTastyUUIDParser` was
added to have a way to check the UUID in the `TASTY` attribute of the
classfile. The `ClassfileParser` could not be used directly because it
eagerly tries to initialize parts of the symbols that are already loaded
from the TASTy file, causing some conflicts.

Open question: should we only check the TASTy UUID under some flag to
avoid loading both the `.tasty` and the `.class` files? The second
commit introduces this check.

Author: nicolasstucki

compiler/src/dotty/tools/backend/jvm/PostProcessor.scala

@@ -23,6 +23,12 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
 
   def postProcessAndSendToDisk(generatedDefs: GeneratedDefs): Unit = {
     val GeneratedDefs(classes, tasty) = generatedDefs
+    if !ctx.settings.YoutputOnlyTasty.value then
+      postProcessClassesAndSendToDisk(classes)
+    postProcessTastyAndSendToDisk(tasty)
+  }
+
+  private def postProcessClassesAndSendToDisk(classes: List[GeneratedClass]): Unit = {
     for (GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated) <- classes) {
       val bytes =
         try
@@ -46,8 +52,10 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
         if clsFile != null then onFileCreated(clsFile)
       }
     }
+  }
 
-    for (GeneratedTasty(classNode, binaryGen) <- tasty){
+  private def postProcessTastyAndSendToDisk(tasty: List[GeneratedTasty]): Unit = {
+    for (GeneratedTasty(classNode, binaryGen) <- tasty) {
       classfileWriter.writeTasty(classNode.name.nn, binaryGen())
     }
   }

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -278,15 +278,17 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile
 
   def findClassFile(className: String): Option[AbstractFile] = {
     val relativePath = FileUtils.dirPath(className)
-    val classFile = new JFile(dir, relativePath + ".class")
-    if (classFile.exists) {
-      Some(classFile.toPath.toPlainFile)
-    }
-    else None
+    val tastyFile = new JFile(dir, relativePath + ".tasty")
+    if tastyFile.exists then Some(tastyFile.toPath.toPlainFile)
+    else
+      val classFile = new JFile(dir, relativePath + ".class")
+      if classFile.exists then  Some(classFile.toPath.toPlainFile)
+      else None
   }
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  protected def isMatchingFile(f: JFile): Boolean = f.isClass
+  protected def isMatchingFile(f: JFile): Boolean =
+    f.isTasty || (f.isClass && f.classToTasty.isEmpty)
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -20,6 +20,10 @@ object FileUtils {
     def isClass: Boolean = !file.isDirectory && file.hasExtension("class") && !file.name.endsWith("$class.class")
       // FIXME: drop last condition when we stop being compatible with Scala 2.11
 
+    def isTasty: Boolean = !file.isDirectory && file.hasExtension("tasty")
+
+    def isScalaBinary: Boolean = file.isClass || file.isTasty
+
     def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java"))
 
     // TODO do we need to check also other files using ZipMagicNumber like in scala.tools.nsc.io.Jar.isJarOrZip?
@@ -30,17 +34,34 @@ object FileUtils {
      * and returning given default value in other case
      */
     def toURLs(default: => Seq[URL] = Seq.empty): Seq[URL] = if (file.file == null) default else Seq(file.toURL)
+
+    /** Returns the tasty file associated with this class file */
+    def classToTasty: Option[AbstractFile] =
+      assert(file.isClass, s"non-class: $file")
+      val tastyName = classNameToTasty(file.name)
+      Option(file.resolveSibling(tastyName))
   }
 
   extension (file: JFile) {
     def isPackage: Boolean = file.isDirectory && mayBeValidPackage(file.getName)
 
-    def isClass: Boolean = file.isFile && file.getName.endsWith(".class") && !file.getName.endsWith("$class.class")
-      // FIXME: drop last condition when we stop being compatible with Scala 2.11
+    def isClass: Boolean = file.isFile && file.getName.endsWith(SUFFIX_CLASS) && !file.getName.endsWith("$class.class")
+    // FIXME: drop last condition when we stop being compatible with Scala 2.11
+
+    def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY)
+
+    /** Returns the tasty file associated with this class file */
+    def classToTasty: Option[JFile] =
+      assert(file.isClass, s"non-class: $file")
+      val tastyName = classNameToTasty(file.getName.stripSuffix(".class"))
+      val tastyPath = file.toPath.resolveSibling(tastyName)
+      if java.nio.file.Files.exists(tastyPath) then Some(tastyPath.toFile) else None
+
   }
 
   private val SUFFIX_CLASS = ".class"
   private val SUFFIX_SCALA = ".scala"
+  private val SUFFIX_TASTY = ".tasty"
   private val SUFFIX_JAVA = ".java"
   private val SUFFIX_SIG = ".sig"
 
@@ -81,4 +102,15 @@ object FileUtils {
   def mkFileFilter(f: JFile => Boolean): FileFilter = new FileFilter {
     def accept(pathname: JFile): Boolean = f(pathname)
   }
+
+  /** Transforms a .class file name to a .tasty file name */
+  private def classNameToTasty(fileName: String): String =
+    val classOrModuleName = fileName.stripSuffix(".class")
+    val className =
+      if classOrModuleName.endsWith("$")
+        && classOrModuleName != "Null$" // scala.runtime.Null$
+        && classOrModuleName != "Nothing$" // scala.runtime.Nothing$
+      then classOrModuleName.stripSuffix("$")
+      else classOrModuleName
+    className + SUFFIX_TASTY
 }

compiler/src/dotty/tools/dotc/classpath/VirtualDirectoryClassPath.scala

@@ -41,12 +41,17 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
   override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl.apply
 
   def findClassFile(className: String): Option[AbstractFile] = {
-    val relativePath = FileUtils.dirPath(className) + ".class"
-    Option(lookupPath(dir)(relativePath.split(java.io.File.separator).toIndexedSeq, directory = false))
+    val pathSeq = FileUtils.dirPath(className).split(java.io.File.separator)
+    val parentDir = lookupPath(dir)(pathSeq.init.toSeq, directory = true)
+    if parentDir == null then return None
+    else
+      Option(lookupPath(parentDir)(pathSeq.last + ".tasty" :: Nil, directory = false))
+        .orElse(Option(lookupPath(parentDir)(pathSeq.last + ".class" :: Nil, directory = false)))
   }
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  protected def isMatchingFile(f: AbstractFile): Boolean = f.isClass
+  protected def isMatchingFile(f: AbstractFile): Boolean =
+    f.isTasty || (f.isClass && f.classToTasty.isEmpty)
 }

compiler/src/dotty/tools/dotc/classpath/ZipAndJarFileLookupFactory.scala

@@ -44,21 +44,21 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
     extends ZipArchiveFileLookup[ClassFileEntryImpl]
     with NoSourcePaths {
 
-    override def findClassFile(className: String): Option[AbstractFile] = {
-      val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className)
-      file(PackageName(pkg), simpleClassName + ".class").map(_.file)
-    }
+    override def findClassFile(className: String): Option[AbstractFile] =
+      findClass(className).map(_.file)
 
     // This method is performance sensitive as it is used by SBT's ExtractDependencies phase.
-    override def findClass(className: String): Option[ClassRepresentation] = {
+    override def findClass(className: String): Option[ClassFileEntryImpl] = {
       val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className)
-      file(PackageName(pkg), simpleClassName + ".class")
+      val binaries = files(PackageName(pkg), simpleClassName + ".tasty", simpleClassName + ".class")
+      binaries.find(_.file.isTasty).orElse(binaries.find(_.file.isClass))
     }
 
     override private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 
     override protected def createFileEntry(file: FileZipArchive#Entry): ClassFileEntryImpl = ClassFileEntryImpl(file)
-    override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isClass
+    override protected def isRequiredFileType(file: AbstractFile): Boolean =
+      file.isTasty || (file.isClass && file.classToTasty.isEmpty)
   }
 
   /**

compiler/src/dotty/tools/dotc/classpath/ZipArchiveFileLookup.scala

@@ -43,6 +43,15 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends Efficie
     }
     yield createFileEntry(entry)
 
+  protected def files(inPackage: PackageName, names: String*): Seq[FileEntryType] =
+    for {
+      dirEntry <- findDirEntry(inPackage).toSeq
+      name <- names
+      entry <- Option(dirEntry.lookupName(name, directory = false))
+      if isRequiredFileType(entry)
+    }
+    yield createFileEntry(entry)
+
   protected def file(inPackage: PackageName, name: String): Option[FileEntryType] =
     for {
       dirEntry <- findDirEntry(inPackage)

compiler/src/dotty/tools/dotc/config/JavaPlatform.scala

@@ -66,4 +66,7 @@ class JavaPlatform extends Platform {
 
   def newClassLoader(bin: AbstractFile)(using Context): SymbolLoader =
     new ClassfileLoader(bin)
+
+  def newTastyLoader(bin: AbstractFile)(using Context): SymbolLoader =
+    new TastyLoader(bin)
 }

compiler/src/dotty/tools/dotc/config/Platform.scala

@@ -36,6 +36,9 @@ abstract class Platform {
   /** Create a new class loader to load class file `bin` */
   def newClassLoader(bin: AbstractFile)(using Context): SymbolLoader
 
+  /** Create a new TASTy loader to load class file `bin` */
+  def newTastyLoader(bin: AbstractFile)(using Context): SymbolLoader
+
   /** The given symbol is a method with the right name and signature to be a runnable program. */
   def isMainMethod(sym: Symbol)(using Context): Boolean
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -369,6 +369,7 @@ private sealed trait YSettings:
   val YnoExperimental: Setting[Boolean] = BooleanSetting("-Yno-experimental", "Disable experimental language features.")
   val YlegacyLazyVals: Setting[Boolean] = BooleanSetting("-Ylegacy-lazy-vals", "Use legacy (pre 3.3.0) implementation of lazy vals.")
   val Yscala2Stdlib: Setting[Boolean] = BooleanSetting("-Yscala2-stdlib", "Used when compiling the Scala 2 standard library.")
+  val YoutputOnlyTasty: Setting[Boolean] = BooleanSetting("-Youtput-only-tasty", "Used to only generate the TASTy file without the classfiles")
 
   val YprofileEnabled: Setting[Boolean] = BooleanSetting("-Yprofile-enabled", "Enable profiling.")
   val YprofileDestination: Setting[String] = StringSetting("-Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "")

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -7,13 +7,14 @@ import java.nio.channels.ClosedByInterruptException
 
 import scala.util.control.NonFatal
 
+import dotty.tools.dotc.classpath.FileUtils.isTasty
 import dotty.tools.io.{ ClassPath, ClassRepresentation, AbstractFile }
 import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions
 
 import Contexts._, Symbols._, Flags._, SymDenotations._, Types._, Scopes._, Names._
 import NameOps._
 import StdNames._
-import classfile.ClassfileParser
+import classfile.{ClassfileParser, ClassfileTastyUUIDParser}
 import Decorators._
 
 import util.Stats
@@ -23,6 +24,7 @@ import ast.desugar
 
 import parsing.JavaParsers.OutlineJavaParser
 import parsing.Parsers.OutlineParser
+import dotty.tools.tasty.TastyHeaderUnpickler
 
 
 object SymbolLoaders {
@@ -192,10 +194,13 @@ object SymbolLoaders {
         if (ctx.settings.verbose.value) report.inform("[symloader] picked up newer source file for " + src.path)
         enterToplevelsFromSource(owner, nameOf(classRep), src)
       case (None, Some(src)) =>
-        if (ctx.settings.verbose.value) report.inform("[symloader] no class, picked up source file for " + src.path)
+        if (ctx.settings.verbose.value) report.inform("[symloader] no class or tasty, picked up source file for " + src.path)
         enterToplevelsFromSource(owner, nameOf(classRep), src)
       case (Some(bin), _) =>
-        enterClassAndModule(owner, nameOf(classRep), ctx.platform.newClassLoader(bin))
+        val completer =
+          if bin.isTasty then ctx.platform.newTastyLoader(bin)
+          else ctx.platform.newClassLoader(bin)
+        enterClassAndModule(owner, nameOf(classRep), completer)
     }
 
   def needCompile(bin: AbstractFile, src: AbstractFile): Boolean =
@@ -404,20 +409,38 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader {
   def description(using Context): String = "class file " + classfile.toString
 
   override def doComplete(root: SymDenotation)(using Context): Unit =
-    load(root)
-
-  def load(root: SymDenotation)(using Context): Unit = {
     val (classRoot, moduleRoot) = rootDenots(root.asClass)
     val classfileParser = new ClassfileParser(classfile, classRoot, moduleRoot)(ctx)
-    val result = classfileParser.run()
-    if (mayLoadTreesFromTasty)
-      result match {
-        case Some(unpickler: tasty.DottyUnpickler) =>
-          classRoot.classSymbol.rootTreeOrProvider = unpickler
-          moduleRoot.classSymbol.rootTreeOrProvider = unpickler
-        case _ =>
-      }
-  }
+    classfileParser.run()
+}
+
+class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
+
+  override def sourceFileOrNull: AbstractFile | Null = tastyFile
+
+  def description(using Context): String = "TASTy file " + tastyFile.toString
+
+  override def doComplete(root: SymDenotation)(using Context): Unit =
+    val (classRoot, moduleRoot) = rootDenots(root.asClass)
+    val tastyBytes = tastyFile.toByteArray
+    val unpickler = new tasty.DottyUnpickler(tastyBytes)
+    unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
+    if mayLoadTreesFromTasty then
+      classRoot.classSymbol.rootTreeOrProvider = unpickler
+      moduleRoot.classSymbol.rootTreeOrProvider = unpickler
+    checkTastyUUID(tastyFile, tastyBytes)
+
+
+  private def checkTastyUUID(tastyFile: AbstractFile, tastyBytes: Array[Byte])(using Context): Unit =
+    var classfile = tastyFile.resolveSibling(tastyFile.name.stripSuffix(".tasty") + ".class")
+    if classfile == null then
+      classfile = tastyFile.resolveSibling(tastyFile.name.stripSuffix(".tasty") + "$.class")
+    if classfile != null then
+      val tastyUUID = new TastyHeaderUnpickler(tastyBytes).readHeader()
+      new ClassfileTastyUUIDParser(classfile)(ctx).checkTastyUUID(tastyUUID)
+    else
+      // This will be the case in any of our tests that compile with `-Youtput-only-tasty`
+      report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID")
 
   private def mayLoadTreesFromTasty(using Context): Boolean =
     ctx.settings.YretainTrees.value || ctx.settings.fromTasty.value

compiler/src/dotty/tools/dotc/core/Symbols.scala

@@ -31,6 +31,7 @@ import io.AbstractFile
 import util.{SourceFile, NoSource, Property, SourcePosition, SrcPos, EqHashMap}
 import scala.annotation.internal.sharable
 import config.Printers.typr
+import dotty.tools.dotc.classpath.FileUtils.isScalaBinary
 
 object Symbols {
 
@@ -151,7 +152,7 @@ object Symbols {
       * symbols defined by the user in a prior run of the REPL, that are still valid.
       */
     final def isDefinedInSource(using Context): Boolean =
-      span.exists && isValidInCurrentRun && associatedFileMatches(_.extension != "class")
+      span.exists && isValidInCurrentRun && associatedFileMatches(!_.isScalaBinary)
 
     /** Is symbol valid in current run? */
     final def isValidInCurrentRun(using Context): Boolean =
@@ -272,7 +273,7 @@ object Symbols {
     /** The class file from which this class was generated, null if not applicable. */
     final def binaryFile(using Context): AbstractFile | Null = {
       val file = associatedFile
-      if (file != null && file.extension == "class") file else null
+      if file != null && file.isScalaBinary then file else null
     }
 
     /** A trap to avoid calling x.symbol on something that is already a symbol.
@@ -285,7 +286,7 @@ object Symbols {
 
     final def source(using Context): SourceFile = {
       def valid(src: SourceFile): SourceFile =
-        if (src.exists && src.file.extension != "class") src
+        if (src.exists && !src.file.isScalaBinary) src
         else NoSource
 
       if (!denot.exists) NoSource
@@ -463,7 +464,7 @@ object Symbols {
       if !mySource.exists && !denot.is(Package) then
         // this allows sources to be added in annotations after `sourceOfClass` is first called
         val file = associatedFile
-        if file != null && file.extension != "class" then
+        if file != null && !file.isScalaBinary then
           mySource = ctx.getSource(file)
         else
           mySource = defn.patchSource(this)