Add attributes section to TASTy and use it for Stdlib TASTy (#18599)

The `Attributes` section contains a list `Attribute` tags.

At this point, the only Scala 2 TASTy is the one from the standard
library. To know that a TASTy contains the definitions of the standard
library, we add the `SCALA2STANDARDLIBRARYattr` attribute to the TASTy
file. We mark all unpickled classes from those TASTy files with `Scala2x
| Scala2Tasty`.

Attributes will also be used to mark files that were compiled with
explicit nulls using the `EXPLICITNULLSattr` attribute.

Author: nicolasstucki

compiler/src/dotty/tools/dotc/core/Flags.scala

@@ -308,8 +308,8 @@ object Flags {
    */
   val (_, StableRealizable @ _, _) = newFlags(24, "<stable>")
 
-  /** A case parameter accessor */
-  val (_, CaseAccessor @ _, _) = newFlags(25, "<caseaccessor>")
+  /** A case parameter accessor / an unpickled Scala 2 TASTy (only for Scala 2 stdlib) */
+  val (_, CaseAccessor @ _, Scala2Tasty @ _) = newFlags(25, "<caseaccessor>", "<scala-2-tasty>")
 
   /** A Scala 2x super accessor / an unpickled Scala 2.x class */
   val (SuperParamAliasOrScala2x @ _, SuperParamAlias @ _, Scala2x @ _) = newFlags(26, "<super-param-alias>", "<scala-2.x>")

compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala

@@ -0,0 +1,26 @@
+package dotty.tools.dotc.core.tasty
+
+import dotty.tools.dotc.ast.{tpd, untpd}
+
+import dotty.tools.tasty.TastyBuffer
+import dotty.tools.tasty.TastyFormat, TastyFormat.AttributesSection
+
+import java.nio.charset.StandardCharsets
+
+object AttributePickler:
+
+  def pickleAttributes(
+    attributes: Attributes,
+    pickler: TastyPickler,
+    buf: TastyBuffer
+  ): Unit =
+    if attributes.scala2StandardLibrary || attributes.explicitNulls then // or any other attribute is set
+      pickler.newSection(AttributesSection, buf)
+      // Pickle attributes
+      if attributes.scala2StandardLibrary then buf.writeNat(TastyFormat.SCALA2STANDARDLIBRARYattr)
+      if attributes.explicitNulls then buf.writeNat(TastyFormat.EXPLICITNULLSattr)
+    end if
+
+  end pickleAttributes
+
+end AttributePickler

compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala

@@ -0,0 +1,33 @@
+package dotty.tools.dotc
+package core.tasty
+
+import scala.language.unsafeNulls
+
+import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer}
+
+import java.nio.charset.StandardCharsets
+
+class AttributeUnpickler(reader: TastyReader):
+  import reader._
+
+  lazy val attributeTags: List[Int] =
+    val listBuilder = List.newBuilder[Int]
+    while !isAtEnd do listBuilder += readNat()
+    listBuilder.result()
+
+  lazy val attributes: Attributes = {
+    var scala2StandardLibrary = false
+    var explicitNulls = false
+    for attributeTag <- attributeTags do
+      attributeTag match
+        case TastyFormat.SCALA2STANDARDLIBRARYattr => scala2StandardLibrary = true
+        case TastyFormat.EXPLICITNULLSattr => explicitNulls = true
+        case attribute =>
+          assert(false, "Unexpected attribute value: " + attribute)
+    Attributes(
+      scala2StandardLibrary,
+      explicitNulls,
+    )
+  }
+
+end AttributeUnpickler

compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala

@@ -0,0 +1,6 @@
+package dotty.tools.dotc.core.tasty
+
+class Attributes(
+  val scala2StandardLibrary: Boolean,
+  val explicitNulls: Boolean,
+)

compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

@@ -13,17 +13,17 @@ import Names.SimpleName
 import TreeUnpickler.UnpickleMode
 
 import dotty.tools.tasty.TastyReader
-import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection}
+import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection, AttributesSection}
 
 object DottyUnpickler {
 
   /** Exception thrown if classfile is corrupted */
   class BadSignature(msg: String) extends RuntimeException(msg)
 
-  class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler])
+  class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler], attributeUnpickler: Option[AttributeUnpickler])
   extends SectionUnpickler[TreeUnpickler](ASTsSection) {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): TreeUnpickler =
-      new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler)
+      new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler, attributeUnpickler)
   }
 
   class PositionsSectionUnpickler extends SectionUnpickler[PositionUnpickler](PositionsSection) {
@@ -35,6 +35,10 @@ object DottyUnpickler {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): CommentUnpickler =
       new CommentUnpickler(reader)
   }
+  class AttributesSectionUnpickler extends SectionUnpickler[AttributeUnpickler](AttributesSection) {
+    def unpickle(reader: TastyReader, nameAtRef: NameTable): AttributeUnpickler =
+      new AttributeUnpickler(reader)
+  }
 }
 
 /** A class for unpickling Tasty trees and symbols.
@@ -48,16 +52,21 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe
   val unpickler: TastyUnpickler = new TastyUnpickler(bytes)
   private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler)
   private val commentUnpicklerOpt = unpickler.unpickle(new CommentsSectionUnpickler)
-  private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)).get
+  private val attributeUnpicklerOpt = unpickler.unpickle(new AttributesSectionUnpickler)
+  private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)).get
 
   /** Enter all toplevel classes and objects into their scopes
    *  @param roots          a set of SymDenotations that should be overwritten by unpickling
    */
   def enter(roots: Set[SymDenotation])(using Context): Unit =
     treeUnpickler.enter(roots)
 
-  protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]): TreeSectionUnpickler =
-    new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)
+  protected def treeSectionUnpickler(
+    posUnpicklerOpt: Option[PositionUnpickler],
+    commentUnpicklerOpt: Option[CommentUnpickler],
+    attributeUnpicklerOpt: Option[AttributeUnpickler]
+  ): TreeSectionUnpickler =
+    new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)
 
   protected def computeRootTrees(using Context): List[Tree] = treeUnpickler.unpickle(mode)
 

compiler/src/dotty/tools/dotc/core/tasty/ScratchData.scala

@@ -10,11 +10,13 @@ class ScratchData:
   val pickledIndices = new mutable.BitSet
 
   val commentBuffer = new TastyBuffer(5000)
+  val attributeBuffer = new TastyBuffer(32)
 
   def reset() =
     assert(delta ne delta1)
     assert(delta.length == delta1.length)
     positionBuffer.reset()
     pickledIndices.clear()
     commentBuffer.reset()
+    attributeBuffer.reset()
 

compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala

@@ -9,7 +9,7 @@ import Contexts.*, Decorators.*
 import Names.Name
 import TastyUnpickler.*
 import util.Spans.offsetToInt
-import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection}
+import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection, AttributesSection}
 import java.nio.file.{Files, Paths}
 import dotty.tools.io.{JarArchive, Path}
 
@@ -84,14 +84,16 @@ class TastyPrinter(bytes: Array[Byte]) {
       case Some(s) => sb.append(s)
       case _ =>
     }
-    sb.append("\n\n")
     unpickle(new PositionSectionUnpickler) match {
-      case Some(s) => sb.append(s)
+      case Some(s) => sb.append("\n\n").append(s)
       case _ =>
     }
-    sb.append("\n\n")
     unpickle(new CommentSectionUnpickler) match {
-      case Some(s) => sb.append(s)
+      case Some(s) => sb.append("\n\n").append(s)
+      case _ =>
+    }
+    unpickle(new AttributesSectionUnpickler) match {
+      case Some(s) => sb.append("\n\n").append(s)
       case _ =>
     }
     sb.result
@@ -222,6 +224,20 @@ class TastyPrinter(bytes: Array[Byte]) {
     }
   }
 
+  class AttributesSectionUnpickler extends SectionUnpickler[String](AttributesSection) {
+  import dotty.tools.tasty.TastyFormat.attributeTagToString
+    private val sb: StringBuilder = new StringBuilder
+
+    def unpickle(reader: TastyReader, tastyName: NameTable): String = {
+      sb.append(s" ${reader.endAddr.index - reader.currentAddr.index}")
+      val attributeTags = new AttributeUnpickler(reader).attributeTags
+      sb.append(s"  attributes bytes:\n")
+      for attributeTag <- attributeTags do
+        sb.append("    ").append(attributeTagToString(attributeTag)).append("\n")
+      sb.result
+    }
+  }
+
   protected def nameStr(str: String): String = str
   protected def treeStr(str: String): String = str
   protected def lengthStr(str: String): String = str

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -52,11 +52,13 @@ import scala.compiletime.uninitialized
  *  @param reader              the reader from which to unpickle
  *  @param posUnpicklerOpt     the unpickler for positions, if it exists
  *  @param commentUnpicklerOpt the unpickler for comments, if it exists
+ *  @param attributeUnpicklerOpt the unpickler for attributes, if it exists
  */
 class TreeUnpickler(reader: TastyReader,
                     nameAtRef: NameTable,
                     posUnpicklerOpt: Option[PositionUnpickler],
-                    commentUnpicklerOpt: Option[CommentUnpickler]) {
+                    commentUnpicklerOpt: Option[CommentUnpickler],
+                    attributeUnpicklerOpt: Option[AttributeUnpickler]) {
   import TreeUnpickler.*
   import tpd.*
 
@@ -97,6 +99,14 @@ class TreeUnpickler(reader: TastyReader,
   /** Was unpickled class compiled with capture checks? */
   private var withCaptureChecks: Boolean = false
 
+  private val unpicklingScala2Library =
+    attributeUnpicklerOpt.exists(_.attributes.scala2StandardLibrary)
+
+  /** This dependency was compiled with explicit nulls enabled */
+  // TODO Use this to tag the symbols of this dependency as compiled with explicit nulls (see use of unpicklingScala2Library).
+  private val explicitNulls =
+    attributeUnpicklerOpt.exists(_.attributes.explicitNulls)
+
   private def registerSym(addr: Addr, sym: Symbol) =
     symAtAddr(addr) = sym
 
@@ -601,7 +611,8 @@ class TreeUnpickler(reader: TastyReader,
       val rhsStart = currentAddr
       val rhsIsEmpty = nothingButMods(end)
       if (!rhsIsEmpty) skipTree()
-      val (givenFlags, annotFns, privateWithin) = readModifiers(end)
+      val (givenFlags0, annotFns, privateWithin) = readModifiers(end)
+      val givenFlags = if isClass && unpicklingScala2Library then givenFlags0 | Scala2x | Scala2Tasty else givenFlags0
       pickling.println(i"creating symbol $name at $start with flags ${givenFlags.flagsString}, isAbsType = $isAbsType, $ttag")
       val flags = normalizeFlags(tag, givenFlags, name, isAbsType, rhsIsEmpty)
       def adjustIfModule(completer: LazyType) =

compiler/src/dotty/tools/dotc/transform/ExtensionMethods.scala

@@ -77,7 +77,7 @@ class ExtensionMethods extends MiniPhase with DenotTransformer with FullParamete
 
           // Create extension methods, except if the class comes from Scala 2
           // because it adds extension methods before pickling.
-          if (!(valueClass.is(Scala2x)))
+          if !valueClass.is(Scala2x, butNot = Scala2Tasty) then
             for (decl <- valueClass.classInfo.decls)
               if isMethodWithExtension(decl) then
                 enterInModuleClass(createExtensionMethod(decl, moduleClassSym.symbol))

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -108,6 +108,12 @@ class Pickler extends Phase {
                 pickler, treePkl.buf.addrOfTree, treePkl.docString, tree,
                 scratch.commentBuffer)
 
+          val attributes = Attributes(
+            scala2StandardLibrary = ctx.settings.YcompileScala2Library.value,
+            explicitNulls = ctx.settings.YexplicitNulls.value,
+          )
+          AttributePickler.pickleAttributes(attributes, pickler, scratch.attributeBuffer)
+
           val pickled = pickler.assembleParts()
 
           def rawBytes = // not needed right now, but useful to print raw format.

scala2-library-tasty-tests/src/Main.scala

@@ -17,6 +17,7 @@ object HelloWorld:
     testScala2ObjectParents()
     testScala2CaseClassUnderscoreMembers()
     testScalaNumberUnderlying()
+    testArrayOps()
     scala.collection.mutable.UnrolledBufferTest.test()
   }
 
@@ -68,3 +69,9 @@ object HelloWorld:
     val _: Object = MyNumber2(BigInt(1)).underlying
     val _: Object = (MyNumber2(BigInt(1)): ScalaNumber).underlying
   }
+
+  def testArrayOps() = {
+    new collection.ArrayOps[String](Array[String]("foo")).exists(x => true)
+  }
+
+end HelloWorld

tasty/src/dotty/tools/tasty/TastyFormat.scala

@@ -265,8 +265,15 @@ All elements of a position section are serialized as Ints
 
 Standard Section: "Comments" Comment*
 ```none
-  Comment       = Length Bytes LongInt      // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates.
+  Comment       = UTF8 LongInt              // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates.
 ```
+
+Standard Section: "Attributes" Attribute*
+```none
+  Attribute     = SCALA2STANDARDLIBRARYattr
+                  EXPLICITNULLSattr
+```
+
 **************************************************************************************/
 
 object TastyFormat {
@@ -361,6 +368,7 @@ object TastyFormat {
   final val ASTsSection = "ASTs"
   final val PositionsSection = "Positions"
   final val CommentsSection = "Comments"
+  final val AttributesSection = "Attributes"
 
   /** Tags used to serialize names, should update [[TastyFormat$.nameTagToString]] if a new constant is added */
   class NameTags {
@@ -597,6 +605,12 @@ object TastyFormat {
   final val firstNatASTTreeTag = IDENT
   final val firstLengthTreeTag = PACKAGE
 
+
+  // Attributes tags
+
+  final val SCALA2STANDARDLIBRARYattr = 1
+  final val EXPLICITNULLSattr = 2
+
   /** Useful for debugging */
   def isLegalTag(tag: Int): Boolean =
     firstSimpleTreeTag <= tag && tag <= SPLITCLAUSE ||
@@ -812,6 +826,11 @@ object TastyFormat {
     case HOLE => "HOLE"
   }
 
+  def attributeTagToString(tag: Int): String = tag match {
+    case SCALA2STANDARDLIBRARYattr => "SCALA2STANDARDLIBRARYattr"
+    case EXPLICITNULLSattr => "EXPLICITNULLSattr"
+  }
+
   /** @return If non-negative, the number of leading references (represented as nats) of a length/trees entry.
    *          If negative, minus the number of leading non-reference trees.
    */