From 2e4e8fec5b44c774eb2968eac6c4d2214dc24379 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 30 Nov 2023 12:00:46 +0100 Subject: [PATCH 1/2] Homogenize TASTy printer formatting * Homogenize the formatting of section names and sizes * Homogenize indentation across sections * Add TASTy header section * Add Names section size --- .../tools/dotc/core/tasty/TastyPrinter.scala | 75 ++++++++++++------- .../dotc/core/tasty/TastyUnpickler.scala | 13 +++- 2 files changed, 60 insertions(+), 28 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index 2b5360349305..3dd80b937919 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -12,6 +12,10 @@ import util.Spans.offsetToInt import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection, AttributesSection} import java.nio.file.{Files, Paths} import dotty.tools.io.{JarArchive, Path} +import dotty.tools.tasty.TastyFormat.header + +import scala.compiletime.uninitialized +import dotty.tools.tasty.TastyBuffer.Addr object TastyPrinter: @@ -62,26 +66,43 @@ class TastyPrinter(bytes: Array[Byte]) { private val sb: StringBuilder = new StringBuilder - private val unpickler: TastyUnpickler = new TastyUnpickler(bytes) + class TastyPrinterUnpickler extends TastyUnpickler(bytes) { + var namesStart: Addr = uninitialized + var namesEnd: Addr = uninitialized + override def readNames() = { + namesStart = reader.currentAddr + super.readNames() + namesEnd = reader.currentAddr + } + } + + private val unpickler: TastyPrinterUnpickler = new TastyPrinterUnpickler import unpickler.{nameAtRef, unpickle} private def nameToString(name: Name): String = name.debugString private def nameRefToString(ref: NameRef): String = nameToString(nameAtRef(ref)) + private def printHeader(): Unit = + val header = unpickler.header + sb.append("Header:\n") + sb.append(s" version: ${header.majorVersion}.${header.minorVersion}.${header.experimentalVersion}\n") + sb.append(" tooling: ").append(header.toolingVersion).append("\n") + sb.append(" UUID: ").append(header.uuid).append("\n") + sb.append("\n") + private def printNames(): Unit = + sb.append(s"Names (${unpickler.namesEnd.index - unpickler.namesStart.index} bytes, starting from ${unpickler.namesStart.index}):\n") for ((name, idx) <- nameAtRef.contents.zipWithIndex) { - val index = nameStr("%4d".format(idx)) + val index = nameStr("%6d".format(idx)) sb.append(index).append(": ").append(nameToString(name)).append("\n") } def showContents(): String = { - sb.append("Names:\n") + printHeader() printNames() - sb.append("\n") - sb.append("Trees:\n") unpickle(new TreeSectionUnpickler) match { - case Some(s) => sb.append(s) + case Some(s) => sb.append("\n\n").append(s) case _ => } unpickle(new PositionSectionUnpickler) match { @@ -108,8 +129,8 @@ class TastyPrinter(bytes: Array[Byte]) { import reader.* var indent = 0 def newLine() = { - val length = treeStr("%5d".format(index(currentAddr) - index(startAddr))) - sb.append(s"\n $length:" + " " * indent) + val length = treeStr("%6d".format(index(currentAddr) - index(startAddr))) + sb.append(s"\n$length:" + " " * indent) } def printNat() = sb.append(treeStr(" " + readNat())) def printName() = { @@ -165,8 +186,7 @@ class TastyPrinter(bytes: Array[Byte]) { } indent -= 2 } - sb.append(s"start = ${reader.startAddr}, base = $base, current = $currentAddr, end = $endAddr\n") - sb.append(s"${endAddr.index - startAddr.index} bytes of AST, base = $currentAddr\n") + sb.append(s"Trees (${endAddr.index - startAddr.index} bytes, starting from $base):") while (!isAtEnd) { printTree() newLine() @@ -180,25 +200,29 @@ class TastyPrinter(bytes: Array[Byte]) { private val sb: StringBuilder = new StringBuilder def unpickle(reader: TastyReader, tastyName: NameTable): String = { + import reader.* val posUnpickler = new PositionUnpickler(reader, tastyName) - sb.append(s" ${reader.endAddr.index - reader.currentAddr.index}") - sb.append(" position bytes:\n") + sb.append(s"Positions (${reader.endAddr.index - reader.startAddr.index} bytes, starting from $base):\n") val lineSizes = posUnpickler.lineSizes - sb.append(s" lines: ${lineSizes.length}\n") - sb.append(posUnpickler.lineSizes.mkString(" line sizes: ", ", ", "\n")) - sb.append(" positions:\n") + sb.append(s" lines: ${lineSizes.length}\n") + sb.append(s" line sizes:\n") + val windowSize = 20 + for window <-posUnpickler.lineSizes.sliding(windowSize, windowSize) do + sb.append(" ").append(window.mkString(", ")).append("\n") + // sb.append(posUnpickler.lineSizes.mkString(" line sizes: ", ", ", "\n")) + sb.append(" positions:\n") val spans = posUnpickler.spans val sorted = spans.toSeq.sortBy(_._1.index) for ((addr, pos) <- sorted) { - sb.append(treeStr("%10d".format(addr.index))) + sb.append(treeStr("%6d".format(addr.index))) sb.append(s": ${offsetToInt(pos.start)} .. ${pos.end}\n") } val sources = posUnpickler.sourcePaths - sb.append(s"\n source paths:\n") + sb.append(s"\n source paths:\n") val sortedPath = sources.toSeq.sortBy(_._1.index) for ((addr, path) <- sortedPath) { - sb.append(treeStr("%10d: ".format(addr.index))) + sb.append(treeStr("%6d: ".format(addr.index))) sb.append(path) sb.append("\n") } @@ -212,14 +236,15 @@ class TastyPrinter(bytes: Array[Byte]) { private val sb: StringBuilder = new StringBuilder def unpickle(reader: TastyReader, tastyName: NameTable): String = { - sb.append(s" ${reader.endAddr.index - reader.currentAddr.index}") + import reader.* val comments = new CommentUnpickler(reader).comments - sb.append(s" comment bytes:\n") - val sorted = comments.toSeq.sortBy(_._1.index) - for ((addr, cmt) <- sorted) { - sb.append(treeStr("%10d".format(addr.index))) - sb.append(s": ${cmt.raw} (expanded = ${cmt.isExpanded})\n") - } + if !comments.isEmpty then + sb.append(s"Comments (${reader.endAddr.index - reader.startAddr.index} bytes, starting from $base):\n") + val sorted = comments.toSeq.sortBy(_._1.index) + for ((addr, cmt) <- sorted) { + sb.append(treeStr("%6d".format(addr.index))) + sb.append(s": ${cmt.raw} (expanded = ${cmt.isExpanded})\n") + } sb.result } } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyUnpickler.scala index f3ec4a603172..6fe648ee98d3 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyUnpickler.scala @@ -11,6 +11,8 @@ import TastyBuffer.NameRef import scala.collection.mutable import Names.{TermName, termName, EmptyTermName} import NameKinds.* +import dotty.tools.tasty.TastyHeader +import dotty.tools.tasty.TastyBuffer.Addr object TastyUnpickler { @@ -61,7 +63,7 @@ object TastyUnpickler { import TastyUnpickler.* -class TastyUnpickler(reader: TastyReader) { +class TastyUnpickler(protected val reader: TastyReader) { import reader.* def this(bytes: Array[Byte]) = this(new TastyReader(bytes)) @@ -121,10 +123,13 @@ class TastyUnpickler(reader: TastyReader) { result } - val header = new TastyHeaderUnpickler(scala3CompilerConfig, reader).readFullHeader() + val header: TastyHeader = + new TastyHeaderUnpickler(scala3CompilerConfig, reader).readFullHeader() - locally { + def readNames(): Unit = until(readEnd()) { nameAtRef.add(readNameContents()) } + + def loadSections(): Unit = { while (!isAtEnd) { val secName = readString() val secEnd = readEnd() @@ -132,6 +137,8 @@ class TastyUnpickler(reader: TastyReader) { goto(secEnd) } } + readNames() + loadSections() def unpickle[R](sec: SectionUnpickler[R]): Option[R] = for (reader <- sectionReader.get(sec.name)) yield From 931eae45373adeeceafa300f1eebc147cdbb4d24 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 30 Nov 2023 10:28:42 +0100 Subject: [PATCH 2/2] Fix TASTy source position printer Now it properly shows that the sources form the position section are references in the name table. This includes the coloring of the indices and referenced names. ```diff source paths: - 0: t/Test.scala + 0: 21 [t/Test.scala] ``` --- .../tools/dotc/core/tasty/PositionUnpickler.scala | 13 ++++++------- .../dotty/tools/dotc/core/tasty/TastyPrinter.scala | 6 +++--- project/scripts/cmdTests | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala index b7f88155e3c8..975264a288dd 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala @@ -18,7 +18,7 @@ class PositionUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName) { private var myLineSizes: Array[Int] = uninitialized private var mySpans: util.HashMap[Addr, Span] = uninitialized - private var mySourcePaths: util.HashMap[Addr, String] = uninitialized + private var mySourceNameRefs: util.HashMap[Addr, NameRef] = uninitialized private var isDefined = false def ensureDefined(): Unit = { @@ -31,15 +31,14 @@ class PositionUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName) { i += 1 mySpans = util.HashMap[Addr, Span]() - mySourcePaths = util.HashMap[Addr, String]() + mySourceNameRefs = util.HashMap[Addr, NameRef]() var curIndex = 0 var curStart = 0 var curEnd = 0 while (!isAtEnd) { val header = readInt() if (header == SOURCE) { - val path = nameAtRef(readNameRef()).toString - mySourcePaths(Addr(curIndex)) = path + mySourceNameRefs(Addr(curIndex)) = readNameRef() } else { val addrDelta = header >> 3 @@ -64,9 +63,9 @@ class PositionUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName) { mySpans } - private[tasty] def sourcePaths: util.ReadOnlyMap[Addr, String] = { + private[tasty] def sourceNameRefs: util.ReadOnlyMap[Addr, NameRef] = { ensureDefined() - mySourcePaths + mySourceNameRefs } private[tasty] def lineSizes: Array[Int] = { @@ -75,5 +74,5 @@ class PositionUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName) { } def spanAt(addr: Addr): Span = spans.getOrElse(addr, NoSpan) - def sourcePathAt(addr: Addr): String = sourcePaths.getOrElse(addr, "") + def sourcePathAt(addr: Addr): String = sourceNameRefs.get(addr).fold("")(nameAtRef(_).toString) } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index 3dd80b937919..8950f4509d3e 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -218,12 +218,12 @@ class TastyPrinter(bytes: Array[Byte]) { sb.append(s": ${offsetToInt(pos.start)} .. ${pos.end}\n") } - val sources = posUnpickler.sourcePaths + val sources = posUnpickler.sourceNameRefs sb.append(s"\n source paths:\n") val sortedPath = sources.toSeq.sortBy(_._1.index) - for ((addr, path) <- sortedPath) { + for ((addr, nameRef) <- sortedPath) { sb.append(treeStr("%6d: ".format(addr.index))) - sb.append(path) + sb.append(nameStr(s"${nameRef.index} [${tastyName(nameRef)}]")) sb.append("\n") } diff --git a/project/scripts/cmdTests b/project/scripts/cmdTests index 3405c06b056f..6208237c324d 100755 --- a/project/scripts/cmdTests +++ b/project/scripts/cmdTests @@ -27,7 +27,7 @@ echo "testing sbt scalac -print-tasty" clear_out "$OUT" "$SBT" ";scalac $SOURCE -d $OUT ;scalac -print-tasty -color:never $TASTY" > "$tmp" grep -qe "0: ASTs" "$tmp" -grep -qe "0: tests/pos/HelloWorld.scala" "$tmp" +grep -qe "0: 41 \[tests/pos/HelloWorld.scala\]" "$tmp" echo "testing that paths SourceFile annotations are relativized" clear_out "$OUT"