Load docstings lazily from TASTy

Now we are always able to load docstings from TASTy, even if `-Yread-docs` is not set.
The `-Yread-docs` flag loads the doc strings eagerly, as it did before.

Fixes #20106

Author: nicolasstucki

compiler/src/dotty/tools/dotc/Driver.scala

@@ -79,7 +79,7 @@ class Driver {
     Positioned.init(using ictx)
 
     inContext(ictx) {
-      if !ctx.settings.YdropComments.value || ctx.settings.YreadComments.value then
+      if !ctx.settings.YdropComments.value then
         ictx.setProperty(ContextDoc, new ContextDocstrings)
       val fileNamesOrNone = command.checkUsage(summary, sourcesRequired)(using ctx.settings)(using ctx.settingsState)
       fileNamesOrNone.map { fileNames =>

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -394,7 +394,7 @@ private sealed trait YSettings:
   val YcheckReentrant: Setting[Boolean] = BooleanSetting(ForkSetting, "Ycheck-reentrant", "Check that compiled program does not contain vars that can be accessed from a global root.")
   val YdropComments: Setting[Boolean] = BooleanSetting(ForkSetting, "Ydrop-docs", "Drop documentation when scanning source files.", aliases = List("-Ydrop-comments"))
   val YcookComments: Setting[Boolean] = BooleanSetting(ForkSetting, "Ycook-docs", "Cook the documentation (type check `@usecase`, etc.)", aliases = List("-Ycook-comments"))
-  val YreadComments: Setting[Boolean] = BooleanSetting(ForkSetting, "Yread-docs", "Read documentation from tasty.")
+  val YreadDocsEagerly: Setting[Boolean] = BooleanSetting(ForkSetting, "Yread-docs", "Read documentation eagerly from TASTy")
   val YforceSbtPhases: Setting[Boolean] = BooleanSetting(ForkSetting, "Yforce-sbt-phases", "Run the phases used by sbt for incremental compilation (ExtractDependencies and ExtractAPI) even if the compiler is ran outside of sbt, for debugging.")
   val YdumpSbtInc: Setting[Boolean] = BooleanSetting(ForkSetting, "Ydump-sbt-inc", "For every compiled foo.scala, output the API representation and dependencies used for sbt incremental compilation in foo.inc, implies -Yforce-sbt-phases.")
   val YcheckAllPatmat: Setting[Boolean] = BooleanSetting(ForkSetting, "Ycheck-all-patmat", "Check exhaustivity and redundancy of all pattern matching (used for testing the algorithm).")

compiler/src/dotty/tools/dotc/core/Comments.scala

@@ -24,18 +24,31 @@ object Comments {
     */
   class ContextDocstrings {
 
-    private val _docstrings: MutableSymbolMap[Comment] = MutableSymbolMap[Comment](512) // FIXME: 2nd [Comment] needed or "not a class type"
+    private val docstrings: MutableSymbolMap[Comment | CommentLoader] = MutableSymbolMap[Comment | CommentLoader](512) // FIXME: 2nd [Comment] needed or "not a class type"
+
+    private class CommentLoader(val load: () => Option[Comment])
 
     val templateExpander: CommentExpander = new CommentExpander
 
-    def docstrings: ReadOnlyMap[Symbol, Comment] = _docstrings
+    def docstring(sym: Symbol): Option[Comment] =
+      docstrings.get(sym).flatMap {
+        case comment: Comment => Some(comment)
+        case commentLoader: CommentLoader =>
+          val commentOpt = commentLoader.load()
+          commentOpt match
+            case Some(comment) => docstrings.update(sym, comment)
+            case None => docstrings.remove(sym)
+          commentOpt
+      }
 
-    def docstring(sym: Symbol): Option[Comment] = _docstrings.get(sym)
+    def registerLoader(sym: Symbol, loader: () => Option[Comment]): Unit =
+      docstrings.update(sym, CommentLoader(loader))
 
     def addDocstring(sym: Symbol, doc: Option[Comment]): Unit =
-      doc.foreach(d => _docstrings.update(sym, d))
+      doc.foreach(d => docstrings.update(sym, d))
   }
 
+
   /**
    * A `Comment` contains the unformatted docstring, it's position and potentially more
    * information that is populated when the comment is "cooked".

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -366,9 +366,10 @@ class TreePickler(pickler: TastyPickler, attributes: Attributes) {
     if sym.is(Method) && sym.owner.isClass then
       profile.recordMethodSize(sym, (currentAddr.index - addr.index) max 1, mdef.span)
     for docCtx <- ctx.docCtx do
-      val comment = docCtx.docstrings.lookup(sym)
-      if comment != null then
-        docStrings(mdef) = comment
+      docCtx.docstring(sym) match
+        case Some(comment) =>
+          docStrings(mdef) = comment
+        case _ =>
   }
 
   def pickleParam(tree: Tree)(using Context): Unit = {

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -981,13 +981,15 @@ class TreeUnpickler(reader: TastyReader,
       if !sym.isType && !sym.is(ParamAccessor) then
         sym.info = ta.avoidPrivateLeaks(sym)
 
-      if (ctx.settings.YreadComments.value) {
-        assert(ctx.docCtx.isDefined, "`-Yread-docs` enabled, but no `docCtx` is set.")
-        commentUnpicklerOpt.foreach { commentUnpickler =>
+      commentUnpicklerOpt.foreach { commentUnpickler =>
+        def loadComment() =
           val comment = commentUnpickler.commentAt(start)
-          ctx.docCtx.get.addDocstring(tree.symbol, comment)
-          tree.setComment(comment)
-        }
+          tree.setComment(comment) // TODO should this be set?
+          comment
+        if ctx.settings.YreadDocsEagerly.value then
+          ctx.docCtx.get.addDocstring(tree.symbol, loadComment())
+        else
+          ctx.docCtx.get.registerLoader(tree.symbol, loadComment)
       }
 
       tree.setDefTree

compiler/src/dotty/tools/dotc/decompiler/IDEDecompilerDriver.scala

@@ -19,7 +19,6 @@ class IDEDecompilerDriver(val settings: List[String]) extends dotc.Driver {
 
   private val myInitCtx: Context = {
     val rootCtx = initCtx.fresh.addMode(Mode.Interactive | Mode.ReadPositions)
-    rootCtx.setSetting(rootCtx.settings.YreadComments, true)
     rootCtx.setSetting(rootCtx.settings.YretainTrees, true)
     rootCtx.setSetting(rootCtx.settings.fromTasty, true)
     val ctx = setup(settings.toArray :+ "dummy.scala", rootCtx).get._2

compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala

@@ -34,7 +34,6 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
     val rootCtx = initCtx.fresh.addMode(Mode.ReadPositions).addMode(Mode.Interactive)
     rootCtx.setSetting(rootCtx.settings.YretainTrees, true)
     rootCtx.setSetting(rootCtx.settings.YcookComments, true)
-    rootCtx.setSetting(rootCtx.settings.YreadComments, true)
     val ctx = setup(settings.toArray, rootCtx) match
       case Some((_, ctx)) => ctx
       case None => rootCtx

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -203,7 +203,6 @@ class Pickler extends Phase {
         super.runOn(units)
     if ctx.settings.YtestPickler.value then
       val ctx2 = ctx.fresh
-        .setSetting(ctx.settings.YreadComments, true)
         .setSetting(ctx.settings.YshowPrintErrors, true)
       testUnpickler(
         using ctx2

compiler/src/dotty/tools/repl/ReplDriver.scala

@@ -83,7 +83,6 @@ class ReplDriver(settings: Array[String],
   private def initialCtx(settings: List[String]) = {
     val rootCtx = initCtx.fresh.addMode(Mode.ReadPositions | Mode.Interactive)
     rootCtx.setSetting(rootCtx.settings.YcookComments, true)
-    rootCtx.setSetting(rootCtx.settings.YreadComments, true)
     setupRootCtx(this.settings ++ settings, rootCtx)
   }
 

compiler/test/dotty/tools/dotc/core/tasty/CommentPicklingTest.scala

@@ -108,10 +108,6 @@ class CommentPicklingTest {
   }
 
   private class UnpicklingDriver extends Driver {
-    override def initCtx =
-      val ctx = super.initCtx.fresh
-      ctx.setSetting(ctx.settings.YreadComments, true)
-      ctx
 
     def unpickle[T](args: Array[String], files: List[File])(fn: (List[tpd.Tree], Context) => T): T = {
       implicit val ctx: Context = setup(args, initCtx).map(_._2).getOrElse(initCtx)

tasty-inspector/src/scala/tasty/inspector/TastyInspector.scala

@@ -100,7 +100,6 @@ object TastyInspector:
         reset()
         val ctx2 = ctx.fresh
             .addMode(Mode.ReadPositions)
-            .setSetting(ctx.settings.YreadComments, true)
         new TASTYRun(this, ctx2)
 
     new InspectorDriver

tests/run-macros/i20106.check

@@ -0,0 +1,16 @@
+docs from same compilation run:
+Some(/**
+ * my doc string for Main
+ *
+ * @param tracks track listing
+ * */)
+docs loaded from tasty:
+Some(/**
+ * my doc string for Main
+ *
+ * @param tracks track listing
+ * */)
+docs from same compilation run:
+Some(/**
+ * Doc of Test
+ * */)

tests/run-macros/i20106/Macro_1.scala

@@ -0,0 +1,12 @@
+import scala.quoted.*
+
+object Doc {
+  inline def of[A]: Option[String] = ${ ofImpl[A] }
+
+  def ofImpl[A: Type](using Quotes): Expr[Option[String]] = {
+    import quotes.reflect.*
+
+    val symbol = TypeRepr.of[A].typeSymbol
+    Expr(symbol.docstring)
+  }
+}

tests/run-macros/i20106/Main_1.scala

@@ -0,0 +1,14 @@
+/**
+ * my doc string for Main
+ *
+ * @param tracks track listing
+ * */
+
+class Main(tracks: String)
+
+object Main {
+  def printdoc(): Unit = {
+    val docString = Doc.of[Main]
+    println(docString)
+  }
+}

tests/run-macros/i20106/Test_2.scala

@@ -0,0 +1,19 @@
+/**
+ * Doc of Test
+ * */
+class Test
+
+object Test {
+  def main(args: Array[String]): Unit = {
+    println("docs from same compilation run:")
+    Main.printdoc()
+
+    println("docs loaded from tasty:")
+    val docString = Doc.of[Main]
+    println(docString)
+
+    println("docs from same compilation run:")
+    val docStringTest = Doc.of[Test]
+    println(docStringTest)
+  }
+}