Fix cooking of docstrings

Author: felixmulder

dottydoc/src/dotty/tools/dottydoc/DottyDoc.scala

@@ -54,7 +54,7 @@ abstract class DocDriver extends Driver {
     val summary = CompilerCommand.distill(args)(ctx)
 
     ctx.setSettings(summary.sstate)
-    ctx.setSetting(ctx.settings.YkeepComments, true)
+    ctx.setSetting(ctx.settings.Ydocrun, true)
 
     val fileNames = CompilerCommand.checkUsage(summary, sourcesRequired)(ctx)
     (fileNames, ctx)

dottydoc/src/dotty/tools/dottydoc/model/comment/CommentExpander.scala

@@ -47,7 +47,7 @@ trait CommentExpander {
     val defines = sections filter { startsWithTag(raw, _, "@define") }
     val usecases = sections filter { startsWithTag(raw, _, "@usecase") }
 
-    val end = startTag(raw, (defines ::: usecases).sortBy(_._1))
+    val end = startTag(raw, (defines /*::: usecases*/).sortBy(_._1))
 
     if (end == raw.length - 2) raw else raw.substring(0, end) + "*/"
   }

dottydoc/test/BaseTest.scala

@@ -17,7 +17,7 @@ trait DottyTest {
     import base.settings._
     val ctx = base.initialCtx.fresh
     ctx.setSetting(ctx.settings.language, List("Scala2"))
-    ctx.setSetting(ctx.settings.YkeepComments, true)
+    ctx.setSetting(ctx.settings.Ydocrun, true)
     base.initialize()(ctx)
     ctx
   }

dottydoc/test/UsecaseTest.scala

@@ -141,6 +141,49 @@ class UsecaseTest extends DottyTest {
     }
   }
 
+  @Test def expandColl = {
+    val source = new SourceFile(
+      "ExpandColl.scala",
+      """
+      |package scala
+      |
+      |/** The trait $Coll
+      | *
+      | *  @define Coll Iterable
+      | */
+      |trait Iterable[A] {
+      |  /** Definition with a "disturbing" signature
+      |   *
+      |   *  @usecase def map[B](f: A => B): $Coll[B]
+      |   */
+      |  def map[B, M[B]](f: A => B): M[B] = ???
+      |}
+      """.stripMargin
+    )
+
+    checkSources(source :: Nil) { packages =>
+      packages("scala") match {
+      case PackageImpl(_, _, List(trt: Trait), _, _) =>
+        val List(map: Def) = trt.members
+
+        val returnValue = map.returnValue match {
+          case ref: TypeReference => ref.title
+          case _ =>
+            assert(
+              false,
+              "Incorrect return value after usecase transformation"
+            )
+            ""
+        }
+
+        assert(
+          returnValue == "Iterable",
+          "Incorrect return type after usecase transformation"
+        )
+      }
+    }
+  }
+
   @Test def checkIterator =
     checkFiles("./scala-scala/src/library/scala/collection/Iterator.scala" :: Nil) { _ =>
       // success if typer throws no errors! :)

src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -170,6 +170,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val Ybuildmanagerdebug = BooleanSetting("-Ybuild-manager-debug", "Generate debug information for the Refined Build Manager compiler.")
   val Ycompletion = BooleanSetting("-Ycompletion-debug", "Trace all tab completion activity.")
   val Ydocdebug = BooleanSetting("-Ydoc-debug", "Trace all scaladoc activity.")
+  val Ydocrun = BooleanSetting("-Ydoc-run", "Current run should be treated as a docrun, enables `@usecase` annotations in comments")
   val Yidedebug = BooleanSetting("-Yide-debug", "Generate, validate and output trees using the interactive compiler.")
   val Yinferdebug = BooleanSetting("-Yinfer-debug", "Trace type inference and implicit search.")
   val Yissuedebug = BooleanSetting("-Yissue-debug", "Print stack traces when a context issues an error.")

src/dotty/tools/dotc/core/Comments.scala

@@ -15,6 +15,7 @@ import dotty.tools.dottydoc.model.comment.CommentUtils._
 object Comments {
 
   case class Comment(pos: Position, raw: String)(implicit ctx: Context) {
+
     val isDocComment = raw.startsWith("/**")
 
     private[this] lazy val sections = tagIndex(raw)
@@ -41,10 +42,10 @@ object Comments {
       val code         = raw.substring(codeStart, codeEnd) + " = ???"
       val codePos      = subPos(codeStart, codeEnd)
       val commentStart = skipLineLead(raw, codeEnd + 1) min end
-      val comment      = "/** " + raw.substring(commentStart, end) + "*/"
+      val commentStr   = "/** " + raw.substring(commentStart, end) + "*/"
       val commentPos   = subPos(commentStart, end)
 
-      UseCase(Comment(commentPos, comment), code, codePos)
+      UseCase(Comment(commentPos, commentStr), code, codePos)
     }
 
     private def subPos(start: Int, end: Int) =
@@ -57,9 +58,6 @@ object Comments {
   }
 
   case class UseCase(comment: Comment, code: String, codePos: Position)(implicit ctx: Context) {
-    /** Entered by Namer */
-    var symbol: Symbol = _
-
     /** Set by typer */
     var tpdCode: tpd.DefDef = _
 
@@ -68,7 +66,7 @@ object Comments {
 
       tree match {
         case tree: untpd.DefDef =>
-          val newName = (tree.name.show + "$" + codePos.start).toTermName
+          val newName = (tree.name.show + "$" + codePos + "$doc").toTermName
           untpd.DefDef(newName, tree.tparams, tree.vparamss, tree.tpt, tree.rhs)
         case _ =>
           ctx.error("proper definition was not found in `@usecase`", codePos)

src/dotty/tools/dotc/core/Contexts.scala

@@ -577,6 +577,8 @@ object Contexts {
     private[this] val _docstrings: mutable.Map[Symbol, Comment] =
       mutable.Map.empty
 
+    def docstrings: Map[Symbol, Comment] = _docstrings.toMap
+
     def docstring(sym: Symbol): Option[Comment] = _docstrings.get(sym)
 
     def addDocstring(sym: Symbol, doc: Option[Comment]): Unit =

src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1515,7 +1515,9 @@ object SymDenotations {
 
     /** Enter a symbol in given `scope` without potentially replacing the old copy. */
     def enterNoReplace(sym: Symbol, scope: MutableScope)(implicit ctx: Context): Unit = {
-      require((sym.denot.flagsUNSAFE is Private) ||  !(this is Frozen) || (scope ne this.unforcedDecls) || sym.hasAnnotation(defn.ScalaStaticAnnot))
+      lazy val isUsecase = ctx.settings.Ydocrun.value && sym.name.show.takeRight(4) == "$doc"
+
+      require((sym.denot.flagsUNSAFE is Private) ||  !(this is Frozen) || (scope ne this.unforcedDecls) || sym.hasAnnotation(defn.ScalaStaticAnnot) || isUsecase)
       scope.enter(sym)
 
       if (myMemberFingerPrint != FingerPrint.unknown)

src/dotty/tools/dotc/parsing/Scanners.scala

@@ -171,7 +171,7 @@ object Scanners {
   }
 
   class Scanner(source: SourceFile, override val startFrom: Offset = 0)(implicit ctx: Context) extends ScannerCommon(source)(ctx) {
-    val keepComments = ctx.settings.YkeepComments.value
+    val keepComments = ctx.settings.YkeepComments.value || ctx.settings.Ydocrun.value
 
     /** All doc comments as encountered, each list contains doc comments from
      *  the same block level. Starting with the deepest level and going upward

src/dotty/tools/dotc/typer/Namer.scala

@@ -4,7 +4,7 @@ package typer
 
 import core._
 import ast._
-import Trees._, Constants._, StdNames._, Scopes._, Denotations._
+import Trees._, Constants._, StdNames._, Scopes._, Denotations._, Comments._
 import Contexts._, Symbols._, Types._, SymDenotations._, Names._, NameOps._, Flags._, Decorators._
 import ast.desugar, ast.desugar._
 import ProtoTypes._
@@ -426,14 +426,7 @@ class Namer { typer: Typer =>
   }
 
   def setDocstring(sym: Symbol, tree: Tree)(implicit ctx: Context) = tree match {
-    case t: MemberDef if t.rawComment.isDefined =>
-      val cmt = t.rawComment
-      ctx.docbase.addDocstring(sym, cmt)
-
-      cmt.get.usecases.foreach { usecase =>
-        usecase.symbol = enterSymbol(createSymbol(usecase.untpdCode))
-      }
-
+    case t: MemberDef if t.rawComment.isDefined => ctx.docbase.addDocstring(sym, t.rawComment)
     case _ => ()
   }
 

src/dotty/tools/dotc/typer/Typer.scala

@@ -11,6 +11,7 @@ import Scopes._
 import Denotations._
 import ProtoTypes._
 import Contexts._
+import Comments._
 import Symbols._
 import Types._
 import SymDenotations._
@@ -1184,7 +1185,8 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
     val dummy = localDummy(cls, impl)
     val body1 = typedStats(impl.body, dummy)(inClassContext(self1.symbol))
 
-    typedUsecases(body1.map(_.symbol), self1.symbol)
+    if (ctx.settings.Ydocrun.value)
+      typedUsecases(body1.map(_.symbol), self1.symbol)
 
     checkNoDoubleDefs(cls)
     val impl1 = cpy.Template(impl)(constr1, parents1, self1, body1)
@@ -1452,16 +1454,37 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit
     traverse(stats)
   }
 
-  def typedUsecases(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit =
-    for {
-      sym <- syms
-      usecase <- ctx.docbase.docstring(sym).map(_.usecases).getOrElse(Nil)
-      List(tpdTree) = typedStats(usecase.untpdCode :: Nil, owner)
-    } yield {
-      if (tpdTree.isInstanceOf[tpd.DefDef])
-        usecase.tpdCode = tpdTree.asInstanceOf[tpd.DefDef]
-      else
-        ctx.error("Couldn't compile `@usecase`", usecase.codePos)
+  def typedUsecases(syms: List[Symbol], owner: Symbol)(implicit ctx: Context): Unit = {
+    val relevantSyms = syms.filter(ctx.docbase.docstring(_).isDefined)
+    relevantSyms.foreach { sym =>
+      expandParentDocs(sym)
+      val usecases = ctx.docbase.docstring(sym).map(_.usecases).getOrElse(Nil)
+
+      usecases.foreach { usecase =>
+        enterSymbol(createSymbol(usecase.untpdCode))
+
+        typedStats(usecase.untpdCode :: Nil, owner) match {
+          case List(df: tpd.DefDef) => usecase.tpdCode = df
+          case _ => ctx.error("`@usecase` was not a valid definition", usecase.codePos)
+        }
+      }
+    }
+  }
+
+  import dotty.tools.dottydoc.model.comment.CommentExpander
+  val expander = new CommentExpander {}
+  def expandParentDocs(sym: Symbol)(implicit ctx: Context): Unit =
+    ctx.docbase.docstring(sym).foreach { cmt =>
+      def expandDoc(owner: Symbol): Unit = {
+        expander.defineVariables(sym)
+        val newCmt = Comment(cmt.pos, expander.expandedDocComment(sym, owner, cmt.raw))
+        ctx.docbase.addDocstring(sym, Some(newCmt))
+      }
+
+      if (sym ne NoSymbol) {
+        expandParentDocs(sym.owner)
+        expandDoc(sym.owner)
+      }
     }
 
   def typedExpr(tree: untpd.Tree, pt: Type = WildcardType)(implicit ctx: Context): Tree =