From 0e6cce669a784ff1bb209372b191d0985e0c1e61 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Mon, 7 Mar 2016 16:02:52 +0100 Subject: [PATCH 01/12] Add initial support for raw docstrings in ASTs --- src/dotty/tools/dotc/ast/Trees.scala | 9 +++++++ src/dotty/tools/dotc/parsing/Parsers.scala | 11 ++++++-- src/dotty/tools/dotc/parsing/Scanners.scala | 30 ++++++++++++++++----- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/dotty/tools/dotc/ast/Trees.scala b/src/dotty/tools/dotc/ast/Trees.scala index d0197b443660..eb5ab15e371c 100644 --- a/src/dotty/tools/dotc/ast/Trees.scala +++ b/src/dotty/tools/dotc/ast/Trees.scala @@ -321,6 +321,10 @@ object Trees { private[ast] def rawMods: Modifiers[T] = if (myMods == null) genericEmptyModifiers else myMods + private[this] var myComment: Option[String] = None + + def rawComment: Option[String] = myComment + def withMods(mods: Modifiers[Untyped]): ThisTree[Untyped] = { val tree = if (myMods == null || (myMods == mods)) this else clone.asInstanceOf[MemberDef[Untyped]] tree.setMods(mods) @@ -329,6 +333,11 @@ object Trees { def withFlags(flags: FlagSet): ThisTree[Untyped] = withMods(Modifiers(flags)) + def withComment(cmt: Option[String]): ThisTree[Untyped] = { + myComment = cmt + asInstanceOf[ThisTree[Untyped]] + } + protected def setMods(mods: Modifiers[T @uncheckedVariance]) = myMods = mods override def envelope: Position = rawMods.pos.union(pos).union(initialPos) diff --git a/src/dotty/tools/dotc/parsing/Parsers.scala b/src/dotty/tools/dotc/parsing/Parsers.scala index 44a70886e01c..a3f15d4b7b88 100644 --- a/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1913,10 +1913,14 @@ object Parsers { val tparams = typeParamClauseOpt(ParamOwner.Class) val cmods = constrModsOpt() val vparamss = paramClauses(name, mods is Case) + makeConstructor(tparams, vparamss).withMods(cmods) } val templ = templateOpt(constr) - TypeDef(name, templ).withMods(mods) + + TypeDef(name, templ) + .withMods(mods) + .withComment(in.getDocString()) } /** ConstrMods ::= AccessModifier @@ -1935,7 +1939,10 @@ object Parsers { def objectDef(mods: Modifiers): ModuleDef = { val name = ident() val template = templateOpt(emptyConstructor()) - ModuleDef(name, template).withMods(mods) + + ModuleDef(name, template) + .withMods(mods) + .withComment(in.getDocString()) } /* -------- TEMPLATES ------------------------------------------- */ diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index 489038f1e264..0529b5cfa1b3 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -182,6 +182,18 @@ object Scanners { */ var revComments: List[Comment] = Nil + /** The currently closest docstring, replaced every time a new docstring is + * encountered + */ + var closestDocString: Option[Comment] = None + + /** Returns `closestDocString`'s raw string and sets it to `None` */ + def getDocString(): Option[String] = { + val current = closestDocString.map(_.chrs) + closestDocString = None + current + } + /** A buffer for comments */ val commentBuf = new StringBuilder @@ -528,8 +540,7 @@ object Scanners { } private def skipComment(): Boolean = { - def appendToComment(ch: Char) = - if (keepComments) commentBuf.append(ch) + def appendToComment(ch: Char) = commentBuf.append(ch) def nextChar() = { appendToComment(ch) Scanner.this.nextChar() @@ -556,11 +567,16 @@ object Scanners { def nestedComment() = { nextChar(); skipComment() } val start = lastCharOffset def finishComment(): Boolean = { - if (keepComments) { - val pos = Position(start, charOffset, start) - nextChar() - revComments = Comment(pos, flushBuf(commentBuf)) :: revComments - } + val pos = Position(start, charOffset, start) + val comment = Comment(pos, flushBuf(commentBuf)) + + closestDocString = + if (comment.isDocComment) Option(comment) + else closestDocString + + if (keepComments) + revComments = comment :: revComments + true } nextChar() From 3b4906d9dc5621d595259254057dd5649e4f9862 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 8 Mar 2016 20:12:08 +0100 Subject: [PATCH 02/12] Add tests for classes and traits --- project/Build.scala | 2 +- src/dotty/tools/dotc/parsing/Parsers.scala | 3 +- test/test/DottyDocTests.scala | 159 +++++++++++++++++++++ test/test/DottyTest.scala | 3 +- 4 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 test/test/DottyDocTests.scala diff --git a/project/Build.scala b/project/Build.scala index 723a7e129d2d..0e6e0a9bbc59 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -182,7 +182,7 @@ object DottyBuild extends Build { } ). settings( - addCommandAlias("partest", ";test:package;package;test:runMain dotc.build;lockPartestFile;test:test;runPartestRunner") ++ + addCommandAlias("partest", ";test:package;package;test:runMain dotc.build;lockPartestFile;test:test;runPartestRunner;test:runMain test.DottyDocTests") ++ addCommandAlias("partest-only", ";test:package;package;test:runMain dotc.build;lockPartestFile;test:test-only dotc.tests;runPartestRunner") ++ addCommandAlias("partest-only-no-bootstrap", ";test:package;package; lockPartestFile;test:test-only dotc.tests;runPartestRunner") ) diff --git a/src/dotty/tools/dotc/parsing/Parsers.scala b/src/dotty/tools/dotc/parsing/Parsers.scala index a3f15d4b7b88..ad50bf476020 100644 --- a/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1909,6 +1909,7 @@ object Parsers { */ def classDef(mods: Modifiers): TypeDef = atPos(tokenRange) { val name = ident().toTypeName + val docstring = in.getDocString() val constr = atPos(in.offset) { val tparams = typeParamClauseOpt(ParamOwner.Class) val cmods = constrModsOpt() @@ -1920,7 +1921,7 @@ object Parsers { TypeDef(name, templ) .withMods(mods) - .withComment(in.getDocString()) + .withComment(docstring) } /** ConstrMods ::= AccessModifier diff --git a/test/test/DottyDocTests.scala b/test/test/DottyDocTests.scala new file mode 100644 index 000000000000..20c10cc4a2f9 --- /dev/null +++ b/test/test/DottyDocTests.scala @@ -0,0 +1,159 @@ +package test + +import dotty.tools.dotc.Compiler +import dotty.tools.dotc.core.Phases.Phase +import dotty.tools.dotc.typer.FrontEnd +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.dotc.ast.tpd +import dotty.tools.dotc.ast.Trees._ + +/** All tests need to extend this trait and define `source` and `assertion` + * + * Instances also need to be placed in the `DottyDocTests::tests` sequence to + * be included in the run + */ +trait DottyDocTest extends DottyTest { self => + /** Source code in string format */ + def source: String + + /** A test to run against the resulting code */ + def assertion: PartialFunction[Tree[Untyped], Unit] + + private def defaultAssertion: PartialFunction[Tree[Untyped], Unit] = { + case x => assert(false, "Couldn't match resulting AST to expected AST in: " + x.show) + } + + private def compiler(assertion: PartialFunction[Tree[Untyped], Unit]) = new Compiler { + override def phases = { + val checker = new Phase { + def phaseName = "assertionChecker" + override def run(implicit ctx: Context): Unit = + (assertion orElse defaultAssertion)(ctx.compilationUnit.untpdTree) + } + + List(List(new FrontEnd)) ::: List(List(checker)) + } + } + + def checkDocString(actual: Option[String], expected: String): Unit = actual match { + case Some(str) => { + assert(str == expected, s"""Docstring: "$str" didn't match expected "$expected"""") + } + case None => + assert(false, s"""No docstring found, expected: "$expected"""") + } + + def run(): Unit = { + val c = compiler(assertion) + c.rootContext(ctx) + c.newRun.compile(source) + println(s"${self.getClass.getSimpleName.split("\\$").last} passed") + } +} + +/** Add tests to the `tests` sequence */ +object DottyDocTests extends DottyTest { + private[this] val tests = Seq( + SingleClassInPackage, + MultipleOpenedOnSingleClassInPackage, + MultipleClassesInPackage, + SingleCaseClassWithoutPackage, + SingleTraitWihoutPackage, + MultipleTraitsWithoutPackage + ) + + def main(args: Array[String]): Unit = { + println("------------ Testing DottyDoc ------------") + tests.foreach(_.run) + println("--------- DottyDoc tests passed! ----------") + } +} + +case object SingleClassInPackage extends DottyDocTest { + override val source = + """ + |package a + | + |/** Hello world! */ + |class Class(val x: String) + """.stripMargin + + override def assertion = { + case PackageDef(_, Seq(t @ TypeDef(name, _))) if name.toString == "Class" => + checkDocString(t.rawComment, "/** Hello world! */") + } +} + +case object MultipleOpenedOnSingleClassInPackage extends DottyDocTest { + override val source = + """ + |package a + | + |/** Hello /* multiple open */ world! */ + |class Class(val x: String) + """.stripMargin + + override def assertion = { + case PackageDef(_, Seq(t @ TypeDef(name, _))) if name.toString == "Class" => + checkDocString(t.rawComment, "/** Hello /* multiple open */ world! */") + } +} + +case object MultipleClassesInPackage extends DottyDocTest { + override val source = + """ + |package a + | + |/** Class1 docstring */ + |class Class1(val x: String) + | + |/** Class2 docstring */ + |class Class2(val x: String) + """.stripMargin + + override def assertion = { + case PackageDef(_, Seq(c1 @ TypeDef(_,_), c2 @ TypeDef(_,_))) => { + checkDocString(c1.rawComment, "/** Class1 docstring */") + checkDocString(c2.rawComment, "/** Class2 docstring */") + } + } +} + +case object SingleCaseClassWithoutPackage extends DottyDocTest { + override val source = + """ + |/** Class without package */ + |case class Class(val x: Int) + """.stripMargin + + override def assertion = { + case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment, "/** Class without package */") + } +} + +case object SingleTraitWihoutPackage extends DottyDocTest { + override val source = "/** Trait docstring */\ntrait Trait" + + override def assertion = { + case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment, "/** Trait docstring */") + } +} + +case object MultipleTraitsWithoutPackage extends DottyDocTest { + //TODO: This will fail if the tratis don't have bodies, because of the Scanner + override val source = + """ + |/** Trait1 docstring */ + |trait Trait1 {} + | + |/** Trait2 docstring */ + |trait Trait2 {} + """.stripMargin + + override def assertion = { + case PackageDef(_, Seq(t1 @ TypeDef(_,_), t2 @ TypeDef(_,_))) => { + checkDocString(t1.rawComment, "/** Trait1 docstring */") + checkDocString(t2.rawComment, "/** Trait2 docstring */") + } + } +} diff --git a/test/test/DottyTest.scala b/test/test/DottyTest.scala index 77642561a8e2..15d82c208b79 100644 --- a/test/test/DottyTest.scala +++ b/test/test/DottyTest.scala @@ -32,7 +32,6 @@ class DottyTest /*extends ContextEscapeDetection*/ { } */ private def compilerWithChecker(phase: String)(assertion:(tpd.Tree, Context) => Unit) = new Compiler { - self => override def phases = { val allPhases = super.phases val targetPhase = allPhases.flatten.find(p => p.phaseName == phase).get @@ -48,7 +47,7 @@ class DottyTest /*extends ContextEscapeDetection*/ { } } - def checkCompile(checkAfterPhase: String, source:String)(assertion:(tpd.Tree, Context) => Unit): Unit = { + def checkCompile(checkAfterPhase: String, source: String)(assertion: (tpd.Tree, Context) => Unit): Unit = { val c = compilerWithChecker(checkAfterPhase)(assertion) c.rootContext(ctx) val run = c.newRun From 7f20aef772c1bdc6e0a699a82f179d93934a1555 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Tue, 8 Mar 2016 21:23:15 +0100 Subject: [PATCH 03/12] Fix traits w/o bodies not getting the correct docstring --- src/dotty/tools/dotc/parsing/Scanners.scala | 15 ++++----- test/test/DottyDocTests.scala | 36 ++++++++++++++++++--- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index 0529b5cfa1b3..438c54398270 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -185,14 +185,12 @@ object Scanners { /** The currently closest docstring, replaced every time a new docstring is * encountered */ - var closestDocString: Option[Comment] = None + val closestDocString: mutable.Queue[Comment] = mutable.Queue.empty /** Returns `closestDocString`'s raw string and sets it to `None` */ - def getDocString(): Option[String] = { - val current = closestDocString.map(_.chrs) - closestDocString = None - current - } + def getDocString(): Option[String] = + if (closestDocString.isEmpty) None + else Some(closestDocString.dequeue.chrs) /** A buffer for comments */ val commentBuf = new StringBuilder @@ -570,9 +568,8 @@ object Scanners { val pos = Position(start, charOffset, start) val comment = Comment(pos, flushBuf(commentBuf)) - closestDocString = - if (comment.isDocComment) Option(comment) - else closestDocString + if (comment.isDocComment) + closestDocString.enqueue(comment) if (keepComments) revComments = comment :: revComments diff --git a/test/test/DottyDocTests.scala b/test/test/DottyDocTests.scala index 20c10cc4a2f9..df53c2ae989a 100644 --- a/test/test/DottyDocTests.scala +++ b/test/test/DottyDocTests.scala @@ -59,7 +59,8 @@ object DottyDocTests extends DottyTest { MultipleClassesInPackage, SingleCaseClassWithoutPackage, SingleTraitWihoutPackage, - MultipleTraitsWithoutPackage + MultipleTraitsWithoutPackage, + MultipleMixedEntitiesWithPackage ) def main(args: Array[String]): Unit = { @@ -140,14 +141,13 @@ case object SingleTraitWihoutPackage extends DottyDocTest { } case object MultipleTraitsWithoutPackage extends DottyDocTest { - //TODO: This will fail if the tratis don't have bodies, because of the Scanner override val source = """ |/** Trait1 docstring */ - |trait Trait1 {} + |trait Trait1 | |/** Trait2 docstring */ - |trait Trait2 {} + |trait Trait2 """.stripMargin override def assertion = { @@ -157,3 +157,31 @@ case object MultipleTraitsWithoutPackage extends DottyDocTest { } } } + +case object MultipleMixedEntitiesWithPackage extends DottyDocTest { + override val source = + """ + |/** Trait1 docstring */ + |trait Trait1 + | + |/** Class2 docstring */ + |class Class2(val x: Int) + | + |/** CaseClass3 docstring */ + |case class CaseClass3() + | + |case class NoComment() + | + |/** AbstractClass4 docstring */ + |abstract class AbstractClass4(val x: Int) + """.stripMargin + + override def assertion = { + case PackageDef(_, Seq(t1 @ TypeDef(_,_), c2 @ TypeDef(_,_), cc3 @ TypeDef(_,_), _, ac4 @ TypeDef(_,_))) => { + checkDocString(t1.rawComment, "/** Trait1 docstring */") + checkDocString(c2.rawComment, "/** Class2 docstring */") + checkDocString(cc3.rawComment, "/** CaseClass3 docstring */") + checkDocString(ac4.rawComment, "/** AbstractClass4 docstring */") + } + } +} From ee5413751183d4ab217cf63fc9c1ac7b33ff4710 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 9 Mar 2016 15:54:25 +0100 Subject: [PATCH 04/12] Add ability to parse nested classes/traits docstrings --- src/dotty/tools/dotc/parsing/Parsers.scala | 34 ++++++++------- src/dotty/tools/dotc/parsing/Scanners.scala | 29 +++++++++---- test/test/DottyDocTests.scala | 48 ++++++++++++++++++++- 3 files changed, 85 insertions(+), 26 deletions(-) diff --git a/src/dotty/tools/dotc/parsing/Parsers.scala b/src/dotty/tools/dotc/parsing/Parsers.scala index ad50bf476020..3aace28cf4ff 100644 --- a/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1888,28 +1888,30 @@ object Parsers { /** TmplDef ::= ([`case'] `class' | `trait') ClassDef * | [`case'] `object' ObjectDef */ - def tmplDef(start: Int, mods: Modifiers): Tree = in.token match { - case TRAIT => - classDef(posMods(start, addFlag(mods, Trait))) - case CLASS => - classDef(posMods(start, mods)) - case CASECLASS => - classDef(posMods(start, mods | Case)) - case OBJECT => - objectDef(posMods(start, mods | Module)) - case CASEOBJECT => - objectDef(posMods(start, mods | Case | Module)) - case _ => - syntaxErrorOrIncomplete("expected start of definition") - EmptyTree + def tmplDef(start: Int, mods: Modifiers): Tree = { + val docstring = in.getDocString() + in.token match { + case TRAIT => + classDef(posMods(start, addFlag(mods, Trait)), docstring) + case CLASS => + classDef(posMods(start, mods), docstring) + case CASECLASS => + classDef(posMods(start, mods | Case), docstring) + case OBJECT => + objectDef(posMods(start, mods | Module)) + case CASEOBJECT => + objectDef(posMods(start, mods | Case | Module)) + case _ => + syntaxErrorOrIncomplete("expected start of definition") + EmptyTree + } } /** ClassDef ::= Id [ClsTypeParamClause] * [ConstrMods] ClsParamClauses TemplateOpt */ - def classDef(mods: Modifiers): TypeDef = atPos(tokenRange) { + def classDef(mods: Modifiers, docstring: Option[String]): TypeDef = atPos(tokenRange) { val name = ident().toTypeName - val docstring = in.getDocString() val constr = atPos(in.offset) { val tparams = typeParamClauseOpt(ParamOwner.Class) val cmods = constrModsOpt() diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index 438c54398270..60042e5a68dc 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -185,12 +185,23 @@ object Scanners { /** The currently closest docstring, replaced every time a new docstring is * encountered */ - val closestDocString: mutable.Queue[Comment] = mutable.Queue.empty + var closestDocString: List[mutable.Queue[Comment]] = mutable.Queue.empty[Comment] :: Nil + + /** Adds level of nesting to docstrings */ + def enterBlock(): Unit = + closestDocString = mutable.Queue.empty[Comment] :: closestDocString + + /** Removes level of nesting for docstrings */ + def exitBlock(): Unit = closestDocString = closestDocString match { + case x :: Nil => mutable.Queue.empty[Comment] :: Nil + case _ => closestDocString.tail + } /** Returns `closestDocString`'s raw string and sets it to `None` */ - def getDocString(): Option[String] = - if (closestDocString.isEmpty) None - else Some(closestDocString.dequeue.chrs) + def getDocString(): Option[String] = closestDocString match { + case x :: _ if !x.isEmpty => Some(x.dequeue.chrs) + case _ => None + } /** A buffer for comments */ val commentBuf = new StringBuilder @@ -497,13 +508,13 @@ object Scanners { case ',' => nextChar(); token = COMMA case '(' => - nextChar(); token = LPAREN + enterBlock(); nextChar(); token = LPAREN case '{' => - nextChar(); token = LBRACE + enterBlock(); nextChar(); token = LBRACE case ')' => - nextChar(); token = RPAREN + exitBlock(); nextChar(); token = RPAREN case '}' => - nextChar(); token = RBRACE + exitBlock(); nextChar(); token = RBRACE case '[' => nextChar(); token = LBRACKET case ']' => @@ -569,7 +580,7 @@ object Scanners { val comment = Comment(pos, flushBuf(commentBuf)) if (comment.isDocComment) - closestDocString.enqueue(comment) + closestDocString.head.enqueue(comment) if (keepComments) revComments = comment :: revComments diff --git a/test/test/DottyDocTests.scala b/test/test/DottyDocTests.scala index df53c2ae989a..af3336cb70c2 100644 --- a/test/test/DottyDocTests.scala +++ b/test/test/DottyDocTests.scala @@ -60,7 +60,9 @@ object DottyDocTests extends DottyTest { SingleCaseClassWithoutPackage, SingleTraitWihoutPackage, MultipleTraitsWithoutPackage, - MultipleMixedEntitiesWithPackage + MultipleMixedEntitiesWithPackage, + NestedClass, + NestedClassThenOuter ) def main(args: Array[String]): Unit = { @@ -185,3 +187,47 @@ case object MultipleMixedEntitiesWithPackage extends DottyDocTest { } } } + +case object NestedClass extends DottyDocTest { + override val source = + """ + |/** Outer docstring */ + |class Outer { + | /** Inner docstring */ + | class Inner(val x: Int) + |} + """.stripMargin + + override def assertion = { + case PackageDef(_, Seq(outer @ TypeDef(_, tpl @ Template(_,_,_,_)))) => + checkDocString(outer.rawComment, "/** Outer docstring */") + tpl.body match { + case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") + case _ => assert(false, "Couldn't find inner class") + } + } +} + +case object NestedClassThenOuter extends DottyDocTest { + override val source = + """ + |/** Outer1 docstring */ + |class Outer1 { + | /** Inner docstring */ + | class Inner(val x: Int) + |} + | + |/** Outer2 docstring */ + |class Outer2 + """.stripMargin + + override def assertion = { + case PackageDef(_, Seq(o1 @ TypeDef(_, tpl @ Template(_,_,_,_)), o2 @ TypeDef(_,_))) => + checkDocString(o1.rawComment, "/** Outer1 docstring */") + checkDocString(o2.rawComment, "/** Outer2 docstring */") + tpl.body match { + case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") + case _ => assert(false, "Couldn't find inner class") + } + } +} From 3aab803d513cec98dab8a58e92924d96662a7d7d Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Wed, 9 Mar 2016 17:24:08 +0100 Subject: [PATCH 05/12] Add package object support --- src/dotty/tools/dotc/parsing/Parsers.scala | 11 +-- test/test/DottyDocTests.scala | 95 +++++++++++++++++++++- 2 files changed, 99 insertions(+), 7 deletions(-) diff --git a/src/dotty/tools/dotc/parsing/Parsers.scala b/src/dotty/tools/dotc/parsing/Parsers.scala index 3aace28cf4ff..3ea504657d2d 100644 --- a/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1898,9 +1898,9 @@ object Parsers { case CASECLASS => classDef(posMods(start, mods | Case), docstring) case OBJECT => - objectDef(posMods(start, mods | Module)) + objectDef(posMods(start, mods | Module), docstring) case CASEOBJECT => - objectDef(posMods(start, mods | Case | Module)) + objectDef(posMods(start, mods | Case | Module), docstring) case _ => syntaxErrorOrIncomplete("expected start of definition") EmptyTree @@ -1939,13 +1939,13 @@ object Parsers { /** ObjectDef ::= Id TemplateOpt */ - def objectDef(mods: Modifiers): ModuleDef = { + def objectDef(mods: Modifiers, docstring: Option[String] = None): ModuleDef = { val name = ident() val template = templateOpt(emptyConstructor()) ModuleDef(name, template) .withMods(mods) - .withComment(in.getDocString()) + .withComment(docstring) } /* -------- TEMPLATES ------------------------------------------- */ @@ -2170,7 +2170,8 @@ object Parsers { if (in.token == PACKAGE) { in.nextToken() if (in.token == OBJECT) { - ts += objectDef(atPos(start, in.skipToken()) { Modifiers(Package) }) + val docstring = in.getDocString() + ts += objectDef(atPos(start, in.skipToken()) { Modifiers(Package) }, docstring) if (in.token != EOF) { acceptStatSep() ts ++= topStatSeq() diff --git a/test/test/DottyDocTests.scala b/test/test/DottyDocTests.scala index af3336cb70c2..2a60e7e7d120 100644 --- a/test/test/DottyDocTests.scala +++ b/test/test/DottyDocTests.scala @@ -4,7 +4,6 @@ import dotty.tools.dotc.Compiler import dotty.tools.dotc.core.Phases.Phase import dotty.tools.dotc.typer.FrontEnd import dotty.tools.dotc.core.Contexts.Context -import dotty.tools.dotc.ast.tpd import dotty.tools.dotc.ast.Trees._ /** All tests need to extend this trait and define `source` and `assertion` @@ -62,7 +61,10 @@ object DottyDocTests extends DottyTest { MultipleTraitsWithoutPackage, MultipleMixedEntitiesWithPackage, NestedClass, - NestedClassThenOuter + NestedClassThenOuter, + Objects, + ObjectsNestedClass, + PackageObject ) def main(args: Array[String]): Unit = { @@ -231,3 +233,92 @@ case object NestedClassThenOuter extends DottyDocTest { } } } + +case object Objects extends DottyDocTest { + override val source = + """ + |package p + | + |/** Object1 docstring */ + |object Object1 + | + |/** Object2 docstring */ + |object Object2 + """.stripMargin + + override def assertion = { + case p @ PackageDef(_, Seq(o1: MemberDef[Untyped], o2: MemberDef[Untyped])) => + assert(o1.name.toString == "Object1") + checkDocString(o1.rawComment, "/** Object1 docstring */") + assert(o2.name.toString == "Object2") + checkDocString(o2.rawComment, "/** Object2 docstring */") + } +} + +case object ObjectsNestedClass extends DottyDocTest { + override val source = + """ + |package p + | + |/** Object1 docstring */ + |object Object1 + | + |/** Object2 docstring */ + |object Object2 { + | class A1 + | /** Inner docstring */ + | class Inner + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case p @ PackageDef(_, Seq(o1: ModuleDef, o2: ModuleDef)) => + assert(o1.name.toString == "Object1") + checkDocString(o1.rawComment, "/** Object1 docstring */") + assert(o2.name.toString == "Object2") + checkDocString(o2.rawComment, "/** Object2 docstring */") + + o2.impl.body match { + case _ :: (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") + case _ => assert(false, "Couldn't find inner class") + } + } +} + +case object PackageObject extends DottyDocTest { + override val source = + """ + |/** Package object docstring */ + |package object foo { + | /** Boo docstring */ + | case class Boo() + | + | /** Trait docstring */ + | trait Trait + | + | /** InnerObject docstring */ + | object InnerObject { + | /** InnerClass docstring */ + | class InnerClass + | } + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case PackageDef(_, Seq(p: ModuleDef)) => { + checkDocString(p.rawComment, "/** Package object docstring */") + + p.impl.body match { + case (b: TypeDef) :: (t: TypeDef) :: (o: ModuleDef) :: Nil => { + checkDocString(b.rawComment, "/** Boo docstring */") + checkDocString(t.rawComment, "/** Trait docstring */") + checkDocString(o.rawComment, "/** InnerObject docstring */") + checkDocString(o.impl.body.head.asInstanceOf[TypeDef].rawComment, "/** InnerClass docstring */") + } + case _ => assert(false, "Incorrect structure inside package object") + } + } + } +} From c66998bc56ba73b5b2cbbf6b10f4d5b83017db13 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 10 Mar 2016 14:00:03 +0100 Subject: [PATCH 06/12] Fix association of docstrings based on posisition This commit fixes errors that would've occurred in this situation: ``` /** Docstring 1 */ <- this one would've been chosen /** Docstring 2 */ /** Docstring 3 */ class Class ``` And this situation: ``` /** Docstring 1 */ trait Trait /** Docstring 2 */ <- this one would've been chosen ``` --- src/dotty/tools/dotc/parsing/Parsers.scala | 4 +- src/dotty/tools/dotc/parsing/Scanners.scala | 27 +++++++---- test/test/DottyDocTests.scala | 51 ++++++++++++++++++++- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/dotty/tools/dotc/parsing/Parsers.scala b/src/dotty/tools/dotc/parsing/Parsers.scala index 3ea504657d2d..053b28a64bf1 100644 --- a/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1889,7 +1889,7 @@ object Parsers { * | [`case'] `object' ObjectDef */ def tmplDef(start: Int, mods: Modifiers): Tree = { - val docstring = in.getDocString() + val docstring = in.getDocString(start) in.token match { case TRAIT => classDef(posMods(start, addFlag(mods, Trait)), docstring) @@ -2170,7 +2170,7 @@ object Parsers { if (in.token == PACKAGE) { in.nextToken() if (in.token == OBJECT) { - val docstring = in.getDocString() + val docstring = in.getDocString(start) ts += objectDef(atPos(start, in.skipToken()) { Modifiers(Package) }, docstring) if (in.token != EOF) { acceptStatSep() diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index 60042e5a68dc..9ad982e45145 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -185,22 +185,33 @@ object Scanners { /** The currently closest docstring, replaced every time a new docstring is * encountered */ - var closestDocString: List[mutable.Queue[Comment]] = mutable.Queue.empty[Comment] :: Nil + var closestDocString: List[List[Comment]] = List(List()) /** Adds level of nesting to docstrings */ def enterBlock(): Unit = - closestDocString = mutable.Queue.empty[Comment] :: closestDocString + closestDocString = Nil ::: closestDocString /** Removes level of nesting for docstrings */ def exitBlock(): Unit = closestDocString = closestDocString match { - case x :: Nil => mutable.Queue.empty[Comment] :: Nil + case x :: xs => List(List()) case _ => closestDocString.tail } - /** Returns `closestDocString`'s raw string and sets it to `None` */ - def getDocString(): Option[String] = closestDocString match { - case x :: _ if !x.isEmpty => Some(x.dequeue.chrs) - case _ => None + /** Returns the closest docstring preceding the position supplied */ + def getDocString(pos: Int): Option[String] = { + def closest(c: Comment, docstrings: List[Comment]): Comment = docstrings match { + case x :: xs if (c.pos.end < x.pos.end && x.pos.end <= pos) => closest(x, xs) + case Nil => c + } + + closestDocString match { + case (list @ (x :: xs)) :: _ => { + val c = closest(x, xs) + closestDocString = list.dropWhile(_ != c).tail :: closestDocString.tail + Some(c.chrs) + } + case _ => None + } } /** A buffer for comments */ @@ -580,7 +591,7 @@ object Scanners { val comment = Comment(pos, flushBuf(commentBuf)) if (comment.isDocComment) - closestDocString.head.enqueue(comment) + closestDocString = (closestDocString.head :+ comment) :: closestDocString.tail if (keepComments) revComments = comment :: revComments diff --git a/test/test/DottyDocTests.scala b/test/test/DottyDocTests.scala index 2a60e7e7d120..5161f98b8ed8 100644 --- a/test/test/DottyDocTests.scala +++ b/test/test/DottyDocTests.scala @@ -53,6 +53,7 @@ trait DottyDocTest extends DottyTest { self => /** Add tests to the `tests` sequence */ object DottyDocTests extends DottyTest { private[this] val tests = Seq( + NoComment, SingleClassInPackage, MultipleOpenedOnSingleClassInPackage, MultipleClassesInPackage, @@ -64,7 +65,9 @@ object DottyDocTests extends DottyTest { NestedClassThenOuter, Objects, ObjectsNestedClass, - PackageObject + PackageObject, + MultipleDocStringsBeforeEntity, + MultipleDocStringsBeforeAndAfter ) def main(args: Array[String]): Unit = { @@ -74,6 +77,16 @@ object DottyDocTests extends DottyTest { } } +case object NoComment extends DottyDocTest { + override val source = "class Class" + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case PackageDef(_, Seq(c: TypeDef)) => + assert(c.rawComment == None, "Should not have a comment, mainly used for exhaustive tests") + } +} + case object SingleClassInPackage extends DottyDocTest { override val source = """ @@ -322,3 +335,39 @@ case object PackageObject extends DottyDocTest { } } } + +case object MultipleDocStringsBeforeEntity extends DottyDocTest { + override val source = + """ + |/** First comment */ + |/** Second comment */ + |/** Real comment */ + |class Class + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case PackageDef(_, Seq(c: TypeDef)) => + checkDocString(c.rawComment, "/** Real comment */") + } +} + +case object MultipleDocStringsBeforeAndAfter extends DottyDocTest { + override val source = + """ + |/** First comment */ + |/** Second comment */ + |/** Real comment */ + |class Class + |/** Following comment 1 */ + |/** Following comment 2 */ + |/** Following comment 3 */ + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case PackageDef(_, Seq(c: TypeDef)) => + checkDocString(c.rawComment, "/** Real comment */") + } + +} From 6c8265c6cc3555e00f82cf4909fea5e95d1a2f94 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 10 Mar 2016 15:03:48 +0100 Subject: [PATCH 07/12] Add docstring support for types, vals, vars and defs --- src/dotty/tools/dotc/parsing/Parsers.scala | 24 ++-- test/test/DottyDocTests.scala | 141 ++++++++++++++++++++- 2 files changed, 150 insertions(+), 15 deletions(-) diff --git a/src/dotty/tools/dotc/parsing/Parsers.scala b/src/dotty/tools/dotc/parsing/Parsers.scala index 053b28a64bf1..dad618381abd 100644 --- a/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1752,13 +1752,13 @@ object Parsers { */ def defOrDcl(start: Int, mods: Modifiers): Tree = in.token match { case VAL => - patDefOrDcl(posMods(start, mods)) + patDefOrDcl(posMods(start, mods), in.getDocString(start)) case VAR => - patDefOrDcl(posMods(start, addFlag(mods, Mutable))) + patDefOrDcl(posMods(start, addFlag(mods, Mutable)), in.getDocString(start)) case DEF => - defDefOrDcl(posMods(start, mods)) + defDefOrDcl(posMods(start, mods), in.getDocString(start)) case TYPE => - typeDefOrDcl(posMods(start, mods)) + typeDefOrDcl(posMods(start, mods), in.getDocString(start)) case _ => tmplDef(start, mods) } @@ -1768,7 +1768,7 @@ object Parsers { * ValDcl ::= Id {`,' Id} `:' Type * VarDcl ::= Id {`,' Id} `:' Type */ - def patDefOrDcl(mods: Modifiers): Tree = { + def patDefOrDcl(mods: Modifiers, docstring: Option[String] = None): Tree = { val lhs = commaSeparated(pattern2) val tpt = typedOpt() val rhs = @@ -1782,8 +1782,10 @@ object Parsers { } } else EmptyTree lhs match { - case (id @ Ident(name: TermName)) :: Nil => cpy.ValDef(id)(name, tpt, rhs).withMods(mods) - case _ => PatDef(mods, lhs, tpt, rhs) + case (id @ Ident(name: TermName)) :: Nil => + cpy.ValDef(id)(name, tpt, rhs).withMods(mods).withComment(docstring) + case _ => + PatDef(mods, lhs, tpt, rhs) } } @@ -1792,7 +1794,7 @@ object Parsers { * DefDcl ::= DefSig `:' Type * DefSig ::= id [DefTypeParamClause] ParamClauses */ - def defDefOrDcl(mods: Modifiers): Tree = atPos(tokenRange) { + def defDefOrDcl(mods: Modifiers, docstring: Option[String] = None): Tree = atPos(tokenRange) { def scala2ProcedureSyntax(resultTypeStr: String) = { val toInsert = if (in.token == LBRACE) s"$resultTypeStr =" @@ -1833,7 +1835,7 @@ object Parsers { accept(EQUALS) expr() } - DefDef(name, tparams, vparamss, tpt, rhs).withMods(mods1) + DefDef(name, tparams, vparamss, tpt, rhs).withMods(mods1).withComment(docstring) } } @@ -1867,7 +1869,7 @@ object Parsers { /** TypeDef ::= type Id [TypeParamClause] `=' Type * TypeDcl ::= type Id [TypeParamClause] TypeBounds */ - def typeDefOrDcl(mods: Modifiers): Tree = { + def typeDefOrDcl(mods: Modifiers, docstring: Option[String] = None): Tree = { newLinesOpt() atPos(tokenRange) { val name = ident().toTypeName @@ -1875,7 +1877,7 @@ object Parsers { in.token match { case EQUALS => in.nextToken() - TypeDef(name, tparams, typ()).withMods(mods) + TypeDef(name, tparams, typ()).withMods(mods).withComment(docstring) case SUPERTYPE | SUBTYPE | SEMI | NEWLINE | NEWLINES | COMMA | RBRACE | EOF => TypeDef(name, tparams, typeBounds()).withMods(mods) case _ => diff --git a/test/test/DottyDocTests.scala b/test/test/DottyDocTests.scala index 5161f98b8ed8..a950095bc595 100644 --- a/test/test/DottyDocTests.scala +++ b/test/test/DottyDocTests.scala @@ -42,11 +42,14 @@ trait DottyDocTest extends DottyTest { self => assert(false, s"""No docstring found, expected: "$expected"""") } - def run(): Unit = { + def run(): Unit = try { val c = compiler(assertion) c.rootContext(ctx) c.newRun.compile(source) println(s"${self.getClass.getSimpleName.split("\\$").last} passed") + } catch { case e: Throwable => + Console.err.println(s"${self.getClass.getSimpleName.split("\\$").last} failed") + throw e } } @@ -54,6 +57,7 @@ trait DottyDocTest extends DottyTest { self => object DottyDocTests extends DottyTest { private[this] val tests = Seq( NoComment, + SingleClassInPackage, MultipleOpenedOnSingleClassInPackage, MultipleClassesInPackage, @@ -67,13 +71,18 @@ object DottyDocTests extends DottyTest { ObjectsNestedClass, PackageObject, MultipleDocStringsBeforeEntity, - MultipleDocStringsBeforeAndAfter + MultipleDocStringsBeforeAndAfter, + + ValuesWithDocString, + VarsWithDocString, + DefsWithDocString, + TypesWithDocString ) def main(args: Array[String]): Unit = { - println("------------ Testing DottyDoc ------------") + println(s"-------------- Testing DottyDoc (${tests.length} tests) --------------") tests.foreach(_.run) - println("--------- DottyDoc tests passed! ----------") + println("--------------- DottyDoc tests passed! -------------------") } } @@ -371,3 +380,127 @@ case object MultipleDocStringsBeforeAndAfter extends DottyDocTest { } } + +case object ValuesWithDocString extends DottyDocTest { + override val source = + """ + |object Object { + | /** val1 */ + | val val1 = 1 + | + | /** val2 */ + | val val2: Int = 2 + | /** bogus docstring */ + | + | /** bogus docstring */ + | /** val3 */ + | val val3: List[Int] = 1 :: 2 :: 3 :: Nil + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case PackageDef(_, Seq(o: ModuleDef)) => + o.impl.body match { + case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { + checkDocString(v1.rawComment, "/** val1 */") + checkDocString(v2.rawComment, "/** val2 */") + checkDocString(v3.rawComment, "/** val3 */") + } + case _ => assert(false, "Incorrect structure inside object") + } + } +} + +case object VarsWithDocString extends DottyDocTest { + override val source = + """ + |object Object { + | /** var1 */ + | var var1 = 1 + | + | /** var2 */ + | var var2: Int = 2 + | /** bogus docstring */ + | + | /** bogus docstring */ + | /** var3 */ + | var var3: List[Int] = 1 :: 2 :: 3 :: Nil + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case PackageDef(_, Seq(o: ModuleDef)) => + o.impl.body match { + case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { + checkDocString(v1.rawComment, "/** var1 */") + checkDocString(v2.rawComment, "/** var2 */") + checkDocString(v3.rawComment, "/** var3 */") + } + case _ => assert(false, "Incorrect structure inside object") + } + } +} + +case object DefsWithDocString extends DottyDocTest { + override val source = + """ + |object Object { + | /** def1 */ + | def def1 = 1 + | + | /** def2 */ + | def def2: Int = 2 + | /** bogus docstring */ + | + | /** bogus docstring */ + | /** def3 */ + | def def3: List[Int] = 1 :: 2 :: 3 :: Nil + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case PackageDef(_, Seq(o: ModuleDef)) => + o.impl.body match { + case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { + checkDocString(v1.rawComment, "/** def1 */") + checkDocString(v2.rawComment, "/** def2 */") + checkDocString(v3.rawComment, "/** def3 */") + } + case _ => assert(false, "Incorrect structure inside object") + } + } +} + +case object TypesWithDocString extends DottyDocTest { + override val source = + """ + |object Object { + | /** type1 */ + | type T1 = Int + | + | /** type2 */ + | type T2 = String + | /** bogus docstring */ + | + | /** bogus docstring */ + | /** type3 */ + | type T3 = T2 + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + override def assertion = { + case PackageDef(_, Seq(o: ModuleDef)) => + o.impl.body match { + case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { + checkDocString(v1.rawComment, "/** type1 */") + checkDocString(v2.rawComment, "/** type2 */") + checkDocString(v3.rawComment, "/** type3 */") + } + case _ => assert(false, "Incorrect structure inside object") + } + } +} From 8513dd692cf690a48b69329b60e1cbb852842f12 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 10 Mar 2016 19:02:01 +0100 Subject: [PATCH 08/12] Change tests to be based on JUnit, rename comments variable --- src/dotty/tools/dotc/parsing/Scanners.scala | 18 +- test/test/DottyDocParsingTests.scala | 454 ++++++++++++++++++ test/test/DottyDocTests.scala | 506 -------------------- 3 files changed, 463 insertions(+), 515 deletions(-) create mode 100644 test/test/DottyDocParsingTests.scala delete mode 100644 test/test/DottyDocTests.scala diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index 9ad982e45145..5ec6823e4104 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -182,19 +182,19 @@ object Scanners { */ var revComments: List[Comment] = Nil - /** The currently closest docstring, replaced every time a new docstring is - * encountered + /** All doc comments as encountered, each list contains doc comments from + * the same block level. Starting with the deepest level and going upward */ - var closestDocString: List[List[Comment]] = List(List()) + var docsPerBlockStack: List[List[Comment]] = List(List()) /** Adds level of nesting to docstrings */ def enterBlock(): Unit = - closestDocString = Nil ::: closestDocString + docsPerBlockStack = Nil ::: docsPerBlockStack /** Removes level of nesting for docstrings */ - def exitBlock(): Unit = closestDocString = closestDocString match { + def exitBlock(): Unit = docsPerBlockStack = docsPerBlockStack match { case x :: xs => List(List()) - case _ => closestDocString.tail + case _ => docsPerBlockStack.tail } /** Returns the closest docstring preceding the position supplied */ @@ -204,10 +204,10 @@ object Scanners { case Nil => c } - closestDocString match { + docsPerBlockStack match { case (list @ (x :: xs)) :: _ => { val c = closest(x, xs) - closestDocString = list.dropWhile(_ != c).tail :: closestDocString.tail + docsPerBlockStack = list.dropWhile(_ != c).tail :: docsPerBlockStack.tail Some(c.chrs) } case _ => None @@ -591,7 +591,7 @@ object Scanners { val comment = Comment(pos, flushBuf(commentBuf)) if (comment.isDocComment) - closestDocString = (closestDocString.head :+ comment) :: closestDocString.tail + docsPerBlockStack = (docsPerBlockStack.head :+ comment) :: docsPerBlockStack.tail if (keepComments) revComments = comment :: revComments diff --git a/test/test/DottyDocParsingTests.scala b/test/test/DottyDocParsingTests.scala new file mode 100644 index 000000000000..2dd5f41d04f7 --- /dev/null +++ b/test/test/DottyDocParsingTests.scala @@ -0,0 +1,454 @@ +package test + +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.dotc.ast.Trees._ + +import org.junit.Assert._ +import org.junit.Test + +class DottyDocParsingTests extends DottyTest { + def checkDocString(actual: Option[String], expected: String): Unit = actual match { + case Some(str) => + assert(str == expected, s"""Docstring: "$str" didn't match expected "$expected"""") + case None => + assert(false, s"""No docstring found, expected: "$expected"""") + } + + private def defaultAssertion: PartialFunction[Tree[Untyped], Unit] = { + case x => assert(false, "Couldn't match resulting AST to expected AST in: " + x.show) + } + + private def checkFrontend(source: String)(docAssert: PartialFunction[Tree[Untyped], Unit]) = { + checkCompile("frontend", source) { (_, ctx) => + implicit val c = ctx + (docAssert orElse defaultAssertion)(ctx.compilationUnit.untpdTree) + } + } + + @Test def noComment() = { + import dotty.tools.dotc.ast.untpd._ + val source = "class Class" + + checkFrontend(source) { + case PackageDef(_, Seq(c: TypeDef)) => + assert(c.rawComment == None, "Should not have a comment, mainly used for exhaustive tests") + } + } + + @Test def singleClassInPackage() = { + val source = + """ + |package a + | + |/** Hello world! */ + |class Class(val x: String) + """.stripMargin + + checkFrontend(source) { + case PackageDef(_, Seq(t @ TypeDef(name, _))) if name.toString == "Class" => + checkDocString(t.rawComment, "/** Hello world! */") + } + } + + @Test def multipleOpenedOnSingleClassInPackage() = { + val source = + """ + |package a + | + |/** Hello /* multiple open */ world! */ + |class Class(val x: String) + """.stripMargin + + checkFrontend(source) { + case PackageDef(_, Seq(t @ TypeDef(name, _))) if name.toString == "Class" => + checkDocString(t.rawComment, "/** Hello /* multiple open */ world! */") + } + } + @Test def multipleClassesInPackage() = { + val source = + """ + |package a + | + |/** Class1 docstring */ + |class Class1(val x: String) + | + |/** Class2 docstring */ + |class Class2(val x: String) + """.stripMargin + + checkCompile("frontend", source) { (_, ctx) => + ctx.compilationUnit.untpdTree match { + case PackageDef(_, Seq(c1 @ TypeDef(_,_), c2 @ TypeDef(_,_))) => { + checkDocString(c1.rawComment, "/** Class1 docstring */") + checkDocString(c2.rawComment, "/** Class2 docstring */") + } + } + } + } + + @Test def singleCaseClassWithoutPackage() = { + val source = + """ + |/** Class without package */ + |case class Class(val x: Int) + """.stripMargin + + checkFrontend(source) { + case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment, "/** Class without package */") + } + } + + @Test def SingleTraitWihoutPackage() = { + val source = "/** Trait docstring */\ntrait Trait" + + checkFrontend(source) { + case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment, "/** Trait docstring */") + } + } + + @Test def multipleTraitsWithoutPackage() = { + val source = + """ + |/** Trait1 docstring */ + |trait Trait1 + | + |/** Trait2 docstring */ + |trait Trait2 + """.stripMargin + + checkFrontend(source) { + case PackageDef(_, Seq(t1 @ TypeDef(_,_), t2 @ TypeDef(_,_))) => { + checkDocString(t1.rawComment, "/** Trait1 docstring */") + checkDocString(t2.rawComment, "/** Trait2 docstring */") + } + } + } + + @Test def multipleMixedEntitiesWithPackage() = { + val source = + """ + |/** Trait1 docstring */ + |trait Trait1 + | + |/** Class2 docstring */ + |class Class2(val x: Int) + | + |/** CaseClass3 docstring */ + |case class CaseClass3() + | + |case class NoComment() + | + |/** AbstractClass4 docstring */ + |abstract class AbstractClass4(val x: Int) + """.stripMargin + + checkFrontend(source) { + case PackageDef(_, Seq(t1 @ TypeDef(_,_), c2 @ TypeDef(_,_), cc3 @ TypeDef(_,_), _, ac4 @ TypeDef(_,_))) => { + checkDocString(t1.rawComment, "/** Trait1 docstring */") + checkDocString(c2.rawComment, "/** Class2 docstring */") + checkDocString(cc3.rawComment, "/** CaseClass3 docstring */") + checkDocString(ac4.rawComment, "/** AbstractClass4 docstring */") + } + } + } + + @Test def nestedClass() = { + val source = + """ + |/** Outer docstring */ + |class Outer { + | /** Inner docstring */ + | class Inner(val x: Int) + |} + """.stripMargin + + checkFrontend(source) { + case PackageDef(_, Seq(outer @ TypeDef(_, tpl @ Template(_,_,_,_)))) => { + checkDocString(outer.rawComment, "/** Outer docstring */") + tpl.body match { + case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") + case _ => assert(false, "Couldn't find inner class") + } + } + } + } + + @Test def nestedClassThenOuter() = { + val source = + """ + |/** Outer1 docstring */ + |class Outer1 { + | /** Inner docstring */ + | class Inner(val x: Int) + |} + | + |/** Outer2 docstring */ + |class Outer2 + """.stripMargin + + checkFrontend(source) { + case PackageDef(_, Seq(o1 @ TypeDef(_, tpl @ Template(_,_,_,_)), o2 @ TypeDef(_,_))) => { + checkDocString(o1.rawComment, "/** Outer1 docstring */") + checkDocString(o2.rawComment, "/** Outer2 docstring */") + tpl.body match { + case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") + case _ => assert(false, "Couldn't find inner class") + } + } + } + } + + @Test def objects() = { + val source = + """ + |package p + | + |/** Object1 docstring */ + |object Object1 + | + |/** Object2 docstring */ + |object Object2 + """.stripMargin + + checkFrontend(source) { + case p @ PackageDef(_, Seq(o1: MemberDef[Untyped], o2: MemberDef[Untyped])) => { + assertEquals(o1.name.toString, "Object1") + checkDocString(o1.rawComment, "/** Object1 docstring */") + assertEquals(o2.name.toString, "Object2") + checkDocString(o2.rawComment, "/** Object2 docstring */") + } + } + } + + @Test def objectsNestedClass() = { + val source = + """ + |package p + | + |/** Object1 docstring */ + |object Object1 + | + |/** Object2 docstring */ + |object Object2 { + | class A1 + | /** Inner docstring */ + | class Inner + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case p @ PackageDef(_, Seq(o1: ModuleDef, o2: ModuleDef)) => { + assert(o1.name.toString == "Object1") + checkDocString(o1.rawComment, "/** Object1 docstring */") + assert(o2.name.toString == "Object2") + checkDocString(o2.rawComment, "/** Object2 docstring */") + + o2.impl.body match { + case _ :: (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") + case _ => assert(false, "Couldn't find inner class") + } + } + } + } + + @Test def packageObject() = { + val source = + """ + |/** Package object docstring */ + |package object foo { + | /** Boo docstring */ + | case class Boo() + | + | /** Trait docstring */ + | trait Trait + | + | /** InnerObject docstring */ + | object InnerObject { + | /** InnerClass docstring */ + | class InnerClass + | } + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case PackageDef(_, Seq(p: ModuleDef)) => { + checkDocString(p.rawComment, "/** Package object docstring */") + + p.impl.body match { + case (b: TypeDef) :: (t: TypeDef) :: (o: ModuleDef) :: Nil => { + checkDocString(b.rawComment, "/** Boo docstring */") + checkDocString(t.rawComment, "/** Trait docstring */") + checkDocString(o.rawComment, "/** InnerObject docstring */") + checkDocString(o.impl.body.head.asInstanceOf[TypeDef].rawComment, "/** InnerClass docstring */") + } + case _ => assert(false, "Incorrect structure inside package object") + } + } + } + } + + @Test def multipleDocStringsBeforeEntity() = { + val source = + """ + |/** First comment */ + |/** Second comment */ + |/** Real comment */ + |class Class + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case PackageDef(_, Seq(c: TypeDef)) => + checkDocString(c.rawComment, "/** Real comment */") + } + } + + @Test def multipleDocStringsBeforeAndAfter() = { + val source = + """ + |/** First comment */ + |/** Second comment */ + |/** Real comment */ + |class Class + |/** Following comment 1 */ + |/** Following comment 2 */ + |/** Following comment 3 */ + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case PackageDef(_, Seq(c: TypeDef)) => + checkDocString(c.rawComment, "/** Real comment */") + } + } + + @Test def valuesWithDocString() = { + val source = + """ + |object Object { + | /** val1 */ + | val val1 = 1 + | + | /** val2 */ + | val val2: Int = 2 + | /** bogus docstring */ + | + | /** bogus docstring */ + | /** val3 */ + | val val3: List[Int] = 1 :: 2 :: 3 :: Nil + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case PackageDef(_, Seq(o: ModuleDef)) => { + o.impl.body match { + case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { + checkDocString(v1.rawComment, "/** val1 */") + checkDocString(v2.rawComment, "/** val2 */") + checkDocString(v3.rawComment, "/** val3 */") + } + case _ => assert(false, "Incorrect structure inside object") + } + } + } + } + + @Test def varsWithDocString() = { + val source = + """ + |object Object { + | /** var1 */ + | var var1 = 1 + | + | /** var2 */ + | var var2: Int = 2 + | /** bogus docstring */ + | + | /** bogus docstring */ + | /** var3 */ + | var var3: List[Int] = 1 :: 2 :: 3 :: Nil + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case PackageDef(_, Seq(o: ModuleDef)) => { + o.impl.body match { + case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { + checkDocString(v1.rawComment, "/** var1 */") + checkDocString(v2.rawComment, "/** var2 */") + checkDocString(v3.rawComment, "/** var3 */") + } + case _ => assert(false, "Incorrect structure inside object") + } + } + } + } + + @Test def defsWithDocString() = { + val source = + """ + |object Object { + | /** def1 */ + | def def1 = 1 + | + | /** def2 */ + | def def2: Int = 2 + | /** bogus docstring */ + | + | /** bogus docstring */ + | /** def3 */ + | def def3: List[Int] = 1 :: 2 :: 3 :: Nil + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case PackageDef(_, Seq(o: ModuleDef)) => { + o.impl.body match { + case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { + checkDocString(v1.rawComment, "/** def1 */") + checkDocString(v2.rawComment, "/** def2 */") + checkDocString(v3.rawComment, "/** def3 */") + } + case _ => assert(false, "Incorrect structure inside object") + } + } + } + } + + @Test def typesWithDocString() = { + val source = + """ + |object Object { + | /** type1 */ + | type T1 = Int + | + | /** type2 */ + | type T2 = String + | /** bogus docstring */ + | + | /** bogus docstring */ + | /** type3 */ + | type T3 = T2 + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case PackageDef(_, Seq(o: ModuleDef)) => { + o.impl.body match { + case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { + checkDocString(v1.rawComment, "/** type1 */") + checkDocString(v2.rawComment, "/** type2 */") + checkDocString(v3.rawComment, "/** type3 */") + } + case _ => assert(false, "Incorrect structure inside object") + } + } + } + } +} /* End class */ diff --git a/test/test/DottyDocTests.scala b/test/test/DottyDocTests.scala deleted file mode 100644 index a950095bc595..000000000000 --- a/test/test/DottyDocTests.scala +++ /dev/null @@ -1,506 +0,0 @@ -package test - -import dotty.tools.dotc.Compiler -import dotty.tools.dotc.core.Phases.Phase -import dotty.tools.dotc.typer.FrontEnd -import dotty.tools.dotc.core.Contexts.Context -import dotty.tools.dotc.ast.Trees._ - -/** All tests need to extend this trait and define `source` and `assertion` - * - * Instances also need to be placed in the `DottyDocTests::tests` sequence to - * be included in the run - */ -trait DottyDocTest extends DottyTest { self => - /** Source code in string format */ - def source: String - - /** A test to run against the resulting code */ - def assertion: PartialFunction[Tree[Untyped], Unit] - - private def defaultAssertion: PartialFunction[Tree[Untyped], Unit] = { - case x => assert(false, "Couldn't match resulting AST to expected AST in: " + x.show) - } - - private def compiler(assertion: PartialFunction[Tree[Untyped], Unit]) = new Compiler { - override def phases = { - val checker = new Phase { - def phaseName = "assertionChecker" - override def run(implicit ctx: Context): Unit = - (assertion orElse defaultAssertion)(ctx.compilationUnit.untpdTree) - } - - List(List(new FrontEnd)) ::: List(List(checker)) - } - } - - def checkDocString(actual: Option[String], expected: String): Unit = actual match { - case Some(str) => { - assert(str == expected, s"""Docstring: "$str" didn't match expected "$expected"""") - } - case None => - assert(false, s"""No docstring found, expected: "$expected"""") - } - - def run(): Unit = try { - val c = compiler(assertion) - c.rootContext(ctx) - c.newRun.compile(source) - println(s"${self.getClass.getSimpleName.split("\\$").last} passed") - } catch { case e: Throwable => - Console.err.println(s"${self.getClass.getSimpleName.split("\\$").last} failed") - throw e - } -} - -/** Add tests to the `tests` sequence */ -object DottyDocTests extends DottyTest { - private[this] val tests = Seq( - NoComment, - - SingleClassInPackage, - MultipleOpenedOnSingleClassInPackage, - MultipleClassesInPackage, - SingleCaseClassWithoutPackage, - SingleTraitWihoutPackage, - MultipleTraitsWithoutPackage, - MultipleMixedEntitiesWithPackage, - NestedClass, - NestedClassThenOuter, - Objects, - ObjectsNestedClass, - PackageObject, - MultipleDocStringsBeforeEntity, - MultipleDocStringsBeforeAndAfter, - - ValuesWithDocString, - VarsWithDocString, - DefsWithDocString, - TypesWithDocString - ) - - def main(args: Array[String]): Unit = { - println(s"-------------- Testing DottyDoc (${tests.length} tests) --------------") - tests.foreach(_.run) - println("--------------- DottyDoc tests passed! -------------------") - } -} - -case object NoComment extends DottyDocTest { - override val source = "class Class" - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case PackageDef(_, Seq(c: TypeDef)) => - assert(c.rawComment == None, "Should not have a comment, mainly used for exhaustive tests") - } -} - -case object SingleClassInPackage extends DottyDocTest { - override val source = - """ - |package a - | - |/** Hello world! */ - |class Class(val x: String) - """.stripMargin - - override def assertion = { - case PackageDef(_, Seq(t @ TypeDef(name, _))) if name.toString == "Class" => - checkDocString(t.rawComment, "/** Hello world! */") - } -} - -case object MultipleOpenedOnSingleClassInPackage extends DottyDocTest { - override val source = - """ - |package a - | - |/** Hello /* multiple open */ world! */ - |class Class(val x: String) - """.stripMargin - - override def assertion = { - case PackageDef(_, Seq(t @ TypeDef(name, _))) if name.toString == "Class" => - checkDocString(t.rawComment, "/** Hello /* multiple open */ world! */") - } -} - -case object MultipleClassesInPackage extends DottyDocTest { - override val source = - """ - |package a - | - |/** Class1 docstring */ - |class Class1(val x: String) - | - |/** Class2 docstring */ - |class Class2(val x: String) - """.stripMargin - - override def assertion = { - case PackageDef(_, Seq(c1 @ TypeDef(_,_), c2 @ TypeDef(_,_))) => { - checkDocString(c1.rawComment, "/** Class1 docstring */") - checkDocString(c2.rawComment, "/** Class2 docstring */") - } - } -} - -case object SingleCaseClassWithoutPackage extends DottyDocTest { - override val source = - """ - |/** Class without package */ - |case class Class(val x: Int) - """.stripMargin - - override def assertion = { - case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment, "/** Class without package */") - } -} - -case object SingleTraitWihoutPackage extends DottyDocTest { - override val source = "/** Trait docstring */\ntrait Trait" - - override def assertion = { - case PackageDef(_, Seq(t @ TypeDef(_,_))) => checkDocString(t.rawComment, "/** Trait docstring */") - } -} - -case object MultipleTraitsWithoutPackage extends DottyDocTest { - override val source = - """ - |/** Trait1 docstring */ - |trait Trait1 - | - |/** Trait2 docstring */ - |trait Trait2 - """.stripMargin - - override def assertion = { - case PackageDef(_, Seq(t1 @ TypeDef(_,_), t2 @ TypeDef(_,_))) => { - checkDocString(t1.rawComment, "/** Trait1 docstring */") - checkDocString(t2.rawComment, "/** Trait2 docstring */") - } - } -} - -case object MultipleMixedEntitiesWithPackage extends DottyDocTest { - override val source = - """ - |/** Trait1 docstring */ - |trait Trait1 - | - |/** Class2 docstring */ - |class Class2(val x: Int) - | - |/** CaseClass3 docstring */ - |case class CaseClass3() - | - |case class NoComment() - | - |/** AbstractClass4 docstring */ - |abstract class AbstractClass4(val x: Int) - """.stripMargin - - override def assertion = { - case PackageDef(_, Seq(t1 @ TypeDef(_,_), c2 @ TypeDef(_,_), cc3 @ TypeDef(_,_), _, ac4 @ TypeDef(_,_))) => { - checkDocString(t1.rawComment, "/** Trait1 docstring */") - checkDocString(c2.rawComment, "/** Class2 docstring */") - checkDocString(cc3.rawComment, "/** CaseClass3 docstring */") - checkDocString(ac4.rawComment, "/** AbstractClass4 docstring */") - } - } -} - -case object NestedClass extends DottyDocTest { - override val source = - """ - |/** Outer docstring */ - |class Outer { - | /** Inner docstring */ - | class Inner(val x: Int) - |} - """.stripMargin - - override def assertion = { - case PackageDef(_, Seq(outer @ TypeDef(_, tpl @ Template(_,_,_,_)))) => - checkDocString(outer.rawComment, "/** Outer docstring */") - tpl.body match { - case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") - case _ => assert(false, "Couldn't find inner class") - } - } -} - -case object NestedClassThenOuter extends DottyDocTest { - override val source = - """ - |/** Outer1 docstring */ - |class Outer1 { - | /** Inner docstring */ - | class Inner(val x: Int) - |} - | - |/** Outer2 docstring */ - |class Outer2 - """.stripMargin - - override def assertion = { - case PackageDef(_, Seq(o1 @ TypeDef(_, tpl @ Template(_,_,_,_)), o2 @ TypeDef(_,_))) => - checkDocString(o1.rawComment, "/** Outer1 docstring */") - checkDocString(o2.rawComment, "/** Outer2 docstring */") - tpl.body match { - case (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") - case _ => assert(false, "Couldn't find inner class") - } - } -} - -case object Objects extends DottyDocTest { - override val source = - """ - |package p - | - |/** Object1 docstring */ - |object Object1 - | - |/** Object2 docstring */ - |object Object2 - """.stripMargin - - override def assertion = { - case p @ PackageDef(_, Seq(o1: MemberDef[Untyped], o2: MemberDef[Untyped])) => - assert(o1.name.toString == "Object1") - checkDocString(o1.rawComment, "/** Object1 docstring */") - assert(o2.name.toString == "Object2") - checkDocString(o2.rawComment, "/** Object2 docstring */") - } -} - -case object ObjectsNestedClass extends DottyDocTest { - override val source = - """ - |package p - | - |/** Object1 docstring */ - |object Object1 - | - |/** Object2 docstring */ - |object Object2 { - | class A1 - | /** Inner docstring */ - | class Inner - |} - """.stripMargin - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case p @ PackageDef(_, Seq(o1: ModuleDef, o2: ModuleDef)) => - assert(o1.name.toString == "Object1") - checkDocString(o1.rawComment, "/** Object1 docstring */") - assert(o2.name.toString == "Object2") - checkDocString(o2.rawComment, "/** Object2 docstring */") - - o2.impl.body match { - case _ :: (inner @ TypeDef(_,_)) :: _ => checkDocString(inner.rawComment, "/** Inner docstring */") - case _ => assert(false, "Couldn't find inner class") - } - } -} - -case object PackageObject extends DottyDocTest { - override val source = - """ - |/** Package object docstring */ - |package object foo { - | /** Boo docstring */ - | case class Boo() - | - | /** Trait docstring */ - | trait Trait - | - | /** InnerObject docstring */ - | object InnerObject { - | /** InnerClass docstring */ - | class InnerClass - | } - |} - """.stripMargin - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case PackageDef(_, Seq(p: ModuleDef)) => { - checkDocString(p.rawComment, "/** Package object docstring */") - - p.impl.body match { - case (b: TypeDef) :: (t: TypeDef) :: (o: ModuleDef) :: Nil => { - checkDocString(b.rawComment, "/** Boo docstring */") - checkDocString(t.rawComment, "/** Trait docstring */") - checkDocString(o.rawComment, "/** InnerObject docstring */") - checkDocString(o.impl.body.head.asInstanceOf[TypeDef].rawComment, "/** InnerClass docstring */") - } - case _ => assert(false, "Incorrect structure inside package object") - } - } - } -} - -case object MultipleDocStringsBeforeEntity extends DottyDocTest { - override val source = - """ - |/** First comment */ - |/** Second comment */ - |/** Real comment */ - |class Class - """.stripMargin - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case PackageDef(_, Seq(c: TypeDef)) => - checkDocString(c.rawComment, "/** Real comment */") - } -} - -case object MultipleDocStringsBeforeAndAfter extends DottyDocTest { - override val source = - """ - |/** First comment */ - |/** Second comment */ - |/** Real comment */ - |class Class - |/** Following comment 1 */ - |/** Following comment 2 */ - |/** Following comment 3 */ - """.stripMargin - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case PackageDef(_, Seq(c: TypeDef)) => - checkDocString(c.rawComment, "/** Real comment */") - } - -} - -case object ValuesWithDocString extends DottyDocTest { - override val source = - """ - |object Object { - | /** val1 */ - | val val1 = 1 - | - | /** val2 */ - | val val2: Int = 2 - | /** bogus docstring */ - | - | /** bogus docstring */ - | /** val3 */ - | val val3: List[Int] = 1 :: 2 :: 3 :: Nil - |} - """.stripMargin - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case PackageDef(_, Seq(o: ModuleDef)) => - o.impl.body match { - case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { - checkDocString(v1.rawComment, "/** val1 */") - checkDocString(v2.rawComment, "/** val2 */") - checkDocString(v3.rawComment, "/** val3 */") - } - case _ => assert(false, "Incorrect structure inside object") - } - } -} - -case object VarsWithDocString extends DottyDocTest { - override val source = - """ - |object Object { - | /** var1 */ - | var var1 = 1 - | - | /** var2 */ - | var var2: Int = 2 - | /** bogus docstring */ - | - | /** bogus docstring */ - | /** var3 */ - | var var3: List[Int] = 1 :: 2 :: 3 :: Nil - |} - """.stripMargin - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case PackageDef(_, Seq(o: ModuleDef)) => - o.impl.body match { - case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { - checkDocString(v1.rawComment, "/** var1 */") - checkDocString(v2.rawComment, "/** var2 */") - checkDocString(v3.rawComment, "/** var3 */") - } - case _ => assert(false, "Incorrect structure inside object") - } - } -} - -case object DefsWithDocString extends DottyDocTest { - override val source = - """ - |object Object { - | /** def1 */ - | def def1 = 1 - | - | /** def2 */ - | def def2: Int = 2 - | /** bogus docstring */ - | - | /** bogus docstring */ - | /** def3 */ - | def def3: List[Int] = 1 :: 2 :: 3 :: Nil - |} - """.stripMargin - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case PackageDef(_, Seq(o: ModuleDef)) => - o.impl.body match { - case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { - checkDocString(v1.rawComment, "/** def1 */") - checkDocString(v2.rawComment, "/** def2 */") - checkDocString(v3.rawComment, "/** def3 */") - } - case _ => assert(false, "Incorrect structure inside object") - } - } -} - -case object TypesWithDocString extends DottyDocTest { - override val source = - """ - |object Object { - | /** type1 */ - | type T1 = Int - | - | /** type2 */ - | type T2 = String - | /** bogus docstring */ - | - | /** bogus docstring */ - | /** type3 */ - | type T3 = T2 - |} - """.stripMargin - - import dotty.tools.dotc.ast.untpd._ - override def assertion = { - case PackageDef(_, Seq(o: ModuleDef)) => - o.impl.body match { - case (v1: MemberDef) :: (v2: MemberDef) :: (v3: MemberDef) :: Nil => { - checkDocString(v1.rawComment, "/** type1 */") - checkDocString(v2.rawComment, "/** type2 */") - checkDocString(v3.rawComment, "/** type3 */") - } - case _ => assert(false, "Incorrect structure inside object") - } - } -} From 844683edb78f45bf37949bd923f56fc98a68c837 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 10 Mar 2016 19:26:05 +0100 Subject: [PATCH 09/12] Implement comments as attachments instead of tree members --- project/Build.scala | 2 +- src/dotty/tools/dotc/ast/Trees.scala | 11 ++++++----- src/dotty/tools/dotc/parsing/Parsers.scala | 18 +++++++----------- src/dotty/tools/dotc/parsing/Scanners.scala | 4 ++-- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index 0e6e0a9bbc59..723a7e129d2d 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -182,7 +182,7 @@ object DottyBuild extends Build { } ). settings( - addCommandAlias("partest", ";test:package;package;test:runMain dotc.build;lockPartestFile;test:test;runPartestRunner;test:runMain test.DottyDocTests") ++ + addCommandAlias("partest", ";test:package;package;test:runMain dotc.build;lockPartestFile;test:test;runPartestRunner") ++ addCommandAlias("partest-only", ";test:package;package;test:runMain dotc.build;lockPartestFile;test:test-only dotc.tests;runPartestRunner") ++ addCommandAlias("partest-only-no-bootstrap", ";test:package;package; lockPartestFile;test:test-only dotc.tests;runPartestRunner") ) diff --git a/src/dotty/tools/dotc/ast/Trees.scala b/src/dotty/tools/dotc/ast/Trees.scala index eb5ab15e371c..be3e2eb5f835 100644 --- a/src/dotty/tools/dotc/ast/Trees.scala +++ b/src/dotty/tools/dotc/ast/Trees.scala @@ -29,6 +29,9 @@ object Trees { /** The total number of created tree nodes, maintained if Stats.enabled */ @sharable var ntrees = 0 + /** Attachment key for trees with documentation strings attached */ + val DocComment = new Attachment.Key[String] + /** Modifiers and annotations for definitions * @param flags The set flags * @param privateWithin If a private or protected has is followed by a @@ -321,9 +324,7 @@ object Trees { private[ast] def rawMods: Modifiers[T] = if (myMods == null) genericEmptyModifiers else myMods - private[this] var myComment: Option[String] = None - - def rawComment: Option[String] = myComment + def rawComment: Option[String] = getAttachment(DocComment) def withMods(mods: Modifiers[Untyped]): ThisTree[Untyped] = { val tree = if (myMods == null || (myMods == mods)) this else clone.asInstanceOf[MemberDef[Untyped]] @@ -333,8 +334,8 @@ object Trees { def withFlags(flags: FlagSet): ThisTree[Untyped] = withMods(Modifiers(flags)) - def withComment(cmt: Option[String]): ThisTree[Untyped] = { - myComment = cmt + def setComment(comment: Option[String]): ThisTree[Untyped] = { + comment.map(putAttachment(DocComment, _)) asInstanceOf[ThisTree[Untyped]] } diff --git a/src/dotty/tools/dotc/parsing/Parsers.scala b/src/dotty/tools/dotc/parsing/Parsers.scala index dad618381abd..cdfc366a7210 100644 --- a/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1782,9 +1782,9 @@ object Parsers { } } else EmptyTree lhs match { - case (id @ Ident(name: TermName)) :: Nil => - cpy.ValDef(id)(name, tpt, rhs).withMods(mods).withComment(docstring) - case _ => + case (id @ Ident(name: TermName)) :: Nil => { + cpy.ValDef(id)(name, tpt, rhs).withMods(mods).setComment(docstring) + } case _ => PatDef(mods, lhs, tpt, rhs) } } @@ -1835,7 +1835,7 @@ object Parsers { accept(EQUALS) expr() } - DefDef(name, tparams, vparamss, tpt, rhs).withMods(mods1).withComment(docstring) + DefDef(name, tparams, vparamss, tpt, rhs).withMods(mods1).setComment(docstring) } } @@ -1877,7 +1877,7 @@ object Parsers { in.token match { case EQUALS => in.nextToken() - TypeDef(name, tparams, typ()).withMods(mods).withComment(docstring) + TypeDef(name, tparams, typ()).withMods(mods).setComment(docstring) case SUPERTYPE | SUBTYPE | SEMI | NEWLINE | NEWLINES | COMMA | RBRACE | EOF => TypeDef(name, tparams, typeBounds()).withMods(mods) case _ => @@ -1923,9 +1923,7 @@ object Parsers { } val templ = templateOpt(constr) - TypeDef(name, templ) - .withMods(mods) - .withComment(docstring) + TypeDef(name, templ).withMods(mods).setComment(docstring) } /** ConstrMods ::= AccessModifier @@ -1945,9 +1943,7 @@ object Parsers { val name = ident() val template = templateOpt(emptyConstructor()) - ModuleDef(name, template) - .withMods(mods) - .withComment(docstring) + ModuleDef(name, template).withMods(mods).setComment(docstring) } /* -------- TEMPLATES ------------------------------------------- */ diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index 5ec6823e4104..4211b6275c7f 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -180,12 +180,12 @@ object Scanners { /** All comments in the reverse order of their position in the source. * set only when `keepComments` is true. */ - var revComments: List[Comment] = Nil + private[this] var revComments: List[Comment] = Nil /** All doc comments as encountered, each list contains doc comments from * the same block level. Starting with the deepest level and going upward */ - var docsPerBlockStack: List[List[Comment]] = List(List()) + private[this] var docsPerBlockStack: List[List[Comment]] = List(List()) /** Adds level of nesting to docstrings */ def enterBlock(): Unit = From f43f520a1a6e60d4a6020af97c52dd6c43ea75cd Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 7 Apr 2016 08:27:43 +0200 Subject: [PATCH 10/12] Add binding between Symbol and Untyped tree in base context This commit also adds a printer for use by dottydoc. --- src/dotty/tools/dotc/config/Printers.scala | 1 + src/dotty/tools/dotc/core/Contexts.scala | 8 +++ src/dotty/tools/dotc/typer/Namer.scala | 16 ++++- src/dotty/tools/dotc/typer/Typer.scala | 2 +- test/test/DottyDocParsingTests.scala | 79 ++++++++++++---------- test/test/DottyDocTest.scala | 30 ++++++++ 6 files changed, 95 insertions(+), 41 deletions(-) create mode 100644 test/test/DottyDocTest.scala diff --git a/src/dotty/tools/dotc/config/Printers.scala b/src/dotty/tools/dotc/config/Printers.scala index 21147fe6fa86..fa36ad12cb4c 100644 --- a/src/dotty/tools/dotc/config/Printers.scala +++ b/src/dotty/tools/dotc/config/Printers.scala @@ -13,6 +13,7 @@ object Printers { } val default: Printer = new Printer + val dottydoc: Printer = noPrinter val core: Printer = noPrinter val typr: Printer = noPrinter val constr: Printer = noPrinter diff --git a/src/dotty/tools/dotc/core/Contexts.scala b/src/dotty/tools/dotc/core/Contexts.scala index a0bb03e505b5..ad3a0057d2f9 100644 --- a/src/dotty/tools/dotc/core/Contexts.scala +++ b/src/dotty/tools/dotc/core/Contexts.scala @@ -550,6 +550,14 @@ object Contexts { def squashed(p: Phase): Phase = { allPhases.find(_.period.containsPhaseId(p.id)).getOrElse(NoPhase) } + + val _docstrings: mutable.Map[Symbol, String] = + mutable.Map.empty + + def docstring(sym: Symbol): Option[String] = _docstrings.get(sym) + + def addDocstring(sym: Symbol, doc: Option[String]): Unit = + doc.map(d => _docstrings += (sym -> d)) } /** The essential mutable state of a context base, collected into a common class */ diff --git a/src/dotty/tools/dotc/typer/Namer.scala b/src/dotty/tools/dotc/typer/Namer.scala index de27333d592d..82b3b56e9714 100644 --- a/src/dotty/tools/dotc/typer/Namer.scala +++ b/src/dotty/tools/dotc/typer/Namer.scala @@ -401,19 +401,29 @@ class Namer { typer: Typer => val pkg = createPackageSymbol(pcl.pid) index(pcl.stats)(ctx.fresh.setOwner(pkg.moduleClass)) invalidateCompanions(pkg, Trees.flatten(pcl.stats map expanded)) + setDocstring(pkg, stat) ctx case imp: Import => importContext(createSymbol(imp), imp.selectors) case mdef: DefTree => - enterSymbol(createSymbol(mdef)) + val sym = enterSymbol(createSymbol(mdef)) + setDocstring(sym, stat) ctx case stats: Thicket => - for (tree <- stats.toList) enterSymbol(createSymbol(tree)) + for (tree <- stats.toList) { + val sym = enterSymbol(createSymbol(tree)) + setDocstring(sym, stat) + } ctx case _ => ctx } + def setDocstring(sym: Symbol, tree: Tree)(implicit ctx: Context) = tree match { + case t: MemberDef => ctx.base.addDocstring(sym, t.rawComment) + case _ => () + } + /** Create top-level symbols for statements and enter them into symbol table */ def index(stats: List[Tree])(implicit ctx: Context): Context = { @@ -859,7 +869,7 @@ class Namer { typer: Typer => WildcardType } paramFn(typedAheadType(mdef.tpt, tptProto).tpe) - } + } /** The type signature of a DefDef with given symbol */ def defDefSig(ddef: DefDef, sym: Symbol)(implicit ctx: Context) = { diff --git a/src/dotty/tools/dotc/typer/Typer.scala b/src/dotty/tools/dotc/typer/Typer.scala index 3b8ada2a8550..84abf85e0f30 100644 --- a/src/dotty/tools/dotc/typer/Typer.scala +++ b/src/dotty/tools/dotc/typer/Typer.scala @@ -586,7 +586,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit case _ => false } - /** The funcion body to be returned in the closure. Can become a TypedSplice + /** The function body to be returned in the closure. Can become a TypedSplice * of a typed expression if this is necessary to infer a parameter type. */ var fnBody = tree.body diff --git a/test/test/DottyDocParsingTests.scala b/test/test/DottyDocParsingTests.scala index 2dd5f41d04f7..32cfaaddf347 100644 --- a/test/test/DottyDocParsingTests.scala +++ b/test/test/DottyDocParsingTests.scala @@ -6,26 +6,9 @@ import dotty.tools.dotc.ast.Trees._ import org.junit.Assert._ import org.junit.Test -class DottyDocParsingTests extends DottyTest { - def checkDocString(actual: Option[String], expected: String): Unit = actual match { - case Some(str) => - assert(str == expected, s"""Docstring: "$str" didn't match expected "$expected"""") - case None => - assert(false, s"""No docstring found, expected: "$expected"""") - } - - private def defaultAssertion: PartialFunction[Tree[Untyped], Unit] = { - case x => assert(false, "Couldn't match resulting AST to expected AST in: " + x.show) - } +class DottyDocParsingTests extends DottyDocTest { - private def checkFrontend(source: String)(docAssert: PartialFunction[Tree[Untyped], Unit]) = { - checkCompile("frontend", source) { (_, ctx) => - implicit val c = ctx - (docAssert orElse defaultAssertion)(ctx.compilationUnit.untpdTree) - } - } - - @Test def noComment() = { + @Test def noComment = { import dotty.tools.dotc.ast.untpd._ val source = "class Class" @@ -35,7 +18,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def singleClassInPackage() = { + @Test def singleClassInPackage = { val source = """ |package a @@ -50,7 +33,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def multipleOpenedOnSingleClassInPackage() = { + @Test def multipleOpenedOnSingleClassInPackage = { val source = """ |package a @@ -64,7 +47,7 @@ class DottyDocParsingTests extends DottyTest { checkDocString(t.rawComment, "/** Hello /* multiple open */ world! */") } } - @Test def multipleClassesInPackage() = { + @Test def multipleClassesInPackage = { val source = """ |package a @@ -86,7 +69,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def singleCaseClassWithoutPackage() = { + @Test def singleCaseClassWithoutPackage = { val source = """ |/** Class without package */ @@ -98,7 +81,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def SingleTraitWihoutPackage() = { + @Test def SingleTraitWihoutPackage = { val source = "/** Trait docstring */\ntrait Trait" checkFrontend(source) { @@ -106,7 +89,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def multipleTraitsWithoutPackage() = { + @Test def multipleTraitsWithoutPackage = { val source = """ |/** Trait1 docstring */ @@ -124,7 +107,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def multipleMixedEntitiesWithPackage() = { + @Test def multipleMixedEntitiesWithPackage = { val source = """ |/** Trait1 docstring */ @@ -152,7 +135,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def nestedClass() = { + @Test def nestedClass = { val source = """ |/** Outer docstring */ @@ -173,7 +156,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def nestedClassThenOuter() = { + @Test def nestedClassThenOuter = { val source = """ |/** Outer1 docstring */ @@ -198,7 +181,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def objects() = { + @Test def objects = { val source = """ |package p @@ -220,7 +203,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def objectsNestedClass() = { + @Test def objectsNestedClass = { val source = """ |package p @@ -252,7 +235,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def packageObject() = { + @Test def packageObject = { val source = """ |/** Package object docstring */ @@ -289,7 +272,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def multipleDocStringsBeforeEntity() = { + @Test def multipleDocStringsBeforeEntity = { val source = """ |/** First comment */ @@ -305,7 +288,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def multipleDocStringsBeforeAndAfter() = { + @Test def multipleDocStringsBeforeAndAfter = { val source = """ |/** First comment */ @@ -324,7 +307,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def valuesWithDocString() = { + @Test def valuesWithDocString = { val source = """ |object Object { @@ -356,7 +339,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def varsWithDocString() = { + @Test def varsWithDocString = { val source = """ |object Object { @@ -388,7 +371,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def defsWithDocString() = { + @Test def defsWithDocString = { val source = """ |object Object { @@ -420,7 +403,7 @@ class DottyDocParsingTests extends DottyTest { } } - @Test def typesWithDocString() = { + @Test def typesWithDocString = { val source = """ |object Object { @@ -451,4 +434,26 @@ class DottyDocParsingTests extends DottyTest { } } } + + @Test def defInnerClass = { + val source = + """ + |object Foo { + | def foo() = { + | /** Innermost */ + | class Innermost + | } + |} + """.stripMargin + + import dotty.tools.dotc.ast.untpd._ + checkFrontend(source) { + case PackageDef(_, Seq(o: ModuleDef)) => + o.impl.body match { + case (foo: MemberDef) :: Nil => + expectNoDocString(foo.rawComment) + case _ => assert(false, "Incorrect structure inside object") + } + } + } } /* End class */ diff --git a/test/test/DottyDocTest.scala b/test/test/DottyDocTest.scala new file mode 100644 index 000000000000..eed5d6cd0c02 --- /dev/null +++ b/test/test/DottyDocTest.scala @@ -0,0 +1,30 @@ +package test + +import dotty.tools.dotc.ast.Trees._ +import dotty.tools.dotc.core.Contexts.Context + +trait DottyDocTest extends DottyTest { + def checkDocString(actual: Option[String], expected: String): Unit = actual match { + case Some(str) => + assert(str == expected, s"""Docstring: "$str" didn't match expected "$expected"""") + case None => + assert(false, s"""No docstring found, expected: "$expected"""") + } + + def expectNoDocString(doc: Option[String]): Unit = + doc.fold(()) { d => assert(false, s"""Expected not to find a docstring, but found: "$d"""") } + + def defaultAssertion: PartialFunction[Any, Unit] = { + case t: Tree[Untyped] => + assert(false, s"Couldn't match resulting AST to expected AST in: ${t.show}") + case x => + assert(false, s"Couldn't match resulting AST to expected AST in: $x") + } + + def checkFrontend(source: String)(docAssert: PartialFunction[Tree[Untyped], Unit]) = { + checkCompile("frontend", source) { (_, ctx) => + implicit val c = ctx + (docAssert orElse defaultAssertion)(ctx.compilationUnit.untpdTree) + } + } +} From c7730ad9ea99ce7fa53e4cddefdd2afd9f60ce20 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 7 Apr 2016 09:58:43 +0200 Subject: [PATCH 11/12] Add commandline argument `-Ykeep-comments` to remove hardcoding --- src/dotty/tools/dotc/config/ScalaSettings.scala | 1 + src/dotty/tools/dotc/parsing/Scanners.scala | 16 +++++++++------- test/test/DottyDocTest.scala | 2 ++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/dotty/tools/dotc/config/ScalaSettings.scala b/src/dotty/tools/dotc/config/ScalaSettings.scala index 07a23fdb694d..f9c10442aa19 100644 --- a/src/dotty/tools/dotc/config/ScalaSettings.scala +++ b/src/dotty/tools/dotc/config/ScalaSettings.scala @@ -159,6 +159,7 @@ class ScalaSettings extends Settings.SettingGroup { val YprintSyms = BooleanSetting("-Yprint-syms", "when printing trees print info in symbols instead of corresponding info in trees.") val YtestPickler = BooleanSetting("-Ytest-pickler", "self-test for pickling functionality; should be used with -Ystop-after:pickler") val YcheckReentrant = BooleanSetting("-Ycheck-reentrant", "check that compiled program does not contain vars that can be accessed from a global root.") + val YkeepComments = BooleanSetting("-Ykeep-comments", "Keep comments when scanning source files.") def stop = YstopAfter /** Area-specific debug output. diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index 4211b6275c7f..9d08948a5a21 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -175,7 +175,7 @@ object Scanners { } class Scanner(source: SourceFile, override val startFrom: Offset = 0)(implicit ctx: Context) extends ScannerCommon(source)(ctx) { - var keepComments = false + val keepComments = ctx.settings.YkeepComments.value /** All comments in the reverse order of their position in the source. * set only when `keepComments` is true. @@ -560,7 +560,8 @@ object Scanners { } private def skipComment(): Boolean = { - def appendToComment(ch: Char) = commentBuf.append(ch) + def appendToComment(ch: Char) = + if (keepComments) commentBuf.append(ch) def nextChar() = { appendToComment(ch) Scanner.this.nextChar() @@ -587,14 +588,15 @@ object Scanners { def nestedComment() = { nextChar(); skipComment() } val start = lastCharOffset def finishComment(): Boolean = { - val pos = Position(start, charOffset, start) - val comment = Comment(pos, flushBuf(commentBuf)) + if (keepComments) { + val pos = Position(start, charOffset, start) + val comment = Comment(pos, flushBuf(commentBuf)) - if (comment.isDocComment) - docsPerBlockStack = (docsPerBlockStack.head :+ comment) :: docsPerBlockStack.tail + if (comment.isDocComment) + docsPerBlockStack = (docsPerBlockStack.head :+ comment) :: docsPerBlockStack.tail - if (keepComments) revComments = comment :: revComments + } true } diff --git a/test/test/DottyDocTest.scala b/test/test/DottyDocTest.scala index eed5d6cd0c02..34269ec64c7b 100644 --- a/test/test/DottyDocTest.scala +++ b/test/test/DottyDocTest.scala @@ -4,6 +4,8 @@ import dotty.tools.dotc.ast.Trees._ import dotty.tools.dotc.core.Contexts.Context trait DottyDocTest extends DottyTest { + ctx = ctx.fresh.setSetting(ctx.settings.YkeepComments, true) + def checkDocString(actual: Option[String], expected: String): Unit = actual match { case Some(str) => assert(str == expected, s"""Docstring: "$str" didn't match expected "$expected"""") From 60379c2eae264547c7fefcbab45b1fca2352e153 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 7 Apr 2016 12:03:22 +0200 Subject: [PATCH 12/12] Remove old datastructure for comments --- src/dotty/tools/dotc/parsing/Scanners.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/dotty/tools/dotc/parsing/Scanners.scala b/src/dotty/tools/dotc/parsing/Scanners.scala index 9d08948a5a21..7ebe6339764a 100644 --- a/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/src/dotty/tools/dotc/parsing/Scanners.scala @@ -177,11 +177,6 @@ object Scanners { class Scanner(source: SourceFile, override val startFrom: Offset = 0)(implicit ctx: Context) extends ScannerCommon(source)(ctx) { val keepComments = ctx.settings.YkeepComments.value - /** All comments in the reverse order of their position in the source. - * set only when `keepComments` is true. - */ - private[this] var revComments: List[Comment] = Nil - /** All doc comments as encountered, each list contains doc comments from * the same block level. Starting with the deepest level and going upward */ @@ -594,8 +589,6 @@ object Scanners { if (comment.isDocComment) docsPerBlockStack = (docsPerBlockStack.head :+ comment) :: docsPerBlockStack.tail - - revComments = comment :: revComments } true