Merge pull request #10170 from Kordyjan/scala3doc/tests-refactor

Cleaning up tests in scala3doc

Author: romanowski

scala3doc-testcases/src/tests/complexNames.scala

@@ -0,0 +1,26 @@
+package tests
+
+package complexNames
+
+abstract class A:
+  def ++(other: A): A
+  def +:(other: Int): A
+  def :+(other: Int): A
+
+  // scala3doc has problems with names in backticks
+  // def `multi word name`: Int
+  // def `*** name with arbitrary chars ^%`: Int
+  // def `mischievous(param:Int)`(otherParam: Int): String
+  // def withMischievousParams(`param: String, param2`: String): String
+
+  def complexName_^*(param: String): A
+
+  def `completelyUnnecessaryBackticks`: Int //expected: def completelyUnnecessaryBackticks: Int
+  def `+++:`(other: Int): A //expected: def +++:(other: Int): A
+  def `:+++`(other: Int): A //expected: def :+++(other: Int): A
+
+  def `abc_^^_&&`: A //expected: def abc_^^_&&: A
+  def `abc_def`: A //expected: def abc_def: A
+  def `abc_def_++`: A //expected: def abc_def_++: A
+  // def `++_abc`: A
+  // def `abc_++_--`: A

scala3doc/README.md

@@ -72,22 +72,36 @@ You can also find the result of building the same sites for latest `master` at:
 
 ### Testing
 
-Most tests rely on comparing signatures (of classes, methods, objects etc.) extracted from the generated documentation
-to signatures found in source files. Such tests are defined using [MultipleFileTest](src/test/scala/dotty/dokka/MultipleFileTest.scala) class
-and its subtypes (such as [SingleFileTest](src/test/scala/dotty/dokka/SingleFileTest.scala))
-
-WARNING: As the classes mentioned above are likely to evolve, the description below might easily get out of date.
-In case of any discrepancies rely on the source files instead.
-
-`MultipleFileTest` requires that you specify the names of the files used to extract signatures,
-the names of directories containing corresponding TASTY files
-and the kinds of signatures from source files (corresponding to keywords used to declare them like `def`, `class`, `object` etc.)
-whose presence in the generated documentation will be checked (other signatures, when missing, will be ignored).
-The mentioned source files should be located directly inside `src/main/scala/tests` directory
-but the file names passed as parameters should contain neither this path prefix nor `.scala` suffix.
-The TASTY folders are expected to be located in `target/${dottyVersion}/classes/tests` (after successful compilation of the project)
-and similarly only their names relative to this path should be provided as tests' parameters.
-For `SingleFileTest` the name of the source file and the TASTY folder are expected to be the same.
+To implement integration tests that inspects state of the model on different stages of
+documentation generation one can use [ScaladocTest](src/test/scala/dotty/dokka/ScaladocTest.scala#L15).
+This class requires providing the name of the test and a the list of assertions.
+
+Name of the test is used to extract symbols that should be included in given test run from
+the TASTY files. All test data is located in [scala3doc-testcases module](../scala3doc-testcases/src).
+It is compiled together with the rest of modules. This solves lots of potential problems with
+invoking the compiler in a separate process such as mismatching versions. Every test is using
+only symbols defined in the package with the same name as the test located inside top level `tests`
+package (including subpackages). This restriction may be relaxed in the future.
+
+Assertions of each test are defined as list of [Assertion enum](src/test/scala/dotty/dokka/ScaladocTest.scala#L63)
+instances. Each of them represents a callback that is invoked in corresponding phase of
+documentation generation. Those callback can inspect the model, log information and raise errors
+using `reportError` method.
+
+#### Signature tests
+
+Some of the tests rely on comparing signatures (of classes, methods, objects etc.) extracted from
+the generated documentation to signatures found in source files. Such tests are defined using
+[SignatureTest](src/test/scala/dotty/dokka/SignatureTest.scala#L16) class, that is a subclass of
+`ScaladocTest` and uses exactly tha same mechanism to find input symbols in the TASTY files.
+
+Signature tests by default assume that sources that were used to generate used TASTY files are
+located in the file with the same name as the name of the test. If this is not the case optional
+parameter `sourceFiles` can be used to pass list of source file names (without `.scala` extension).
+
+Those tests also requires specifying kinds of ignatures from source files (corresponding to
+keywords used to declare them like `def`, `class`, `object` etc.) whose presence in the generated
+documentation will be checked (other signatures, when missing, will be ignored).
 
 By default it's expected that all signatures from the source files will be present in the documentation
 but not vice versa (because the documentation can contain also inherited signatures).

scala3doc/test/dotty/dokka/DottyTestRunner.scala

@@ -0,0 +1,95 @@
+package dotty.dokka
+
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.model.{DModule, WithChildren}
+import org.jetbrains.dokka.pages.RootPageNode
+import org.jetbrains.dokka.testApi.testRunner.{DokkaTestGenerator, TestMethods}
+import org.jetbrains.dokka.testApi.logger.TestLogger
+import org.jetbrains.dokka.utilities.DokkaConsoleLogger
+import org.jetbrains.dokka.DokkaConfiguration
+import scala.jdk.CollectionConverters.{ListHasAsScala, SeqHasAsJava}
+import org.junit.{Test, Rule}
+import org.junit.rules.{TemporaryFolder, ErrorCollector}
+import java.io.File
+
+abstract class ScaladocTest(val name: String):
+  def assertions: Seq[Assertion]
+
+  private def getTempDir() : TemporaryFolder =
+    val folder = new TemporaryFolder()
+    folder.create()
+    folder
+
+  private def args = Args(
+      name = "test",
+      tastyRoots = Nil ,
+      classpath =  System.getProperty("java.class.path"),
+      None,
+      output = getTempDir().getRoot,
+      projectVersion = "1.0",
+      projectTitle = None,
+      projectLogo = None,
+      defaultSyntax = None,
+      sourceLinks = Nil
+    )
+
+  private def tastyFiles =
+    def listFilesSafe(dir: File) = Option(dir.listFiles).getOrElse {
+      throw AssertionError(s"$dir not found. The test name is incorrect or scala3doc-testcases were not recompiled.")
+    }
+    def collectFiles(dir: File): List[String] = listFilesSafe(dir).toList.flatMap {
+        case f if f.isDirectory => collectFiles(f)
+        case f if f.getName endsWith ".tasty" => f.getAbsolutePath :: Nil
+        case _ => Nil
+      }
+    collectFiles(File(s"${BuildInfo.test_testcasesOutputDir}/tests/$name"))
+
+  @Rule
+  def collector = _collector
+  private val _collector = new ErrorCollector();
+  def reportError(msg: String) = collector.addError(new AssertionError(msg))
+
+  @Test
+  def executeTest =
+    DokkaTestGenerator(
+      DottyDokkaConfig(DocConfiguration.Standalone(args, tastyFiles, Nil)),
+      TestLogger(DokkaConsoleLogger.INSTANCE),
+      assertions.asTestMethods,
+      Nil.asJava
+    ).generate()
+
+end ScaladocTest
+
+type Validator = () => Unit
+
+/**
+ * Those assertions map 1-1 to their dokka counterparts. Some of them may be irrelevant in scala3doc.
+ */
+enum Assertion:
+  case AfterPluginSetup(fn: DokkaContext => Unit)
+  case DuringValidation(fn: Validator => Unit)
+  case AfterDocumentablesCreation(fn: Seq[DModule] => Unit)
+  case AfterPreMergeDocumentablesTransformation(fn: Seq[DModule] => Unit)
+  case AfterDocumentablesMerge(fn: DModule => Unit)
+  case AfterDocumentablesTransformation(fn: DModule => Unit)
+  case AfterPagesGeneration(fn: RootPageNode => Unit)
+  case AfterPagesTransformation(fn: RootPageNode => Unit)
+  case AfterRendering(fn: (RootPageNode, DokkaContext) => Unit)
+
+extension (s: Seq[Assertion]):
+  def asTestMethods: TestMethods =
+    import Assertion._
+    TestMethods(
+      (context => s.collect { case AfterPluginSetup(fn) => fn(context) }.kUnit),
+      (validator => s.collect { case DuringValidation(fn) => fn(() => validator.invoke()) }.kUnit),
+      (modules => s.collect { case AfterDocumentablesCreation(fn) => fn(modules.asScala.toSeq) }.kUnit),
+      (modules => s.collect { case AfterPreMergeDocumentablesTransformation(fn) => fn(modules.asScala.toSeq) }.kUnit),
+      (module => s.collect { case AfterDocumentablesMerge(fn) => fn(module)}.kUnit),
+      (module => s.collect { case AfterDocumentablesTransformation(fn) => fn(module) }.kUnit),
+      (root => s.collect { case AfterPagesGeneration(fn) => fn(root) }.kUnit),
+      (root => s.collect { case AfterPagesTransformation(fn) => fn(root) }.kUnit),
+      ((root, context) => s.collect { case AfterRendering(fn) => fn(root, context)}.kUnit)
+    )
+
+extension [T] (s: T):
+  private def kUnit = kotlin.Unit.INSTANCE