Do not depend on dokka anymore

FIx some test failures

Author: romanowski

project/Build.scala

@@ -1544,19 +1544,13 @@ object Build {
       def joinProducts(products: Seq[java.io.File]): String =
         products.iterator.map(_.getAbsolutePath.toString).mkString(" ")
 
-      val dokkaVersion = "1.4.10.2"
       val flexmarkVersion = "0.42.12"
 
       project.settings(commonBootstrappedSettings).
         dependsOn(`scala3-compiler-bootstrapped`).
         dependsOn(`scala3-tasty-inspector`).
         settings(
-          // Needed to download dokka and its dependencies
-          resolvers += Resolver.jcenterRepo,
           libraryDependencies ++= Seq(
-            "org.jetbrains.dokka" % "dokka-core" % dokkaVersion,
-            "org.jetbrains.dokka" % "dokka-base" % dokkaVersion,
-            "org.jetbrains.kotlinx" % "kotlinx-html-jvm" % "0.7.2", // Needs update when dokka version changes
             "com.vladsch.flexmark" % "flexmark" % flexmarkVersion,
             "com.vladsch.flexmark" % "flexmark-html-parser" % flexmarkVersion,
             "com.vladsch.flexmark" % "flexmark-ext-anchorlink" % flexmarkVersion,
@@ -1572,15 +1566,12 @@ object Build {
             "args4j" % "args4j" % "2.33",
             Dependencies.`jackson-dataformat-yaml`,
 
-            "org.jetbrains.dokka" % "dokka-test-api" % dokkaVersion % "test",
             "com.novocode" % "junit-interface" % "0.11" % "test",
           ),
           Test / test := (Test / test).dependsOn(compile.in(Compile).in(`scala3doc-testcases`)).value,
           testcasesOutputDir.in(Test) := joinProducts((`scala3doc-testcases`/Compile/products).value),
           testcasesSourceRoot.in(Test) := (baseDirectory.in(`scala3doc-testcases`).value / "src").getAbsolutePath.toString,
           Compile / mainClass := Some("dotty.dokka.Main"),
-          // There is a bug in dokka that prevents parallel tests withing the same jvm
-          fork.in(test) := true,
           baseDirectory.in(run) := baseDirectory.in(ThisBuild).value,
           generateSelfDocumentation := Def.taskDyn {
             generateDocumentation(
@@ -1665,8 +1656,6 @@ object Build {
           BuildInfoPlugin.buildInfoScopedSettings(Test),
           BuildInfoPlugin.buildInfoScopedSettings(Compile),
           BuildInfoPlugin.buildInfoDefaultSettings,
-          // Uncomment to debug dokka processing (require to run debug in listen mode on 5005 port)
-          // javaOptions.in(run) += "-agentlib:jdwp=transport=dt_socket,server=n,address=localhost:5005,suspend=y"
         )
     }
 

scala3doc/README.md

@@ -2,8 +2,7 @@
 
 Scala3doc (name subject to change) is the documentation tool for
 [Dotty](https://github.com/lampepfl/dotty), which is scheduled to become
-Scala 3. It's based on [Dokka](https://github.com/Kotlin/dokka), the
-documentation tool for Kotlin. It uses the TastyInspector to access definitions,
+Scala 3.  It uses the TastyInspector to access definitions,
 which is an officially supported way to access Dotty's perspective of a
 codebase.
 
@@ -48,9 +47,7 @@ CLI command for running our tool is in form: `sbt main -n <name> -o <output> -t
 - `<classpath>`: classpath that was used to generate tasty files
 - `<sources>`: links to source files of module that are used to link symbols on pages to their source file. They need to be supplied in form:
   `local_dir=remote_dir#line_suffix` e.g. `src/main/scala=https://github.com/lampepfl/scala3doc/tree/master/src/main/scala#L`
-- `<documentation>`: directory of static documentation that you would like to render with API documentation. This feature is provided by dokka-site plugin:
-  - [GitHub](https://github.com/VirtusLab/dokka-site)
-  - [Documentation](https://virtuslab.github.io/dokka-site/index.html)
+- `<documentation>`: directory of static documentation that you would like to render with API documentation.
 
 ## Developing
 
@@ -164,16 +161,6 @@ Make sure all the tests pass (simply run `sbt test` to verify that).
 
 ## FAQ
 
-### Why depend on Dokka?
-
-We have two primary reasons for depending on Dokka. One of them is division of
-labour - Dokka already has a team of maintainers, and it supports an excellent
-API which already allowed us to quite easily generate documentation with it. By
-depending on Dokka, we will be able to share a large portion of the maintenance
-burden. The second reason is very pragmatic - on our own, it'd be difficult for
-us to reach even feature parity with Scaladoc, simply because of workforce
-constraints. Meanwhile, Dokka maintainers from VirtusLab reached out to us with
-an offer of help, which we were happy to take.
 
 ### Why use TASTy?
 

scala3doc/src/dotty/dokka/Main.scala

@@ -12,17 +12,7 @@ import dotty.tools.dotc.config.Settings._
 import dotty.tools.dotc.config.CommonScalaSettings
 import dotty.tools.dotc.core.Contexts._
 
-/** Main class for the doctool.
-  *
-  * The `main` method is mostly responsible just for parsing arguments and
-  * configuring Dokka. After that, we hand control to Dokka.
-  *
-  * Other important classes:
-  *
-  * - [](package.DottyDokkaPlugin) is our class that Dokka calls back and which
-  *   actually generates the documentation.
-  * - [](package.DocContext) is our config for Dokka
-  */
+/** Main class for the doctool when used from cli. */
 object Main:
   def main(args: Array[String]): Unit =
     try

scala3doc/src/dotty/dokka/ScalaModuleCreator.scala

@@ -3,7 +3,6 @@ package dotty.dokka
 import dotty.dokka.tasty.DokkaTastyInspector
 import dotty.dokka.model.api._
 import collection.JavaConverters._
-import kotlin.coroutines.Continuation
 
 case class Module(rootPackage: Member, members: Map[DRI, Member])
 

scala3doc/src/dotty/dokka/compat.scala

@@ -3,7 +3,6 @@ package dotty.dokka
 import java.util.stream.Stream // comment out - wrong error!
 import java.util.stream.Collectors
 import java.util.Collections
-import kotlin.jvm.JvmClassMappingKt.getKotlinClass
 import java.nio.file.Path
 import com.vladsch.flexmark.util.ast.{Node => MdNode}
 import dotty.dokka.tasty.comments.wiki.WikiDocElement

scala3doc/src/dotty/dokka/site/SidebarParser.scala

@@ -20,7 +20,8 @@ object Sidebar:
     def setUrl(u: String) = this.url = u
     def setSubsection(l: JList[RawInput]) = this.subsection = l
 
-  private object RawTypeRef extends TypeReference[JMap[String, JList[RawInput]]]
+  type RawInnerTpe = JMap[String, JList[RawInput]]
+  private object RawTypeRef extends TypeReference[RawInnerTpe]
 
   private def toSidebar(r: RawInput): Sidebar = r match
     case RawInput(title, url, list) if title.nonEmpty && url.nonEmpty && list.isEmpty() =>
@@ -30,6 +31,6 @@ object Sidebar:
 
   def load(content: String): Seq[Sidebar] =
     val mapper = ObjectMapper(YAMLFactory())
-    val raw = mapper.readValue(content, RawTypeRef)
+    val raw: RawInnerTpe = mapper.readValue(content, RawTypeRef)
 
     raw.get("sidebar").asScala.toList.map(toSidebar)

scala3doc/src/dotty/dokka/tasty/ScalaDocSupport.scala

@@ -3,7 +3,6 @@ package dotty.dokka.tasty
 import scala.jdk.CollectionConverters._
 
 import dotty.dokka.Scala3doc.CommentSyntax
-import comments.{kt, dkk}
 import dotty.dokka.tasty.comments.Comment
 
 trait ScaladocSupport { self: TastyParser =>

scala3doc/src/dotty/dokka/tasty/comments/package.scala

@@ -3,53 +3,8 @@ package tasty.comments
 
 import scala.jdk.CollectionConverters._
 
-import org.jetbrains.dokka.model.{doc => dkkd}
 import com.vladsch.flexmark.util.{ast => mdu}
 
-object kt:
-  import kotlin.collections.builders.{ListBuilder => KtListBuilder, MapBuilder => KtMapBuilder}
-
-  def emptyList[T] = new KtListBuilder[T]().build()
-  def emptyMap[A, B] = new KtMapBuilder[A, B]().build()
-
-object dkk:
-  def p(children: dkkd.DocTag*) =
-    dkkd.P(children.asJava, Map.empty.asJava)
-  def p(params: (String, String)*)(children: dkkd.DocTag*) =
-    dkkd.P(children.asJava, params.toMap.asJava)
-
-  def text(str: String) = dkkd.Text(str, JList(), Map.empty.asJava)
-
-  def a(children: dkkd.DocTag*) =
-    dkkd.A(children.asJava, Map.empty.asJava)
-  def a(params: (String, String)*)(children: dkkd.DocTag*) =
-    dkkd.A(children.asJava, params.toMap.asJava)
-
-  def pre(params: (String, String)*)(children: dkkd.DocTag*) =
-    dkkd.Pre(children.asJava, params.toMap.asJava)
-
-  def codeInline(children: dkkd.DocTag*) =
-    dkkd.CodeInline(children.asJava, Map.empty.asJava)
-  def codeInline(params: (String, String)*)(children: dkkd.DocTag*) =
-    dkkd.CodeInline(children.asJava, params.toMap.asJava)
-  def codeBlock(children: dkkd.DocTag*) =
-    dkkd.CodeBlock(children.asJava, Map.empty.asJava)
-  def codeBlock(params: (String, String)*)(children: dkkd.DocTag*) =
-    dkkd.CodeBlock(children.asJava, params.toMap.asJava)
-
-  def ul(children: dkkd.DocTag*) =
-    dkkd.Ul(children.asJava, Map.empty.asJava)
-  def ul(params: (String, String)*)(children: dkkd.DocTag*) =
-    dkkd.Ul(children.asJava, params.toMap.asJava)
-  def ol(children: dkkd.DocTag*) =
-    dkkd.Ol(children.asJava, Map.empty.asJava)
-  def ol(params: (String, String)*)(children: dkkd.DocTag*) =
-    dkkd.Ol(children.asJava, params.toMap.asJava)
-  def li(children: dkkd.DocTag*) =
-    dkkd.Li(children.asJava, Map.empty.asJava)
-  def li(params: (String, String)*)(children: dkkd.DocTag*) =
-    dkkd.Li(children.asJava, params.toMap.asJava)
-
 object dbg:
   case class See(n: mdu.Node, c: Seq[See]) {
     def show(sb: StringBuilder, indent: Int): Unit = {

scala3doc/src/dotty/renderers/HtmlRenderer.scala

@@ -66,7 +66,8 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
     case m: Member =>
       val signatureRenderer = new SignatureRenderer:
         def currentDri: DRI = page.link.dri
-        def link(dri: DRI): Option[String] = Some(pathToPage(dri, page.link.dri))
+        def link(dri: DRI): Option[String] =
+          Some(pathToPage(currentDri, dri)).filter(_ != UnresolvedLocationLink)
 
       MemberRenderer(signatureRenderer).fullMember(m)
     case t: ResolvedTemplate => siteContent(page.link.dri, t)

scala3doc/src/dotty/renderers/Locations.scala

@@ -15,20 +15,21 @@ import java.nio.file.Files
 import java.io.File
 import scala.util.matching._
 
+val UnresolvedLocationLink = "#"
+
 trait Locations(using ctx: DocContext):
   def members: Map[DRI, Member]
 
   // TODO verify if location exisits
-  def rawLocation(dri: DRI): Seq[String] = dri match
-    case `docsDRI` => List("docs", "index")
-    case `docsRootDRI` => List("index")
-    case `apiPageDRI` => List("api", "index")
-    case dri if dri.isStaticFile =>
-      Paths.get(dri.location).iterator.asScala.map(_.toString).toList
-    case dri =>
-      val loc = dri.location
-      if loc == null then List("index") // Dokka leftovers
-      else
+  def rawLocation(dri: DRI): Seq[String] =
+    dri match
+      case `docsDRI` => List("docs", "index")
+      case `docsRootDRI` => List("index")
+      case `apiPageDRI` => List("api", "index")
+      case dri if dri.isStaticFile =>
+        Paths.get(dri.location).iterator.asScala.map(_.toString).toList
+      case dri =>
+        val loc = dri.location
         val fqn = loc.split(Array('.')).toList match
           case List("<empty>") => List("index")
           case other => other
@@ -38,12 +39,12 @@ trait Locations(using ctx: DocContext):
   private def unknownPage(dri: DRI): String =
     // TODO we should switch that to warning probably or has dedicated setting
     report.inform(s"Unrecognized page for ${dri.location} ($dri)")
-    "#"
+    UnresolvedLocationLink
 
   def pathToPage(from: DRI, to: DRI): String =
     if to.isStaticFile || members.contains(to) then
       val anchor = if to.anchor.isEmpty then "" else "#" + to.anchor
-      pathTo(rawLocation(from), rawLocation(to)) +".html" + anchor
+      pathToRaw(rawLocation(from), rawLocation(to)) +".html" + anchor
     else
       to.origin match
         case "" =>
@@ -55,11 +56,11 @@ trait Locations(using ctx: DocContext):
 
 
 
-  def pathTo(to: Seq[String], fullFrom: Seq[String]): String =
-    val from = fullFrom.dropRight(1)
-    val commonPaths = to.zip(from).takeWhile{ case (a, b) => a == b }.size
+  def pathToRaw(from: Seq[String], to: Seq[String]): String =
+    val fromDir = from.dropRight(1)
+    val commonPaths = to.zip(fromDir).takeWhile{ case (a, b) => a == b }.size
 
-    val contextPath = from.drop(commonPaths).map(_ => "..")
+    val contextPath = fromDir.drop(commonPaths).map(_ => "..")
     val nodePath = to.drop(commonPaths) match
         case Nil if contextPath.isEmpty && to.nonEmpty=> Seq("..", to.last)
         case Nil => to.lastOption.fold(Seq("index"))(".." :: _ :: Nil)
@@ -68,7 +69,7 @@ trait Locations(using ctx: DocContext):
     (contextPath ++ nodePath).mkString("/")
 
   def resolveRoot(from: Seq[String], to: String): String =
-    pathTo(to.split("/").toList, from)
+    pathToRaw(from, to.split("/").toList)
 
   def resolveRoot(dri: DRI, path: String): String = resolveRoot(rawLocation(dri), path)
   def absolutePath(dri: DRI): String = rawLocation(dri).mkString("", "/", ".html")

scala3doc/src/dotty/renderers/SignatureRenderer.scala

@@ -5,12 +5,6 @@ import collection.JavaConverters._
 import java.net.URI
 import java.net.URL
 import java.util.{List => JList, Set => JSet}
-import kotlinx.html.FlowContent
-import kotlinx.html.stream.StreamKt
-import kotlinx.html.Gen_consumer_tagsKt
-import kotlinx.html.ApiKt
-import kotlinx.html.HTMLTag
-import kotlinx.html.DIV
 import dotty.dokka.model.api.Link
 import dotty.dokka.model.api.HierarchyGraph
 import scala.util.Try

scala3doc/src/dotty/renderers/SiteRenderer.scala

@@ -36,7 +36,7 @@ trait SiteRenderer(using DocContext) extends Locations:
 
       val res = ctx.driForLink(content.template.templateFile, path).filter(driExisits)
       if res.isEmpty then report.warn(s"Unable to resolve link '$str'", content.template.file)
-      res.headOption.fold(str)(pathToPage(_, pageDri) + prefix)
+      res.headOption.fold(str)(pathToPage(pageDri, _) + prefix)
 
     def processLocalLink(str: String): String =
       if str.startsWith("#") || str.isEmpty then str

scala3doc/test/dotty/dokka/ExternalLocationProviderIntegrationTest.scala

@@ -64,7 +64,7 @@ abstract class ExternalLocationProviderIntegrationTest(
         linksBuilder ++= hrefValues
       }
 
-
+    println(output)
     IO.foreachFileIn(output, processFile)
     val links = linksBuilder.result
     val errors = expectedLinks.flatMap(expect => Option.when(!links.contains(expect))(expect))

scala3doc/test/dotty/dokka/renderers/LocationTests.scala

@@ -19,9 +19,9 @@ class LocationTests:
   @Test
   def testLinks() =
     def path(from: String, to: String) =
-      locations.pathTo(to.split('/').toList, from.split('/').toList)
+      locations.pathToRaw(from.split('/').toList, to.split('/').toList)
 
-    assertEquals("a/b", locations.pathTo(Seq("a", "b"), Nil))
+    assertEquals("a/b", locations.pathToRaw(Nil, Seq("a", "b")))
 
     assertEquals(
       "../comments",

scala3doc/test/dotty/dokka/tasty/comments/DocFlexmarkParserTests.scala

@@ -2,6 +2,7 @@ package dotty.dokka.tasty.comments
 
 import org.junit.{Test, Rule}
 import org.junit.Assert.{assertSame, assertTrue, assertEquals}
+import dotty.dokka.tasty.comments.markdown.DocFlexmarkParser
 
 class DocFlexmarkParserTests {
   @Test def test(): Unit = {