Add scaladoc feature for preprocessing markdown files and publishing docs.scala-lang

Author: BarkingBad

project/Build.scala

@@ -29,6 +29,7 @@ import sbtbuildinfo.BuildInfoPlugin.autoImport._
 import scala.util.Properties.isJavaAtLeast
 
 import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._
+import org.scalajs.linker.interface.ModuleInitializer
 
 object DottyJSPlugin extends AutoPlugin {
   import Build._
@@ -329,7 +330,6 @@ object Build {
         ".*java.*::javadoc::https://docs.oracle.com/javase/8/docs/api/",
       "-skip-by-regex:.+\\.internal($|\\..+)",
       "-skip-by-regex:.+\\.impl($|\\..+)",
-      "-format", "md",
       "-project-logo", "docs/logo.svg",
       "-social-links:" +
         "github::https://github.com/lampepfl/dotty," +
@@ -1245,6 +1245,7 @@ object Build {
   // Note: the two tasks below should be one, but a bug in Tasty prevents that
   val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib")
   val generateTestcasesDocumentation  = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests")
+  val copyTheScaladocJsOutput = inputKey[Unit]("Copy the output of the scaladoc js files")
 
   lazy val `scaladoc-testcases` = project.in(file("scaladoc-testcases")).
     dependsOn(`scala3-compiler-bootstrapped`).
@@ -1253,8 +1254,12 @@ object Build {
     enablePlugins(DottyJSPlugin).
     dependsOn(`scala3-library-bootstrappedJS`).
     settings(
+      Compile / scalaJSMainModuleInitializer := (sys.env.get("scaladoc.projectFormat") match {
+        case Some("md") => Some(ModuleInitializer.mainMethod("dotty.tools.scaladoc.Main", "markdownMain"))
+        case _ => Some(ModuleInitializer.mainMethod("dotty.tools.scaladoc.Main", "main"))
+      }),
       Test / fork := false,
-      scalaJSUseMainModuleInitializer := true,
+      Compile / scalaJSUseMainModuleInitializer := true,
       libraryDependencies += ("org.scala-js" %%% "scalajs-dom" % "1.1.0").cross(CrossVersion.for3Use2_13)
     )
 
@@ -1285,7 +1290,6 @@ object Build {
         scalaSrcLink(stdLibVersion, srcManaged(dottyNonBootstrappedVersion, "scala") + "="),
         dottySrcLink(referenceVersion, srcManaged(dottyNonBootstrappedVersion, "dotty") + "=", "#library/src"),
         dottySrcLink(referenceVersion),
-        "-Ygenerate-inkuire",
       ) ++ scalacOptionsDocSettings ++ revision ++ params ++ targets
       import _root_.scala.sys.process._
       val escapedCmd = cmd.map(arg => if(arg.contains(" ")) s""""$arg"""" else arg)
@@ -1398,6 +1402,13 @@ object Build {
         )
       }.value,
 
+      copyTheScaladocJsOutput := Def.inputTask {
+        val extraArgs = spaceDelimited("<arg>").parsed
+        val dest = extraArgs.lift(0).getOrElse("output")
+        val jsDestinationFile: File = Paths.get(dest).toFile
+        sbt.IO.copyFile((`scaladoc-js` / Compile / fullOptJS).value.data, jsDestinationFile)
+      }.evaluated,
+
       Test / buildInfoKeys := Seq[BuildInfoKey](
         (Test / Build.testcasesOutputDir),
         (Test / Build.testcasesSourceRoot),

project/scripts/genDocsScalaLang

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+shopt -s extglob # needed for rm everything but x
+echo "Working directory: $PWD"
+
+GENDOC_EXTRA_ARGS=$@
+GIT_HEAD=$(git rev-parse HEAD) # save current head for commit message in gh-pages
+PREVIOUS_SNAPSHOTS_DIR="$PWD/../prev_snapshots"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)"
+SITE_OUT_DIR="$PWD/docs/_site"
+
+DOCS_SCALA_LANG_DIR="$PWD/docsScalaLang"
+
+rm -rf $DOCS_SCALA_LANG_DIR
+mkdir -pv $DOCS_SCALA_LANG_DIR
+git clone "https://github.com/scala/docs.scala-lang.git" $DOCS_SCALA_LANG_DIR
+
+SBT="$SCRIPT_DIR/sbt"
+mkdir -pv $SITE_OUT_DIR
+env "scaladoc.projectFormat=md" "$SBT" "scaladoc/copyTheScaladocJsOutput $DOCS_SCALA_LANG_DIR/scripts/searchbar.js"
+"dist/target/pack/bin/scaladoc" "-d" "$SITE_OUT_DIR" "-format" "md" "-siteroot" "docs" "/dev/null"
+
+if [ ! -d "$SITE_OUT_DIR" ]; then
+  echo "Output directory did not exist: $SITE_OUT_DIR" 1>&2
+  exit 1
+fi
+
+cp -rf "$SITE_OUT_DIR/docs/reference"/* "$DOCS_SCALA_LANG_DIR/_scala3-reference"
+cp -f "$SITE_OUT_DIR/resources/my.css" "$DOCS_SCALA_LANG_DIR/scripts/my.css"
+cp -f "$SITE_OUT_DIR/resources/footer.html" "$DOCS_SCALA_LANG_DIR/_includes/footer.html"

scaladoc-js/src/Main.scala

@@ -1,9 +1,22 @@
 package dotty.tools.scaladoc
 
-object Main extends App {
-  Searchbar()
-  SocialLinks()
-  CodeSnippets()
-  DropdownHandler()
-  Ux()
-}
+object Main:
+
+  private def common(): Unit =
+    CodeSnippets()
+
+  def main(): Unit =
+    Searchbar()
+    SocialLinks()
+    DropdownHandler()
+    Ux()
+    common()
+
+  /**
+   * This main is conditionally enabled by system env variable `scaladoc.projectFormat=md`
+   * passed in ./projects/scripts/genDocsScalaLang
+   * The reason why we have to pass the condition by env variable is because js is build before scaladoc,
+   * so we cannot access its args
+   */
+  def markdownMain(): Unit =
+    common()

scaladoc/src/dotty/tools/scaladoc/Scaladoc.scala

@@ -234,6 +234,10 @@ object Scaladoc:
     given docContext: DocContext = new DocContext(args, ctx)
     val module = ScalaModuleProvider.mkModule()
 
-    new dotty.tools.scaladoc.renderers.HtmlRenderer(module.rootPackage, module.members).render()
+    val renderer = args.projectFormat match
+      case "html" => new dotty.tools.scaladoc.renderers.HtmlRenderer(module.rootPackage, module.members)
+      case "md" => new dotty.tools.scaladoc.renderers.MarkdownRenderer(module.rootPackage, module.members)
+      
+    renderer.render()
     report.inform("generation completed successfully")
     docContext

scaladoc/src/dotty/tools/scaladoc/renderers/HtmlRenderer.scala

@@ -14,122 +14,21 @@ import java.nio.file.Files
 import java.nio.file.FileVisitOption
 import java.io.File
 
-case class Page(link: Link, content: Member | ResolvedTemplate | String, children: Seq[Page]):
-  def withNewChildren(newChildren: Seq[Page]) = copy(children = children ++ newChildren)
-
-  def withTitle(newTitle: String) = copy(link = link.copy(name = newTitle))
-
-  def hasFrame = content match
-    case t: ResolvedTemplate => t.hasFrame
-    case _ => true
-
-class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx: DocContext)
-  extends SiteRenderer, Resources, Locations, Writer:
-  private val args = summon[DocContext].args
-  val staticSite = summon[DocContext].staticSiteContext
-
-  val effectiveMembers = members
-
-  private def memberPage(member: Member): Page =
-    val childrenPages = member.members.filter(_.needsOwnPage)
-    Page(Link(member.name, member.dri), member, childrenPages.map(memberPage))
-
-  val navigablePage: Page =
-    val rootPckPage = memberPage(rootPackage)
-    staticSite match
-      case None => rootPckPage.withTitle(args.name)
-      case Some(siteContext) =>
-        val (indexes, templates) = siteContext.templates.partition(f =>
-          f.templateFile.isIndexPage() && f.file.toPath.getParent() == siteContext.docsPath)
-        if (indexes.size > 1)
-          val msg = s"ERROR: Multiple index pages for doc found ${indexes.map(_.file)}"
-          report.error(msg)
-
-        val templatePages = templates.map(templateToPage(_, siteContext))
-
-        indexes.headOption match
-          case None if templatePages.isEmpty=>
-            rootPckPage.withTitle(args.name)
-          case None =>
-            Page(Link(args.name, docsRootDRI),"", templatePages :+ rootPckPage.withTitle("API"))
-          case Some(indexPage) =>
-            val newChildren = templatePages :+ rootPckPage.withTitle("API")
-            templateToPage(indexPage, siteContext).withNewChildren(newChildren)
-
-  val hiddenPages: Seq[Page] =
-    staticSite match
-      case None =>
-        Seq(navigablePage.copy( // Add index page that is a copy of api/index.html
-          link = navigablePage.link.copy(dri = docsRootDRI),
-          children = Nil
-        ))
-      case Some(siteContext) =>
-        // In case that we do not have an index page and we do not have any API entries
-        // we want to create empty index page, so there is one
-        val actualIndexTemplate = siteContext.indexTemplate() match {
-            case None if effectiveMembers.isEmpty => Seq(siteContext.emptyIndexTemplate)
-            case templates => templates.toSeq
-          }
-
-          (siteContext.orphanedTemplates ++ actualIndexTemplate).map(templateToPage(_, siteContext))
-
-  /**
-   * Here we have to retrive index pages from hidden pages and replace fake index pages in navigable page tree.
-   */
-  val allPages: Seq[Page] =
-    def traversePages(page: Page): (Page, Seq[Page]) =
-      val (newChildren, newPagesToRemove): (Seq[Page], Seq[Page]) = page.children.map(traversePages(_)).foldLeft((Seq[Page](), Seq[Page]())) {
-        case ((pAcc, ptrAcc), (p, ptr)) => (pAcc :+ p, ptrAcc ++ ptr)
-      }
-      hiddenPages.find(_.link == page.link) match
-        case None =>
-          (page.copy(children = newChildren), newPagesToRemove)
-        case Some(newPage) =>
-          (newPage.copy(children = newChildren), newPagesToRemove :+ newPage)
-
-    val (newNavigablePage, pagesToRemove) = traversePages(navigablePage)
-
-    val all = newNavigablePage +: hiddenPages.filterNot(pagesToRemove.contains)
-    // We need to check for conflicts only if we have top-level member called blog or docs
-    val hasPotentialConflict =
-      rootPackage.members.exists(m => m.name.startsWith("docs") || m.name.startsWith("blog"))
-
-    if hasPotentialConflict then
-      def walk(page: Page): Unit =
-        if page.link.dri.isStaticFile then
-          val dest = absolutePath(page.link.dri)
-          if apiPaths.contains(dest) then
-            report.error(s"Conflict between static page and API member for $dest. $pathsConflictResoultionMsg")
-          page.children.foreach(walk)
-
-      all.foreach (walk)
-
-    all
-
-  def renderContent(page: Page) = page.content match
-    case m: Member =>
-      val signatureRenderer = new SignatureRenderer:
-        def currentDri: DRI = page.link.dri
-        def link(dri: DRI): Option[String] =
-          Some(pathToPage(currentDri, dri)).filter(_ != UnresolvedLocationLink)
-
-      MemberRenderer(signatureRenderer).fullMember(m)
-    case t: ResolvedTemplate => siteContent(page.link.dri, t)
-    case a: String =>  raw(a)
-
-
-  def renderPage(page: Page, parents: Vector[Link]): Seq[String] =
-    val newParents = parents :+ page.link
-    val content = ctx.args.projectFormat match
-      case "html" => html(
-        mkHead(page),
-        body(
-          if !page.hasFrame then renderContent(page)
-          else mkFrame(page.link, newParents, renderContent(page))
-        )
+class HtmlRenderer(rootPackage: Member, members: Map[DRI, Member])(using ctx: DocContext)
+  extends Renderer(rootPackage, members, extension = "html"):
+
+  override def pageContent(page: Page, parents: Vector[Link]): AppliedTag =
+    html(
+      mkHead(page),
+      body(
+        if !page.hasFrame then renderContent(page)
+        else mkFrame(page.link, parents, renderContent(page))
       )
-      case "md" => renderContent(page)
-    write(page.link.dri, content) +: page.children.flatMap(renderPage(_, newParents))
+    )
+
+  override def render(): Unit =
+    val renderedResources = renderResources()
+    super.render()
 
   private def specificResources(page: Page): Set[String] =
     page.children.toSet.flatMap(specificResources) ++ (page.content match
@@ -159,10 +58,6 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
     val resources = siteResourcesPaths.toSeq.map(pathToResource) ++ allResources(allPages) ++ onlyRenderedResources
     resources.flatMap(renderResource)
 
-  def render(): Unit =
-    val renderedResources = renderResources()
-    val sites = allPages.map(renderPage(_, Vector.empty))
-
   def mkHead(page: Page): AppliedTag =
     val resources = page.content match
       case t: ResolvedTemplate =>
@@ -226,14 +121,6 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
 
     renderNested(navigablePage, toplevel = true)._2
 
-  private def canonicalUrl(l: String): AppliedTag | String =
-    val canon = args.docCanonicalBaseUrl
-    if !canon.isEmpty then
-      val canonicalUrl = if canon.endsWith("/") then canon else canon + "/"
-      link(rel := "canonical", href := canonicalUrl + l)
-    else
-      "" // return empty tag
-
   private def hasSocialLinks = !args.socialLinks.isEmpty
 
   private def socialLinks(whiteIcon: Boolean = true) =

scaladoc/src/dotty/tools/scaladoc/renderers/Locations.scala

@@ -21,7 +21,7 @@ trait Locations(using ctx: DocContext):
 
   // We generate this collection only if there may be a conflict with resources.
   // Potentially can be quite big.
-  lazy val apiPaths = effectiveMembers.keySet.filterNot(_.isStaticFile).map(absolutePath)
+  lazy val apiPaths = effectiveMembers.keySet.filterNot(_.isStaticFile).map(absolutePath(_))
 
   var cache = new JHashMap[DRI, Seq[String]]()
 
@@ -80,7 +80,7 @@ trait Locations(using ctx: DocContext):
     pathToRaw(from, to.split("/").toList)
 
   def resolveRoot(dri: DRI, path: String): String = resolveRoot(rawLocation(dri), path)
-  def absolutePath(dri: DRI): String = rawLocation(dri).mkString("", "/", ".html")
+  def absolutePath(dri: DRI, extension: String = "html"): String = rawLocation(dri).mkString("", "/", s".$extension")
 
   def resolveLink(dri: DRI, url: String): String =
     if URI(url).isAbsolute then url else resolveRoot(dri, url)

scaladoc/src/dotty/tools/scaladoc/renderers/MarkdownRenderer.scala

@@ -0,0 +1,48 @@
+package dotty.tools.scaladoc
+package renderers
+
+import util.HTML._
+import collection.JavaConverters._
+import java.net.URI
+import java.net.URL
+import dotty.tools.scaladoc.site._
+import scala.util.Try
+import org.jsoup.Jsoup
+import java.nio.file.Paths
+import java.nio.file.Path
+import java.nio.file.Files
+import java.nio.file.FileVisitOption
+import java.io.File
+
+class MarkdownRenderer(rootPackage: Member, members: Map[DRI, Member])(using ctx: DocContext)
+  extends Renderer(rootPackage, members, extension = "md"):
+
+  override def render(): Unit =
+    renderResources()
+    super.render()
+
+  override def pageContent(page: Page, parents: Vector[Link]): AppliedTag =
+    renderContent(page)
+
+  private def renderResources(): Seq[String] =
+
+    // TODO REMOVE THIS CODE AND RESOURCES UNDER docs/resources
+    // when we sort out the css classes
+    def siteRoot = staticSite.get.root.toPath
+    def pathToResource(p: String) = Resource.File(p, siteRoot.resolve(p))
+
+    def harvestResources(path: String) =
+      val siteImgPath = siteRoot.resolve(path)
+      if !Files.exists(siteImgPath) then Nil
+      else
+        val allPaths = Files.walk(siteImgPath, FileVisitOption.FOLLOW_LINKS)
+        val files = allPaths.filter(Files.isRegularFile(_)).iterator().asScala
+        files.map(p => siteRoot.relativize(p).toString).toList
+
+    val staticResources = staticSite.toSeq.flatMap { _ =>
+      harvestResources("images") ++ harvestResources("resources")
+    }
+    staticResources.map(pathToResource).flatMap(renderResource)
+    // END TODO REMOVE THIS CODE AND RESOURCES UNDER docs/resources
+
+    allResources(Nil).flatMap(renderResource)