Merge pull request #10274 from romanowski/scala3doc/source-links1

New format of source links

Author: romanowski

project/Build.scala

@@ -1455,8 +1455,10 @@ object Build {
 
     def asScala3doc: Project = {
       def generateDocumentation(targets: String, name: String, outDir: String, params: String = "") = Def.taskDyn {
-          val sourceMapping = "=https://github.com/lampepfl/dotty/tree/master#L"
-          run.in(Compile).toTask(s""" -d output/$outDir -t $targets -n "$name" -s $sourceMapping $params""")
+          val sourcesAndRevision = "-s github://lampepfl/dotty --revision master"
+          run.in(Compile).toTask(
+            s""" -d output/$outDir -t $targets -n "$name" $sourcesAndRevision $params"""
+            )
       }
 
       def joinProducts(products: Seq[java.io.File]): String =

scala3doc/src/dotty/dokka/DottyDokkaConfig.scala

@@ -19,10 +19,11 @@ case class DottyDokkaConfig(docConfiguration: DocConfiguration) extends DokkaCon
 
   lazy val staticSiteContext = docConfiguration.args.docsRoot.map(path => StaticSiteContext(File(path).getAbsoluteFile(), Set(mkSourceSet.asInstanceOf[SourceSetWrapper])))
 
+  lazy val sourceLinks: SourceLinks = SourceLinks.load(docConfiguration)
+
   override def getPluginsConfiguration: JList[DokkaConfiguration.PluginConfiguration] = JList()
 
   lazy val mkSourceSet: DokkaSourceSet =
-    val sourceLinks:Set[SourceLinkDefinitionImpl] = docConfiguration.args.sourceLinks.map(SourceLinkDefinitionImpl.Companion.parseSourceLinkDefinition(_)).toSet
     new DokkaSourceSetImpl(
       /*displayName=*/ docConfiguration.args.name,
       /*sourceSetID=*/ new DokkaSourceSetID(docConfiguration.args.name, "main"),
@@ -36,7 +37,7 @@ case class DottyDokkaConfig(docConfiguration: DocConfiguration) extends DokkaCon
       /*skipEmptyPackages=*/ false, // Now all our packages are empty from dokka perspective
       /*skipDeprecated=*/ true,
       /*jdkVersion=*/ 8,
-      /*sourceLinks=*/ sourceLinks.asJava,
+      /*sourceLinks=*/ JSet(),
       /*perPackageOptions=*/ JList(),
       /*externalDocumentationLinks=*/ JSet(),
       /*languageVersion=*/ null,

scala3doc/src/dotty/dokka/DottyDokkaPlugin.scala

@@ -4,6 +4,7 @@ import org.jetbrains.dokka.plugability._
 import org.jetbrains.dokka.transformers.sources._
 import org.jetbrains.dokka.transformers.documentation.PreMergeDocumentableTransformer
 import org.jetbrains.dokka.transformers.pages.PageTransformer
+import org.jetbrains.dokka.transformers.documentation.DocumentableToPageTranslator
 
 import org.jetbrains.dokka.DokkaConfiguration
 import org.jetbrains.dokka.{ DokkaConfiguration$DokkaSourceSet => DokkaSourceSet }
@@ -82,11 +83,16 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
 
   val scalaDocumentableToPageTranslator = extend(
     _.extensionPoint(CoreExtensions.INSTANCE.getDocumentableToPageTranslator)
-      .fromRecipe(ctx => ScalaDocumentableToPageTranslator(
-            ctx.single(dokkaBase.getCommentsToContentConverter),
-            ctx.single(dokkaBase.getSignatureProvider),
-            ctx.getLogger
-      ))
+      .fromRecipe(ctx =>
+          new DocumentableToPageTranslator {
+            override def invoke(module: DModule): ModulePageNode = ScalaPageCreator(
+              ctx.single(dokkaBase.getCommentsToContentConverter),
+              ctx.single(dokkaBase.getSignatureProvider),
+              ctx.getConfiguration.asInstanceOf[DottyDokkaConfig].sourceLinks,
+              ctx.getLogger
+            ).pageForModule(module)
+          }
+      )
       .overrideExtension(dokkaBase.getDocumentableToPageTranslator)
   )
 
@@ -102,17 +108,6 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
       .name("inheritanceTransformer")
   )
 
-  val ourSourceLinksTransformer = extend(
-    _.extensionPoint(CoreExtensions.INSTANCE.getDocumentableTransformer)
-      .fromRecipe(ctx => ScalaSourceLinksTransformer(
-            ctx,
-            ctx.single(dokkaBase.getCommentsToContentConverter),
-            ctx.single(dokkaBase.getSignatureProvider),
-            ctx.getLogger
-        )
-      )
-  )
-
   val ourRenderer = extend(
     _.extensionPoint(CoreExtensions.INSTANCE.getRenderer)
       .fromRecipe(ScalaHtmlRenderer(_))
@@ -131,15 +126,6 @@ class DottyDokkaPlugin extends DokkaJavaPlugin:
       .name("implicitMembersExtensionTransformer")
   )
 
-  val muteDefaultSourceLinksTransformer = extend(
-    _.extensionPoint(CoreExtensions.INSTANCE.getPageTransformer)
-    .fromInstance(new PageTransformer {
-      override def invoke(root: RootPageNode) = root
-    })
-    .overrideExtension(dokkaBase.getSourceLinksTransformer)
-    .name("muteDefaultSourceLinksTransformer")
-  )
-
   val customDocumentationProvider = extend(
     _.extensionPoint(dokkaBase.getHtmlPreprocessors)
       .fromRecipe(c => SitePagesCreator(c.siteContext))

scala3doc/src/dotty/dokka/Main.scala

@@ -47,6 +47,9 @@ class RawArgs:
     @COption(name="--syntax")
     protected var syntax: String = null
 
+    @COption(name="--revision")
+    protected var revision: String = null
+
     def toArgs =
       val parsedSyntax = syntax match
         case null => None
@@ -66,7 +69,8 @@ class RawArgs:
         Option(projectTitle),
         Option(projectLogo),
         parsedSyntax,
-        Option(sourceLinks).map(_.asScala.toList).getOrElse(List.empty)
+        Option(sourceLinks).map(_.asScala.toList).getOrElse(List.empty),
+        Option(revision)
       )
 
 
@@ -80,7 +84,8 @@ case class Args(
   projectTitle: Option[String],
   projectLogo: Option[String],
   defaultSyntax: Option[Args.CommentSyntax],
-  sourceLinks: List[String]
+  sourceLinks: List[String],
+  revision: Option[String]
 )
 
 object Args:

scala3doc/src/dotty/dokka/SourceLinks.scala

@@ -0,0 +1,139 @@
+package dotty.dokka
+
+import java.nio.file.Path
+import java.nio.file.Paths
+import liqp.Template
+import dotty.dokka.model.api._
+
+case class SourceLink(val path: Option[Path], val urlTemplate: Template)
+
+object SourceLink:
+  val SubPath = "([^=]+)=(.+)".r
+  val KnownProvider = raw"(\w+):\/\/([^\/]+)\/([^\/]+)".r
+  val BrokenKnownProvider = raw"(\w+):\/\/.+".r
+  val ScalaDocPatten = raw"€\{(TPL_NAME|TPL_NAME|FILE_PATH|FILE_EXT|FILE_LINE|FILE_PATH_EXT)\}".r
+  val SupportedScalaDocPatternReplacements = Map(
+    "€{FILE_PATH_EXT}" -> "{{ path }}",
+    "€{FILE_LINE}" -> "{{ line }}"
+  )
+
+  def githubTemplate(organization: String, repo: String)(revision: String) =
+    s"""https://github.com/$organization/$repo/{{ operation | replace: "view", "blob" }}/$revision/{{ path }}#L{{ line }}""".stripMargin
+
+  def gitlabTemplate(organization: String, repo: String)(revision: String) =
+    s"""https://gitlab.com/$organization/$repo/-/{{ operation | replace: "view", "blob" }}/$revision/{{ path }}#L{{ line }}"""
+
+
+  private def parseLinkDefinition(s: String): Option[SourceLink] = ???
+
+  def parse(string: String, revision: Option[String]): Either[String, SourceLink] =
+    def asTemplate(template: String) =
+       try Right(SourceLink(None,Template.parse(template))) catch
+            case e: RuntimeException =>
+              Left(s"Failed to parse template: ${e.getMessage}")
+
+    string match
+      case KnownProvider(name, organization, repo) =>
+        def withRevision(template: String => String) =
+            revision.fold(Left(s"No revision provided"))(rev => Right(SourceLink(None, Template.parse(template(rev)))))
+
+        name match
+          case "github" =>
+            withRevision(githubTemplate(organization, repo))
+          case "gitlab" =>
+            withRevision(gitlabTemplate(organization, repo))
+          case other =>
+            Left(s"'$other' is not a known provider, please provide full source path template.")
+
+      case SubPath(prefix, config) =>
+        parse(config, revision) match
+          case l: Left[String, _] => l
+          case Right(SourceLink(Some(prefix), _)) =>
+            Left(s"Source path $string has duplicated subpath setting (scm template can not contains '=')")
+          case Right(SourceLink(None, template)) =>
+            Right(SourceLink(Some(Paths.get(prefix)), template))
+      case BrokenKnownProvider("gitlab" | "github") =>
+        Left(s"Does not match known provider syntax: `<name>://organization/repository`")
+      case scaladocSetting if ScalaDocPatten.findFirstIn(scaladocSetting).nonEmpty =>
+        val all = ScalaDocPatten.findAllIn(scaladocSetting)
+        val (supported, unsupported) = all.partition(SupportedScalaDocPatternReplacements.contains)
+        if unsupported.nonEmpty then Left(s"Unsupported patterns from scaladoc format are used: ${unsupported.mkString(" ")}")
+        else asTemplate(supported.foldLeft(string)((template, pattern) =>
+          template.replace(pattern, SupportedScalaDocPatternReplacements(pattern))))
+
+      case template => asTemplate(template)
+
+
+type Operation = "view" | "edit"
+
+case class SourceLinks(links: Seq[SourceLink], projectRoot: Path):
+  def pathTo(rawPath: Path, line: Option[Int] = None, operation: Operation = "view"): Option[String] =
+    def resolveRelativePath(path: Path) =
+      links.find(_.path.forall(p => path.startsWith(p))).map { link =>
+        val config = java.util.HashMap[String, Object]()
+        val pathString = path.toString.replace('\\', '/')
+        config.put("path", pathString)
+        line.foreach(l => config.put("line", l.toString))
+        config.put("operation", operation)
+
+        link.urlTemplate.render(config)
+      }
+
+    if rawPath.isAbsolute then
+      if rawPath.startsWith(projectRoot) then resolveRelativePath(projectRoot.relativize(rawPath))
+      else None
+    else resolveRelativePath(rawPath)
+
+  def pathTo(member: Member): Option[String] =
+    member.sources.flatMap(s => pathTo(Paths.get(s.path), Option(s.lineNumber)))
+
+object SourceLinks:
+
+  val usage =
+    """Source links provide a mapping between file in documentation and code repositry (usual)." +
+      |Accepted formats:
+      |<sub-path>=<source-link>
+      |<source-link>
+      |
+      |where <source-link> is one of following:
+      | - `github://<organization>/<repository>` (requires revision to be specified as argument for scala3doc)
+      | - `gitlab://<organization>/<repository>` (requires revision to be specified as argument for scala3doc)
+      | - <scaladoc-template>
+      | - <template>
+      |
+      |<scaladoc-template> is a format for `doc-source-url` parameter scaladoc.
+      |NOTE: We only supports `€{FILE_PATH_EXT}` and €{FILE_LINE} patterns
+      |
+      |<template> is a liqid template string that can accepts follwoing arguments:
+      | - `operation`: either "view" or "edit"
+      | - `path`: relative path of file to provide link to
+      | - `line`: optional parameter that specify line number within a file
+      |
+      |
+      |Template can defined only by subset of sources defined by path prefix represented by `<sub-path>`""".stripMargin
+
+  def load(configs: Seq[String], revision: Option[String], projectRoot: Path): SourceLinks =
+    // TODO ...
+    val mappings = configs.map(str => str -> SourceLink.parse(str, revision))
+
+    val errors = mappings.collect {
+      case (template, Left(message)) =>
+        s"'$template': $message"
+    }.mkString("\n")
+
+    if errors.nonEmpty then println(
+      s"""Following templates has invalid format:
+         |$errors
+         |
+         |$usage
+         |""".stripMargin
+      )
+
+    SourceLinks(mappings.collect {case (_, Right(link)) => link}, projectRoot)
+
+  def load(config: DocConfiguration): SourceLinks =
+    load(
+      config.args.sourceLinks,
+      config.args.revision,
+      Paths.get("").toAbsolutePath
+    )

scala3doc/src/dotty/dokka/model/api/api.scala

@@ -129,6 +129,7 @@ extension[T] (member: Member):
   def kind: Kind = memberExt.fold(Kind.Unknown)(_.kind)
   def origin: Origin =  memberExt.fold(Origin.DefinedWithin)(_.origin)
   def annotations: List[Annotation] = memberExt.fold(Nil)(_.annotations)
+  def sources: Option[TastyDocumentableSource] = memberExt.fold(None)(_.sources)
   def name = member.getName
   def dri = member.getDri
 
@@ -143,3 +144,5 @@ extension[T] (member: Member):
 
 extension (module: DModule):
   def driMap: Map[DRI, Member] = ModuleExtension.getFrom(module).fold(Map.empty)(_.driMap)
+
+case class TastyDocumentableSource(val path: String, val lineNumber: Int)

scala3doc/src/dotty/dokka/model/api/internalExtensions.scala

@@ -28,13 +28,14 @@ private [model] case class MemberExtension(
   kind: Kind,
   val annotations: List[Annotation],
   signature: Signature,
+  sources: Option[TastyDocumentableSource] = None,
   origin: Origin = Origin.DefinedWithin,
   graph: HierarchyGraph = HierarchyGraph.empty,
 ) extends ExtraProperty[Documentable]:
  override def getKey = MemberExtension
 
 object MemberExtension extends BaseKey[Documentable, MemberExtension]:
-  val empty = MemberExtension(Visibility.Unrestricted, Nil, Kind.Unknown, Nil, Nil)
+  val empty = MemberExtension(Visibility.Unrestricted, Nil, Kind.Unknown, Nil, Signature(), None)
 
 case class CompositeMemberExtension(
   members : Seq[Member] = Nil,

scala3doc/src/dotty/dokka/model/extras.scala

@@ -34,20 +34,6 @@ case class ClasslikeExtension(
 
 object ClasslikeExtension extends BaseKey[DClasslike, ClasslikeExtension]
 
-
-case class SourceLinks(
-  links: Map[DokkaConfiguration$DokkaSourceSet, String]
-) extends ExtraProperty[Documentable]:
-  override def getKey = SourceLinks
-
-object SourceLinks extends BaseKey[Documentable, SourceLinks]
-
-// case class ImplicitConversions(val conversions: List[ImplicitConversion]) extends ExtraProperty[WithScope]:
-//   override def getKey = ImplicitConversions
-
-// object ImplicitConversions extends BaseKey[WithScope, ImplicitConversions]
-
-
 case class IsInherited(flag: Boolean) extends ExtraProperty[Documentable]:
   override def getKey = IsInherited
 

scala3doc/src/dotty/dokka/model/scalaModel.scala

@@ -11,10 +11,6 @@ import org.jetbrains.dokka.pages._
 import dotty.dokka.model.api.Signature
 import dotty.dokka.model.api.HierarchyGraph
 
-case class TastyDocumentableSource(val path: String, val lineNumber: Int) extends DocumentableSource {
-    override def getPath = path
-}
-
 enum TableStyle extends org.jetbrains.dokka.pages.Style:
   case Borderless
   case DescriptionList

scala3doc/src/dotty/dokka/tasty/BasicSupport.scala

@@ -5,6 +5,7 @@ import org.jetbrains.dokka.model._
 import collection.JavaConverters._
 import dotty.dokka._
 import dotty.dokka.model.api.Annotation
+import dotty.dokka.model.api.TastyDocumentableSource
 
 trait BasicSupport:
   self: TastyParser =>
@@ -41,10 +42,7 @@ trait BasicSupport:
 
     def source(using ctx: Context) =
       val path = Some(sym.pos.sourceFile.jpath).filter(_ != null).map(_.toAbsolutePath).map(_.toString)
-      path match{
-        case Some(p) => Map(sourceSet -> TastyDocumentableSource(p, sym.pos.startLine))
-        case None => Map.empty
-      }
+      path.map(TastyDocumentableSource(_, sym.pos.startLine))
 
     def getAnnotations(): List[Annotation] =
     sym.annots.filterNot(_.symbol.packageName.startsWith("scala.annotation.internal")).map(parseAnnotation).reverse