From d8ff7d8d169665a2789e3295a7e6495b4dc773c0 Mon Sep 17 00:00:00 2001 From: Aleksander Boruch-Gruszecki Date: Mon, 30 Nov 2020 15:40:09 +0100 Subject: [PATCH 1/7] Scala3doc: fix incremental compilation issue --- .../dokka/tasty/comments/wiki/Converter.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scala3doc/src/dotty/dokka/tasty/comments/wiki/Converter.scala b/scala3doc/src/dotty/dokka/tasty/comments/wiki/Converter.scala index 2bf589764fa8..7e846e1d0218 100644 --- a/scala3doc/src/dotty/dokka/tasty/comments/wiki/Converter.scala +++ b/scala3doc/src/dotty/dokka/tasty/comments/wiki/Converter.scala @@ -31,13 +31,16 @@ class Converter(val repr: Repr) extends BaseConverter { block match { case Title(text, level) => val content = convertInline(text) + // NOTE: these aren't strictly necessary, but if you inline them, incremental compilation will break + val jContent = content.asJava : java.util.List[_ <: dkkd.DocTag] + val jAtt = kt.emptyMap[String, String] emit(level match { - case 1 => dkkd.H1(content.asJava, kt.emptyMap) - case 2 => dkkd.H2(content.asJava, kt.emptyMap) - case 3 => dkkd.H3(content.asJava, kt.emptyMap) - case 4 => dkkd.H4(content.asJava, kt.emptyMap) - case 5 => dkkd.H5(content.asJava, kt.emptyMap) - case 6 => dkkd.H6(content.asJava, kt.emptyMap) + case 1 => dkkd.H1(jContent, jAtt) + case 2 => dkkd.H2(jContent, jAtt) + case 3 => dkkd.H3(jContent, jAtt) + case 4 => dkkd.H4(jContent, jAtt) + case 5 => dkkd.H5(jContent, jAtt) + case 6 => dkkd.H6(jContent, jAtt) }) case Paragraph(text) => From e06a3812257a8892fd3e0c04d4cc81d4c631b10a Mon Sep 17 00:00:00 2001 From: Aleksander Boruch-Gruszecki Date: Fri, 27 Nov 2020 11:56:19 +0100 Subject: [PATCH 2/7] Scala3doc: document stdlib together with Dotty extensions --- project/Build.scala | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index 916b7f2e4719..94f17b7f9081 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -841,6 +841,16 @@ object Build { settings( moduleName := "scala-library", javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value, + Compile/scalacOptions += "-Yerased-terms", + Compile/scalacOptions ++= { + Seq( + "-sourcepath", + Seq( + (Compile/sourceManaged).value / "scala-library-src", + (Compile/sourceManaged).value / "dotty-library-src", + ).mkString(File.pathSeparator), + ) + }, scalacOptions -= "-Xfatal-warnings", ivyConfigurations += SourceDeps.hide, transitiveClassifiers := Seq("sources"), @@ -870,6 +880,30 @@ object Build { ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet } (Set(scalaLibrarySourcesJar)).toSeq }.taskValue, + sourceGenerators in Compile += Def.task { + val s = streams.value + val cacheDir = s.cacheDirectory + val trgDir = (sourceManaged in Compile).value / "dotty-library-src" + + // NOTE `sourceDirectory` is used for actual copying, + // but `sources` are used as cache keys + val dottyLibSourceDir = (`scala3-library-bootstrapped`/sourceDirectory).value + val dottyLibSources = (`scala3-library-bootstrapped`/Compile/sources).value + + val cachedFun = FileFunction.cached( + cacheDir / s"copyDottyLibrarySrc", + FilesInfo.lastModified, + FilesInfo.exists, + ) { _ => + s.log.info(s"Copying scala3-library sources from $dottyLibSourceDir to $trgDir...") + if (trgDir.exists) IO.delete(trgDir) + IO.copyDirectory(dottyLibSourceDir, trgDir) + + ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet + } + + cachedFun(dottyLibSources.toSet).toSeq + }.taskValue, sources in Compile ~= (_.filterNot(file => // sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux file.getPath.endsWith("scala-library-src/scala/Any.scala") || @@ -1489,7 +1523,7 @@ object Build { } def joinProducts(products: Seq[java.io.File]): String = - products.iterator.map(_.getAbsolutePath.toString).mkString(java.io.File.pathSeparator) + products.iterator.map(_.getAbsolutePath.toString).mkString(" ") val dokkaVersion = "1.4.10.2" @@ -1537,7 +1571,7 @@ object Build { (`scala3-library-bootstrapped`/Compile/products).value, ).flatten - val roots = dottyJars.mkString(" ") + val roots = joinProducts(dottyJars) if (dottyJars.isEmpty) Def.task { streams.value.log.error("Dotty lib wasn't found") } else Def.task{ @@ -1554,6 +1588,8 @@ object Build { generateScala3StdlibDocumentation:= Def.taskDyn { val dottyJars: Seq[java.io.File] = Seq( (`stdlib-bootstrapped`/Compile/products).value, + (`scala3-interfaces`/Compile/products).value, + (`tasty-core-bootstrapped`/Compile/products).value, ).flatten val roots = joinProducts(dottyJars) From 3fa8fb9b435c7fac0523f3d7c4ac88530d180dc9 Mon Sep 17 00:00:00 2001 From: Aleksander Boruch-Gruszecki Date: Fri, 27 Nov 2020 15:13:07 +0100 Subject: [PATCH 3/7] Scala3doc: avoid documenting sources of stdlib patches --- .../src/dotty/dokka/tasty/TastyParser.scala | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scala3doc/src/dotty/dokka/tasty/TastyParser.scala b/scala3doc/src/dotty/dokka/tasty/TastyParser.scala index 7ab56780182c..6b2cf4404d42 100644 --- a/scala3doc/src/dotty/dokka/tasty/TastyParser.scala +++ b/scala3doc/src/dotty/dokka/tasty/TastyParser.scala @@ -30,17 +30,22 @@ case class DokkaTastyInspector(parser: Parser)(using ctx: DocContext) extends Do private val topLevels = Seq.newBuilder[Documentable] def processCompilationUnit(using q: Quotes)(root: q.reflect.Tree): Unit = - val parser = new TastyParser(q, this) - - def driFor(link: String): Option[DRI] = - val symOps = new SymOps[q.type](q) - import symOps._ - Try(QueryParser(link).readQuery()).toOption.flatMap(q => - MemberLookup.lookupOpt(q, None).map{ case (sym, _) => sym.dri} - ) + // NOTE we avoid documenting definitions in the magical stdLibPatches directory; + // the symbols there are "patched" through dark Dotty magic onto other stdlib + // definitions, so if we documented their origin, we'd get defs with duplicate DRIs + if !root.symbol.show.startsWith("scala.runtime.stdLibPatches") then + val parser = new TastyParser(q, this) + + def driFor(link: String): Option[DRI] = + val symOps = new SymOps[q.type](q) + import symOps._ + Try(QueryParser(link).readQuery()).toOption.flatMap(q => + MemberLookup.lookupOpt(q, None).map{ case (sym, _) => sym.dri} + ) - ctx.staticSiteContext.foreach(_.memberLinkResolver = driFor) - topLevels ++= parser.parseRootTree(root.asInstanceOf[parser.qctx.reflect.Tree]) + ctx.staticSiteContext.foreach(_.memberLinkResolver = driFor) + topLevels ++= parser.parseRootTree(root.asInstanceOf[parser.qctx.reflect.Tree]) + end processCompilationUnit def result(): List[DPackage] = topLevels.clear() From 3071a53748ef82a0b4f0ed45a024d47e307d8246 Mon Sep 17 00:00:00 2001 From: Aleksander Boruch-Gruszecki Date: Mon, 30 Nov 2020 15:00:37 +0100 Subject: [PATCH 4/7] Scala3doc: manually document patched definitions --- .../dotty/dokka/tasty/ClassLikeSupport.scala | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/scala3doc/src/dotty/dokka/tasty/ClassLikeSupport.scala b/scala3doc/src/dotty/dokka/tasty/ClassLikeSupport.scala index 842aaee1791a..d1e0c4e78275 100644 --- a/scala3doc/src/dotty/dokka/tasty/ClassLikeSupport.scala +++ b/scala3doc/src/dotty/dokka/tasty/ClassLikeSupport.scala @@ -69,11 +69,13 @@ trait ClassLikeSupport: val fullExtra = if (signatureOnly) baseExtra - else baseExtra.plus(CompositeMemberExtension( - classDef.extractMembers, - classDef.getParents.map(_.dokkaType.asSignature), - supertypes, - Nil)) + else + baseExtra.plus(CompositeMemberExtension( + classDef.extractPatchedMembers, + classDef.getParents.map(_.dokkaType.asSignature), + supertypes, + Nil)) + end if new DClass( dri, @@ -203,6 +205,31 @@ trait ClassLikeSupport: inherited.flatMap(s => parseInheritedMember(s)) } + /** Extracts members while taking Dotty logic for patching the stdlib into account. */ + def extractPatchedMembers: Seq[Member] = { + val ownMembers = c.extractMembers + def extractPatchMembers(sym: Symbol) = { + // NOTE for some reason scala.language$.experimental$ class doesn't show up here, so we manually add the name + val ownMemberDRIs = ownMembers.iterator.map(_.name).toSet + "experimental$" + sym.tree.asInstanceOf[ClassDef] + .membersToDocument.filterNot(m => ownMemberDRIs.contains(m.symbol.name)) + .flatMap(parseMember) + } + c.symbol.show match { + case "scala.Predef$" => + ownMembers ++ + extractPatchMembers(qctx.reflect.Symbol.requiredClass("scala.runtime.stdLibPatches.Predef$")) + case "scala.language$" => + ownMembers ++ + extractPatchMembers(qctx.reflect.Symbol.requiredModule("scala.runtime.stdLibPatches.language").moduleClass) + case "scala.language$.experimental$" => + ownMembers ++ + extractPatchMembers(qctx.reflect.Symbol.requiredModule("scala.runtime.stdLibPatches.language.experimental").moduleClass) + case _ => ownMembers + } + + } + def getParents: List[Tree] = for parentTree <- c.parents if isValidPos(parentTree.pos) // We assume here that order is correct @@ -417,4 +444,3 @@ trait ClassLikeSupport: valDef.symbol.source )) ) - From 0d8d6b7be34d825152e907ccc1f9b7191bc3abda Mon Sep 17 00:00:00 2001 From: Aleksander Boruch-Gruszecki Date: Wed, 2 Dec 2020 11:50:12 +0100 Subject: [PATCH 5/7] Scala3doc: remove generateScala3StdlibDocumentation task A separate task is no longer necessary. --- project/Build.scala | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index 94f17b7f9081..6a191c5312c4 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -1216,7 +1216,6 @@ object Build { val generateSelfDocumentation = taskKey[Unit]("Generate example documentation") // Note: the two tasks below should be one, but a bug in Tasty prevents that val generateScala3Documentation = inputKey[Unit]("Generate documentation for dotty lib") - val generateScala3StdlibDocumentation = taskKey[Unit]("Generate documentation for Scala3 standard library") val generateTestcasesDocumentation = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests") lazy val `scala3doc` = project.in(file("scala3doc")).asScala3doc lazy val `scala3doc-testcases` = project.in(file("scala3doc-testcases")).asScala3docTestcases @@ -1566,9 +1565,9 @@ object Build { val majorVersion = (scalaBinaryVersion in LocalProject("scala3-library-bootstrapped")).value val dottyJars: Seq[java.io.File] = Seq( + (`stdlib-bootstrapped`/Compile/products).value, (`scala3-interfaces`/Compile/products).value, (`tasty-core-bootstrapped`/Compile/products).value, - (`scala3-library-bootstrapped`/Compile/products).value, ).flatten val roots = joinProducts(dottyJars) @@ -1584,23 +1583,6 @@ object Build { "-siteroot scala3doc/scala3-docs -project-logo scala3doc/scala3-docs/logo.svg")) }.evaluated, - - generateScala3StdlibDocumentation:= Def.taskDyn { - val dottyJars: Seq[java.io.File] = Seq( - (`stdlib-bootstrapped`/Compile/products).value, - (`scala3-interfaces`/Compile/products).value, - (`tasty-core-bootstrapped`/Compile/products).value, - ).flatten - - val roots = joinProducts(dottyJars) - - if (dottyJars.isEmpty) Def.task { streams.value.log.error("Dotty lib wasn't found") } - else generateDocumentation( - roots, "Scala 3", "scala3doc/output/scala3-stdlib", "maser", - "-siteroot scala3doc/scala3-docs -comment-syntax wiki -project-logo scala3doc/scala3-docs/logo.svg " - ) - }.value, - generateTestcasesDocumentation := Def.taskDyn { generateDocumentation(Build.testcasesOutputDir.in(Test).value, "Scala3doc testcases", "scala3doc/output/testcases", "master") }.value, From 454bc22710d920d208d97d1e0d65ef7534b19e6f Mon Sep 17 00:00:00 2001 From: Aleksander Boruch-Gruszecki Date: Wed, 2 Dec 2020 12:11:15 +0100 Subject: [PATCH 6/7] Scala3doc: remove generateScala3StdlibDocumentation task from CI --- .github/workflows/scala3doc.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/scala3doc.yaml b/.github/workflows/scala3doc.yaml index 4d4fe76379ea..ad9d29a3c298 100644 --- a/.github/workflows/scala3doc.yaml +++ b/.github/workflows/scala3doc.yaml @@ -45,9 +45,6 @@ jobs: - name: Generate Scala 3 documentation run: ./project/scripts/sbt scala3doc/generateScala3Documentation - - name: Generate Scala 3 stdlib documentation - run: ./project/scripts/sbt scala3doc/generateScala3StdlibDocumentation - - name: Generate documentation for example project using dotty-sbt run: ./project/scripts/sbt "sbt-dotty/scripted sbt-dotty/scala3doc" From 8b7467cbe85fa67e5626bf804b966ff906169715 Mon Sep 17 00:00:00 2001 From: Aleksander Boruch-Gruszecki Date: Wed, 2 Dec 2020 17:18:57 +0100 Subject: [PATCH 7/7] Scala3doc: make wiki the default syntax for stdlib --- library/src/scala/quoted/Quotes.scala | 2 ++ project/Build.scala | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/library/src/scala/quoted/Quotes.scala b/library/src/scala/quoted/Quotes.scala index a8dd2317e5d8..3c311ba7821f 100644 --- a/library/src/scala/quoted/Quotes.scala +++ b/library/src/scala/quoted/Quotes.scala @@ -176,6 +176,8 @@ trait Quotes { self: runtime.QuoteUnpickler & runtime.QuoteMatching => * +- Flags * * ``` + * + * @syntax markdown */ trait Reflection { self: reflect.type => diff --git a/project/Build.scala b/project/Build.scala index 6a191c5312c4..e3e4a4ff7c9f 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -1580,7 +1580,7 @@ object Build { IO.write(dest / "CNAME", "dotty.epfl.ch") }.dependsOn(generateDocumentation( roots, "Scala 3", dest.getAbsolutePath, "master", - "-siteroot scala3doc/scala3-docs -project-logo scala3doc/scala3-docs/logo.svg")) + "-comment-syntax wiki -siteroot scala3doc/scala3-docs -project-logo scala3doc/scala3-docs/logo.svg")) }.evaluated, generateTestcasesDocumentation := Def.taskDyn {