WIP: test fix for stdlib loading

compiler/src/dotty/tools/dotc/typer/TypeAssigner.scala

@@ -299,6 +299,12 @@ trait TypeAssigner {
           else fntpe.resultType
         else
           errorType(i"wrong number of arguments at ${ctx.phase.prev} for $fntpe: ${fn.tpe}, expected: ${fntpe.paramInfos.length}, found: ${args.length}", tree.srcPos)
+      // case t if args.isEmpty =>
+      //   // NOTE this case is necessary when loading the stdlib from Tasty. Our extensions are compiled with stdlib loaded
+      //   // from Scala2 classfiles while they are loaded with stdlib loaded from Tasty. This results in `def toString` being
+      //   // typed as MethodType in the first case and ExprType in the second case. In the second case (and, I think, only then)
+      //   // we need to handle ExprType being present here.
+      //   t
       case t =>
         if (ctx.settings.Ydebug.value) new FatalError("").printStackTrace()
         errorType(err.takesNoParamsStr(fn, ""), tree.srcPos)

project/Build.scala

@@ -841,6 +841,16 @@ object Build {
     settings(
       moduleName := "scala-library",
       javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value,
+      Compile/scalacOptions += "-Yerased-terms",
+      Compile/scalacOptions ++= {
+        Seq(
+          "-sourcepath",
+          Seq(
+            (Compile/sourceManaged).value / "scala-library-src",
+            (Compile/sourceManaged).value / "dotty-library-src",
+          ).mkString(File.pathSeparator),
+        )
+      },
       scalacOptions -= "-Xfatal-warnings",
       ivyConfigurations += SourceDeps.hide,
       transitiveClassifiers := Seq("sources"),
@@ -870,6 +880,31 @@ object Build {
           ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
         } (Set(scalaLibrarySourcesJar)).toSeq
       }.taskValue,
+      sourceGenerators in Compile += Def.task {
+        val s = streams.value
+        val cacheDir = s.cacheDirectory
+        val trgDir = (sourceManaged in Compile).value / "dotty-library-src"
+
+        // NOTE `sourceDirectory` is used for actual copying,
+        // but `sources` are used as cache keys
+        val dottyLibSourceDir = (`scala3-library-bootstrapped`/sourceDirectory).value
+        val dottyLibSources = (`scala3-library-bootstrapped`/Compile/sources).value
+
+        val cachedFun = FileFunction.cached(
+          cacheDir / s"copyDottyLibrarySrc",
+          FilesInfo.lastModified,
+          FilesInfo.exists,
+        ) { _ =>
+          s.log.info(s"Copying scala3-library sources from $dottyLibSourceDir to $trgDir...")
+          if (trgDir.exists) IO.delete(trgDir)
+          // IO.createDirectory(trgDir)
+          IO.copyDirectory(dottyLibSourceDir, trgDir)
+
+          ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
+        }
+
+        cachedFun(dottyLibSources.toSet).toSeq
+      }.taskValue,
       sources in Compile ~= (_.filterNot(file =>
         // sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux
         file.getPath.endsWith("scala-library-src/scala/Any.scala") ||
@@ -1491,7 +1526,7 @@ object Build {
       }
 
       def joinProducts(products: Seq[java.io.File]): String =
-        products.iterator.map(_.getAbsolutePath.toString).mkString(java.io.File.pathSeparator)
+        products.iterator.map(_.getAbsolutePath.toString).mkString(" ")
 
       val dokkaVersion = "1.4.10.2"
 
@@ -1539,7 +1574,7 @@ object Build {
               (`scala3-library-bootstrapped`/Compile/products).value,
             ).flatten
 
-            val roots = dottyJars.mkString(" ")
+            val roots = joinProducts(dottyJars)
 
             if (dottyJars.isEmpty) Def.task { streams.value.log.error("Dotty lib wasn't found") }
             else Def.task{
@@ -1556,6 +1591,8 @@ object Build {
           generateScala3StdlibDocumentation:= Def.taskDyn {
             val dottyJars: Seq[java.io.File] = Seq(
               (`stdlib-bootstrapped`/Compile/products).value,
+              (`scala3-interfaces`/Compile/products).value,
+              (`tasty-core-bootstrapped`/Compile/products).value,
             ).flatten
 
             val roots = joinProducts(dottyJars)

scala3doc/src/dotty/dokka/tasty/ClassLikeSupport.scala

@@ -69,11 +69,13 @@ trait ClassLikeSupport:
 
       val fullExtra =
         if (signatureOnly) baseExtra
-        else baseExtra.plus(CompositeMemberExtension(
-          classDef.extractMembers,
-          classDef.getParents.map(_.dokkaType.asSignature),
-          supertypes,
-          Nil))
+        else
+          baseExtra.plus(CompositeMemberExtension(
+            classDef.extractPatchedMembers,
+            classDef.getParents.map(_.dokkaType.asSignature),
+            supertypes,
+            Nil))
+        end if
 
       new DClass(
           dri,
@@ -109,9 +111,9 @@ trait ClassLikeSupport:
       case dd: DefDef if !dd.symbol.isHiddenByVisibility && !dd.symbol.isSyntheticFunc && dd.symbol.isExtensionMethod =>
         dd.symbol.extendedSymbol.map { extSym =>
           val target = ExtensionTarget(
-            extSym.symbol.normalizedName, 
-            extSym.tpt.dokkaType.asSignature, 
-            extSym.tpt.symbol.dri, 
+            extSym.symbol.normalizedName,
+            extSym.tpt.dokkaType.asSignature,
+            extSym.tpt.symbol.dri,
             extSym.symbol.pos.start
           )
           parseMethod(dd.symbol, kind = Kind.Extension(target))
@@ -203,6 +205,31 @@ trait ClassLikeSupport:
         inherited.flatMap(s => parseInheritedMember(s))
     }
 
+    /** Extracts members while taking Dotty logic for patching the stdlib into account. */
+    def extractPatchedMembers: Seq[Member] = {
+      val ownMembers = c.extractMembers
+      def extractPatchMembers(sym: Symbol) = {
+        // NOTE for some reason scala.language$.experimental$ class doesn't show up here, so we manually add the name
+        val ownMemberDRIs = ownMembers.iterator.map(_.name).toSet + "experimental$"
+        sym.tree.asInstanceOf[ClassDef]
+          .membersToDocument.filterNot(m => ownMemberDRIs.contains(m.symbol.name))
+          .flatMap(parseMember)
+      }
+      c.symbol.show match {
+        case "scala.Predef$" =>
+          ownMembers ++
+          extractPatchMembers(qctx.reflect.Symbol.requiredClass("scala.runtime.stdLibPatches.Predef$"))
+        case "scala.language$" =>
+          ownMembers ++
+          extractPatchMembers(qctx.reflect.Symbol.requiredModule("scala.runtime.stdLibPatches.language").moduleClass)
+        case "scala.language$.experimental$" =>
+          ownMembers ++
+          extractPatchMembers(qctx.reflect.Symbol.requiredModule("scala.runtime.stdLibPatches.language.experimental").moduleClass)
+        case _ => ownMembers
+      }
+
+    }
+
     def getParents: List[Tree] =
       for
         parentTree <- c.parents if isValidPos(parentTree.pos)  // We assume here that order is correct
@@ -417,4 +444,3 @@ trait ClassLikeSupport:
           valDef.symbol.source
       ))
     )
-

scala3doc/src/dotty/dokka/tasty/TastyParser.scala

@@ -33,17 +33,21 @@ case class DokkaTastyInspector(
   private val topLevels = Seq.newBuilder[Documentable]
 
   def processCompilationUnit(using q: Quotes)(root: q.reflect.Tree): Unit =
-    val parser = new TastyParser(q, this, config)
-
-    def driFor(link: String): Option[DRI] =
-      val symOps = new SymOps[q.type](q)
-      import symOps._
-      Try(QueryParser(link).readQuery()).toOption.flatMap(q =>
-        MemberLookup.lookupOpt(q, None).map{ case (sym, _) => sym.dri}
-      )
+    // NOTE we avoid documenting definitions in the magical stdLibPatches directory;
+    // the symbols there are "patched" through dark Dotty magic onto other stdlib
+    // definitions, so if we documented their origin, we'd get defs with duplicate DRIs
+    if !root.symbol.show.startsWith("scala.runtime.stdLibPatches") then
+      val parser = new TastyParser(q, this, config)
+
+      def driFor(link: String): Option[DRI] =
+        val symOps = new SymOps[q.type](q)
+        import symOps._
+        Try(QueryParser(link).readQuery()).toOption.flatMap(q =>
+          MemberLookup.lookupOpt(q, None).map{ case (sym, _) => sym.dri}
+        )
 
-    config.staticSiteContext.foreach(_.memberLinkResolver = driFor)
-    topLevels ++= parser.parseRootTree(root.asInstanceOf[parser.qctx.reflect.Tree])
+      config.staticSiteContext.foreach(_.memberLinkResolver = driFor)
+      topLevels ++= parser.parseRootTree(root.asInstanceOf[parser.qctx.reflect.Tree])
 
   def result(): List[DPackage] =
     topLevels.clear()

scala3doc/src/dotty/dokka/tasty/comments/wiki/Converter.scala

@@ -31,13 +31,16 @@ class Converter(val repr: Repr) extends BaseConverter {
     block match {
       case Title(text, level) =>
         val content = convertInline(text)
+        // NOTE: these aren't strictly necessary, but if you inline them, incremental compilation will break
+        val jContent = content.asJava : java.util.List[_ <: dkkd.DocTag]
+        val jAtt = kt.emptyMap[String, String]
         emit(level match {
-          case 1 => dkkd.H1(content.asJava, kt.emptyMap)
-          case 2 => dkkd.H2(content.asJava, kt.emptyMap)
-          case 3 => dkkd.H3(content.asJava, kt.emptyMap)
-          case 4 => dkkd.H4(content.asJava, kt.emptyMap)
-          case 5 => dkkd.H5(content.asJava, kt.emptyMap)
-          case 6 => dkkd.H6(content.asJava, kt.emptyMap)
+          case 1 => dkkd.H1(jContent, jAtt)
+          case 2 => dkkd.H2(jContent, jAtt)
+          case 3 => dkkd.H3(jContent, jAtt)
+          case 4 => dkkd.H4(jContent, jAtt)
+          case 5 => dkkd.H5(jContent, jAtt)
+          case 6 => dkkd.H6(jContent, jAtt)
         })
 
       case Paragraph(text) =>