Support the -Yjava-tasty flag.

- Keep Java compilation units up to Pickler phase if -Yjava-tasty.
  Skip phases for Java when not needed.
- Add JAVAattr and OUTLINEattr TASTy attributes, ELIDED tree tag.
  ELIDED trees are pickled as rhs of java term definitions.
  ELIDED trees can only be unpickled if OUTLINEattr is present.
  For now OUTLINEattr implies JAVAattr. In the future we might
  expand OUTLINEattr to include outline Scala typing.
- write java tasty files to a special jar, set with -Yjava-tasty-output
  this option is for testing purposes only.

Author: bishabosha

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -31,6 +31,11 @@ class CompilationUnit protected (val source: SourceFile) {
   /** Is this the compilation unit of a Java file */
   def isJava: Boolean = source.file.name.endsWith(".java")
 
+  /** Is this the compilation unit of a Java file, or TASTy derived from a Java file */
+  def typedAsJava = isJava || tastyAttributes.exists(_.isJava)
+
+  var tastyAttributes: Option[tasty.Attributes] = None
+
   /** The source version for this unit, as determined by a language import */
   var sourceVersion: Option[SourceVersion] = None
 

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -17,6 +17,7 @@ import reporting.*
 import annotation.constructorOnly
 import printing.Formatting.hl
 import config.Printers
+import parsing.Parsers
 
 import scala.annotation.internal.sharable
 import scala.annotation.threadUnsafe
@@ -143,8 +144,13 @@ object desugar {
 
   /** A value definition copied from `vdef` with a tpt typetree derived from it */
   def derivedTermParam(vdef: ValDef)(using Context): ValDef =
+    derivedTermParam(vdef, vdef.unforcedRhs)
+
+  def derivedTermParam(vdef: ValDef, rhs: LazyTree)(using Context): ValDef =
     cpy.ValDef(vdef)(
-      tpt = DerivedFromParamTree().withSpan(vdef.tpt.span).watching(vdef))
+      tpt = DerivedFromParamTree().withSpan(vdef.tpt.span).watching(vdef),
+      rhs = rhs
+    )
 
 // ----- Desugar methods -------------------------------------------------
 
@@ -544,8 +550,11 @@ object desugar {
       constrTparams.zipWithConserve(impliedTparams)((tparam, impliedParam) =>
         derivedTypeParam(tparam).withAnnotations(impliedParam.mods.annotations))
     val derivedVparamss =
-      constrVparamss.nestedMap(vparam =>
-        derivedTermParam(vparam).withAnnotations(Nil))
+      constrVparamss.nestedMap: vparam =>
+        val derived =
+          if ctx.compilationUnit.isJava then derivedTermParam(vparam, Parsers.unimplementedExpr)
+          else derivedTermParam(vparam)
+        derived.withAnnotations(Nil)
 
     val constr = cpy.DefDef(constr1)(paramss = joinParams(constrTparams, constrVparamss))
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -8,7 +8,7 @@ import dotty.tools.dotc.config.Settings.{Setting, SettingGroup}
 import dotty.tools.dotc.config.SourceVersion
 import dotty.tools.dotc.core.Contexts.*
 import dotty.tools.dotc.rewrites.Rewrites
-import dotty.tools.io.{AbstractFile, Directory, JDK9Reflectors, PlainDirectory}
+import dotty.tools.io.{AbstractFile, Directory, JDK9Reflectors, PlainDirectory, NoAbstractFile}
 import Setting.ChoiceWithHelp
 
 import scala.util.chaining.*
@@ -433,4 +433,8 @@ private sealed trait YSettings:
   val YforceInlineWhileTyping: Setting[Boolean] = BooleanSetting("-Yforce-inline-while-typing", "Make non-transparent inline methods inline when typing. Emulates the old inlining behavior of 3.0.0-M3.")
 
   val YdebugMacros: Setting[Boolean] = BooleanSetting("-Ydebug-macros", "Show debug info when quote pattern match fails")
+
+  // Pipeline compilation options
+  val YjavaTasty: Setting[Boolean] = BooleanSetting("-Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools")
+  val YjavaTastyOutput: Setting[AbstractFile] = OutputSetting("-Yjava-tasty-output", "directory|jar", "(Internal use only!) destination for generated .tasty files containing Java type signatures.", NoAbstractFile)
 end YSettings

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -992,6 +992,7 @@ class Definitions {
   @tu lazy val AnnotationClass: ClassSymbol = requiredClass("scala.annotation.Annotation")
   @tu lazy val StaticAnnotationClass: ClassSymbol = requiredClass("scala.annotation.StaticAnnotation")
   @tu lazy val RefiningAnnotationClass: ClassSymbol = requiredClass("scala.annotation.RefiningAnnotation")
+  @tu lazy val JavaAnnotationClass: ClassSymbol = requiredClass("java.lang.annotation.Annotation")
 
   // Annotation classes
   @tu lazy val AllowConversionsAnnot: ClassSymbol = requiredClass("scala.annotation.allowConversions")

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -333,16 +333,25 @@ object Phases {
     def subPhases: List[Run.SubPhase] = Nil
     final def traversals: Int = if subPhases.isEmpty then 1 else subPhases.length
 
+    /** skip the phase for a Java compilation unit, may depend on -Yjava-tasty */
+    def skipIfJava(using Context): Boolean = true
+
     /** @pre `isRunnable` returns true */
     def run(using Context): Unit
 
     /** @pre `isRunnable` returns true */
     def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] =
       val buf = List.newBuilder[CompilationUnit]
+      // factor out typedAsJava check when not needed
+      val doSkipJava = ctx.settings.YjavaTasty.value && this <= picklerPhase && skipIfJava
       for unit <- units do
         given unitCtx: Context = runCtx.fresh.setPhase(this.start).setCompilationUnit(unit).withRootImports
         if ctx.run.enterUnit(unit) then
-          try run
+          try
+            if doSkipJava && unit.typedAsJava then
+              ()
+            else
+              run
           catch case ex: Throwable if !ctx.run.enrichedErrorMessage =>
             println(ctx.run.enrichErrorMessage(s"unhandled exception while running $phaseName on $unit"))
             throw ex

compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala

@@ -14,11 +14,13 @@ object AttributePickler:
     pickler: TastyPickler,
     buf: TastyBuffer
   ): Unit =
-    if attributes.scala2StandardLibrary || attributes.explicitNulls then // or any other attribute is set
+    if attributes.nonEmpty then
       pickler.newSection(AttributesSection, buf)
       // Pickle attributes
       if attributes.scala2StandardLibrary then buf.writeNat(TastyFormat.SCALA2STANDARDLIBRARYattr)
       if attributes.explicitNulls then buf.writeNat(TastyFormat.EXPLICITNULLSattr)
+      if attributes.isJava then buf.writeNat(TastyFormat.JAVAattr)
+      if attributes.isOutline then buf.writeNat(TastyFormat.OUTLINEattr)
     end if
 
   end pickleAttributes

compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala

@@ -18,15 +18,21 @@ class AttributeUnpickler(reader: TastyReader):
   lazy val attributes: Attributes = {
     var scala2StandardLibrary = false
     var explicitNulls = false
+    var isJava = false
+    var isOutline = false
     for attributeTag <- attributeTags do
       attributeTag match
         case TastyFormat.SCALA2STANDARDLIBRARYattr => scala2StandardLibrary = true
         case TastyFormat.EXPLICITNULLSattr => explicitNulls = true
+        case TastyFormat.JAVAattr => isJava = true
+        case TastyFormat.OUTLINEattr => isOutline = true
         case attribute =>
           assert(false, "Unexpected attribute value: " + attribute)
     Attributes(
       scala2StandardLibrary,
       explicitNulls,
+      isJava,
+      isOutline,
     )
   }
 

compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala

@@ -3,4 +3,12 @@ package dotty.tools.dotc.core.tasty
 class Attributes(
   val scala2StandardLibrary: Boolean,
   val explicitNulls: Boolean,
-)
+  val isJava: Boolean,
+  val isOutline: Boolean,
+) {
+  def nonEmpty: Boolean = scala2StandardLibrary || explicitNulls || isJava || isOutline
+}
+
+object Attributes {
+  lazy val empty = new Attributes(false, false, false, false)
+}

compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

@@ -36,6 +36,7 @@ object DottyUnpickler {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): CommentUnpickler =
       new CommentUnpickler(reader)
   }
+
   class AttributesSectionUnpickler extends SectionUnpickler[AttributeUnpickler](AttributesSection) {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): AttributeUnpickler =
       new AttributeUnpickler(reader)
@@ -57,14 +58,16 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe
   private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)).get
 
   def tastyAttributes: Attributes =
-    attributeUnpicklerOpt.map(_.attributes).getOrElse(Attributes(false, false))
+    attributeUnpicklerOpt.map(_.attributes).getOrElse(Attributes.empty)
 
   /** Enter all toplevel classes and objects into their scopes
    *  @param roots          a set of SymDenotations that should be overwritten by unpickling
    */
   def enter(roots: Set[SymDenotation])(using Context): Unit =
     treeUnpickler.enter(roots)
 
+  def attributes: Option[Attributes] = attributeUnpicklerOpt.map(_.attributes)
+
   protected def treeSectionUnpickler(
     posUnpicklerOpt: Option[PositionUnpickler],
     commentUnpicklerOpt: Option[CommentUnpickler],

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -24,7 +24,7 @@ import quoted.QuotePatterns
 object TreePickler:
   class StackSizeExceeded(val mdef: tpd.MemberDef) extends Exception
 
-class TreePickler(pickler: TastyPickler) {
+class TreePickler(pickler: TastyPickler, attributes: Attributes) {
   val buf: TreeBuffer = new TreeBuffer
   pickler.newSection(ASTsSection, buf)
   import buf.*
@@ -323,6 +323,11 @@ class TreePickler(pickler: TastyPickler) {
     if (!tree.isEmpty) pickleTree(tree)
   }
 
+  def pickleElidedUnlessEmpty(tree: Tree, tp: Type)(using Context): Unit =
+    if !tree.isEmpty then
+      writeByte(ELIDED)
+      pickleType(tp)
+
   def pickleDef(tag: Int, mdef: MemberDef, tpt: Tree, rhs: Tree = EmptyTree, pickleParams: => Unit = ())(using Context): Unit = {
     val sym = mdef.symbol
 
@@ -338,7 +343,12 @@ class TreePickler(pickler: TastyPickler) {
           case _: Template | _: Hole => pickleTree(tpt)
           case _ if tpt.isType => pickleTpt(tpt)
         }
-        pickleTreeUnlessEmpty(rhs)
+        if attributes.isOutline && sym.isTerm && attributes.isJava then
+          // TODO: if we introduce outline typing for Scala definitions
+          // then we will need to update the check here
+          pickleElidedUnlessEmpty(rhs, tpt.tpe)
+        else
+          pickleTreeUnlessEmpty(rhs)
         pickleModifiers(sym, mdef)
       }
     catch

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -47,6 +47,7 @@ import dotty.tools.tasty.TastyFormat.*
 import scala.annotation.constructorOnly
 import scala.annotation.internal.sharable
 import scala.compiletime.uninitialized
+import dotty.tools.tasty.UnpickleException
 
 /** Unpickler for typed trees
  *  @param reader              the reader from which to unpickle
@@ -107,6 +108,11 @@ class TreeUnpickler(reader: TastyReader,
   private val explicitNulls =
     attributeUnpicklerOpt.exists(_.attributes.explicitNulls)
 
+  private val unpicklingJava =
+    attributeUnpicklerOpt.exists(_.attributes.isJava)
+
+  private val isOutline = attributeUnpicklerOpt.exists(_.attributes.isOutline)
+
   private def registerSym(addr: Addr, sym: Symbol) =
     symAtAddr(addr) = sym
 
@@ -115,6 +121,9 @@ class TreeUnpickler(reader: TastyReader,
    */
   def enter(roots: Set[SymDenotation])(using Context): Unit = {
     this.roots = roots
+    if isOutline && !unpicklingJava then
+      // TODO: overly cautious here, eventually we could enable this with 2-pass compilation.
+      throw UnpickleException("Outline TASTy is not supported for Scala sources.")
     val rdr = new TreeReader(reader).fork
     ownerTree = new OwnerTree(NoAddr, 0, rdr.fork, reader.endAddr)
     if (rdr.isTopLevel)
@@ -612,7 +621,10 @@ class TreeUnpickler(reader: TastyReader,
       val rhsIsEmpty = nothingButMods(end)
       if (!rhsIsEmpty) skipTree()
       val (givenFlags0, annotFns, privateWithin) = readModifiers(end)
-      val givenFlags = if isClass && unpicklingScala2Library then givenFlags0 | Scala2x | Scala2Tasty else givenFlags0
+      val givenFlags =
+        if isClass && unpicklingScala2Library then givenFlags0 | Scala2x | Scala2Tasty
+        else if unpicklingJava then givenFlags0 | JavaDefined
+        else givenFlags0
       pickling.println(i"creating symbol $name at $start with flags ${givenFlags.flagsString}, isAbsType = $isAbsType, $ttag")
       val flags = normalizeFlags(tag, givenFlags, name, isAbsType, rhsIsEmpty)
       def adjustIfModule(completer: LazyType) =
@@ -1045,6 +1057,8 @@ class TreeUnpickler(reader: TastyReader,
       val parentReader = fork
       val parents = readParents(withArgs = false)(using parentCtx)
       val parentTypes = parents.map(_.tpe.dealias)
+      if cls.is(JavaDefined) && parentTypes.exists(_.derivesFrom(defn.JavaAnnotationClass)) then
+        cls.setFlag(JavaAnnotation)
       val self =
         if (nextByte == SELFDEF) {
           readByte()
@@ -1205,7 +1219,12 @@ class TreeUnpickler(reader: TastyReader,
 
       def completeSelect(name: Name, sig: Signature, target: Name): Select =
         val qual = readTree()
-        val denot = accessibleDenot(qual.tpe.widenIfUnstable, name, sig, target)
+        val denot0 = accessibleDenot(qual.tpe.widenIfUnstable, name, sig, target)
+        val denot =
+          if unpicklingJava && name == tpnme.Object && denot0.symbol == defn.ObjectClass then
+            defn.FromJavaObjectType.denot
+          else
+            denot0
         makeSelect(qual, name, denot)
 
       def readQualId(): (untpd.Ident, TypeRef) =
@@ -1224,6 +1243,11 @@ class TreeUnpickler(reader: TastyReader,
           forkAt(readAddr()).readTree()
         case IDENT =>
           untpd.Ident(readName()).withType(readType())
+        case ELIDED =>
+          if !isOutline then
+            report.error(
+              s"Illegal elided tree in unpickler without ${attributeTagToString(OUTLINEattr)}, ${ctx.source}")
+          untpd.Ident(nme.WILDCARD).withType(readType())
         case IDENTtpt =>
           untpd.Ident(readName().toTypeName).withType(readType())
         case SELECT =>

compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala

@@ -46,9 +46,14 @@ class ReadTasty extends Phase {
             case unpickler: tasty.DottyUnpickler =>
               if (cls.rootTree.isEmpty) None
               else {
-                val unit = CompilationUnit(cls, cls.rootTree, forceTrees = true)
-                unit.pickled += (cls -> (() => unpickler.unpickler.bytes))
-                Some(unit)
+                if unpickler.attributes.exists(_.isJava) && !ctx.settings.YjavaTasty.value then
+                  // filter out Java compilation units if -Yjava-tasty is not set
+                  None
+                else
+                  val unit = CompilationUnit(cls, cls.rootTree, forceTrees = true)
+                  unit.pickled += (cls -> (() => unpickler.unpickler.bytes))
+                  unit.tastyAttributes = unpickler.attributes
+                  Some(unit)
               }
             case tree: Tree[?] =>
               // TODO handle correctly this case correctly to get the tree or avoid it completely.

compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala

@@ -136,7 +136,7 @@ object JavaParsers {
       ValDef(name, tpt, EmptyTree).withMods(Modifiers(Flags.JavaDefined | Flags.Param))
 
     def makeConstructor(formals: List[Tree], tparams: List[TypeDef], flags: FlagSet = Flags.JavaDefined): DefDef = {
-      val vparams = formals.zipWithIndex.map { case (p, i) => makeSyntheticParam(i + 1, p) }
+      val vparams = formals.zipWithIndex.map { case (p, i) => makeSyntheticParam(i + 1, p).withMods(Modifiers(flags)) }
       DefDef(nme.CONSTRUCTOR, joinParams(tparams, List(vparams)), TypeTree(), EmptyTree).withMods(Modifiers(flags))
     }
 

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -97,6 +97,9 @@ object Parsers {
   private val InCond: Region => Region = Scanners.InParens(LPAREN, _)
   private val InFor : Region => Region = Scanners.InBraces(_)
 
+  def unimplementedExpr(using Context): Select =
+    Select(scalaDot(nme.Predef), nme.???)
+
   abstract class ParserCommon(val source: SourceFile)(using Context) {
 
     val in: ScannerCommon
@@ -164,9 +167,6 @@ object Parsers {
      */
     def syntaxError(msg: Message, span: Span): Unit =
       report.error(msg, source.atSpan(span))
-
-    def unimplementedExpr(using Context): Select =
-      Select(scalaDot(nme.Predef), nme.???)
   }
 
   trait OutlineParserCommon extends ParserCommon {

compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala

@@ -9,7 +9,7 @@ import dotty.tools.dotc.core.Flags.*
 import dotty.tools.dotc.core.Mode
 import dotty.tools.dotc.core.Symbols.*
 import dotty.tools.dotc.core.Types.*
-import dotty.tools.dotc.core.tasty.{ PositionPickler, TastyPickler, TastyPrinter, TreePickler }
+import dotty.tools.dotc.core.tasty.{ PositionPickler, TastyPickler, TastyPrinter, TreePickler, Attributes }
 import dotty.tools.dotc.core.tasty.DottyUnpickler
 import dotty.tools.dotc.core.tasty.TreeUnpickler.UnpickleMode
 import dotty.tools.dotc.report
@@ -217,7 +217,7 @@ object PickledQuotes {
   private def pickle(tree: Tree)(using Context): Array[Byte] = {
     quotePickling.println(i"**** pickling quote of\n$tree")
     val pickler = new TastyPickler(defn.RootClass)
-    val treePkl = new TreePickler(pickler)
+    val treePkl = new TreePickler(pickler, Attributes.empty)
     treePkl.pickle(tree :: Nil)
     treePkl.compactify()
     if tree.span.exists then

compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -55,6 +55,9 @@ class ExtractAPI extends Phase {
   // Check no needed. Does not transform trees
   override def isCheckable: Boolean = false
 
+  // when `-Yjava-tasty` is set we actually want to run this phase on Java sources
+  override def skipIfJava(using Context): Boolean = false
+
   // SuperAccessors need to be part of the API (see the scripted test
   // `trait-super` for an example where this matters), this is only the case
   // after `PostTyper` (unlike `ExtractDependencies`, the simplication to trees

compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala

@@ -64,6 +64,9 @@ class ExtractDependencies extends Phase {
   // Check no needed. Does not transform trees
   override def isCheckable: Boolean = false
 
+  // when `-Yjava-tasty` is set we actually want to run this phase on Java sources
+  override def skipIfJava(using Context): Boolean = false
+
   // This phase should be run directly after `Frontend`, if it is run after
   // `PostTyper`, some dependencies will be lost because trees get simplified.
   // See the scripted test `constants` for an example where this matters.