Denotation#signature: Compute the Java-ness of the prefix more precisely

Instead of relying on `prefix.classSymbol.is(JavaDefined)`, we now
explicitly check if every part of the prefix is Java-defined.

Being precise like this doesn't seem necessary in practice (we support
having multiple denotations with the same signature since the
introduction of SELECTin in tasty), but it's important that we stay
consistent across versions for tasty compatibility so it's better to be
explicit than rely on the implementation details of `classSymbol`.

Author: smarter

compiler/src/dotty/tools/dotc/core/Denotations.scala

@@ -586,11 +586,19 @@ object Denotations {
       if (isType) Signature.NotAMethod // don't force info if this is a type SymDenotation
       else info match {
         case info: MethodOrPoly =>
-          val isJava =
-            if prefix eq NoPrefix then
-              symbol.is(JavaDefined)
-            else
-              prefix.classSymbol.is(JavaDefined)
+          def isJavaPrefix(tp: Type): Boolean = tp match
+            case tp: ThisType  => tp.cls.is(JavaDefined)
+            case tp: ClassInfo => tp.cls.is(JavaDefined)
+            case tp: AndOrType => isJavaPrefix(tp.tp1) && isJavaPrefix(tp.tp2)
+            case tp: TypeRef if tp.symbol.isClass => tp.symbol.is(JavaDefined)
+            case tp: TypeProxy => isJavaPrefix(tp.underlying)
+            case _: JavaArrayType => true
+            case _ => false
+
+          // Use the Java signature of this method if all parts of its prefix
+          // are Java-defined or if it doesn't have a prefix (i.e., it is a
+          // SymDenotation) and is defined in a Java class.
+          val isJava = if prefix eq NoPrefix then symbol.is(JavaDefined) else isJavaPrefix(prefix)
           try info.signature(isJava)
           catch { // !!! DEBUG
             case scala.util.control.NonFatal(ex) =>

compiler/test/dotty/tools/SignatureTest.scala

@@ -10,6 +10,8 @@ import dotc.core.Contexts._
 import dotc.core.Phases._
 import dotc.core.Types._
 import dotc.core.Symbols._
+import dotc.core.Signature
+import dotc.core.TypeErasure.sigName
 
 import java.io.File
 import java.nio.file._
@@ -38,3 +40,40 @@ class SignatureTest:
              |${ref.denot.signature}""".stripMargin)
       }
     }
+
+  @Test def prefixDependentJavaness: Unit =
+    inCompilerContext(TestConfiguration.basicClasspath, separateRun = false,
+    """trait MyComparable[T]  { def compareTo(x: Array[T]): Int }
+      |trait MyComparable2[T] { def compareTo(x: Array[T]): Int }
+      |class Foo[T] {
+      |  val jj: Comparable[Array[T]] & Serializable = ???
+      |  val js: Comparable[Array[T]] & MyComparable[T] = ???
+      |  val sj: MyComparable[T] & Comparable[Array[T]] = ???
+      |  val ss: MyComparable[T] & MyComparable2[T] = ???
+      |}""".stripMargin) {
+
+      atPhase(picklerPhase) {
+        val cls = requiredClass("Foo")
+
+        def checkSignature(prefix: String, expectedSig: Signature) =
+          val meth = cls.requiredMethodRef(prefix)
+          val denot = meth.select("compareTo".toTermName)
+          val actualSig = denot.signature
+          assert(actualSig == expectedSig,
+            s"""Incorrect signature found for $denot with prefix $meth
+               |Expected: $expectedSig
+               |Actual  : $actualSig""")
+
+        val objSigName = defn.ObjectClass.fullName.asTypeName
+        val resSigName = defn.IntClass.fullName.asTypeName
+        val javaSig = Signature(List(objSigName ++ "[]"), resSigName)
+        val scalaSig = Signature(List(objSigName), resSigName)
+
+        // If all parts of the prefix are Java-defined, we use the Java
+        // signature, otherwise we use the Scala signature.
+        checkSignature("jj", javaSig)
+        checkSignature("js", scalaSig)
+        checkSignature("sj", scalaSig)
+        checkSignature("ss", scalaSig)
+      }
+    }

tests/pos/i8615b.scala

@@ -5,3 +5,14 @@ class ArrayOrdering[N] extends Comparable[Array[N]] {
 class ArrayIntOrdering extends Comparable[Array[Int]] {
   override def compareTo(x: Array[Int]): Int = 0
 }
+
+class ArrayOrdering2[N] extends Comparable[Array[N]] {
+  self: Serializable =>
+  override def compareTo(x: Array[N]): Int = 0
+}
+
+class ArrayIntOrdering2 extends Comparable[Array[Int]] {
+  self: Serializable =>
+
+  override def compareTo(x: Array[Int]): Int = 0
+}