Fix spurious subtype check pruning when both sides have unions

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -1479,9 +1479,27 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
 
     /** Like tp1 <:< tp2, but returns false immediately if we know that
      *  the case was covered previously during subtyping.
+     *
+     *  A type has been covered previously in subtype checking if it
+     *  is some combination of TypeRefs that point to classes, where the
+     *  combiners are AppliedTypes, RefinedTypes, RecTypes, And/Or-Types or AnnotatedTypes.
+     *
+     *  The exception is that if both sides contain OrTypes, the check hasn't been covered.
+     *  See #17465.
      */
     def isNewSubType(tp1: Type): Boolean =
-      if (isCovered(tp1) && isCovered(tp2))
+      def isCovered(tp: Type): CoveredStatus =
+        tp.dealiasKeepRefiningAnnots.stripTypeVar match
+          case tp: TypeRef if tp.symbol.isClass && tp.symbol != NothingClass && tp.symbol != NullClass => CoveredStatus.Covered
+          case tp: AppliedType => isCovered(tp.tycon)
+          case tp: RefinedOrRecType => isCovered(tp.parent)
+          case tp: AndType => isCovered(tp.tp1) combinedWith isCovered(tp.tp2)
+          case tp: OrType  => isCovered(tp.tp1) combinedWith isCovered(tp.tp2)
+          case _ => CoveredStatus.Uncovered
+
+      val covered1 = isCovered(tp1)
+      val covered2 = isCovered(tp2)
+      if CoveredStatus.bothCovered(covered1, covered2) && !CoveredStatus.bothHaveOr(covered1, covered2) then
         //println(s"useless subtype: $tp1 <:< $tp2")
         false
       else isSubType(tp1, tp2, approx.addLow)
@@ -2091,19 +2109,6 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
             tp1.parent.asInstanceOf[RefinedType],
             tp2.parent.asInstanceOf[RefinedType], limit))
 
-  /** A type has been covered previously in subtype checking if it
-   *  is some combination of TypeRefs that point to classes, where the
-   *  combiners are AppliedTypes, RefinedTypes, RecTypes, And/Or-Types or AnnotatedTypes.
-   */
-  private def isCovered(tp: Type): Boolean = tp.dealiasKeepRefiningAnnots.stripTypeVar match {
-    case tp: TypeRef => tp.symbol.isClass && tp.symbol != NothingClass && tp.symbol != NullClass
-    case tp: AppliedType => isCovered(tp.tycon)
-    case tp: RefinedOrRecType => isCovered(tp.parent)
-    case tp: AndType => isCovered(tp.tp1) && isCovered(tp.tp2)
-    case tp: OrType  => isCovered(tp.tp1) && isCovered(tp.tp2)
-    case _ => false
-  }
-
   /** Defer constraining type variables when compared against prototypes */
   def isMatchedByProto(proto: ProtoType, tp: Type): Boolean = tp.stripTypeVar match {
     case tp: TypeParamRef if constraint contains tp => true
@@ -2991,6 +2996,31 @@ object TypeComparer {
   end ApproxState
   type ApproxState = ApproxState.Repr
 
+  /** Result of `isCovered` check. */
+  object CoveredStatus:
+    opaque type Repr = Int
+
+    private inline val IsCovered = 2
+    private inline val NotHasOr = 1
+
+    /** The type is not covered. */
+    val Uncovered: Repr = 1
+
+    /** The type is covered and contains OrTypes. */
+    val CoveredWithOr: Repr = 2
+
+    /** The type is covered and free from OrTypes. */
+    val Covered: Repr = 3
+
+    object Repr:
+      extension (s: Repr)
+        inline def combinedWith(that: Repr): Repr = s min that
+
+    inline def bothHaveOr(s1: Repr, s2: Repr): Boolean = ~(s1 | s2 & NotHasOr) != 0
+    inline def bothCovered(s1: Repr, s2: Repr): Boolean = (s1 & s2 & IsCovered) != 0
+  end CoveredStatus
+  type CoveredStatus = CoveredStatus.Repr
+
   def topLevelSubType(tp1: Type, tp2: Type)(using Context): Boolean =
     comparing(_.topLevelSubType(tp1, tp2))
 

tests/pos/i17465.scala

@@ -0,0 +1,45 @@
+def test1[A, B]: Unit = {
+  def f[T](x: T{ def *(y: Int): T }): T = ???
+  def test = f[scala.collection.StringOps | String]("Hello")
+  locally:
+    val test1 : (scala.collection.StringOps | String) { def *(y: Int): (scala.collection.StringOps | String) } = ???
+    val test2 : (scala.collection.StringOps | String) { def *(y: Int): (scala.collection.StringOps | String) } = test1
+
+  locally:
+    val test1 : (Int | String) { def foo(x: Int): Int } = ???
+    val test2 : (Int | String) { def foo(x: Int): Int } = test1
+
+  locally:
+    val test1 : ((Int | String) & Any) { def foo(): Int } = ???
+    val test2 : ((Int | String) & Any) { def foo(): Int } = test1
+
+  locally:
+    val test1 : Int { def foo(): Int } = ???
+    val test2 : Int { def foo(): Int } = test1
+
+  locally:
+    val test1 : (Int | String) { def foo(): Int } = ???
+    val test2 : (Int | String) & Any = test1
+
+  locally:
+    val test1 : (Int | B) { def *(y: Int): Int } = ???
+    val test2 : (Int | B) { def *(y: Int): Int } = test1
+
+  locally:
+    val test1 : (Int | String) = ???
+    val test2 : (Int | String) = test1
+
+  type Foo = Int | String
+  locally:
+    val test1 : Foo { type T = Int } = ???
+    val test2 : (Int | String) = test1
+}
+
+def test2: Unit = {
+  import reflect.Selectable.reflectiveSelectable
+
+  trait A[T](x: T{ def *(y: Int): T }):
+    def f: T = x * 2
+
+  class B extends A("Hello")
+}