Fix 10256: Avoid crashing in LazyRef.ref when checking for cycles

compiler/src/dotty/tools/dotc/core/Mode.scala

@@ -41,6 +41,7 @@ object Mode {
    */
   val TypevarsMissContext: Mode = newMode(4, "TypevarsMissContext")
 
+  /** Are we looking for cyclic references? */
   val CheckCyclic: Mode = newMode(5, "CheckCyclic")
 
   /** We are in a pattern alternative */

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -2680,7 +2680,7 @@ object Types {
     }
   }
 
-  case class LazyRef(private var refFn: Context ?=> Type, reportCycles: Boolean = false) extends UncachedProxyType with ValueType {
+  case class LazyRef(private var refFn: Context ?=> Type) extends UncachedProxyType with ValueType {
     private var myRef: Type = null
     private var computed = false
 
@@ -2689,7 +2689,7 @@ object Types {
         if myRef == null then
           // if errors were reported previously handle this by throwing a CyclicReference
           // instead of crashing immediately. A test case is neg/i6057.scala.
-          assert(reportCycles || ctx.reporter.errorsReported)
+          assert(ctx.mode.is(Mode.CheckCyclic) || ctx.reporter.errorsReported)
           throw CyclicReference(NoDenotation)
       else
         computed = true
@@ -4515,7 +4515,9 @@ object Types {
             else defn.AnyType         // dummy type in case of errors
           def refineSelfType(selfType: Type) =
             RefinedType(selfType, sym.name,
-              TypeAlias(LazyRef(force, reportCycles = true)))
+              TypeAlias(
+                withMode(Mode.CheckCyclic)(
+                  LazyRef(force))))
           cinfo.selfInfo match
             case self: Type =>
               cinfo.derivedClassInfo(

compiler/src/dotty/tools/dotc/typer/Checking.scala

@@ -406,21 +406,19 @@ object Checking {
       for (parent <- parents; mbr <- parent.abstractTypeMembers if qualifies(mbr.symbol))
       yield mbr.name.asTypeName
 
-    for (name <- abstractTypeNames)
-      try {
-        val mbr = joint.member(name)
-        mbr.info match {
-          case bounds: TypeBounds =>
-            !checkNonCyclic(mbr.symbol, bounds, reportErrors = true).isError
-          case _ =>
-            true
-        }
-      }
-      catch {
-        case ex: RecursionOverflow =>
+    withMode(Mode.CheckCyclic) {
+      for name <- abstractTypeNames do
+        try
+          val mbr = joint.member(name)
+          mbr.info match
+            case bounds: TypeBounds =>
+              !checkNonCyclic(mbr.symbol, bounds, reportErrors = true).isError
+            case _ =>
+              true
+        catch case _: RecursionOverflow | _: CyclicReference =>
           report.error(em"cyclic reference involving type $name", pos)
           false
-      }
+    }
   }
 
   /** Check that symbol's definition is well-formed. */

tests/neg/i10256.scala

@@ -0,0 +1,7 @@
+trait Foo[T <: Foo[T]] {
+  type I <: Foo[I]
+}
+
+trait Bar[T <: Foo[T]] extends Foo[T] { // error: cyclic
+  self: T =>
+}