Better handling for exponential-sized types in misc places

Mostly to fix ui/issues/issue-37311-type-length-limit/issue-37311.rs.

Most parts of the compiler can handle deeply nested types with a lot
of duplicates just fine, but some parts still attempt to naively
traverse type tree.

Before such problems were caught by type length limit check,
but now these places will have to be changed to handle
duplicated types gracefully.

Author: VFLashM

compiler/rustc_infer/src/infer/combine.rs

@@ -31,6 +31,7 @@ use super::unify_key::replace_if_possible;
 use super::unify_key::{ConstVarValue, ConstVariableValue};
 use super::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use super::{InferCtxt, MiscVariable, TypeTrace};
+use rustc_data_structures::fx::FxHashMap;
 
 use crate::traits::{Obligation, PredicateObligations};
 
@@ -379,6 +380,7 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
             needs_wf: false,
             root_ty: ty,
             param_env: self.param_env,
+            cache: FxHashMap::default(),
         };
 
         let ty = match generalize.relate(ty, ty) {
@@ -438,6 +440,8 @@ struct Generalizer<'cx, 'tcx> {
     root_ty: Ty<'tcx>,
 
     param_env: ty::ParamEnv<'tcx>,
+
+    cache: FxHashMap<(Ty<'tcx>, Ty<'tcx>), RelateResult<'tcx, Ty<'tcx>>>,
 }
 
 /// Result from a generalization operation. This includes
@@ -535,13 +539,17 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
     fn tys(&mut self, t: Ty<'tcx>, t2: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> {
         assert_eq!(t, t2); // we are abusing TypeRelation here; both LHS and RHS ought to be ==
 
+        let cache_key = (t, t2);
+        if let Some(result) = self.cache.get(&cache_key) {
+            return result.clone();
+        }
         debug!("generalize: t={:?}", t);
 
         // Check to see whether the type we are generalizing references
         // any other type variable related to `vid` via
         // subtyping. This is basically our "occurs check", preventing
         // us from creating infinitely sized types.
-        match *t.kind() {
+        let result = match *t.kind() {
             ty::Infer(ty::TyVar(vid)) => {
                 let vid = self.infcx.inner.borrow_mut().type_variables().root_var(vid);
                 let sub_vid = self.infcx.inner.borrow_mut().type_variables().sub_root_var(vid);
@@ -598,7 +606,10 @@ impl TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
                 Ok(t)
             }
             _ => relate::super_relate_tys(self, t, t),
-        }
+        };
+
+        self.cache.insert(cache_key, result.clone());
+        return result;
     }
 
     fn regions(

compiler/rustc_middle/src/ty/print/mod.rs

@@ -4,6 +4,7 @@ use crate::ty::{self, DefIdTree, Ty, TyCtxt};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::def_id::{CrateNum, DefId};
 use rustc_hir::definitions::{DefPathData, DisambiguatedDefPathData};
+use rustc_middle::ty::walk::MiniSet;
 
 // `pretty` is a separate module only for organization.
 mod pretty;
@@ -263,21 +264,33 @@ pub trait Printer<'tcx>: Sized {
 /// function tries to find a "characteristic `DefId`" for a
 /// type. It's just a heuristic so it makes some questionable
 /// decisions and we may want to adjust it later.
-pub fn characteristic_def_id_of_type(ty: Ty<'_>) -> Option<DefId> {
+///
+/// Visited set is needed in to avoid full iteration over
+/// deeply nested tuples that have no DefId.
+fn characteristic_def_id_of_type_cached<'a>(
+    ty: Ty<'a>,
+    visited: &mut MiniSet<Ty<'a>>,
+) -> Option<DefId> {
     match *ty.kind() {
         ty::Adt(adt_def, _) => Some(adt_def.did),
 
         ty::Dynamic(data, ..) => data.principal_def_id(),
 
-        ty::Array(subty, _) | ty::Slice(subty) => characteristic_def_id_of_type(subty),
+        ty::Array(subty, _) | ty::Slice(subty) => {
+            characteristic_def_id_of_type_cached(subty, visited)
+        }
 
-        ty::RawPtr(mt) => characteristic_def_id_of_type(mt.ty),
+        ty::RawPtr(mt) => characteristic_def_id_of_type_cached(mt.ty, visited),
 
-        ty::Ref(_, ty, _) => characteristic_def_id_of_type(ty),
+        ty::Ref(_, ty, _) => characteristic_def_id_of_type_cached(ty, visited),
 
-        ty::Tuple(ref tys) => {
-            tys.iter().find_map(|ty| characteristic_def_id_of_type(ty.expect_ty()))
-        }
+        ty::Tuple(ref tys) => tys.iter().find_map(|ty| {
+            let ty = ty.expect_ty();
+            if visited.insert(ty) {
+                return characteristic_def_id_of_type_cached(ty, visited);
+            }
+            return None;
+        }),
 
         ty::FnDef(def_id, _)
         | ty::Closure(def_id, _)
@@ -302,6 +315,9 @@ pub fn characteristic_def_id_of_type(ty: Ty<'_>) -> Option<DefId> {
         | ty::Float(_) => None,
     }
 }
+pub fn characteristic_def_id_of_type(ty: Ty<'_>) -> Option<DefId> {
+    characteristic_def_id_of_type_cached(ty, &mut MiniSet::new())
+}
 
 impl<'tcx, P: Printer<'tcx>> Print<'tcx, P> for ty::RegionKind {
     type Output = P::Region;

compiler/rustc_middle/src/ty/print/pretty.rs

@@ -1264,6 +1264,7 @@ pub struct FmtPrinterData<'a, 'tcx, F> {
     used_region_names: FxHashSet<Symbol>,
     region_index: usize,
     binder_depth: usize,
+    printed_type_count: usize,
 
     pub region_highlight_mode: RegionHighlightMode,
 
@@ -1294,6 +1295,7 @@ impl<F> FmtPrinter<'a, 'tcx, F> {
             used_region_names: Default::default(),
             region_index: 0,
             binder_depth: 0,
+            printed_type_count: 0,
             region_highlight_mode: RegionHighlightMode::default(),
             name_resolver: None,
         }))
@@ -1411,8 +1413,14 @@ impl<F: fmt::Write> Printer<'tcx> for FmtPrinter<'_, 'tcx, F> {
         self.pretty_print_region(region)
     }
 
-    fn print_type(self, ty: Ty<'tcx>) -> Result<Self::Type, Self::Error> {
-        self.pretty_print_type(ty)
+    fn print_type(mut self, ty: Ty<'tcx>) -> Result<Self::Type, Self::Error> {
+        if self.tcx.sess.type_length_limit().value_within_limit(self.printed_type_count) {
+            self.printed_type_count += 1;
+            self.pretty_print_type(ty)
+        } else {
+            write!(self, "...")?;
+            Ok(self)
+        }
     }
 
     fn print_dyn_existential(

compiler/rustc_mir/src/monomorphize/collector.rs

@@ -418,6 +418,29 @@ fn record_accesses<'a, 'tcx: 'a>(
     inlining_map.lock_mut().record_accesses(caller, &accesses);
 }
 
+// Shrinks string by keeping prefix and suffix of given sizes.
+fn shrink(s: String, before: usize, after: usize) -> String {
+    // An iterator of all byte positions including the end of the string.
+    let positions = || s.char_indices().map(|(i, _)| i).chain(iter::once(s.len()));
+
+    let shrunk = format!(
+        "{before}...{after}",
+        before = &s[..positions().nth(before).unwrap_or(s.len())],
+        after = &s[positions().rev().nth(after).unwrap_or(0)..],
+    );
+
+    // Only use the shrunk version if it's really shorter.
+    // This also avoids the case where before and after slices overlap.
+    if shrunk.len() < s.len() { shrunk } else { s }
+}
+
+// Format instance name that is already known to be too long for rustc.
+// Show only the first and last 32 characters to avoid blasting
+// the user's terminal with thousands of lines of type-name.
+fn shrunk_instance_name(instance: &Instance<'tcx>) -> String {
+    shrink(instance.to_string(), 32, 32)
+}
+
 fn check_recursion_limit<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
@@ -440,7 +463,10 @@ fn check_recursion_limit<'tcx>(
     // more than the recursion limit is assumed to be causing an
     // infinite expansion.
     if !tcx.sess.recursion_limit().value_within_limit(adjusted_recursion_depth) {
-        let error = format!("reached the recursion limit while instantiating `{}`", instance);
+        let error = format!(
+            "reached the recursion limit while instantiating `{}`",
+            shrunk_instance_name(&instance),
+        );
         let mut err = tcx.sess.struct_span_fatal(span, &error);
         err.span_note(
             tcx.def_span(def_id),
@@ -474,26 +500,9 @@ fn check_type_length_limit<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
     //
     // Bail out in these cases to avoid that bad user experience.
     if !tcx.sess.type_length_limit().value_within_limit(type_length) {
-        // The instance name is already known to be too long for rustc.
-        // Show only the first and last 32 characters to avoid blasting
-        // the user's terminal with thousands of lines of type-name.
-        let shrink = |s: String, before: usize, after: usize| {
-            // An iterator of all byte positions including the end of the string.
-            let positions = || s.char_indices().map(|(i, _)| i).chain(iter::once(s.len()));
-
-            let shrunk = format!(
-                "{before}...{after}",
-                before = &s[..positions().nth(before).unwrap_or(s.len())],
-                after = &s[positions().rev().nth(after).unwrap_or(0)..],
-            );
-
-            // Only use the shrunk version if it's really shorter.
-            // This also avoids the case where before and after slices overlap.
-            if shrunk.len() < s.len() { shrunk } else { s }
-        };
         let msg = format!(
             "reached the type-length limit while instantiating `{}`",
-            shrink(instance.to_string(), 32, 32)
+            shrunk_instance_name(&instance),
         );
         let mut diag = tcx.sess.struct_span_fatal(tcx.def_span(instance.def_id()), &msg);
         diag.note(&format!(

src/test/ui/infinite/infinite-instantiation.stderr

@@ -1,4 +1,4 @@
-error: reached the recursion limit while instantiating `function::<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<usize>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`
+error: reached the recursion limit while instantiating `function::<Option<Option<Option<...>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`
   --> $DIR/infinite-instantiation.rs:21:9
    |
 LL |         function(counter - 1, t.to_option());

src/test/ui/issues/issue-37311-type-length-limit/issue-37311.rs

@@ -12,8 +12,8 @@ trait Foo {
 
 impl<T> Foo for T {
     #[allow(unconditional_recursion)]
-    fn recurse(&self) { //~ ERROR reached the type-length limit
-        (self, self).recurse();
+    fn recurse(&self) {
+        (self, self).recurse(); //~ ERROR reached the recursion limit
     }
 }
 

src/test/ui/issues/issue-37311-type-length-limit/issue-37311.stderr

@@ -1,10 +1,14 @@
-error: reached the type-length limit while instantiating `<(&(&(&(&(&(&(&(&(&(&(&(&(&(&(&(...))))))))))))))) as Foo>::recurse`
+error: reached the recursion limit while instantiating `<(&(&(&(&(&(&(&(&(&(&(&(&(&(&(&(.....), ...), ...) as Foo>::recurse`
+  --> $DIR/issue-37311.rs:16:9
+   |
+LL |         (self, self).recurse();
+   |         ^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: `<T as Foo>::recurse` defined here
   --> $DIR/issue-37311.rs:15:5
    |
 LL |     fn recurse(&self) {
    |     ^^^^^^^^^^^^^^^^^
-   |
-   = note: consider adding a `#![type_length_limit="2097149"]` attribute to your crate
 
 error: aborting due to previous error
 

src/test/ui/issues/issue-67552.stderr

@@ -1,4 +1,4 @@
-error: reached the recursion limit while instantiating `rec::<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Empty>`
+error: reached the recursion limit while instantiating `rec::<&mut &mut &mut &mut &mut &... &mut &mut &mut &mut &mut Empty>`
   --> $DIR/issue-67552.rs:27:9
    |
 LL |         rec(identity(&mut it))

src/test/ui/issues/issue-8727.stderr

@@ -9,7 +9,7 @@ LL |     generic::<Option<T>>();
    = note: `#[warn(unconditional_recursion)]` on by default
    = help: a `loop` may express intention better if this is on purpose
 
-error: reached the recursion limit while instantiating `generic::<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<Option<i32>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`
+error: reached the recursion limit while instantiating `generic::<Option<Option<Option<O...>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`
   --> $DIR/issue-8727.rs:7:5
    |
 LL |     generic::<Option<T>>();

src/test/ui/recursion/issue-38591-non-regular-dropck-recursion.stderr

@@ -1,4 +1,4 @@
-error: reached the recursion limit while instantiating `drop_in_place::<S<fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(u32))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))>> - shim(Some(S<fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(fn(u32))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))>))`
+error: reached the recursion limit while instantiating `drop_in_place::<S<fn(fn(fn(fn(fn...)))))))))))))))))))))))))))))>))`
   --> $SRC_DIR/core/src/ptr/mod.rs:LL:COL
    |
 LL | / pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {

src/test/ui/recursion/recursion.stderr

@@ -1,4 +1,4 @@
-error: reached the recursion limit while instantiating `test::<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Cons<Nil>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`
+error: reached the recursion limit while instantiating `test::<Cons<Cons<Cons<Cons<Cons<...>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`
   --> $DIR/recursion.rs:17:11
    |
 LL |     _ => {test (n-1, i+1, Cons {head:2*i+1, tail:first}, Cons{head:i*i, tail:second})}

src/test/ui/type_length_limit.stderr

@@ -1,4 +1,4 @@
-error: reached the type-length limit while instantiating `std::mem::drop::<Option<((((((G,... G), (G, G, G), (G, G, G))))))>>`
+error: reached the type-length limit while instantiating `std::mem::drop::<Option<((((...,....., ...), ..., ...), ..., ...)>>`
   --> $SRC_DIR/core/src/mem/mod.rs:LL:COL
    |
 LL | pub fn drop<T>(_x: T) {}