Unrolled build for #137725

Rollup merge of #137725 - oli-obk:i-want-to-move-it-move-it, r=compiler-errors,traviscross

Add `iter` macro

See related discussion in https://rust-lang.zulipchat.com/#narrow/channel/481571-t-lang.2Fgen/topic/iter!.20macro/near/500784563

very little error case testing so far, but the success path works.

There is also no `IterFn` trait yet, as T-lang didn't consider it something urgently needed I think we can implement it in follow-up PRs.

r? lang for the tests, `@compiler-errors` for the impl

Author: rust-timer

compiler/rustc_ast_lowering/src/expr.rs

@@ -1,4 +1,3 @@
-use std::assert_matches::assert_matches;
 use std::ops::ControlFlow;
 use std::sync::Arc;
 
@@ -1199,11 +1198,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let closure_def_id = self.local_def_id(closure_id);
         let (binder_clause, generic_params) = self.lower_closure_binder(binder);
 
-        assert_matches!(
-            coroutine_kind,
-            CoroutineKind::Async { .. },
-            "only async closures are supported currently"
-        );
+        let coroutine_desugaring = match coroutine_kind {
+            CoroutineKind::Async { .. } => hir::CoroutineDesugaring::Async,
+            CoroutineKind::Gen { .. } => hir::CoroutineDesugaring::Gen,
+            CoroutineKind::AsyncGen { span, .. } => {
+                span_bug!(span, "only async closures and `iter!` closures are supported currently")
+            }
+        };
 
         let body = self.with_new_scopes(fn_decl_span, |this| {
             let inner_decl =
@@ -1247,7 +1248,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             // Lower this as a `CoroutineClosure`. That will ensure that HIR typeck
             // knows that a `FnDecl` output type like `-> &str` actually means
             // "coroutine that returns &str", rather than directly returning a `&str`.
-            kind: hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Async),
+            kind: hir::ClosureKind::CoroutineClosure(coroutine_desugaring),
             constness: hir::Constness::NotConst,
         });
         hir::ExprKind::Closure(c)

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -477,11 +477,12 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
         for span in spans {
             if (!visitor.features.coroutines() && !span.allows_unstable(sym::coroutines))
                 && (!visitor.features.gen_blocks() && !span.allows_unstable(sym::gen_blocks))
+                && (!visitor.features.yield_expr() && !span.allows_unstable(sym::yield_expr))
             {
                 #[allow(rustc::untranslatable_diagnostic)]
-                // Don't know which of the two features to include in the
-                // error message, so I am arbitrarily picking one.
-                feature_err(&visitor.sess, sym::coroutines, *span, "yield syntax is experimental")
+                // Emit yield_expr as the error, since that will be sufficient. You can think of it
+                // as coroutines and gen_blocks imply yield_expr.
+                feature_err(&visitor.sess, sym::yield_expr, *span, "yield syntax is experimental")
                     .emit();
             }
         }

compiler/rustc_borrowck/src/type_check/input_output.rs

@@ -52,7 +52,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             assert_matches!(
                 self.tcx().coroutine_kind(self.tcx().coroutine_for_closure(mir_def_id)),
                 Some(hir::CoroutineKind::Desugared(
-                    hir::CoroutineDesugaring::Async,
+                    hir::CoroutineDesugaring::Async | hir::CoroutineDesugaring::Gen,
                     hir::CoroutineSource::Closure
                 )),
                 "this needs to be modified if we're lowering non-async closures"

compiler/rustc_builtin_macros/src/iter.rs

@@ -0,0 +1,53 @@
+use rustc_ast::ptr::P;
+use rustc_ast::tokenstream::TokenStream;
+use rustc_ast::{CoroutineKind, DUMMY_NODE_ID, Expr, ast, token};
+use rustc_errors::PResult;
+use rustc_expand::base::{self, DummyResult, ExpandResult, ExtCtxt, MacroExpanderResult};
+use rustc_span::Span;
+
+pub(crate) fn expand<'cx>(
+    cx: &'cx mut ExtCtxt<'_>,
+    sp: Span,
+    tts: TokenStream,
+) -> MacroExpanderResult<'cx> {
+    let closure = match parse_closure(cx, sp, tts) {
+        Ok(parsed) => parsed,
+        Err(err) => {
+            return ExpandResult::Ready(DummyResult::any(sp, err.emit()));
+        }
+    };
+
+    ExpandResult::Ready(base::MacEager::expr(closure))
+}
+
+fn parse_closure<'a>(
+    cx: &mut ExtCtxt<'a>,
+    span: Span,
+    stream: TokenStream,
+) -> PResult<'a, P<Expr>> {
+    let mut closure_parser = cx.new_parser_from_tts(stream);
+
+    let coroutine_kind = Some(CoroutineKind::Gen {
+        span,
+        closure_id: DUMMY_NODE_ID,
+        return_impl_trait_id: DUMMY_NODE_ID,
+    });
+
+    let mut closure = closure_parser.parse_expr()?;
+    match &mut closure.kind {
+        ast::ExprKind::Closure(c) => {
+            if let Some(kind) = c.coroutine_kind {
+                cx.dcx().span_err(kind.span(), "only plain closures allowed in `iter!`");
+            }
+            c.coroutine_kind = coroutine_kind;
+            if closure_parser.token != token::Eof {
+                closure_parser.unexpected()?;
+            }
+            Ok(closure)
+        }
+        _ => {
+            cx.dcx().span_err(closure.span, "`iter!` body must be a closure");
+            Err(closure_parser.unexpected().unwrap_err())
+        }
+    }
+}

compiler/rustc_builtin_macros/src/lib.rs

@@ -47,6 +47,7 @@ mod errors;
 mod format;
 mod format_foreign;
 mod global_allocator;
+mod iter;
 mod log_syntax;
 mod pattern_type;
 mod source_util;
@@ -95,6 +96,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
         include: source_util::expand_include,
         include_bytes: source_util::expand_include_bytes,
         include_str: source_util::expand_include_str,
+        iter: iter::expand,
         line: source_util::expand_line,
         log_syntax: log_syntax::expand_log_syntax,
         module_path: source_util::expand_mod,

compiler/rustc_const_eval/src/check_consts/check.rs

@@ -589,12 +589,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
             Rvalue::Aggregate(kind, ..) => {
                 if let AggregateKind::Coroutine(def_id, ..) = kind.as_ref()
-                    && let Some(
-                        coroutine_kind @ hir::CoroutineKind::Desugared(
-                            hir::CoroutineDesugaring::Async,
-                            _,
-                        ),
-                    ) = self.tcx.coroutine_kind(def_id)
+                    && let Some(coroutine_kind) = self.tcx.coroutine_kind(def_id)
                 {
                     self.check_op(ops::Coroutine(coroutine_kind));
                 }

compiler/rustc_const_eval/src/check_consts/ops.rs

@@ -486,24 +486,25 @@ impl<'tcx> NonConstOp<'tcx> for IntrinsicUnstable {
 pub(crate) struct Coroutine(pub hir::CoroutineKind);
 impl<'tcx> NonConstOp<'tcx> for Coroutine {
     fn status_in_item(&self, _: &ConstCx<'_, 'tcx>) -> Status {
-        if let hir::CoroutineKind::Desugared(
-            hir::CoroutineDesugaring::Async,
-            hir::CoroutineSource::Block,
-        ) = self.0
-        {
-            Status::Unstable {
+        match self.0 {
+            hir::CoroutineKind::Desugared(
+                hir::CoroutineDesugaring::Async,
+                hir::CoroutineSource::Block,
+            )
+            // FIXME(coroutines): eventually we want to gate const coroutine coroutines behind a
+            // different feature.
+            | hir::CoroutineKind::Coroutine(_) => Status::Unstable {
                 gate: sym::const_async_blocks,
                 gate_already_checked: false,
                 safe_to_expose_on_stable: false,
                 is_function_call: false,
-            }
-        } else {
-            Status::Forbidden
+            },
+            _ => Status::Forbidden,
         }
     }
 
     fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> Diag<'tcx> {
-        let msg = format!("{:#}s are not allowed in {}s", self.0, ccx.const_kind());
+        let msg = format!("{} are not allowed in {}s", self.0.to_plural_string(), ccx.const_kind());
         if let Status::Unstable { gate, .. } = self.status_in_item(ccx) {
             ccx.tcx.sess.create_feature_err(errors::UnallowedOpInConstContext { span, msg }, gate)
         } else {

compiler/rustc_hir/src/hir.rs

@@ -2061,12 +2061,19 @@ impl CoroutineKind {
             CoroutineKind::Coroutine(mov) => mov,
         }
     }
-}
 
-impl CoroutineKind {
     pub fn is_fn_like(self) -> bool {
         matches!(self, CoroutineKind::Desugared(_, CoroutineSource::Fn))
     }
+
+    pub fn to_plural_string(&self) -> String {
+        match self {
+            CoroutineKind::Desugared(d, CoroutineSource::Fn) => format!("{d:#}fn bodies"),
+            CoroutineKind::Desugared(d, CoroutineSource::Block) => format!("{d:#}blocks"),
+            CoroutineKind::Desugared(d, CoroutineSource::Closure) => format!("{d:#}closure bodies"),
+            CoroutineKind::Coroutine(_) => "coroutines".to_string(),
+        }
+    }
 }
 
 impl fmt::Display for CoroutineKind {

compiler/rustc_hir_typeck/src/closure.rs

@@ -204,14 +204,19 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 )
             }
             hir::ClosureKind::CoroutineClosure(kind) => {
-                // async closures always return the type ascribed after the `->` (if present),
-                // and yield `()`.
                 let (bound_return_ty, bound_yield_ty) = match kind {
+                    hir::CoroutineDesugaring::Gen => {
+                        // `iter!` closures always return unit and yield the `Iterator::Item` type
+                        // that we have to infer.
+                        (tcx.types.unit, self.infcx.next_ty_var(expr_span))
+                    }
                     hir::CoroutineDesugaring::Async => {
+                        // async closures always return the type ascribed after the `->` (if present),
+                        // and yield `()`.
                         (bound_sig.skip_binder().output(), tcx.types.unit)
                     }
-                    hir::CoroutineDesugaring::Gen | hir::CoroutineDesugaring::AsyncGen => {
-                        todo!("`gen` and `async gen` closures not supported yet")
+                    hir::CoroutineDesugaring::AsyncGen => {
+                        todo!("`async gen` closures not supported yet")
                     }
                 };
                 // Compute all of the variables that will be used to populate the coroutine.
@@ -465,7 +470,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             if let Some(trait_def_id) = trait_def_id {
                 let found_kind = match closure_kind {
-                    hir::ClosureKind::Closure => self.tcx.fn_trait_kind_from_def_id(trait_def_id),
+                    hir::ClosureKind::Closure
+                    // FIXME(iter_macro): Someday we'll probably want iterator closures instead of
+                    // just using Fn* for iterators.
+                    | hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Gen) => {
+                        self.tcx.fn_trait_kind_from_def_id(trait_def_id)
+                    }
                     hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Async) => self
                         .tcx
                         .async_fn_trait_kind_from_def_id(trait_def_id)

compiler/rustc_parse/src/parser/stmt.rs

@@ -713,7 +713,7 @@ impl<'a> Parser<'a> {
 
     /// Parses the rest of a block expression or function body.
     /// Precondition: already parsed the '{'.
-    pub(crate) fn parse_block_tail(
+    pub fn parse_block_tail(
         &mut self,
         lo: Span,
         s: BlockCheckMode,

compiler/rustc_trait_selection/messages.ftl

@@ -72,8 +72,6 @@ trait_selection_adjust_signature_remove_borrow = consider adjusting the signatur
 
 trait_selection_ascribe_user_type_prove_predicate = ...so that the where clause holds
 
-trait_selection_async_closure_not_fn = async closure does not implement `{$kind}` because it captures state from its environment
-
 trait_selection_await_both_futures = consider `await`ing on both `Future`s
 trait_selection_await_future = consider `await`ing on the `Future`
 trait_selection_await_note = calling an async function returns a future
@@ -123,6 +121,8 @@ trait_selection_closure_kind_requirement = the requirement to implement `{$trait
 
 trait_selection_compare_impl_item_obligation = ...so that the definition in impl matches the definition from the trait
 trait_selection_consider_specifying_length = consider specifying the actual array length
+trait_selection_coro_closure_not_fn = {$coro_kind}closure does not implement `{$kind}` because it captures state from its environment
+
 trait_selection_data_flows = ...but data{$label_var1_exists ->
     [true] {" "}from `{$label_var1}`
     *[false] {""}

compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs

@@ -42,9 +42,7 @@ use super::{
 use crate::error_reporting::TypeErrCtxt;
 use crate::error_reporting::infer::TyCategory;
 use crate::error_reporting::traits::report_dyn_incompatibility;
-use crate::errors::{
-    AsyncClosureNotFn, ClosureFnMutLabel, ClosureFnOnceLabel, ClosureKindMismatch,
-};
+use crate::errors::{ClosureFnMutLabel, ClosureFnOnceLabel, ClosureKindMismatch, CoroClosureNotFn};
 use crate::infer::{self, InferCtxt, InferCtxtExt as _};
 use crate::traits::query::evaluate_obligation::InferCtxtExt as _;
 use crate::traits::{
@@ -886,9 +884,18 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
         // is unimplemented is because async closures don't implement `Fn`/`FnMut`
         // if they have captures.
         if has_self_borrows && expected_kind != ty::ClosureKind::FnOnce {
-            let mut err = self.dcx().create_err(AsyncClosureNotFn {
+            let coro_kind = match self
+                .tcx
+                .coroutine_kind(self.tcx.coroutine_for_closure(closure_def_id))
+                .unwrap()
+            {
+                rustc_hir::CoroutineKind::Desugared(desugaring, _) => desugaring.to_string(),
+                coro => coro.to_string(),
+            };
+            let mut err = self.dcx().create_err(CoroClosureNotFn {
                 span: self.tcx.def_span(closure_def_id),
                 kind: expected_kind.as_str(),
+                coro_kind,
             });
             self.note_obligation_cause(&mut err, &obligation);
             return Some(err.emit());

compiler/rustc_trait_selection/src/errors.rs

@@ -201,11 +201,12 @@ pub struct ClosureFnMutLabel {
 }
 
 #[derive(Diagnostic)]
-#[diag(trait_selection_async_closure_not_fn)]
-pub(crate) struct AsyncClosureNotFn {
+#[diag(trait_selection_coro_closure_not_fn)]
+pub(crate) struct CoroClosureNotFn {
     #[primary_span]
     pub span: Span,
     pub kind: &'static str,
+    pub coro_kind: String,
 }
 
 #[derive(Diagnostic)]

compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs

@@ -11,7 +11,7 @@ use std::ops::ControlFlow;
 use hir::LangItem;
 use hir::def_id::DefId;
 use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
-use rustc_hir as hir;
+use rustc_hir::{self as hir, CoroutineDesugaring, CoroutineKind};
 use rustc_infer::traits::{Obligation, PolyTraitObligation, SelectionError};
 use rustc_middle::ty::fast_reject::DeepRejectCtxt;
 use rustc_middle::ty::{self, Ty, TypeVisitableExt, TypingMode, elaborate};
@@ -438,6 +438,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip(self, candidates))]
     fn assemble_async_closure_candidates(
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,
@@ -446,15 +447,30 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let goal_kind =
             self.tcx().async_fn_trait_kind_from_def_id(obligation.predicate.def_id()).unwrap();
 
+        debug!("self_ty = {:?}", obligation.self_ty().skip_binder().kind());
         match *obligation.self_ty().skip_binder().kind() {
-            ty::CoroutineClosure(_, args) => {
+            ty::CoroutineClosure(def_id, args) => {
                 if let Some(closure_kind) =
                     args.as_coroutine_closure().kind_ty().to_opt_closure_kind()
                     && !closure_kind.extends(goal_kind)
                 {
                     return;
                 }
-                candidates.vec.push(AsyncClosureCandidate);
+
+                // Make sure this is actually an async closure.
+                let Some(coroutine_kind) =
+                    self.tcx().coroutine_kind(self.tcx().coroutine_for_closure(def_id))
+                else {
+                    bug!("coroutine with no kind");
+                };
+
+                debug!(?coroutine_kind);
+                match coroutine_kind {
+                    CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => {
+                        candidates.vec.push(AsyncClosureCandidate);
+                    }
+                    _ => (),
+                }
             }
             // Closures and fn pointers implement `AsyncFn*` if their return types
             // implement `Future`, which is checked later.

library/core/src/iter/mod.rs

@@ -420,6 +420,8 @@ pub use self::adapters::{Intersperse, IntersperseWith};
     issue = "42168"
 )]
 pub use self::range::Step;
+#[unstable(feature = "iter_macro", issue = "none", reason = "generators are unstable")]
+pub use self::sources::iter;
 #[stable(feature = "iter_empty", since = "1.2.0")]
 pub use self::sources::{Empty, empty};
 #[unstable(

library/core/src/iter/sources.rs

@@ -1,6 +1,7 @@
 mod empty;
 mod from_coroutine;
 mod from_fn;
+mod generator;
 mod once;
 mod once_with;
 mod repeat;
@@ -18,6 +19,8 @@ pub use self::empty::{Empty, empty};
 pub use self::from_coroutine::{FromCoroutine, from_coroutine};
 #[stable(feature = "iter_from_fn", since = "1.34.0")]
 pub use self::from_fn::{FromFn, from_fn};
+#[unstable(feature = "iter_macro", issue = "none", reason = "generators are unstable")]
+pub use self::generator::iter;
 #[stable(feature = "iter_once", since = "1.2.0")]
 pub use self::once::{Once, once};
 #[stable(feature = "iter_once_with", since = "1.43.0")]