Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 132433d

Browse files
borsbors
committed
Auto merge of rust-lang#138582 - scottmcm:option-ssa-2, r=<try>
Don't require `alloca`s for consuming simple enums For example, if you pass an `Option<u32>` to a function, don't immediately write it to an `alloca` then read it again.
2 parents 9c67cec + 3477a13 commit 132433d

File tree

9 files changed

+312
-119
lines changed

9 files changed

+312
-119
lines changed

‎compiler/rustc_codegen_ssa/src/mir/analyze.rs

Lines changed: 79 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ use rustc_data_structures::graph::dominators::Dominators;
55
use rustc_index::bit_set::DenseBitSet;
66
use rustc_index::{IndexSlice, IndexVec};
77
use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor};
8-
use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind, traversal};
9-
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
8+
use rustc_middle::mir::{self, DefLocation, Location, PlaceElem, TerminatorKind, traversal};
9+
use rustc_middle::ty::layout::LayoutOf;
1010
use rustc_middle::{bug, span_bug};
1111
use tracing::debug;
1212

@@ -96,65 +96,92 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
9696
fn process_place(
9797
&mut self,
9898
place_ref: &mir::PlaceRef<'tcx>,
99-
context: PlaceContext,
99+
mut context: PlaceContext,
100100
location: Location,
101101
) {
102-
let cx = self.fx.cx;
103-
104-
if let Some((place_base, elem)) = place_ref.last_projection() {
105-
let mut base_context = if context.is_mutating_use() {
106-
PlaceContext::MutatingUse(MutatingUseContext::Projection)
107-
} else {
108-
PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection)
109-
};
110-
111-
// Allow uses of projections that are ZSTs or from scalar fields.
112-
let is_consume = matches!(
113-
context,
114-
PlaceContext::NonMutatingUse(
115-
NonMutatingUseContext::Copy | NonMutatingUseContext::Move,
116-
)
102+
let maybe_local = if place_ref.is_indirect_first_projection() {
103+
// After we deref a pointer, the local *of that pointer* is no
104+
// longer interesting for the rest of the projection chain.
105+
self.visit_local(
106+
place_ref.local,
107+
PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy),
108+
location,
117109
);
118-
if is_consume {
119-
let base_ty = place_base.ty(self.fx.mir, cx.tcx());
120-
let base_ty = self.fx.monomorphize(base_ty);
121-
122-
// ZSTs don't require any actual memory access.
123-
let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty;
124-
let span = self.fx.mir.local_decls[place_ref.local].source_info.span;
125-
if cx.spanned_layout_of(elem_ty, span).is_zst() {
126-
return;
127-
}
128-
129-
if let mir::ProjectionElem::Field(..) = elem {
130-
let layout = cx.spanned_layout_of(base_ty.ty, span);
131-
if cx.is_backend_immediate(layout) || cx.is_backend_scalar_pair(layout) {
132-
// Recurse with the same context, instead of `Projection`,
133-
// potentially stopping at non-operand projections,
134-
// which would trigger `not_ssa` on locals.
135-
base_context = context;
136-
}
137-
}
138-
}
139-
140-
if let mir::ProjectionElem::Deref = elem {
141-
// Deref projections typically only read the pointer.
142-
base_context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy);
143-
}
110+
None
111+
} else {
112+
Some(place_ref.local)
113+
};
144114

145-
self.process_place(&place_base, base_context, location);
146-
// HACK(eddyb) this emulates the old `visit_projection_elem`, this
147-
// entire `visit_place`-like `process_place` method should be rewritten,
148-
// now that we have moved to the "slice of projections" representation.
149-
if let mir::ProjectionElem::Index(local) = elem {
115+
let mut projection: &[PlaceElem<'tcx>] = place_ref.projection;
116+
loop {
117+
// Index projections are the only ones with another local, so handle
118+
// that special case before the normal projection match.
119+
if let [PlaceElem::Index(index_local), ..] = *projection {
150120
self.visit_local(
151-
local,
121+
index_local,
152122
PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy),
153123
location,
154124
);
155125
}
156-
} else {
157-
self.visit_local(place_ref.local, context, location);
126+
127+
projection = match projection {
128+
// No more projections means we're done looping.
129+
[] => break,
130+
// The only deref allowed in a Runtime-phase place is at the
131+
// beginning, which we checked before the loop.
132+
[PlaceElem::Deref, rest @ ..] => {
133+
assert_eq!(maybe_local, None);
134+
rest
135+
}
136+
// Making SSA locals useful for non-primitives heavily depends on
137+
// not forcing stack allocation for basic newtypes and simple
138+
// enums like `Option<u32>` or `Result<bool, Box<MyError>>`.
139+
[PlaceElem::Downcast { .. }, PlaceElem::Field { .. }, rest @ ..]
140+
| [PlaceElem::Field { .. }, rest @ ..] => {
141+
if let PlaceContext::NonMutatingUse(
142+
NonMutatingUseContext::Copy
143+
| NonMutatingUseContext::Move
144+
| NonMutatingUseContext::Inspect,
145+
) = context
146+
{
147+
// Reading fields (or pseudo-fields) in operands can stay SSA
148+
} else {
149+
// But storing into a projection needs memory, especially for function returns
150+
context = PlaceContext::MutatingUse(MutatingUseContext::Projection);
151+
}
152+
rest
153+
}
154+
[PlaceElem::Downcast { .. }, ..] => {
155+
span_bug!(self.fx.mir.span, "Non-field downcast in {place_ref:?}");
156+
}
157+
// FIXME: These are type-changing, but not layout-affecting, so
158+
// they probably needn't force memory, but for now they do since
159+
// `maybe_codegen_consume_direct` doesn't handle them.
160+
[
161+
PlaceElem::OpaqueCast { .. }
162+
| PlaceElem::UnwrapUnsafeBinder { .. }
163+
| PlaceElem::Subtype { .. },
164+
rest @ ..,
165+
] => {
166+
context = PlaceContext::MutatingUse(MutatingUseContext::Projection);
167+
rest
168+
}
169+
// The various types of indexing use address arithmetic, so we
170+
// need to force the local to Memory like a borrow would.
171+
[
172+
PlaceElem::Index { .. }
173+
| PlaceElem::ConstantIndex { .. }
174+
| PlaceElem::Subslice { .. },
175+
rest @ ..,
176+
] => {
177+
context = PlaceContext::MutatingUse(MutatingUseContext::Projection);
178+
rest
179+
}
180+
};
181+
}
182+
183+
if let Some(local) = maybe_local {
184+
self.visit_local(local, context, location);
158185
}
159186
}
160187
}

‎compiler/rustc_codegen_ssa/src/mir/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
135135
}
136136
}
137137

138+
#[derive(Debug)]
138139
enum LocalRef<'tcx, V> {
139140
Place(PlaceRef<'tcx, V>),
140141
/// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place).

‎compiler/rustc_codegen_ssa/src/mir/operand.rs

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ pub struct OperandRef<'tcx, V> {
142142
pub layout: TyAndLayout<'tcx>,
143143
}
144144

145-
impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
145+
impl<V: fmt::Debug> fmt::Debug for OperandRef<'_, V> {
146146
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
147147
write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout)
148148
}
@@ -372,16 +372,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
372372
)
373373
})
374374
} else {
375-
let (in_scalar, imm) = match (self.val, self.layout.backend_repr) {
375+
let (imm, in_scalar, in_bty) = match (self.val, self.layout.backend_repr) {
376376
// Extract a scalar component from a pair.
377377
(OperandValue::Pair(a_llval, b_llval), BackendRepr::ScalarPair(a, b)) => {
378-
if offset.bytes() == 0 {
378+
// This needs to look at `offset`, rather than `i`, because
379+
// for a type like `Option<u32>`, the first thing in the pair
380+
// is the tag, so `(_2 as Some).0` needs to read the *second*
381+
// thing in the pair despite it being "field zero".
382+
if offset == Size::ZERO {
379383
assert_eq!(field.size, a.size(bx.cx()));
380-
(Some(a), a_llval)
384+
let bty = bx.scalar_pair_element_backend_type(self.layout, 0, false);
385+
(a_llval, a, bty)
381386
} else {
382387
assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi));
383388
assert_eq!(field.size, b.size(bx.cx()));
384-
(Some(b), b_llval)
389+
let bty = bx.scalar_pair_element_backend_type(self.layout, 1, false);
390+
(b_llval, b, bty)
385391
}
386392
}
387393

@@ -392,23 +398,13 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
392398
OperandValue::Immediate(match field.backend_repr {
393399
BackendRepr::SimdVector { .. } => imm,
394400
BackendRepr::Scalar(out_scalar) => {
395-
let Some(in_scalar) = in_scalar else {
396-
span_bug!(
397-
fx.mir.span,
398-
"OperandRef::extract_field({:?}): missing input scalar for output scalar",
399-
self
400-
)
401-
};
402-
if in_scalar != out_scalar {
403-
// If the backend and backend_immediate types might differ,
404-
// flip back to the backend type then to the new immediate.
405-
// This avoids nop truncations, but still handles things like
406-
// Bools in union fields needs to be truncated.
407-
let backend = bx.from_immediate(imm);
408-
bx.to_immediate_scalar(backend, out_scalar)
409-
} else {
410-
imm
411-
}
401+
// For a type like `Result<usize, &u32>` the layout is `Pair(i64, ptr)`.
402+
// But if we're reading the `Ok` payload, we need to turn that `ptr`
403+
// back into an integer. To avoid repeating logic we do that by
404+
// calling the transmute code, which is legal thanks to the size
405+
// assert we did when pulling it out of the pair.
406+
let out_bty = bx.backend_type(field);
407+
fx.transmute_immediate(bx, imm, in_scalar, in_bty, out_scalar, out_bty)
412408
}
413409
BackendRepr::ScalarPair(_, _) | BackendRepr::Memory { .. } => bug!(),
414410
})
@@ -712,7 +708,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
712708
LocalRef::Operand(mut o) => {
713709
// Moves out of scalar and scalar pair fields are trivial.
714710
for elem in place_ref.projection.iter() {
715-
match elem {
711+
match *elem {
716712
mir::ProjectionElem::Field(f, _) => {
717713
assert!(
718714
!o.layout.ty.is_any_ptr(),
@@ -721,6 +717,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
721717
);
722718
o = o.extract_field(self, bx, f.index());
723719
}
720+
mir::ProjectionElem::Downcast(_sym, variant_idx) => {
721+
let layout = o.layout.for_variant(bx.cx(), variant_idx);
722+
// The transmute here handles cases like `Result<bool, u8>`
723+
// where the immediate values need to change for
724+
// the specific types in the cast-to variant.
725+
let Some(val) = self.codegen_transmute_operand(bx, o, layout) else {
726+
bug!("Couldn't transmute in downcast to {variant_idx:?} of {o:?}");
727+
};
728+
o = OperandRef { val, layout };
729+
}
724730
mir::ProjectionElem::Index(_)
725731
| mir::ProjectionElem::ConstantIndex { .. } => {
726732
// ZSTs don't require any actual memory access.

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -281,13 +281,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
281281
&& in_a.size(self.cx) == out_a.size(self.cx)
282282
&& in_b.size(self.cx) == out_b.size(self.cx)
283283
{
284-
let in_a_ibty = bx.scalar_pair_element_backend_type(operand.layout, 0, false);
285-
let in_b_ibty = bx.scalar_pair_element_backend_type(operand.layout, 1, false);
286-
let out_a_ibty = bx.scalar_pair_element_backend_type(cast, 0, false);
287-
let out_b_ibty = bx.scalar_pair_element_backend_type(cast, 1, false);
284+
let in_a_bty = bx.scalar_pair_element_backend_type(operand.layout, 0, false);
285+
let in_b_bty = bx.scalar_pair_element_backend_type(operand.layout, 1, false);
286+
let out_a_bty = bx.scalar_pair_element_backend_type(cast, 0, false);
287+
let out_b_bty = bx.scalar_pair_element_backend_type(cast, 1, false);
288288
Some(OperandValue::Pair(
289-
self.transmute_immediate(bx, imm_a, in_a, in_a_ibty, out_a, out_a_ibty),
290-
self.transmute_immediate(bx, imm_b, in_b, in_b_ibty, out_b, out_b_ibty),
289+
self.transmute_immediate(bx, imm_a, in_a, in_a_bty, out_a, out_a_bty),
290+
self.transmute_immediate(bx, imm_b, in_b, in_b_bty, out_b, out_b_bty),
291291
))
292292
} else {
293293
None
@@ -353,7 +353,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
353353
///
354354
/// `to_backend_ty` must be the *non*-immediate backend type (so it will be
355355
/// `i8`, not `i1`, for `bool`-like types.)
356-
fn transmute_immediate(
356+
pub(crate) fn transmute_immediate(
357357
&self,
358358
bx: &mut Bx,
359359
mut imm: Bx::Value,
@@ -371,8 +371,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
371371
return imm;
372372
}
373373

374-
use abi::Primitive::*;
375374
imm = bx.from_immediate(imm);
375+
debug_assert_eq!(bx.cx().val_ty(imm), from_backend_ty);
376376

377377
// If we have a scalar, we must already know its range. Either
378378
//
@@ -385,8 +385,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
385385
// <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182>
386386
self.assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
387387

388+
use abi::Primitive::*;
388389
imm = match (from_scalar.primitive(), to_scalar.primitive()) {
389-
(Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty),
390+
(Int(..), Int(..)) | (Float(_), Float(_)) => imm,
391+
(Int(..), Float(_)) | (Float(_), Int(..)) => bx.bitcast(imm, to_backend_ty),
390392
(Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty),
391393
(Int(..), Pointer(..)) => bx.ptradd(bx.const_null(bx.type_ptr()), imm),
392394
(Pointer(..), Int(..)) => {
@@ -411,6 +413,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
411413
// constraint that the `transmute` introduced is to `assume` it.
412414
self.assume_scalar_range(bx, imm, to_scalar, to_backend_ty);
413415

416+
debug_assert_eq!(bx.cx().val_ty(imm), to_backend_ty);
414417
imm = bx.to_immediate_scalar(imm, to_scalar);
415418
imm
416419
}
@@ -1157,8 +1160,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
11571160
mir::AggregateKind::Coroutine(..) | mir::AggregateKind::CoroutineClosure(..) => false,
11581161
};
11591162
allowed_kind && {
1160-
let ty = rvalue.ty(self.mir, self.cx.tcx());
1161-
let ty = self.monomorphize(ty);
1163+
let ty = rvalue.ty(self.mir, self.cx.tcx());
1164+
let ty = self.monomorphize(ty);
11621165
let layout = self.cx.spanned_layout_of(ty, span);
11631166
!self.cx.is_backend_ref(layout)
11641167
}

‎tests/codegen/common_prim_int_ptr.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,13 @@ pub unsafe fn extract_int(x: Result<usize, Box<()>>) -> usize {
4040
}
4141

4242
// CHECK-LABEL: @extract_box
43-
// CHECK-SAME: (i{{[0-9]+}} {{[^%]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^%]+}} [[PAYLOAD:%[0-9]+]])
43+
// CHECK-SAME: (i{{[0-9]+}} {{[^%]+}} [[DISCRIMINANT:%x.0]], ptr {{[^%]+}} [[PAYLOAD:%x.1]])
4444
#[no_mangle]
4545
pub unsafe fn extract_box(x: Result<usize, Box<i32>>) -> Box<i32> {
46+
// CHECK: [[NOT_OK:%.+]] = icmp ne i{{[0-9]+}} [[DISCRIMINANT]], 0
47+
// CHECK: call void @llvm.assume(i1 [[NOT_OK]])
48+
// CHECK: [[NOT_NULL:%.+]] = icmp ne ptr [[PAYLOAD]], null
49+
// CHECK: call void @llvm.assume(i1 [[NOT_NULL]])
4650
// CHECK: ret ptr [[PAYLOAD]]
4751
match x {
4852
Ok(_) => std::intrinsics::unreachable(),

0 commit comments

Comments
 (0)