Skip to content

Commit b741351

Browse files
oli-obkoli-obk
committed
Generalize the Assume intrinsic statement to a general Intrinsic statement
1 parent 3f07645 commit b741351

File tree

28 files changed

+166
-143
lines changed

28 files changed

+166
-143
lines changed

‎compiler/rustc_borrowck/src/dataflow.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -391,8 +391,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
391391
| mir::StatementKind::Retag { .. }
392392
| mir::StatementKind::AscribeUserType(..)
393393
| mir::StatementKind::Coverage(..)
394-
| mir::StatementKind::CopyNonOverlapping(..)
395-
| mir::StatementKind::Assume(..)
394+
| mir::StatementKind::Intrinsic(..)
396395
| mir::StatementKind::Nop => {}
397396
}
398397
}

‎compiler/rustc_borrowck/src/invalidation.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use rustc_data_structures::graph::dominators::Dominators;
22
use rustc_middle::mir::visit::Visitor;
3-
use rustc_middle::mir::{BasicBlock, Body, Location, Place, Rvalue};
3+
use rustc_middle::mir::{self, BasicBlock, Body, Location, NonDivergingIntrinsic, Place, Rvalue};
44
use rustc_middle::mir::{BorrowKind, Mutability, Operand};
55
use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind};
66
use rustc_middle::mir::{Statement, StatementKind};
@@ -63,11 +63,14 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
6363
StatementKind::FakeRead(box (_, _)) => {
6464
// Only relevant for initialized/liveness/safety checks.
6565
}
66-
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
66+
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
67+
self.consume_operand(location, op);
68+
}
69+
StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
6770
ref src,
6871
ref dst,
6972
ref count,
70-
}) => {
73+
})) => {
7174
self.consume_operand(location, src);
7275
self.consume_operand(location, dst);
7376
self.consume_operand(location, count);
@@ -76,8 +79,6 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
7679
StatementKind::AscribeUserType(..)
7780
// Doesn't have any language semantics
7881
| StatementKind::Coverage(..)
79-
// Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
80-
| StatementKind::Assume(..)
8182
// Does not actually affect borrowck
8283
| StatementKind::StorageLive(..) => {}
8384
StatementKind::StorageDead(local) => {

‎compiler/rustc_borrowck/src/lib.rs

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ use rustc_index::bit_set::ChunkedBitSet;
2626
use rustc_index::vec::IndexVec;
2727
use rustc_infer::infer::{DefiningAnchor, InferCtxt, TyCtxtInferExt};
2828
use rustc_middle::mir::{
29-
traversal, Body, ClearCrossCrate, Local, Location, Mutability, Operand, Place, PlaceElem,
30-
PlaceRef, VarDebugInfoContents,
29+
traversal, Body, ClearCrossCrate, Local, Location, Mutability, NonDivergingIntrinsic, Operand,
30+
Place, PlaceElem, PlaceRef, VarDebugInfoContents,
3131
};
3232
use rustc_middle::mir::{AggregateKind, BasicBlock, BorrowCheckResult, BorrowKind};
3333
use rustc_middle::mir::{Field, ProjectionElem, Promoted, Rvalue, Statement, StatementKind};
@@ -591,10 +591,10 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
591591
flow_state,
592592
);
593593
}
594-
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
595-
..
596-
}) => {
597-
span_bug!(
594+
StatementKind::Intrinsic(box ref kind) => match kind {
595+
// Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
596+
NonDivergingIntrinsic::Assume(..) => {},
597+
NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
598598
span,
599599
"Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",
600600
)
@@ -603,8 +603,6 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
603603
StatementKind::AscribeUserType(..)
604604
// Doesn't have any language semantics
605605
| StatementKind::Coverage(..)
606-
// Takes a `bool` argument, and has no return value, thus being irrelevant for borrowck
607-
| StatementKind::Assume(..)
608606
// Does not actually affect borrowck
609607
| StatementKind::StorageLive(..) => {}
610608
StatementKind::StorageDead(local) => {

‎compiler/rustc_borrowck/src/type_check/mod.rs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1302,14 +1302,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
13021302
);
13031303
}
13041304
}
1305-
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
1306-
..
1307-
}) => span_bug!(
1308-
stmt.source_info.span,
1309-
"Unexpected StatementKind::CopyNonOverlapping, should only appear after lowering_intrinsics",
1310-
),
1305+
StatementKind::Intrinsic(box ref kind) => match kind {
1306+
NonDivergingIntrinsic::Assume(..) => {}
1307+
NonDivergingIntrinsic::CopyNonOverlapping(..) => span_bug!(
1308+
stmt.source_info.span,
1309+
"Unexpected NonDivergingIntrinsic::CopyNonOverlapping, should only appear after lowering_intrinsics",
1310+
),
1311+
},
13111312
StatementKind::FakeRead(..)
1312-
| StatementKind::Assume(..)
13131313
| StatementKind::StorageLive(..)
13141314
| StatementKind::StorageDead(..)
13151315
| StatementKind::Retag { .. }

‎compiler/rustc_codegen_cranelift/src/base.rs

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -791,25 +791,34 @@ fn codegen_stmt<'tcx>(
791791
| StatementKind::Nop
792792
| StatementKind::FakeRead(..)
793793
| StatementKind::Retag { .. }
794-
// We ignore `assume` intrinsics, they are only useful for optimizations
795-
| StatementKind::Assume(..)
796794
| StatementKind::AscribeUserType(..) => {}
797795

798796
StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
799-
StatementKind::CopyNonOverlapping(inner) => {
800-
let dst = codegen_operand(fx, &inner.dst);
801-
let pointee = dst
802-
.layout()
803-
.pointee_info_at(fx, rustc_target::abi::Size::ZERO)
804-
.expect("Expected pointer");
805-
let dst = dst.load_scalar(fx);
806-
let src = codegen_operand(fx, &inner.src).load_scalar(fx);
807-
let count = codegen_operand(fx, &inner.count).load_scalar(fx);
808-
let elem_size: u64 = pointee.size.bytes();
809-
let bytes =
810-
if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
811-
fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
812-
}
797+
StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic {
798+
// We ignore `assume` intrinsics, they are only useful for optimizations
799+
NonDivergingIntrinsic::Assume(_) => {}
800+
NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
801+
src,
802+
dst,
803+
count,
804+
}) => {
805+
let dst = codegen_operand(fx, dst);
806+
let pointee = dst
807+
.layout()
808+
.pointee_info_at(fx, rustc_target::abi::Size::ZERO)
809+
.expect("Expected pointer");
810+
let dst = dst.load_scalar(fx);
811+
let src = codegen_operand(fx, src).load_scalar(fx);
812+
let count = codegen_operand(fx, count).load_scalar(fx);
813+
let elem_size: u64 = pointee.size.bytes();
814+
let bytes = if elem_size != 1 {
815+
fx.bcx.ins().imul_imm(count, elem_size as i64)
816+
} else {
817+
count
818+
};
819+
fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
820+
}
821+
},
813822
}
814823
}
815824

‎compiler/rustc_codegen_cranelift/src/constant.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -536,11 +536,12 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
536536
{
537537
return None;
538538
}
539-
StatementKind::CopyNonOverlapping(_) => {
540-
return None;
541-
} // conservative handling
539+
StatementKind::Intrinsic(ref intrinsic) => match **intrinsic {
540+
NonDivergingIntrinsic::CopyNonOverlapping(..) => return None,
541+
NonDivergingIntrinsic::Assume(..) => {}
542+
},
543+
// conservative handling
542544
StatementKind::Assign(_)
543-
| StatementKind::Assume(_)
544545
| StatementKind::FakeRead(_)
545546
| StatementKind::SetDiscriminant { .. }
546547
| StatementKind::Deinit(_)

‎compiler/rustc_codegen_ssa/src/mir/statement.rs

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use rustc_middle::mir;
2+
use rustc_middle::mir::NonDivergingIntrinsic;
23

34
use super::FunctionCx;
45
use super::LocalRef;
@@ -73,11 +74,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
7374
self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope);
7475
bx
7576
}
76-
mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
77-
ref src,
78-
ref dst,
79-
ref count,
80-
}) => {
77+
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
78+
let op_val = self.codegen_operand(&mut bx, op);
79+
bx.assume(op_val.immediate());
80+
bx
81+
}
82+
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
83+
mir::CopyNonOverlapping { ref count, ref src, ref dst },
84+
)) => {
8185
let dst_val = self.codegen_operand(&mut bx, dst);
8286
let src_val = self.codegen_operand(&mut bx, src);
8387
let count = self.codegen_operand(&mut bx, count).immediate();
@@ -93,11 +97,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
9397
bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
9498
bx
9599
}
96-
mir::StatementKind::Assume(box ref op) => {
97-
let op_val = self.codegen_operand(&mut bx, op);
98-
bx.assume(op_val.immediate());
99-
bx
100-
}
101100
mir::StatementKind::FakeRead(..)
102101
| mir::StatementKind::Retag { .. }
103102
| mir::StatementKind::AscribeUserType(..)

‎compiler/rustc_const_eval/src/interpret/step.rs

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
//!
33
//! The main entry point is the `step` method.
44
5-
use rustc_middle::mir;
65
use rustc_middle::mir::interpret::{InterpResult, Scalar};
6+
use rustc_middle::mir::{self, NonDivergingIntrinsic};
77
use rustc_middle::ty::layout::LayoutOf;
88

99
use super::{InterpCx, Machine};
@@ -114,22 +114,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
114114
M::retag(self, *kind, &dest)?;
115115
}
116116

117-
// Call CopyNonOverlapping
118-
CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { src, dst, count }) => {
119-
let src = self.eval_operand(src, None)?;
120-
let dst = self.eval_operand(dst, None)?;
121-
let count = self.eval_operand(count, None)?;
122-
self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
123-
}
124-
125-
// Call Assume
126-
Assume(box op) => {
117+
Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
127118
let op = self.eval_operand(op, None)?;
128119
let cond = self.read_scalar(&op)?.to_bool()?;
129120
if !cond {
130121
throw_ub_format!("`assume` called with `false`");
131122
}
132123
}
124+
Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
125+
ref count,
126+
ref src,
127+
ref dst,
128+
})) => {
129+
let src = self.eval_operand(src, None)?;
130+
let dst = self.eval_operand(dst, None)?;
131+
let count = self.eval_operand(count, None)?;
132+
self.copy_intrinsic(&src, &dst, &count, /* nonoverlapping */ true)?;
133+
}
133134

134135
// Statements we do not track.
135136
AscribeUserType(..) => {}

‎compiler/rustc_const_eval/src/transform/check_consts/check.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -678,8 +678,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
678678
| StatementKind::Retag { .. }
679679
| StatementKind::AscribeUserType(..)
680680
| StatementKind::Coverage(..)
681-
| StatementKind::CopyNonOverlapping(..)
682-
| StatementKind::Assume(..)
681+
| StatementKind::Intrinsic(..)
683682
| StatementKind::Nop => {}
684683
}
685684
}

‎compiler/rustc_const_eval/src/transform/validate.rs

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@ use rustc_middle::mir::interpret::Scalar;
77
use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
88
use rustc_middle::mir::visit::{PlaceContext, Visitor};
99
use rustc_middle::mir::{
10-
traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, Local, Location,
11-
MirPass, MirPhase, Operand, Place, PlaceElem, PlaceRef, ProjectionElem, RuntimePhase, Rvalue,
12-
SourceScope, Statement, StatementKind, Terminator, TerminatorKind, UnOp, START_BLOCK,
10+
traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping,
11+
Local, Location, MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef,
12+
ProjectionElem, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator,
13+
TerminatorKind, UnOp, START_BLOCK,
1314
};
1415
use rustc_middle::ty::fold::BottomUpFolder;
1516
use rustc_middle::ty::subst::Subst;
@@ -636,7 +637,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
636637
);
637638
}
638639
}
639-
StatementKind::Assume(box ref op) => {
640+
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
640641
let ty = op.ty(&self.body.local_decls, self.tcx);
641642
if !ty.is_bool() {
642643
self.fail(
@@ -645,11 +646,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
645646
);
646647
}
647648
}
648-
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
649-
ref src,
650-
ref dst,
651-
ref count,
652-
}) => {
649+
StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
650+
CopyNonOverlapping { src, dst, count },
651+
)) => {
653652
let src_ty = src.ty(&self.body.local_decls, self.tcx);
654653
let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) {
655654
src_deref.ty

‎compiler/rustc_middle/src/mir/mod.rs

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1370,14 +1370,7 @@ impl Debug for Statement<'_> {
13701370
write!(fmt, "Coverage::{:?} for {:?}", kind, rgn)
13711371
}
13721372
Coverage(box ref coverage) => write!(fmt, "Coverage::{:?}", coverage.kind),
1373-
CopyNonOverlapping(box crate::mir::CopyNonOverlapping {
1374-
ref src,
1375-
ref dst,
1376-
ref count,
1377-
}) => {
1378-
write!(fmt, "copy_nonoverlapping(src={:?}, dst={:?}, count={:?})", src, dst, count)
1379-
}
1380-
Assume(box ref cond) => write!(fmt, "assume({:?})", cond),
1373+
Intrinsic(box ref intrinsic) => write!(fmt, "{intrinsic}"),
13811374
Nop => write!(fmt, "nop"),
13821375
}
13831376
}

‎compiler/rustc_middle/src/mir/spanview.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -249,8 +249,7 @@ pub fn statement_kind_name(statement: &Statement<'_>) -> &'static str {
249249
Retag(..) => "Retag",
250250
AscribeUserType(..) => "AscribeUserType",
251251
Coverage(..) => "Coverage",
252-
CopyNonOverlapping(..) => "CopyNonOverlapping",
253-
Assume(..) => "Assume",
252+
Intrinsic(..) => "Intrinsic",
254253
Nop => "Nop",
255254
}
256255
}

‎compiler/rustc_middle/src/mir/syntax.rs

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,34 @@ pub enum StatementKind<'tcx> {
327327
/// executed.
328328
Coverage(Box<Coverage>),
329329

330+
/// Denotes a call to an intrinsic that does not require an unwind path. This avoids
331+
/// adding a new block and a terminator for simple intrinsics.
332+
Intrinsic(Box<NonDivergingIntrinsic<'tcx>>),
333+
334+
/// No-op. Useful for deleting instructions without affecting statement indices.
335+
Nop,
336+
}
337+
338+
#[derive(
339+
Clone,
340+
TyEncodable,
341+
TyDecodable,
342+
Debug,
343+
PartialEq,
344+
Hash,
345+
HashStable,
346+
TypeFoldable,
347+
TypeVisitable
348+
)]
349+
pub enum NonDivergingIntrinsic<'tcx> {
350+
/// Denotes a call to the intrinsic function `assume`.
351+
///
352+
/// The operand must be a boolean. Optimizers may use the value of the boolean to backtrack its
353+
/// computation to infer information about other variables. So if the boolean came from a
354+
/// `x < y` operation, subsequent operations on `x` and `y` could elide various bound checks.
355+
/// If the argument is `false`, this operation is equivalent to `TerminatorKind::Unreachable`.
356+
Assume(Operand<'tcx>),
357+
330358
/// Denotes a call to the intrinsic function `copy_nonoverlapping`.
331359
///
332360
/// First, all three operands are evaluated. `src` and `dest` must each be a reference, pointer,
@@ -340,18 +368,18 @@ pub enum StatementKind<'tcx> {
340368
///
341369
/// **Needs clarification**: Is this typed or not, ie is there a typed load and store involved?
342370
/// I vaguely remember Ralf saying somewhere that he thought it should not be.
343-
CopyNonOverlapping(Box<CopyNonOverlapping<'tcx>>),
344-
345-
/// Denotes a call to the intrinsic function `assume`.
346-
///
347-
/// The operand must be a boolean. Optimizers may use the value of the boolean to backtrack its
348-
/// computation to infer information about other variables. So if the boolean came from a
349-
/// `x < y` operation, subsequent operations on `x` and `y` could elide various bound checks.
350-
/// If the argument is `false`, this operation is equivalent to `TerminatorKind::Unreachable`.
351-
Assume(Box<Operand<'tcx>>),
371+
CopyNonOverlapping(CopyNonOverlapping<'tcx>),
372+
}
352373

353-
/// No-op. Useful for deleting instructions without affecting statement indices.
354-
Nop,
374+
impl std::fmt::Display for NonDivergingIntrinsic<'_> {
375+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
376+
match self {
377+
Self::Assume(op) => write!(f, "assume({op:?})"),
378+
Self::CopyNonOverlapping(CopyNonOverlapping { src, dst, count }) => {
379+
write!(f, "copy_nonoverlapping(dst = {dst:?}, src = {src:?}, count = {count:?})")
380+
}
381+
}
382+
}
355383
}
356384

357385
/// Describes what kind of retag is to be performed.

0 commit comments

Comments
 (0)