Skip to content

Commit 5723c99

Browse files
beetreesbeetrees
committed
Fix RISC-V C function ABI when passing/returning structs containing floats
1 parent 68ac5ab commit 5723c99

File tree

13 files changed

+537
-134
lines changed

13 files changed

+537
-134
lines changed

‎compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

Lines changed: 42 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,18 @@ fn apply_attrs_to_abi_param(param: AbiParam, arg_attrs: ArgAttributes) -> AbiPar
4040
}
4141
}
4242

43-
fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
43+
fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[(Size, AbiParam); 2]> {
44+
if let Some(offset_from_start) = cast.rest_offset {
45+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
46+
assert_eq!(cast.rest.unit.size, cast.rest.total);
47+
let first = cast.prefix[0].unwrap();
48+
let second = cast.rest.unit;
49+
return smallvec![
50+
(Size::ZERO, reg_to_abi_param(first)),
51+
(offset_from_start, reg_to_abi_param(second))
52+
];
53+
}
54+
4455
let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 {
4556
(0, 0)
4657
} else {
@@ -55,25 +66,32 @@ fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
5566
// different types in Cranelift IR. Instead a single array of primitive types is used.
5667

5768
// Create list of fields in the main structure
58-
let mut args = cast
69+
let args = cast
5970
.prefix
6071
.iter()
6172
.flatten()
6273
.map(|&reg| reg_to_abi_param(reg))
63-
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)))
64-
.collect::<SmallVec<_>>();
74+
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)));
75+
76+
let mut res = SmallVec::new();
77+
let mut offset = Size::ZERO;
78+
79+
for arg in args {
80+
res.push((offset, arg));
81+
offset += Size::from_bytes(arg.value_type.bytes());
82+
}
6583

6684
// Append final integer
6785
if rem_bytes != 0 {
6886
// Only integers can be really split further.
6987
assert_eq!(cast.rest.unit.kind, RegKind::Integer);
70-
args.push(reg_to_abi_param(Reg {
71-
kind: RegKind::Integer,
72-
size: Size::from_bytes(rem_bytes),
73-
}));
88+
res.push((
89+
offset,
90+
reg_to_abi_param(Reg { kind: RegKind::Integer, size: Size::from_bytes(rem_bytes) }),
91+
));
7492
}
7593

76-
args
94+
res
7795
}
7896

7997
impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
@@ -104,7 +122,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
104122
},
105123
PassMode::Cast { ref cast, pad_i32 } => {
106124
assert!(!pad_i32, "padding support not yet implemented");
107-
cast_target_to_abi_params(cast)
125+
cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
108126
}
109127
PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
110128
if on_stack {
@@ -160,9 +178,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
160178
}
161179
_ => unreachable!("{:?}", self.layout.backend_repr),
162180
},
163-
PassMode::Cast { ref cast, .. } => {
164-
(None, cast_target_to_abi_params(cast).into_iter().collect())
165-
}
181+
PassMode::Cast { ref cast, .. } => (
182+
None,
183+
cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(),
184+
),
166185
PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
167186
assert!(!on_stack);
168187
(
@@ -187,12 +206,14 @@ pub(super) fn to_casted_value<'tcx>(
187206
) -> SmallVec<[Value; 2]> {
188207
let (ptr, meta) = arg.force_stack(fx);
189208
assert!(meta.is_none());
190-
let mut offset = 0;
191209
cast_target_to_abi_params(cast)
192210
.into_iter()
193-
.map(|param| {
194-
let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new());
195-
offset += i64::from(param.value_type.bytes());
211+
.map(|(offset, param)| {
212+
let val = ptr.offset_i64(fx, offset.bytes() as i64).load(
213+
fx,
214+
param.value_type,
215+
MemFlags::new(),
216+
);
196217
val
197218
})
198219
.collect()
@@ -205,7 +226,7 @@ pub(super) fn from_casted_value<'tcx>(
205226
cast: &CastTarget,
206227
) -> CValue<'tcx> {
207228
let abi_params = cast_target_to_abi_params(cast);
208-
let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
229+
let abi_param_size: u32 = abi_params.iter().map(|(_, param)| param.value_type.bytes()).sum();
209230
let layout_size = u32::try_from(layout.size.bytes()).unwrap();
210231
let ptr = fx.create_stack_slot(
211232
// Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`.
@@ -214,16 +235,13 @@ pub(super) fn from_casted_value<'tcx>(
214235
std::cmp::max(abi_param_size, layout_size),
215236
u32::try_from(layout.align.abi.bytes()).unwrap(),
216237
);
217-
let mut offset = 0;
218238
let mut block_params_iter = block_params.iter().copied();
219-
for param in abi_params {
220-
let val = ptr.offset_i64(fx, offset).store(
239+
for (offset, _) in abi_params {
240+
ptr.offset_i64(fx, offset.bytes() as i64).store(
221241
fx,
222242
block_params_iter.next().unwrap(),
223243
MemFlags::new(),
224-
);
225-
offset += i64::from(param.value_type.bytes());
226-
val
244+
)
227245
}
228246
assert_eq!(block_params_iter.next(), None, "Leftover block param");
229247
CValue::by_ref(ptr, layout)

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -626,7 +626,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
626626
bx.lifetime_start(llscratch, scratch_size);
627627

628628
// ... where we first store the value...
629-
bx.store(val, llscratch, scratch_align);
629+
rustc_codegen_ssa::mir::store_cast(bx, cast, val, llscratch, scratch_align);
630630

631631
// ... and then memcpy it to the intended destination.
632632
bx.memcpy(

‎compiler/rustc_codegen_llvm/src/abi.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
229229
let llscratch = bx.alloca(scratch_size, scratch_align);
230230
bx.lifetime_start(llscratch, scratch_size);
231231
// ...store the value...
232-
bx.store(val, llscratch, scratch_align);
232+
rustc_codegen_ssa::mir::store_cast(bx, cast, val, llscratch, scratch_align);
233233
// ... and then memcpy it to the intended destination.
234234
bx.memcpy(
235235
dst.val.llval,

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::cmp;
22

3-
use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, Size, WrappingRange};
3+
use rustc_abi::{Align, BackendRepr, ExternAbi, HasDataLayout, Reg, Size, WrappingRange};
44
use rustc_ast as ast;
55
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
66
use rustc_data_structures::packed::Pu128;
@@ -13,7 +13,7 @@ use rustc_middle::{bug, span_bug};
1313
use rustc_session::config::OptLevel;
1414
use rustc_span::Span;
1515
use rustc_span::source_map::Spanned;
16-
use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
16+
use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, PassMode};
1717
use tracing::{debug, info};
1818

1919
use super::operand::OperandRef;
@@ -558,8 +558,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
558558
}
559559
ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
560560
};
561-
let ty = bx.cast_backend_type(cast_ty);
562-
bx.load(ty, llslot, self.fn_abi.ret.layout.align.abi)
561+
load_cast(bx, cast_ty, llslot, self.fn_abi.ret.layout.align.abi)
563562
}
564563
};
565564
bx.ret(llval);
@@ -1618,8 +1617,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
16181617
MemFlags::empty(),
16191618
);
16201619
// ...and then load it with the ABI type.
1621-
let cast_ty = bx.cast_backend_type(cast);
1622-
llval = bx.load(cast_ty, llscratch, scratch_align);
1620+
llval = load_cast(bx, cast, llscratch, scratch_align);
16231621
bx.lifetime_end(llscratch, scratch_size);
16241622
} else {
16251623
// We can't use `PlaceRef::load` here because the argument
@@ -1969,3 +1967,47 @@ enum ReturnDest<'tcx, V> {
19691967
/// Store a direct return value to an operand local place.
19701968
DirectOperand(mir::Local),
19711969
}
1970+
1971+
fn load_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
1972+
bx: &mut Bx,
1973+
cast: &CastTarget,
1974+
ptr: Bx::Value,
1975+
align: Align,
1976+
) -> Bx::Value {
1977+
let cast_ty = bx.cast_backend_type(cast);
1978+
if let Some(offset_from_start) = cast.rest_offset {
1979+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
1980+
assert_eq!(cast.rest.unit.size, cast.rest.total);
1981+
let first_ty = bx.reg_backend_type(&cast.prefix[0].unwrap());
1982+
let second_ty = bx.reg_backend_type(&cast.rest.unit);
1983+
let first = bx.load(first_ty, ptr, align);
1984+
let second_ptr = bx.inbounds_ptradd(ptr, bx.const_usize(offset_from_start.bytes()));
1985+
let second = bx.load(second_ty, second_ptr, align.restrict_for_offset(offset_from_start));
1986+
let res = bx.cx().const_poison(cast_ty);
1987+
let res = bx.insert_value(res, first, 0);
1988+
bx.insert_value(res, second, 1)
1989+
} else {
1990+
bx.load(cast_ty, ptr, align)
1991+
}
1992+
}
1993+
1994+
pub fn store_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
1995+
bx: &mut Bx,
1996+
cast: &CastTarget,
1997+
value: Bx::Value,
1998+
ptr: Bx::Value,
1999+
align: Align,
2000+
) {
2001+
if let Some(offset_from_start) = cast.rest_offset {
2002+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
2003+
assert_eq!(cast.rest.unit.size, cast.rest.total);
2004+
assert!(cast.prefix[0].is_some());
2005+
let first = bx.extract_value(value, 0);
2006+
let second = bx.extract_value(value, 1);
2007+
bx.store(first, ptr, align);
2008+
let second_ptr = bx.inbounds_ptradd(ptr, bx.const_usize(offset_from_start.bytes()));
2009+
bx.store(second, second_ptr, align.restrict_for_offset(offset_from_start));
2010+
} else {
2011+
bx.store(value, ptr, align);
2012+
};
2013+
}

‎compiler/rustc_codegen_ssa/src/mir/mod.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub mod place;
2626
mod rvalue;
2727
mod statement;
2828

29+
pub use self::block::store_cast;
2930
use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo};
3031
use self::operand::{OperandRef, OperandValue};
3132
use self::place::PlaceRef;
@@ -259,7 +260,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
259260
}
260261
PassMode::Cast { ref cast, .. } => {
261262
debug!("alloc: {:?} (return place) -> place", local);
262-
let size = cast.size(&start_bx);
263+
let size = cast.size(&start_bx).max(layout.size);
263264
return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));
264265
}
265266
_ => {}

‎compiler/rustc_target/src/callconv/mips64.rs

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@ use rustc_abi::{
22
BackendRepr, FieldsShape, Float, HasDataLayout, Primitive, Reg, Size, TyAbiInterface,
33
};
44

5-
use crate::callconv::{
6-
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode, Uniform,
7-
};
5+
use crate::callconv::{ArgAbi, ArgExtension, CastTarget, FnAbi, PassMode, Uniform};
86

97
fn extend_integer_width_mips<Ty>(arg: &mut ArgAbi<'_, Ty>, bits: u64) {
108
// Always sign extend u32 values on 64-bit mips
@@ -140,16 +138,7 @@ where
140138

141139
// Extract first 8 chunks as the prefix
142140
let rest_size = size - Size::from_bytes(8) * prefix_index as u64;
143-
arg.cast_to(CastTarget {
144-
prefix,
145-
rest: Uniform::new(Reg::i64(), rest_size),
146-
attrs: ArgAttributes {
147-
regular: ArgAttribute::default(),
148-
arg_ext: ArgExtension::None,
149-
pointee_size: Size::ZERO,
150-
pointee_align: None,
151-
},
152-
});
141+
arg.cast_to(CastTarget::prefixed(prefix, Uniform::new(Reg::i64(), rest_size)));
153142
}
154143

155144
pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)

0 commit comments

Comments
 (0)