Skip to content

Commit 65698ae

Browse files
rcvallebjorn3
rcvalle
and
bjorn3
committed
Add LLVM KCFI support to the Rust compiler
This commit adds LLVM Kernel Control Flow Integrity (KCFI) support to the Rust compiler. It initially provides forward-edge control flow protection for operating systems kernels for Rust-compiled code only by aggregating function pointers in groups identified by their return and parameter types. (See llvm/llvm-project@cff5bef.) Forward-edge control flow protection for C or C++ and Rust -compiled code "mixed binaries" (i.e., for when C or C++ and Rust -compiled code share the same virtual address space) will be provided in later work as part of this project by identifying C char and integer type uses at the time types are encoded (see Type metadata in the design document in the tracking issue #89653). LLVM KCFI can be enabled with -Zsanitizer=kcfi. Co-authored-by: bjorn3 <17426603+bjorn3@users.noreply.github.com>
1 parent b7bc90f commit 65698ae

File tree

26 files changed

+231
-28
lines changed

26 files changed

+231
-28
lines changed

‎Cargo.lock

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4269,6 +4269,7 @@ dependencies = [
42694269
"rustc_span",
42704270
"rustc_target",
42714271
"tracing",
4272+
"twox-hash",
42724273
]
42734274

42744275
[[package]]
@@ -5197,6 +5198,17 @@ dependencies = [
51975198
"tracing-subscriber",
51985199
]
51995200

5201+
[[package]]
5202+
name = "twox-hash"
5203+
version = "1.6.3"
5204+
source = "registry+https://github.com/rust-lang/crates.io-index"
5205+
checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
5206+
dependencies = [
5207+
"cfg-if 1.0.0",
5208+
"rand 0.8.5",
5209+
"static_assertions",
5210+
]
5211+
52005212
[[package]]
52015213
name = "type-map"
52025214
version = "0.4.0"

‎compiler/rustc_codegen_gcc/src/type_.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,4 +300,8 @@ impl<'gcc, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
300300
// Unsupported.
301301
self.context.new_rvalue_from_int(self.int_type, 0)
302302
}
303+
304+
fn set_kcfi_type_metadata(&self, _function: RValue<'gcc>, _kcfi_typeid: u32) {
305+
// Unsupported.
306+
}
303307
}

‎compiler/rustc_codegen_llvm/src/allocator.rs

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ pub(crate) unsafe fn codegen(
8888
callee,
8989
args.as_ptr(),
9090
args.len() as c_uint,
91-
None,
91+
[].as_ptr(),
92+
0 as c_uint,
9293
);
9394
llvm::LLVMSetTailCall(ret, True);
9495
if output.is_some() {
@@ -132,8 +133,15 @@ pub(crate) unsafe fn codegen(
132133
.enumerate()
133134
.map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint))
134135
.collect::<Vec<_>>();
135-
let ret =
136-
llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None);
136+
let ret = llvm::LLVMRustBuildCall(
137+
llbuilder,
138+
ty,
139+
callee,
140+
args.as_ptr(),
141+
args.len() as c_uint,
142+
[].as_ptr(),
143+
0 as c_uint,
144+
);
137145
llvm::LLVMSetTailCall(ret, True);
138146
llvm::LLVMBuildRetVoid(llbuilder);
139147
llvm::LLVMDisposeBuilder(llbuilder);

‎compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use rustc_middle::ty::layout::{
2020
};
2121
use rustc_middle::ty::{self, Ty, TyCtxt};
2222
use rustc_span::Span;
23+
use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi;
2324
use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange};
2425
use rustc_target::spec::{HasTargetSpec, Target};
2526
use std::borrow::Cow;
@@ -225,9 +226,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
225226
debug!("invoke {:?} with args ({:?})", llfn, args);
226227

227228
let args = self.check_call("invoke", llty, llfn, args);
228-
let bundle = funclet.map(|funclet| funclet.bundle());
229-
let bundle = bundle.as_ref().map(|b| &*b.raw);
229+
let funclet_bundle = funclet.map(|funclet| funclet.bundle());
230+
let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
231+
let mut bundles = vec![funclet_bundle];
232+
233+
// Set KCFI operand bundle
234+
let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
235+
let kcfi_bundle =
236+
if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call {
237+
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap());
238+
Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
239+
} else {
240+
None
241+
};
242+
if kcfi_bundle.is_some() {
243+
let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
244+
bundles.push(kcfi_bundle);
245+
}
230246

247+
bundles.retain(|bundle| bundle.is_some());
231248
let invoke = unsafe {
232249
llvm::LLVMRustBuildInvoke(
233250
self.llbuilder,
@@ -237,7 +254,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
237254
args.len() as c_uint,
238255
then,
239256
catch,
240-
bundle,
257+
bundles.as_ptr(),
258+
bundles.len() as c_uint,
241259
UNNAMED,
242260
)
243261
};
@@ -1143,7 +1161,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
11431161
llfn,
11441162
args.as_ptr() as *const &llvm::Value,
11451163
args.len() as c_uint,
1146-
None,
1164+
[].as_ptr(),
1165+
0 as c_uint,
11471166
);
11481167
}
11491168
}
@@ -1159,17 +1178,34 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
11591178
debug!("call {:?} with args ({:?})", llfn, args);
11601179

11611180
let args = self.check_call("call", llty, llfn, args);
1162-
let bundle = funclet.map(|funclet| funclet.bundle());
1163-
let bundle = bundle.as_ref().map(|b| &*b.raw);
1181+
let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1182+
let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
1183+
let mut bundles = vec![funclet_bundle];
1184+
1185+
// Set KCFI operand bundle
1186+
let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
1187+
let kcfi_bundle =
1188+
if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call {
1189+
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap());
1190+
Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1191+
} else {
1192+
None
1193+
};
1194+
if kcfi_bundle.is_some() {
1195+
let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
1196+
bundles.push(kcfi_bundle);
1197+
}
11641198

1199+
bundles.retain(|bundle| bundle.is_some());
11651200
let call = unsafe {
11661201
llvm::LLVMRustBuildCall(
11671202
self.llbuilder,
11681203
llty,
11691204
llfn,
11701205
args.as_ptr() as *const &llvm::Value,
11711206
args.len() as c_uint,
1172-
bundle,
1207+
bundles.as_ptr(),
1208+
bundles.len() as c_uint,
11731209
)
11741210
};
11751211
if let Some(fn_abi) = fn_abi {

‎compiler/rustc_codegen_llvm/src/context.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,11 @@ pub unsafe fn create_module<'ll>(
250250
);
251251
}
252252

253+
if sess.is_sanitizer_kcfi_enabled() {
254+
let kcfi = "kcfi\0".as_ptr().cast();
255+
llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1);
256+
}
257+
253258
// Control Flow Guard is currently only supported by the MSVC linker on Windows.
254259
if sess.target.is_like_msvc {
255260
match sess.opts.cg.control_flow_guard {

‎compiler/rustc_codegen_llvm/src/declare.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use crate::type_::Type;
2020
use crate::value::Value;
2121
use rustc_codegen_ssa::traits::TypeMembershipMethods;
2222
use rustc_middle::ty::Ty;
23-
use rustc_symbol_mangling::typeid::typeid_for_fnabi;
23+
use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi};
2424
use smallvec::SmallVec;
2525

2626
/// Declare a function.
@@ -136,6 +136,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
136136
self.set_type_metadata(llfn, typeid);
137137
}
138138

139+
if self.tcx.sess.is_sanitizer_kcfi_enabled() {
140+
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi);
141+
self.set_kcfi_type_metadata(llfn, kcfi_typeid);
142+
}
143+
139144
llfn
140145
}
141146

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ pub enum MetadataType {
427427
MD_type = 19,
428428
MD_vcall_visibility = 28,
429429
MD_noundef = 29,
430+
MD_kcfi_type = 36,
430431
}
431432

432433
/// LLVMRustAsmDialect
@@ -1060,6 +1061,7 @@ extern "C" {
10601061
pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
10611062
pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
10621063
pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata;
1064+
pub fn LLVMIsAFunction(Val: &Value) -> Option<&Value>;
10631065

10641066
// Operations on constants of any type
10651067
pub fn LLVMConstNull(Ty: &Type) -> &Value;
@@ -1270,7 +1272,8 @@ extern "C" {
12701272
NumArgs: c_uint,
12711273
Then: &'a BasicBlock,
12721274
Catch: &'a BasicBlock,
1273-
Bundle: Option<&OperandBundleDef<'a>>,
1275+
OpBundles: *const Option<&OperandBundleDef<'a>>,
1276+
NumOpBundles: c_uint,
12741277
Name: *const c_char,
12751278
) -> &'a Value;
12761279
pub fn LLVMBuildLandingPad<'a>(
@@ -1640,7 +1643,8 @@ extern "C" {
16401643
Fn: &'a Value,
16411644
Args: *const &'a Value,
16421645
NumArgs: c_uint,
1643-
Bundle: Option<&OperandBundleDef<'a>>,
1646+
OpBundles: *const Option<&OperandBundleDef<'a>>,
1647+
NumOpBundles: c_uint,
16441648
) -> &'a Value;
16451649
pub fn LLVMRustBuildMemCpy<'a>(
16461650
B: &Builder<'a>,

‎compiler/rustc_codegen_llvm/src/type_.rs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,4 +316,19 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> {
316316
)
317317
}
318318
}
319+
320+
fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) {
321+
let kcfi_type_metadata = self.const_u32(kcfi_typeid);
322+
unsafe {
323+
llvm::LLVMGlobalSetMetadata(
324+
function,
325+
llvm::MD_kcfi_type as c_uint,
326+
llvm::LLVMMDNodeInContext2(
327+
self.llcx,
328+
&llvm::LLVMValueAsMetadata(kcfi_type_metadata),
329+
1,
330+
),
331+
)
332+
}
333+
}
319334
}

‎compiler/rustc_codegen_ssa/src/traits/type_.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> {
122122
pub trait TypeMembershipMethods<'tcx>: Backend<'tcx> {
123123
fn set_type_metadata(&self, function: Self::Function, typeid: String);
124124
fn typeid_metadata(&self, typeid: String) -> Self::Value;
125+
fn set_kcfi_type_metadata(&self, function: Self::Function, typeid: u32);
125126
}
126127

127128
pub trait ArgAbiMethods<'tcx>: HasCodegen<'tcx> {

‎compiler/rustc_feature/src/builtin_attrs.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
394394
ungated!(instruction_set, Normal, template!(List: "set"), ErrorPreceding),
395395
gated!(
396396
no_sanitize, Normal,
397-
template!(List: "address, memory, thread"), DuplicatesOk,
397+
template!(List: "address, kcfi, memory, thread"), DuplicatesOk,
398398
experimental!(no_sanitize)
399399
),
400400
gated!(no_coverage, Normal, template!(Word), WarnFollowing, experimental!(no_coverage)),

‎compiler/rustc_hir_analysis/src/collect.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1859,6 +1859,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
18591859
codegen_fn_attrs.no_sanitize |= SanitizerSet::ADDRESS;
18601860
} else if item.has_name(sym::cfi) {
18611861
codegen_fn_attrs.no_sanitize |= SanitizerSet::CFI;
1862+
} else if item.has_name(sym::kcfi) {
1863+
codegen_fn_attrs.no_sanitize |= SanitizerSet::KCFI;
18621864
} else if item.has_name(sym::memory) {
18631865
codegen_fn_attrs.no_sanitize |= SanitizerSet::MEMORY;
18641866
} else if item.has_name(sym::memtag) {
@@ -1872,7 +1874,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
18721874
} else {
18731875
tcx.sess
18741876
.struct_span_err(item.span(), "invalid argument for `no_sanitize`")
1875-
.note("expected one of: `address`, `cfi`, `hwaddress`, `memory`, `memtag`, `shadow-call-stack`, or `thread`")
1877+
.note("expected one of: `address`, `cfi`, `hwaddress`, `kcfi`, `memory`, `memtag`, `shadow-call-stack`, or `thread`")
18761878
.emit();
18771879
}
18781880
}

‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1460,13 +1460,13 @@ extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) {
14601460

14611461
extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
14621462
LLVMValueRef *Args, unsigned NumArgs,
1463-
OperandBundleDef *Bundle) {
1463+
OperandBundleDef **OpBundles,
1464+
unsigned NumOpBundles) {
14641465
Value *Callee = unwrap(Fn);
14651466
FunctionType *FTy = unwrap<FunctionType>(Ty);
1466-
unsigned Len = Bundle ? 1 : 0;
1467-
ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len);
14681467
return wrap(unwrap(B)->CreateCall(
1469-
FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles));
1468+
FTy, Callee, makeArrayRef(unwrap(Args), NumArgs),
1469+
makeArrayRef(*OpBundles, NumOpBundles)));
14701470
}
14711471

14721472
extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) {
@@ -1506,14 +1506,14 @@ extern "C" LLVMValueRef
15061506
LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
15071507
LLVMValueRef *Args, unsigned NumArgs,
15081508
LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
1509-
OperandBundleDef *Bundle, const char *Name) {
1509+
OperandBundleDef **OpBundles, unsigned NumOpBundles,
1510+
const char *Name) {
15101511
Value *Callee = unwrap(Fn);
15111512
FunctionType *FTy = unwrap<FunctionType>(Ty);
1512-
unsigned Len = Bundle ? 1 : 0;
1513-
ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len);
15141513
return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch),
15151514
makeArrayRef(unwrap(Args), NumArgs),
1516-
Bundles, Name));
1515+
makeArrayRef(*OpBundles, NumOpBundles),
1516+
Name));
15171517
}
15181518

15191519
extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B,

‎compiler/rustc_session/src/options.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ mod desc {
368368
pub const parse_opt_panic_strategy: &str = parse_panic_strategy;
369369
pub const parse_oom_strategy: &str = "either `panic` or `abort`";
370370
pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`";
371-
pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`";
371+
pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `kcfi`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`";
372372
pub const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2";
373373
pub const parse_cfguard: &str =
374374
"either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`";
@@ -675,6 +675,7 @@ mod parse {
675675
*slot |= match s {
676676
"address" => SanitizerSet::ADDRESS,
677677
"cfi" => SanitizerSet::CFI,
678+
"kcfi" => SanitizerSet::KCFI,
678679
"leak" => SanitizerSet::LEAK,
679680
"memory" => SanitizerSet::MEMORY,
680681
"memtag" => SanitizerSet::MEMTAG,

‎compiler/rustc_session/src/session.rs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -683,6 +683,10 @@ impl Session {
683683
self.opts.unstable_opts.sanitizer.contains(SanitizerSet::CFI)
684684
}
685685

686+
pub fn is_sanitizer_kcfi_enabled(&self) -> bool {
687+
self.opts.unstable_opts.sanitizer.contains(SanitizerSet::KCFI)
688+
}
689+
686690
/// Check whether this compile session and crate type use static crt.
687691
pub fn crt_static(&self, crate_type: Option<CrateType>) -> bool {
688692
if !self.target.crt_static_respected {
@@ -1530,6 +1534,14 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
15301534
}
15311535
}
15321536

1537+
// LLVM CFI and KCFI are mutually exclusive
1538+
if sess.is_sanitizer_cfi_enabled() && sess.is_sanitizer_kcfi_enabled() {
1539+
sess.emit_err(CannotMixAndMatchSanitizers {
1540+
first: "cfi".to_string(),
1541+
second: "kcfi".to_string(),
1542+
});
1543+
}
1544+
15331545
if sess.opts.unstable_opts.stack_protector != StackProtector::None {
15341546
if !sess.target.options.supports_stack_protector {
15351547
sess.emit_warning(StackProtectorNotSupportedForTarget {

‎compiler/rustc_span/src/symbol.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -828,6 +828,7 @@ symbols! {
828828
item_like_imports,
829829
iter,
830830
iter_repeat,
831+
kcfi,
831832
keyword,
832833
kind,
833834
kreg,

‎compiler/rustc_symbol_mangling/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ bitflags = "1.2.1"
1010
tracing = "0.1"
1111
punycode = "0.4.0"
1212
rustc-demangle = "0.1.21"
13+
twox-hash = "1.6.3"
1314

1415
rustc_span = { path = "../rustc_span" }
1516
rustc_middle = { path = "../rustc_middle" }

0 commit comments

Comments
 (0)