From 4923e856be61bfb8faac18eaf1f2b0d8e70ea3be Mon Sep 17 00:00:00 2001 From: Zalathar Date: Fri, 25 Oct 2024 13:06:57 +1100 Subject: [PATCH 1/3] coverage: Emit `llvm.instrprof.increment` using the normal helper method --- compiler/rustc_codegen_llvm/src/builder.rs | 26 ++----------------- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 - .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 11 -------- 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index dbf5298d64ba3..8e718226a9a3f 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1165,6 +1165,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); } + #[instrument(level = "debug", skip(self))] fn instrprof_increment( &mut self, fn_name: &'ll Value, @@ -1172,30 +1173,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { num_counters: &'ll Value, index: &'ll Value, ) { - debug!( - "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", - fn_name, hash, num_counters, index - ); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], - self.cx.type_void(), - ); - let args = &[fn_name, hash, num_counters, index]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]); } fn call( diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 50e6c1494a8d6..a52004bacd929 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1615,7 +1615,6 @@ unsafe extern "C" { pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions - pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value; pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value; diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 910c27da95463..ed9db20593325 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1531,17 +1531,6 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, ArrayRef(OpBundles))); } -extern "C" LLVMValueRef -LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { -#if LLVM_VERSION_GE(20, 0) - return wrap(llvm::Intrinsic::getOrInsertDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_increment)); -#else - return wrap(llvm::Intrinsic::getDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_increment)); -#endif -} - extern "C" LLVMValueRef LLVMRustGetInstrProfMCDCParametersIntrinsic(LLVMModuleRef M) { #if LLVM_VERSION_LT(19, 0) From b3d65852c3addf4e8e6be96965ead86c2d4fb8be Mon Sep 17 00:00:00 2001 From: Zalathar Date: Fri, 25 Oct 2024 13:25:19 +1100 Subject: [PATCH 2/3] coverage: Emit MC/DC intrinsics using the normal helper method --- compiler/rustc_codegen_llvm/src/builder.rs | 55 +++---------------- compiler/rustc_codegen_llvm/src/context.rs | 4 ++ .../src/coverageinfo/mod.rs | 1 + compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 3 - .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 28 ---------- 5 files changed, 14 insertions(+), 77 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 8e718226a9a3f..f4463e037c945 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1654,40 +1654,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { /// /// [`CodeGenPGO::emitMCDCParameters`]: /// https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124 + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_parameters( &mut self, fn_name: &'ll Value, hash: &'ll Value, bitmap_bits: &'ll Value, ) { - debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bits); - assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" ); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], - self.cx.type_void(), - ); - let args = &[fn_name, hash, bitmap_bits]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]); } + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_tvbitmap_update( &mut self, fn_name: &'ll Value, @@ -1695,39 +1676,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { bitmap_index: &'ll Value, mcdc_temp: &'ll Value, ) { - debug!( - "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?})", - fn_name, hash, bitmap_index, mcdc_temp - ); assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" ); - - let llfn = - unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_ptr()], - self.cx.type_void(), - ); let args = &[fn_name, hash, bitmap_index, mcdc_temp]; - let args = self.check_call("call", llty, llfn, args); - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args); + } + + #[instrument(level = "debug", skip(self))] + pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) { self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); } + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) { - debug!("mcdc_condbitmap_update() with args ({:?}, {:?})", cond_index, mcdc_temp); assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 2f830d6f941e7..fb845c0087b1f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -1099,6 +1099,10 @@ impl<'ll> CodegenCx<'ll, '_> { if self.sess().instrument_coverage() { ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void); + if crate::llvm_util::get_version() >= (19, 0, 0) { + ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void); + ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void); + } } ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index b2956945911b1..36b1747f2fdc8 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -208,6 +208,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let hash = bx.const_u64(function_coverage_info.function_source_hash); let bitmap_index = bx.const_u32(bitmap_idx); bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap); + bx.mcdc_condbitmap_reset(cond_bitmap); } } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index a52004bacd929..10e55a4f7f659 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1615,9 +1615,6 @@ unsafe extern "C" { pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions - pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustBuildCall<'a>( B: &Builder<'a>, Ty: &'a Type, diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index ed9db20593325..cb75888abd76d 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1531,34 +1531,6 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, ArrayRef(OpBundles))); } -extern "C" LLVMValueRef -LLVMRustGetInstrProfMCDCParametersIntrinsic(LLVMModuleRef M) { -#if LLVM_VERSION_LT(19, 0) - report_fatal_error("LLVM 19.0 is required for mcdc intrinsic functions"); -#endif -#if LLVM_VERSION_GE(20, 0) - return wrap(llvm::Intrinsic::getOrInsertDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters)); -#else - return wrap(llvm::Intrinsic::getDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters)); -#endif -} - -extern "C" LLVMValueRef -LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(LLVMModuleRef M) { -#if LLVM_VERSION_LT(19, 0) - report_fatal_error("LLVM 19.0 is required for mcdc intrinsic functions"); -#endif -#if LLVM_VERSION_GE(20, 0) - return wrap(llvm::Intrinsic::getOrInsertDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update)); -#else - return wrap(llvm::Intrinsic::getDeclaration( - unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update)); -#endif -} - extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign, LLVMValueRef Src, unsigned SrcAlign, From 8f075145200aef04b36f2e2239f09b796c6ac8b8 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Fri, 25 Oct 2024 13:57:46 +1100 Subject: [PATCH 3/3] coverage: SSA doesn't need to know about `instrprof_increment` --- compiler/rustc_codegen_gcc/src/builder.rs | 10 -------- compiler/rustc_codegen_llvm/src/builder.rs | 23 ++++++++++--------- .../rustc_codegen_ssa/src/traits/builder.rs | 8 ------- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 457380685093f..7c52cba096b40 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -1725,16 +1725,6 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn fptosi_sat(&mut self, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> { self.fptoint_sat(true, val, dest_ty) } - - fn instrprof_increment( - &mut self, - _fn_name: RValue<'gcc>, - _hash: RValue<'gcc>, - _num_counters: RValue<'gcc>, - _index: RValue<'gcc>, - ) { - unimplemented!(); - } } impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index f4463e037c945..8702532c36eee 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1165,17 +1165,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); } - #[instrument(level = "debug", skip(self))] - fn instrprof_increment( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - num_counters: &'ll Value, - index: &'ll Value, - ) { - self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]); - } - fn call( &mut self, llty: &'ll Type, @@ -1645,6 +1634,18 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { kcfi_bundle } + /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation. + #[instrument(level = "debug", skip(self))] + pub(crate) fn instrprof_increment( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + num_counters: &'ll Value, + index: &'ll Value, + ) { + self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]); + } + /// Emits a call to `llvm.instrprof.mcdc.parameters`. /// /// This doesn't produce any code directly, but is used as input by diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index c0c1085e949e2..50a5171414695 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -437,14 +437,6 @@ pub trait BuilderMethods<'a, 'tcx>: /// Called for `StorageDead` fn lifetime_end(&mut self, ptr: Self::Value, size: Size); - fn instrprof_increment( - &mut self, - fn_name: Self::Value, - hash: Self::Value, - num_counters: Self::Value, - index: Self::Value, - ); - fn call( &mut self, llty: Self::Type,