Skip to content

Commit ddd3d95

Browse files
bjorn3bjorn3
committed
Make atomic instructions atomic using a global lock
1 parent a3f3787 commit ddd3d95

File tree

6 files changed

+147
-2
lines changed

6 files changed

+147
-2
lines changed

‎config.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,12 @@ fi
1313
TARGET_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
1414

1515
export RUSTFLAGS='-Cpanic=abort -Cdebuginfo=2 -Zpanic-abort-tests -Zcodegen-backend='$(pwd)'/target/'$CHANNEL'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$(pwd)'/build_sysroot/sysroot'
16+
17+
# FIXME remove once the atomic shim is gone
18+
if [[ `uname` == 'Darwin' ]]; then
19+
export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
20+
fi
21+
1622
RUSTC="rustc $RUSTFLAGS -L crate=target/out --out-dir target/out"
1723
export RUSTC_LOG=warn # display metadata load errors
1824

‎src/atomic_shim.rs

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
//! Atomic intrinsics are implemented using a global lock for now, as Cranelift doesn't support
2+
//! atomic operations yet.
3+
4+
// FIXME implement atomic instructions in Cranelift.
5+
6+
use crate::prelude::*;
7+
8+
#[no_mangle]
9+
pub static mut __cg_clif_global_atomic_mutex: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER;
10+
11+
pub fn init_global_lock(module: &mut Module<impl Backend>, bcx: &mut FunctionBuilder<'_>) {
12+
if std::env::var("SHOULD_RUN").is_ok () {
13+
// When using JIT, dylibs won't find the __cg_clif_global_atomic_mutex data object defined here,
14+
// so instead define it in the cg_clif dylib.
15+
16+
return;
17+
}
18+
19+
let mut data_ctx = DataContext::new();
20+
data_ctx.define_zeroinit(1024); // 1024 bytes should be big enough on all platforms.
21+
let atomic_mutex = module.declare_data(
22+
"__cg_clif_global_atomic_mutex",
23+
Linkage::Export,
24+
true,
25+
Some(16),
26+
).unwrap();
27+
module.define_data(atomic_mutex, &data_ctx).unwrap();
28+
29+
let pthread_mutex_init = module.declare_function("pthread_mutex_init", Linkage::Import, &cranelift_codegen::ir::Signature {
30+
call_conv: module.target_config().default_call_conv,
31+
params: vec![
32+
AbiParam::new(module.target_config().pointer_type() /* *mut pthread_mutex_t */),
33+
AbiParam::new(module.target_config().pointer_type() /* *const pthread_mutex_attr_t */),
34+
],
35+
returns: vec![AbiParam::new(types::I32 /* c_int */)],
36+
}).unwrap();
37+
38+
let pthread_mutex_init = module.declare_func_in_func(pthread_mutex_init, bcx.func);
39+
40+
let atomic_mutex = module.declare_data_in_func(atomic_mutex, bcx.func);
41+
let atomic_mutex = bcx.ins().global_value(module.target_config().pointer_type(), atomic_mutex);
42+
43+
let nullptr = bcx.ins().iconst(module.target_config().pointer_type(), 0);
44+
45+
bcx.ins().call(pthread_mutex_init, &[atomic_mutex, nullptr]);
46+
}
47+
48+
pub fn lock_global_lock(fx: &mut FunctionCx<'_, '_, impl Backend>) {
49+
let atomic_mutex = fx.module.declare_data(
50+
"__cg_clif_global_atomic_mutex",
51+
Linkage::Import,
52+
true,
53+
None,
54+
).unwrap();
55+
56+
let pthread_mutex_lock = fx.module.declare_function("pthread_mutex_lock", Linkage::Import, &cranelift_codegen::ir::Signature {
57+
call_conv: fx.module.target_config().default_call_conv,
58+
params: vec![
59+
AbiParam::new(fx.module.target_config().pointer_type() /* *mut pthread_mutex_t */),
60+
],
61+
returns: vec![AbiParam::new(types::I32 /* c_int */)],
62+
}).unwrap();
63+
64+
let pthread_mutex_lock = fx.module.declare_func_in_func(pthread_mutex_lock, fx.bcx.func);
65+
66+
let atomic_mutex = fx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
67+
let atomic_mutex = fx.bcx.ins().global_value(fx.module.target_config().pointer_type(), atomic_mutex);
68+
69+
fx.bcx.ins().call(pthread_mutex_lock, &[atomic_mutex]);
70+
}
71+
72+
pub fn unlock_global_lock(fx: &mut FunctionCx<'_, '_, impl Backend>) {
73+
let atomic_mutex = fx.module.declare_data(
74+
"__cg_clif_global_atomic_mutex",
75+
Linkage::Import,
76+
true,
77+
None,
78+
).unwrap();
79+
80+
let pthread_mutex_unlock = fx.module.declare_function("pthread_mutex_unlock", Linkage::Import, &cranelift_codegen::ir::Signature {
81+
call_conv: fx.module.target_config().default_call_conv,
82+
params: vec![
83+
AbiParam::new(fx.module.target_config().pointer_type() /* *mut pthread_mutex_t */),
84+
],
85+
returns: vec![AbiParam::new(types::I32 /* c_int */)],
86+
}).unwrap();
87+
88+
let pthread_mutex_unlock = fx.module.declare_func_in_func(pthread_mutex_unlock, fx.bcx.func);
89+
90+
let atomic_mutex = fx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
91+
let atomic_mutex = fx.bcx.ins().global_value(fx.module.target_config().pointer_type(), atomic_mutex);
92+
93+
fx.bcx.ins().call(pthread_mutex_unlock, &[atomic_mutex]);
94+
}

‎src/driver.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@ pub fn codegen_crate(
3636
fn run_jit(tcx: TyCtxt<'_>) -> ! {
3737
use cranelift_simplejit::{SimpleJITBackend, SimpleJITBuilder};
3838

39+
// Rustc opens us without the RTLD_GLOBAL flag, so __cg_clif_global_atomic_mutex will not be
40+
// exported. We fix this by opening ourself again as global.
41+
// FIXME remove once atomic_shim is gone
42+
let cg_dylib = std::ffi::OsString::from(&tcx.sess.opts.debugging_opts.codegen_backend.as_ref().unwrap());
43+
std::mem::forget(libloading::os::unix::Library::open(Some(cg_dylib), libc::RTLD_NOW | libc::RTLD_GLOBAL).unwrap());
44+
45+
3946
let imported_symbols = load_imported_symbols_for_jit(tcx);
4047

4148
let mut jit_builder = SimpleJITBuilder::with_isa(

‎src/intrinsics/mod.rs

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,14 +104,20 @@ macro call_intrinsic_match {
104104
}
105105

106106
macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
107+
crate::atomic_shim::lock_global_lock($fx);
108+
107109
let clif_ty = $fx.clif_type($T).unwrap();
108110
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
109111
let new = $fx.bcx.ins().$op(old, $src);
110112
$fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
111113
$ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
114+
115+
crate::atomic_shim::unlock_global_lock($fx);
112116
}
113117

114118
macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
119+
crate::atomic_shim::lock_global_lock($fx);
120+
115121
// Read old
116122
let clif_ty = $fx.clif_type($T).unwrap();
117123
let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
@@ -125,6 +131,8 @@ macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $
125131

126132
let ret_val = CValue::by_val(old, $ret.layout());
127133
$ret.write_cvalue($fx, ret_val);
134+
135+
crate::atomic_shim::unlock_global_lock($fx);
128136
}
129137

130138
fn lane_type_and_count<'tcx>(
@@ -845,19 +853,35 @@ pub fn codegen_intrinsic_call<'tcx>(
845853
ret.write_cvalue(fx, caller_location);
846854
};
847855

848-
_ if intrinsic.starts_with("atomic_fence"), () {};
849-
_ if intrinsic.starts_with("atomic_singlethreadfence"), () {};
856+
_ if intrinsic.starts_with("atomic_fence"), () {
857+
crate::atomic_shim::lock_global_lock(fx);
858+
crate::atomic_shim::unlock_global_lock(fx);
859+
};
860+
_ if intrinsic.starts_with("atomic_singlethreadfence"), () {
861+
crate::atomic_shim::lock_global_lock(fx);
862+
crate::atomic_shim::unlock_global_lock(fx);
863+
};
850864
_ if intrinsic.starts_with("atomic_load"), (c ptr) {
865+
crate::atomic_shim::lock_global_lock(fx);
866+
851867
let inner_layout =
852868
fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
853869
let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
854870
ret.write_cvalue(fx, val);
871+
872+
crate::atomic_shim::unlock_global_lock(fx);
855873
};
856874
_ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
875+
crate::atomic_shim::lock_global_lock(fx);
876+
857877
let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
858878
dest.write_cvalue(fx, val);
879+
880+
crate::atomic_shim::unlock_global_lock(fx);
859881
};
860882
_ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
883+
crate::atomic_shim::lock_global_lock(fx);
884+
861885
// Read old
862886
let clif_ty = fx.clif_type(T).unwrap();
863887
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
@@ -866,8 +890,12 @@ pub fn codegen_intrinsic_call<'tcx>(
866890
// Write new
867891
let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
868892
dest.write_cvalue(fx, src);
893+
894+
crate::atomic_shim::unlock_global_lock(fx);
869895
};
870896
_ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, v test_old, v new) { // both atomic_cxchg_* and atomic_cxchgweak_*
897+
crate::atomic_shim::lock_global_lock(fx);
898+
871899
// Read old
872900
let clif_ty = fx.clif_type(T).unwrap();
873901
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
@@ -881,6 +909,8 @@ pub fn codegen_intrinsic_call<'tcx>(
881909

882910
let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
883911
ret.write_cvalue(fx, ret_val);
912+
913+
crate::atomic_shim::unlock_global_lock(fx);
884914
};
885915

886916
_ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, v amount) {
@@ -893,12 +923,16 @@ pub fn codegen_intrinsic_call<'tcx>(
893923
atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
894924
};
895925
_ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, v src) {
926+
crate::atomic_shim::lock_global_lock(fx);
927+
896928
let clif_ty = fx.clif_type(T).unwrap();
897929
let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
898930
let and = fx.bcx.ins().band(old, src);
899931
let new = fx.bcx.ins().bnot(and);
900932
fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
901933
ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
934+
935+
crate::atomic_shim::unlock_global_lock(fx);
902936
};
903937
_ if intrinsic.starts_with("atomic_or"), <T> (v ptr, v src) {
904938
atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);

‎src/lib.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#![allow(intra_doc_link_resolution_failure)]
33

44
extern crate flate2;
5+
extern crate libc;
56
extern crate tempfile;
67
extern crate rustc;
78
extern crate rustc_codegen_ssa;
@@ -36,6 +37,7 @@ mod abi;
3637
mod allocator;
3738
mod analyze;
3839
mod archive;
40+
mod atomic_shim;
3941
mod base;
4042
mod backend;
4143
mod cast;

‎src/main_shim.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ pub fn maybe_create_entry_wrapper(tcx: TyCtxt<'_>, module: &mut Module<impl Back
6767
let arg_argc = bcx.append_ebb_param(ebb, m.target_config().pointer_type());
6868
let arg_argv = bcx.append_ebb_param(ebb, m.target_config().pointer_type());
6969

70+
crate::atomic_shim::init_global_lock(m, &mut bcx);
71+
7072
let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
7173

7274
let call_inst = if use_start_lang_item {

0 commit comments

Comments
 (0)