diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 4bb2a78d9..8c4af0eff 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -19,7 +19,7 @@ run() { -e CARGO_HOME=/cargo \ -e CARGO_TARGET_DIR=/target \ -e RUST_COMPILER_RT_ROOT \ - -v $(dirname $(dirname `which cargo`)):/cargo \ + -v "${HOME}/.cargo":/cargo \ -v `pwd`/target:/target \ -v `pwd`:/checkout:ro \ -v `rustc --print sysroot`:/rust:ro \ diff --git a/src/arm.rs b/src/arm.rs index 3660825aa..95bde5116 100644 --- a/src/arm.rs +++ b/src/arm.rs @@ -3,251 +3,181 @@ use core::intrinsics; -// NOTE This function and the ones below are implemented using assembly because they are using a -// custom calling convention which can't be implemented using a normal Rust function. -// NOTE The only difference between the iOS and non-iOS versions of those functions is that the iOS -// versions use 3 leading underscores in the names of called functions instead of 2. -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_uidivmod() { - core::arch::asm!( - "push {{lr}}", - "sub sp, sp, #4", - "mov r2, sp", - "bl __udivmodsi4", - "ldr r1, [sp]", - "add sp, sp, #4", - "pop {{pc}}", - options(noreturn) - ); -} - +// iOS symbols have a leading underscore. #[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_uidivmod() { - core::arch::asm!( - "push {{lr}}", - "sub sp, sp, #4", - "mov r2, sp", - "bl ___udivmodsi4", - "ldr r1, [sp]", - "add sp, sp, #4", - "pop {{pc}}", - options(noreturn) - ); +macro_rules! bl { + ($func:literal) => { + concat!("bl _", $func) + }; } - #[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_uldivmod() { - core::arch::asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - "bl __udivmoddi4", - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - options(noreturn) - ); -} - -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_uldivmod() { - core::arch::asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - "bl ___udivmoddi4", - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - options(noreturn) - ); -} - -#[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_idivmod() { - core::arch::asm!( - "push {{r0, r1, r4, lr}}", - "bl __aeabi_idiv", - "pop {{r1, r2}}", - "muls r2, r2, r0", - "subs r1, r1, r2", - "pop {{r4, pc}}", - options(noreturn) - ); -} - -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_idivmod() { - core::arch::asm!( - "push {{r0, r1, r4, lr}}", - "bl ___aeabi_idiv", - "pop {{r1, r2}}", - "muls r2, r2, r0", - "subs r1, r1, r2", - "pop {{r4, pc}}", - options(noreturn) - ); -} +macro_rules! bl { + ($func:literal) => { + concat!("bl ", $func) + }; +} + +intrinsics! { + // NOTE This function and the ones below are implemented using assembly because they are using a + // custom calling convention which can't be implemented using a normal Rust function. + #[naked] + #[cfg(not(target_env = "msvc"))] + pub unsafe extern "C" fn __aeabi_uidivmod() { + core::arch::asm!( + "push {{lr}}", + "sub sp, sp, #4", + "mov r2, sp", + bl!("__udivmodsi4"), + "ldr r1, [sp]", + "add sp, sp, #4", + "pop {{pc}}", + options(noreturn) + ); + } -#[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_ldivmod() { - core::arch::asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - "bl __divmoddi4", - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - options(noreturn) - ); -} + #[naked] + pub unsafe extern "C" fn __aeabi_uldivmod() { + core::arch::asm!( + "push {{r4, lr}}", + "sub sp, sp, #16", + "add r4, sp, #8", + "str r4, [sp]", + bl!("__udivmodsi4"), + "ldr r2, [sp, #8]", + "ldr r3, [sp, #12]", + "add sp, sp, #16", + "pop {{r4, pc}}", + options(noreturn) + ); + } -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_ldivmod() { - core::arch::asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - "bl ___divmoddi4", - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - options(noreturn) - ); -} + #[naked] + pub unsafe extern "C" fn __aeabi_idivmod() { + core::arch::asm!( + "push {{r0, r1, r4, lr}}", + bl!("__aeabi_idiv"), + "pop {{r1, r2}}", + "muls r2, r2, r0", + "subs r1, r1, r2", + "pop {{r4, pc}}", + options(noreturn) + ); + } -// The following functions use weak linkage to allow users to override -// with custom implementation. -// FIXME: The `*4` and `*8` variants should be defined as aliases. + #[naked] + pub unsafe extern "C" fn __aeabi_ldivmod() { + core::arch::asm!( + "push {{r4, lr}}", + "sub sp, sp, #16", + "add r4, sp, #8", + "str r4, [sp]", + bl!("__divmoddi4"), + "ldr r2, [sp, #8]", + "ldr r3, [sp, #12]", + "add sp, sp, #16", + "pop {{r4, pc}}", + options(noreturn) + ); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) { - ::mem::memcpy(dest, src, n); -} + // The following functions use weak linkage to allow users to override + // with custom implementation. + // FIXME: The `*4` and `*8` variants should be defined as aliases. -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, mut n: usize) { - // We are guaranteed 4-alignment, so accessing at u32 is okay. - let mut dest = dest as *mut u32; - let mut src = src as *mut u32; + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) { + ::mem::memcpy(dest, src, n); + } - while n >= 4 { - *dest = *src; - dest = dest.offset(1); - src = src.offset(1); - n -= 4; + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, n: usize) { + // We are guaranteed 4-alignment, so accessing at u32 is okay. + let mut dest = dest as *mut u32; + let mut src = src as *mut u32; + let mut n = n; + + while n >= 4 { + *dest = *src; + dest = dest.offset(1); + src = src.offset(1); + n -= 4; + } + + __aeabi_memcpy(dest as *mut u8, src as *const u8, n); } - __aeabi_memcpy(dest as *mut u8, src as *const u8, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memcpy4(dest, src, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memcpy4(dest, src, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) { + ::mem::memmove(dest, src, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) { - ::mem::memmove(dest, src, n); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memmove(dest, src, n); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memmove(dest, src, n); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memmove(dest, src, n); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memmove(dest, src, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) { + // Note the different argument order + ::mem::memset(dest, c, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) { - // Note the different argument order - ::mem::memset(dest, c, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, n: usize, c: i32) { + let mut dest = dest as *mut u32; + let mut n = n; -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, mut n: usize, c: i32) { - let mut dest = dest as *mut u32; + let byte = (c as u32) & 0xff; + let c = (byte << 24) | (byte << 16) | (byte << 8) | byte; - let byte = (c as u32) & 0xff; - let c = (byte << 24) | (byte << 16) | (byte << 8) | byte; + while n >= 4 { + *dest = c; + dest = dest.offset(1); + n -= 4; + } - while n >= 4 { - *dest = c; - dest = dest.offset(1); - n -= 4; + __aeabi_memset(dest as *mut u8, n, byte as i32); } - __aeabi_memset(dest as *mut u8, n, byte as i32); -} - -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) { - __aeabi_memset4(dest, n, c); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) { + __aeabi_memset4(dest, n, c); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) { - __aeabi_memset(dest, n, 0); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) { + __aeabi_memset(dest, n, 0); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) { - __aeabi_memset4(dest, n, 0); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) { + __aeabi_memset4(dest, n, 0); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) { - __aeabi_memset4(dest, n, 0); + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) { + __aeabi_memset4(dest, n, 0); + } } diff --git a/src/arm_linux.rs b/src/arm_linux.rs index e710c1ab9..df1723d99 100644 --- a/src/arm_linux.rs +++ b/src/arm_linux.rs @@ -90,17 +90,19 @@ unsafe fn atomic_cmpxchg(ptr: *mut T, oldval: u32, newval: u32) -> u32 { macro_rules! atomic_rmw { ($name:ident, $ty:ty, $op:expr) => { - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty { - atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty + intrinsics! { + pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty { + atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty + } } }; } macro_rules! atomic_cmpxchg { ($name:ident, $ty:ty) => { - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty { - atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty + intrinsics! { + pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty { + atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty + } } }; } @@ -205,7 +207,8 @@ atomic_cmpxchg!(__sync_val_compare_and_swap_1, u8); atomic_cmpxchg!(__sync_val_compare_and_swap_2, u16); atomic_cmpxchg!(__sync_val_compare_and_swap_4, u32); -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __sync_synchronize() { - __kuser_memory_barrier(); +intrinsics! { + pub unsafe extern "C" fn __sync_synchronize() { + __kuser_memory_barrier(); + } } diff --git a/src/macros.rs b/src/macros.rs index 214f0795f..6926feac0 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -76,7 +76,7 @@ macro_rules! intrinsics { ( #[maybe_use_optimized_c_shim] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -84,9 +84,9 @@ macro_rules! intrinsics { ) => ( #[cfg($name = "optimized-c")] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { extern $abi { - fn $name($($argname: $ty),*) -> $ret; + fn $name($($argname: $ty),*) $(-> $ret)?; } unsafe { $name($($argname),*) @@ -96,7 +96,7 @@ macro_rules! intrinsics { #[cfg(not($name = "optimized-c"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -110,7 +110,7 @@ macro_rules! intrinsics { ( #[aapcs_on_arm] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -119,7 +119,7 @@ macro_rules! intrinsics { #[cfg(target_arch = "arm")] intrinsics! { $(#[$($attr)*])* - pub extern "aapcs" fn $name( $($argname: $ty),* ) -> $ret { + pub extern "aapcs" fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -127,7 +127,7 @@ macro_rules! intrinsics { #[cfg(not(target_arch = "arm"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -140,7 +140,7 @@ macro_rules! intrinsics { ( #[unadjusted_on_win64] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -149,7 +149,7 @@ macro_rules! intrinsics { #[cfg(all(windows, target_pointer_width = "64"))] intrinsics! { $(#[$($attr)*])* - pub extern "unadjusted" fn $name( $($argname: $ty),* ) -> $ret { + pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -157,7 +157,7 @@ macro_rules! intrinsics { #[cfg(not(all(windows, target_pointer_width = "64")))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -175,7 +175,7 @@ macro_rules! intrinsics { ( #[win64_128bit_abi_hack] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -183,7 +183,7 @@ macro_rules! intrinsics { ) => ( #[cfg(all(windows, target_arch = "x86_64"))] $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } @@ -193,7 +193,7 @@ macro_rules! intrinsics { pub extern $abi fn $name( $($argname: $ty),* ) -> ::macros::win64_128bit_abi_hack::U64x2 { - let e: $ret = super::$name($($argname),*); + let e: $($ret)? = super::$name($($argname),*); ::macros::win64_128bit_abi_hack::U64x2::from(e) } } @@ -201,7 +201,7 @@ macro_rules! intrinsics { #[cfg(not(all(windows, target_arch = "x86_64")))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -216,21 +216,21 @@ macro_rules! intrinsics { ( #[arm_aeabi_alias = $alias:ident] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( #[cfg(target_arch = "arm")] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } #[cfg(target_arch = "arm")] pub mod $name { #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { super::$name($($argname),*) } } @@ -238,7 +238,7 @@ macro_rules! intrinsics { #[cfg(target_arch = "arm")] pub mod $alias { #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub extern "aapcs" fn $alias( $($argname: $ty),* ) -> $ret { + pub extern "aapcs" fn $alias( $($argname: $ty),* ) $(-> $ret)? { super::$name($($argname),*) } } @@ -246,7 +246,57 @@ macro_rules! intrinsics { #[cfg(not(target_arch = "arm"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + } + + intrinsics!($($rest)*); + ); + + // C mem* functions are only generated when the "mem" feature is enabled. + ( + #[mem_builtin] + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + $(#[$($attr)*])* + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + + #[cfg(feature = "mem")] + pub mod $name { + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) + } + } + + intrinsics!($($rest)*); + ); + + // Naked functions are special: we can't generate wrappers for them since + // they use a custom calling convention. + ( + #[naked] + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + pub mod $name { + #[naked] + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -268,21 +318,46 @@ macro_rules! intrinsics { // input we were given. ( $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + $(#[$($attr)*])* + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + + pub mod $name { + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) + } + } + + intrinsics!($($rest)*); + ); + + // Same as the above for unsafe functions. + ( + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } pub mod $name { $(#[$($attr)*])* #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { super::$name($($argname),*) } } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 2f9a9fd94..dce4d87e0 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -19,53 +19,55 @@ use core::ops::{BitOr, Shl}; )] mod impls; -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - impls::copy_forward(dest, src, n); - dest -} - -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - let delta = (dest as usize).wrapping_sub(src as usize); - if delta >= n { - // We can copy forwards because either dest is far enough ahead of src, - // or src is ahead of dest (and delta overflowed). +intrinsics! { + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { impls::copy_forward(dest, src, n); - } else { - impls::copy_backward(dest, src, n); + dest } - dest -} -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn memset(s: *mut u8, c: c_int, n: usize) -> *mut u8 { - impls::set_bytes(s, c as u8, n); - s -} + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + let delta = (dest as usize).wrapping_sub(src as usize); + if delta >= n { + // We can copy forwards because either dest is far enough ahead of src, + // or src is ahead of dest (and delta overflowed). + impls::copy_forward(dest, src, n); + } else { + impls::copy_backward(dest, src, n); + } + dest + } -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - let mut i = 0; - while i < n { - let a = *s1.add(i); - let b = *s2.add(i); - if a != b { - return a as i32 - b as i32; + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memset(s: *mut u8, c: crate::mem::c_int, n: usize) -> *mut u8 { + impls::set_bytes(s, c as u8, n); + s + } + + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + let mut i = 0; + while i < n { + let a = *s1.add(i); + let b = *s2.add(i); + if a != b { + return a as i32 - b as i32; + } + i += 1; } - i += 1; + 0 } - 0 -} -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - memcmp(s1, s2, n) + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + memcmp(s1, s2, n) + } } // `bytes` must be a multiple of `mem::size_of::()` @@ -133,65 +135,65 @@ where intrinsics! { #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } } diff --git a/src/x86.rs b/src/x86.rs index abcc2bdb3..fd1f32e3a 100644 --- a/src/x86.rs +++ b/src/x86.rs @@ -8,82 +8,78 @@ use core::intrinsics; // NOTE These functions are never mangled as they are not tested against compiler-rt // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn ___chkstk_ms() { - core::arch::asm!( - "push %ecx", - "push %eax", - "cmp $0x1000,%eax", - "lea 12(%esp),%ecx", - "jb 1f", - "2:", - "sub $0x1000,%ecx", - "test %ecx,(%ecx)", - "sub $0x1000,%eax", - "cmp $0x1000,%eax", - "ja 2b", - "1:", - "sub %eax,%ecx", - "test %ecx,(%ecx)", - "pop %eax", - "pop %ecx", - "ret", - options(noreturn, att_syntax) - ); -} +intrinsics! { + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk_ms() { + core::arch::asm!( + "push %ecx", + "push %eax", + "cmp $0x1000,%eax", + "lea 12(%esp),%ecx", + "jb 1f", + "2:", + "sub $0x1000,%ecx", + "test %ecx,(%ecx)", + "sub $0x1000,%eax", + "cmp $0x1000,%eax", + "ja 2b", + "1:", + "sub %eax,%ecx", + "test %ecx,(%ecx)", + "pop %eax", + "pop %ecx", + "ret", + options(noreturn, att_syntax) + ); + } -// FIXME: __alloca should be an alias to __chkstk -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn __alloca() { - core::arch::asm!( - "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" - options(noreturn, att_syntax) - ); -} + // FIXME: __alloca should be an alias to __chkstk + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn __alloca() { + core::arch::asm!( + "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" + options(noreturn, att_syntax) + ); + } -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn ___chkstk() { - core::arch::asm!( - "push %ecx", - "cmp $0x1000,%eax", - "lea 8(%esp),%ecx", // esp before calling this routine -> ecx - "jb 1f", - "2:", - "sub $0x1000,%ecx", - "test %ecx,(%ecx)", - "sub $0x1000,%eax", - "cmp $0x1000,%eax", - "ja 2b", - "1:", - "sub %eax,%ecx", - "test %ecx,(%ecx)", - "lea 4(%esp),%eax", // load pointer to the return address into eax - "mov %ecx,%esp", // install the new top of stack pointer into esp - "mov -4(%eax),%ecx", // restore ecx - "push (%eax)", // push return address onto the stack - "sub %esp,%eax", // restore the original value in eax - "ret", - options(noreturn, att_syntax) - ); + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk() { + core::arch::asm!( + "push %ecx", + "cmp $0x1000,%eax", + "lea 8(%esp),%ecx", // esp before calling this routine -> ecx + "jb 1f", + "2:", + "sub $0x1000,%ecx", + "test %ecx,(%ecx)", + "sub $0x1000,%eax", + "cmp $0x1000,%eax", + "ja 2b", + "1:", + "sub %eax,%ecx", + "test %ecx,(%ecx)", + "lea 4(%esp),%eax", // load pointer to the return address into eax + "mov %ecx,%esp", // install the new top of stack pointer into esp + "mov -4(%eax),%ecx", // restore ecx + "push (%eax)", // push return address onto the stack + "sub %esp,%eax", // restore the original value in eax + "ret", + options(noreturn, att_syntax) + ); + } } diff --git a/src/x86_64.rs b/src/x86_64.rs index ea3c99497..393eeddd8 100644 --- a/src/x86_64.rs +++ b/src/x86_64.rs @@ -8,89 +8,87 @@ use core::intrinsics; // NOTE These functions are never mangled as they are not tested against compiler-rt // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn ___chkstk_ms() { - core::arch::asm!( - "push %rcx", - "push %rax", - "cmp $0x1000,%rax", - "lea 24(%rsp),%rcx", - "jb 1f", - "2:", - "sub $0x1000,%rcx", - "test %rcx,(%rcx)", - "sub $0x1000,%rax", - "cmp $0x1000,%rax", - "ja 2b", - "1:", - "sub %rax,%rcx", - "test %rcx,(%rcx)", - "pop %rax", - "pop %rcx", - "ret", - options(noreturn, att_syntax) - ); -} +intrinsics! { + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk_ms() { + core::arch::asm!( + "push %rcx", + "push %rax", + "cmp $0x1000,%rax", + "lea 24(%rsp),%rcx", + "jb 1f", + "2:", + "sub $0x1000,%rcx", + "test %rcx,(%rcx)", + "sub $0x1000,%rax", + "cmp $0x1000,%rax", + "ja 2b", + "1:", + "sub %rax,%rcx", + "test %rcx,(%rcx)", + "pop %rax", + "pop %rcx", + "ret", + options(noreturn, att_syntax) + ); + } -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn __alloca() { - core::arch::asm!( - "mov %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx - "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" - options(noreturn, att_syntax) - ); -} + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn __alloca() { + core::arch::asm!( + "mov %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx + "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" + options(noreturn, att_syntax) + ); + } -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn ___chkstk() { - core::arch::asm!( - "push %rcx", - "cmp $0x1000,%rax", - "lea 16(%rsp),%rcx", // rsp before calling this routine -> rcx - "jb 1f", - "2:", - "sub $0x1000,%rcx", - "test %rcx,(%rcx)", - "sub $0x1000,%rax", - "cmp $0x1000,%rax", - "ja 2b", - "1:", - "sub %rax,%rcx", - "test %rcx,(%rcx)", - "lea 8(%rsp),%rax", // load pointer to the return address into rax - "mov %rcx,%rsp", // install the new top of stack pointer into rsp - "mov -8(%rax),%rcx", // restore rcx - "push (%rax)", // push return address onto the stack - "sub %rsp,%rax", // restore the original value in rax - "ret", - options(noreturn, att_syntax) - ); + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk() { + core::arch::asm!( + "push %rcx", + "cmp $0x1000,%rax", + "lea 16(%rsp),%rcx", // rsp before calling this routine -> rcx + "jb 1f", + "2:", + "sub $0x1000,%rcx", + "test %rcx,(%rcx)", + "sub $0x1000,%rax", + "cmp $0x1000,%rax", + "ja 2b", + "1:", + "sub %rax,%rcx", + "test %rcx,(%rcx)", + "lea 8(%rsp),%rax", // load pointer to the return address into rax + "mov %rcx,%rsp", // install the new top of stack pointer into rsp + "mov -8(%rax),%rcx", // restore rcx + "push (%rax)", // push return address onto the stack + "sub %rsp,%rax", // restore the original value in rax + "ret", + options(noreturn, att_syntax) + ); + } } // HACK(https://github.com/rust-lang/rust/issues/62785): x86_64-unknown-uefi needs special LLVM // support unless we emit the _fltused -#[no_mangle] -#[used] -#[cfg(target_os = "uefi")] -static _fltused: i32 = 0; +mod _fltused { + #[no_mangle] + #[used] + #[cfg(target_os = "uefi")] + static _fltused: i32 = 0; +}