From e4c4124872d50d128e1f4008e7c6970c050eff41 Mon Sep 17 00:00:00 2001 From: Mahmut Bulut Date: Thu, 21 May 2020 23:31:23 +0200 Subject: [PATCH 1/3] Implementation for Aarch64 TME intrinsics --- crates/core_arch/src/aarch64/mod.rs | 3 + crates/core_arch/src/aarch64/tme.rs | 124 +++++++++++++++++++++++ crates/std_detect/tests/cpu-detection.rs | 1 + 3 files changed, 128 insertions(+) create mode 100644 crates/core_arch/src/aarch64/tme.rs diff --git a/crates/core_arch/src/aarch64/mod.rs b/crates/core_arch/src/aarch64/mod.rs index 190383df21..700fa87e46 100644 --- a/crates/core_arch/src/aarch64/mod.rs +++ b/crates/core_arch/src/aarch64/mod.rs @@ -15,6 +15,9 @@ pub use self::neon::*; mod crypto; pub use self::crypto::*; +mod tme; +pub use self::tme::*; + mod crc; pub use self::crc::*; diff --git a/crates/core_arch/src/aarch64/tme.rs b/crates/core_arch/src/aarch64/tme.rs new file mode 100644 index 0000000000..7f541d39a9 --- /dev/null +++ b/crates/core_arch/src/aarch64/tme.rs @@ -0,0 +1,124 @@ +//! ARM's Transactional Memory Extensions (TME). +//! +//! This CPU feature is available on Aarch64 – ARMv8-A arch. onwards. +//! This feature is in the non-neon feature set. TME specific vendor documentation can +//! be found [TME Intrinsics Introduction][tme_intrinsics_intro]. +//! +//! The reference is [ACLE Q4 2019][acle_q4_2019_ref]. +//! +//! ACLE has a section for TME extensions and state masks for aborts and failure codes. +//! In addition to that [LLVM Aarch64 Intrinsics][llvm_aarch64_int] are +//! self explanatory for what needs to be exported. +//! +//! [acle_q4_2019_ref]: https://static.docs.arm.com/101028/0010/ACLE_2019Q4_release-0010.pdf +//! [tme_intrinsics_intro]: https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics +//! [llvm_aarch64_int]: https://github.com/llvm/llvm-project/commit/a36d31478c182903523e04eb271bbf102bfab2cc#diff-ff24e1c35f4d54f1110ce5d90c709319R626-R646 + +#[cfg(test)] +use stdarch_test::assert_instr; + +extern "C" { + #[link_name = "llvm.aarch64.tstart"] + fn aarch64_tstart() -> i32; + #[link_name = "llvm.aarch64.tcommit"] + fn aarch64_tcommit() -> (); + #[link_name = "llvm.aarch64.tcancel"] + fn aarch64_tcancel(imm0: i64) -> (); + #[link_name = "llvm.aarch64.ttest"] + fn aarch64_ttest() -> i32; +} + +/// Transaction successfully started. +pub const _TMSTART_SUCCESS: u32 = 0x00_u32; + +/// Extraction mask for failure reason +pub const _TMFAILURE_REASON: u32 = 0x00007FFF_u32; + +/// Transaction retry is possible. +pub const _TMFAILURE_RTRY: u32 = 1 << 15; + +/// Transaction cancelled. +pub const _TMFAILURE_CNCL: u32 = 1 << 16; + +/// Transaction cancelled due to high memory usage. +pub const _TMFAILURE_MEM: u32 = 1 << 17; + +/// +pub const _TMFAILURE_IMP: u32 = 1 << 18; + +/// +pub const _TMFAILURE_ERR: u32 = 1 << 19; + +/// +pub const _TMFAILURE_SIZE: u32 = 1 << 20; + +/// Transaction abort in a inner nested transaction. +pub const _TMFAILURE_NEST: u32 = 1 << 21; + +/// Transaction abort due to a debug trap. +pub const _TMFAILURE_DBG: u32 = 1 << 22; + +/// +pub const _TMFAILURE_INT: u32 = 1 << 23; + +/// +pub const _TMFAILURE_TRIVIAL: u32 = 1 << 24; + + +/// Starts a new transaction. When the transaction starts successfully the return value is 0. +/// If the transaction fails, all state modifications are discarded and a cause of the failure +/// is encoded in the return value. +/// +/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). +#[inline] +#[target_feature(enable = "tme")] +#[cfg_attr(test, assert_instr(tstart))] +pub unsafe fn __tstart() -> u32 { + aarch64_tstart() as _ +} + +/// Commits the current transaction. For a nested transaction, the only effect is that the +/// transactional nesting depth is decreased. For an outer transaction, the state modifications +/// performed transactionally are committed to the architectural state. +/// +/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). +#[inline] +#[target_feature(enable = "tme")] +#[cfg_attr(test, assert_instr(tcommit))] +pub unsafe fn __tcommit() { + aarch64_tcommit() +} + +/// Cancels the current transaction and discards all state modifications that were performed transactionally. +/// +/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). +#[inline] +#[target_feature(enable = "tme")] +#[cfg_attr(test, assert_instr(tcancel, imm0 = 0x0))] +#[rustc_args_required_const(0)] +pub unsafe fn __tcancel(imm0: u32) { + macro_rules! call { + ($imm0:expr) => { + aarch64_tcancel($imm0) + }; + } + constify_imm8!(imm0, call) +} + +/// Tests if executing inside a transaction. If no transaction is currently executing, +/// the return value is 0. Otherwise, this intrinsic returns the depth of the transaction. +/// +/// [ARM TME Intrinsics](https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics). +#[inline] +#[target_feature(enable = "tme")] +#[cfg_attr(test, assert_instr(ttest))] +pub unsafe fn __ttest() -> u32 { + aarch64_ttest() as _ +} + +/// Encodes cancellation reason, which is the parameter passed to [`__tcancel`] +/// Takes cancellation reason flags and retry-ability. +#[inline] +pub const fn _tcancel_code(reason: u32, retryable: bool) -> u32 { + (retryable as i32) << 15 | (reason & _TMFAILURE_REASON) +} diff --git a/crates/std_detect/tests/cpu-detection.rs b/crates/std_detect/tests/cpu-detection.rs index b59ed32e12..a95b1e7393 100644 --- a/crates/std_detect/tests/cpu-detection.rs +++ b/crates/std_detect/tests/cpu-detection.rs @@ -45,6 +45,7 @@ fn aarch64_linux() { println!("rdm: {}", is_aarch64_feature_detected!("rdm")); println!("rcpc: {}", is_aarch64_feature_detected!("rcpc")); println!("dotprod: {}", is_aarch64_feature_detected!("dotprod")); + println!("tme: {}", is_aarch64_feature_detected!("tme")); } #[test] From 9fb45e78532e7d951faeb4c2e0e78c472d833f6c Mon Sep 17 00:00:00 2001 From: Mahmut Bulut Date: Mon, 25 May 2020 12:43:55 +0200 Subject: [PATCH 2/3] Fix cancellation code arithmetic --- crates/core_arch/src/aarch64/tme.rs | 97 +++++++++++++++++++++++------ crates/stdarch-verify/tests/arm.rs | 1 + 2 files changed, 80 insertions(+), 18 deletions(-) diff --git a/crates/core_arch/src/aarch64/tme.rs b/crates/core_arch/src/aarch64/tme.rs index 7f541d39a9..704416ae88 100644 --- a/crates/core_arch/src/aarch64/tme.rs +++ b/crates/core_arch/src/aarch64/tme.rs @@ -1,18 +1,18 @@ //! ARM's Transactional Memory Extensions (TME). //! -//! This CPU feature is available on Aarch64 – ARMv8-A arch. onwards. +//! This CPU feature is available on Aarch64 - A architecture profile. //! This feature is in the non-neon feature set. TME specific vendor documentation can //! be found [TME Intrinsics Introduction][tme_intrinsics_intro]. //! //! The reference is [ACLE Q4 2019][acle_q4_2019_ref]. //! //! ACLE has a section for TME extensions and state masks for aborts and failure codes. -//! In addition to that [LLVM Aarch64 Intrinsics][llvm_aarch64_int] are -//! self explanatory for what needs to be exported. +//! [ARM A64 Architecture Register Datasheet][a_profile_future] also describes possible failure code scenarios. //! //! [acle_q4_2019_ref]: https://static.docs.arm.com/101028/0010/ACLE_2019Q4_release-0010.pdf //! [tme_intrinsics_intro]: https://developer.arm.com/docs/101028/0010/transactional-memory-extension-tme-intrinsics //! [llvm_aarch64_int]: https://github.com/llvm/llvm-project/commit/a36d31478c182903523e04eb271bbf102bfab2cc#diff-ff24e1c35f4d54f1110ce5d90c709319R626-R646 +//! [a_profile_future]: https://static.docs.arm.com/ddi0601/a/SysReg_xml_futureA-2019-04.pdf?_ga=2.116560387.441514988.1590524918-1110153136.1588469296 #[cfg(test)] use stdarch_test::assert_instr; @@ -37,34 +37,33 @@ pub const _TMFAILURE_REASON: u32 = 0x00007FFF_u32; /// Transaction retry is possible. pub const _TMFAILURE_RTRY: u32 = 1 << 15; -/// Transaction cancelled. +/// Transaction executed a TCANCEL instruction pub const _TMFAILURE_CNCL: u32 = 1 << 16; -/// Transaction cancelled due to high memory usage. +/// Transaction aborted because a conflict occurred pub const _TMFAILURE_MEM: u32 = 1 << 17; -/// +/// Fallback error type for any other reason pub const _TMFAILURE_IMP: u32 = 1 << 18; -/// +/// Transaction aborted because a non-permissible operation was attempted pub const _TMFAILURE_ERR: u32 = 1 << 19; -/// +/// Transaction aborted due to read or write set limit was exceeded pub const _TMFAILURE_SIZE: u32 = 1 << 20; -/// Transaction abort in a inner nested transaction. +/// Transaction aborted due to transactional nesting level was exceeded pub const _TMFAILURE_NEST: u32 = 1 << 21; -/// Transaction abort due to a debug trap. +/// Transaction aborted due to a debug trap. pub const _TMFAILURE_DBG: u32 = 1 << 22; -/// +/// Transaction failed from interrupt pub const _TMFAILURE_INT: u32 = 1 << 23; -/// +/// Indicates a TRIVIAL version of TM is available pub const _TMFAILURE_TRIVIAL: u32 = 1 << 24; - /// Starts a new transaction. When the transaction starts successfully the return value is 0. /// If the transaction fails, all state modifications are discarded and a cause of the failure /// is encoded in the return value. @@ -116,9 +115,71 @@ pub unsafe fn __ttest() -> u32 { aarch64_ttest() as _ } -/// Encodes cancellation reason, which is the parameter passed to [`__tcancel`] -/// Takes cancellation reason flags and retry-ability. -#[inline] -pub const fn _tcancel_code(reason: u32, retryable: bool) -> u32 { - (retryable as i32) << 15 | (reason & _TMFAILURE_REASON) +#[cfg(test)] +mod tests { + use stdarch_test::simd_test; + + use crate::core_arch::aarch64::*; + + #[simd_test(enable = "tme")] + unsafe fn test_tstart() { + let mut x = 0; + for i in 0..10 { + let code = tme::__tstart(); + if code == _TMSTART_SUCCESS { + x += 1; + assert_eq!(x, i+1); + break; + } + assert_eq!(x, 0); + } + } + + #[simd_test(enable = "tme")] + unsafe fn test_tcommit() { + let mut x = 0; + for i in 0..10 { + let code = tme::__tstart(); + if code == _TMSTART_SUCCESS { + x += 1; + assert_eq!(x, i+1); + tme::__tcommit(); + } + assert_eq!(x, i+1); + } + } + + #[simd_test(enable = "tme")] + unsafe fn test_tcancel() { + let reason = 0x123; + let cancel_code = (0 | (reason & _TMFAILURE_REASON) as i32) as u32; + let mut x = 0; + + for i in 0..10 { + let code = tme::__tstart(); + if code == _TMSTART_SUCCESS { + x += 1; + assert_eq!(x, i+1); + tme::__tcancel(cancel_code); + break; + } + } + + assert_eq!(x, 0); + } + + #[simd_test(enable = "tme")] + unsafe fn test_ttest() { + let reason = 0x123; + let cancel_code = (0 | (reason & _TMFAILURE_REASON) as i32) as u32; + for _ in 0..10 { + let code = tme::__tstart(); + if code == _TMSTART_SUCCESS { + if tme::__ttest() == 2 { + tme::__tcancel(cancel_code); + break; + } + } + } + } } diff --git a/crates/stdarch-verify/tests/arm.rs b/crates/stdarch-verify/tests/arm.rs index 83ba480e07..b877b1e869 100644 --- a/crates/stdarch-verify/tests/arm.rs +++ b/crates/stdarch-verify/tests/arm.rs @@ -366,6 +366,7 @@ fn verify_all_signatures() { && !rust.file.ends_with("v6.rs\"") && !rust.file.ends_with("v7.rs\"") && !rust.file.ends_with("v8.rs\"") + && !rust.file.ends_with("tme.rs\"") { println!( "missing arm definition for {:?} in {}", From 99e6f75659cb5bbfe05ee2235d6c75c6a369017e Mon Sep 17 00:00:00 2001 From: Mahmut Bulut Date: Fri, 29 May 2020 02:28:14 +0200 Subject: [PATCH 3/3] feature detection --- .../aarch64-unknown-linux-gnu/Dockerfile | 2 +- crates/core_arch/src/aarch64/tme.rs | 60 +++++++++---------- crates/simd-test-macro/src/lib.rs | 5 +- crates/std_detect/src/detect/arch/aarch64.rs | 2 + crates/std_detect/src/detect/os/aarch64.rs | 1 + .../src/detect/os/freebsd/aarch64.rs | 1 + 6 files changed, 38 insertions(+), 33 deletions(-) diff --git a/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/ci/docker/aarch64-unknown-linux-gnu/Dockerfile index 41ff4729ac..09ead2c017 100644 --- a/ci/docker/aarch64-unknown-linux-gnu/Dockerfile +++ b/ci/docker/aarch64-unknown-linux-gnu/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ ca-certificates \ diff --git a/crates/core_arch/src/aarch64/tme.rs b/crates/core_arch/src/aarch64/tme.rs index 704416ae88..f0ff867813 100644 --- a/crates/core_arch/src/aarch64/tme.rs +++ b/crates/core_arch/src/aarch64/tme.rs @@ -19,50 +19,50 @@ use stdarch_test::assert_instr; extern "C" { #[link_name = "llvm.aarch64.tstart"] - fn aarch64_tstart() -> i32; + fn aarch64_tstart() -> u64; #[link_name = "llvm.aarch64.tcommit"] fn aarch64_tcommit() -> (); #[link_name = "llvm.aarch64.tcancel"] - fn aarch64_tcancel(imm0: i64) -> (); + fn aarch64_tcancel(imm0: u64) -> (); #[link_name = "llvm.aarch64.ttest"] - fn aarch64_ttest() -> i32; + fn aarch64_ttest() -> u64; } /// Transaction successfully started. -pub const _TMSTART_SUCCESS: u32 = 0x00_u32; +pub const _TMSTART_SUCCESS: u64 = 0x00_u64; /// Extraction mask for failure reason -pub const _TMFAILURE_REASON: u32 = 0x00007FFF_u32; +pub const _TMFAILURE_REASON: u64 = 0x00007FFF_u64; /// Transaction retry is possible. -pub const _TMFAILURE_RTRY: u32 = 1 << 15; +pub const _TMFAILURE_RTRY: u64 = 1 << 15; /// Transaction executed a TCANCEL instruction -pub const _TMFAILURE_CNCL: u32 = 1 << 16; +pub const _TMFAILURE_CNCL: u64 = 1 << 16; /// Transaction aborted because a conflict occurred -pub const _TMFAILURE_MEM: u32 = 1 << 17; +pub const _TMFAILURE_MEM: u64 = 1 << 17; /// Fallback error type for any other reason -pub const _TMFAILURE_IMP: u32 = 1 << 18; +pub const _TMFAILURE_IMP: u64 = 1 << 18; /// Transaction aborted because a non-permissible operation was attempted -pub const _TMFAILURE_ERR: u32 = 1 << 19; +pub const _TMFAILURE_ERR: u64 = 1 << 19; /// Transaction aborted due to read or write set limit was exceeded -pub const _TMFAILURE_SIZE: u32 = 1 << 20; +pub const _TMFAILURE_SIZE: u64 = 1 << 20; /// Transaction aborted due to transactional nesting level was exceeded -pub const _TMFAILURE_NEST: u32 = 1 << 21; +pub const _TMFAILURE_NEST: u64 = 1 << 21; /// Transaction aborted due to a debug trap. -pub const _TMFAILURE_DBG: u32 = 1 << 22; +pub const _TMFAILURE_DBG: u64 = 1 << 22; /// Transaction failed from interrupt -pub const _TMFAILURE_INT: u32 = 1 << 23; +pub const _TMFAILURE_INT: u64 = 1 << 23; /// Indicates a TRIVIAL version of TM is available -pub const _TMFAILURE_TRIVIAL: u32 = 1 << 24; +pub const _TMFAILURE_TRIVIAL: u64 = 1 << 24; /// Starts a new transaction. When the transaction starts successfully the return value is 0. /// If the transaction fails, all state modifications are discarded and a cause of the failure @@ -72,8 +72,8 @@ pub const _TMFAILURE_TRIVIAL: u32 = 1 << 24; #[inline] #[target_feature(enable = "tme")] #[cfg_attr(test, assert_instr(tstart))] -pub unsafe fn __tstart() -> u32 { - aarch64_tstart() as _ +pub unsafe fn __tstart() -> u64 { + aarch64_tstart() } /// Commits the current transaction. For a nested transaction, the only effect is that the @@ -95,7 +95,7 @@ pub unsafe fn __tcommit() { #[target_feature(enable = "tme")] #[cfg_attr(test, assert_instr(tcancel, imm0 = 0x0))] #[rustc_args_required_const(0)] -pub unsafe fn __tcancel(imm0: u32) { +pub unsafe fn __tcancel(imm0: u64) { macro_rules! call { ($imm0:expr) => { aarch64_tcancel($imm0) @@ -111,8 +111,8 @@ pub unsafe fn __tcancel(imm0: u32) { #[inline] #[target_feature(enable = "tme")] #[cfg_attr(test, assert_instr(ttest))] -pub unsafe fn __ttest() -> u32 { - aarch64_ttest() as _ +pub unsafe fn __ttest() -> u64 { + aarch64_ttest() } #[cfg(test)] @@ -121,6 +121,8 @@ mod tests { use crate::core_arch::aarch64::*; + const CANCEL_CODE: u64 = (0 | (0x123 & _TMFAILURE_REASON) as u64) as u64; + #[simd_test(enable = "tme")] unsafe fn test_tstart() { let mut x = 0; @@ -128,7 +130,7 @@ mod tests { let code = tme::__tstart(); if code == _TMSTART_SUCCESS { x += 1; - assert_eq!(x, i+1); + assert_eq!(x, i + 1); break; } assert_eq!(x, 0); @@ -142,25 +144,23 @@ mod tests { let code = tme::__tstart(); if code == _TMSTART_SUCCESS { x += 1; - assert_eq!(x, i+1); + assert_eq!(x, i + 1); tme::__tcommit(); } - assert_eq!(x, i+1); + assert_eq!(x, i + 1); } } #[simd_test(enable = "tme")] unsafe fn test_tcancel() { - let reason = 0x123; - let cancel_code = (0 | (reason & _TMFAILURE_REASON) as i32) as u32; let mut x = 0; for i in 0..10 { let code = tme::__tstart(); if code == _TMSTART_SUCCESS { x += 1; - assert_eq!(x, i+1); - tme::__tcancel(cancel_code); + assert_eq!(x, i + 1); + tme::__tcancel(CANCEL_CODE); break; } } @@ -170,14 +170,12 @@ mod tests { #[simd_test(enable = "tme")] unsafe fn test_ttest() { - let reason = 0x123; - let cancel_code = (0 | (reason & _TMFAILURE_REASON) as i32) as u32; for _ in 0..10 { let code = tme::__tstart(); if code == _TMSTART_SUCCESS { if tme::__ttest() == 2 { - tme::__tcancel(cancel_code); - break; + tme::__tcancel(CANCEL_CODE); + break; } } } diff --git a/crates/simd-test-macro/src/lib.rs b/crates/simd-test-macro/src/lib.rs index d4ebb787b9..6df48b1e6a 100644 --- a/crates/simd-test-macro/src/lib.rs +++ b/crates/simd-test-macro/src/lib.rs @@ -156,5 +156,8 @@ fn find_name(item: TokenStream) -> Ident { } } - tokens.next().and_then(get_ident).expect("failed to find function name") + tokens + .next() + .and_then(get_ident) + .expect("failed to find function name") } diff --git a/crates/std_detect/src/detect/arch/aarch64.rs b/crates/std_detect/src/detect/arch/aarch64.rs index 154207e5ad..761cde9583 100644 --- a/crates/std_detect/src/detect/arch/aarch64.rs +++ b/crates/std_detect/src/detect/arch/aarch64.rs @@ -33,4 +33,6 @@ features! { /// Release consistent Processor consistent (RcPc) @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] dotprod: "dotprod"; /// Vector Dot-Product (ASIMDDP) + @FEATURE: #[unstable(feature = "stdsimd", issue = "27731")] tme: "tme"; + /// Transactional Memory Extensions (TME) } diff --git a/crates/std_detect/src/detect/os/aarch64.rs b/crates/std_detect/src/detect/os/aarch64.rs index 56fe7958d6..c95b688970 100644 --- a/crates/std_detect/src/detect/os/aarch64.rs +++ b/crates/std_detect/src/detect/os/aarch64.rs @@ -44,6 +44,7 @@ pub(crate) fn detect_features() -> cache::Initializer { enable_feature(Feature::pmull, pmull); // Crypto is specified as AES + PMULL + SHA1 + SHA2 per LLVM/hosts.cpp enable_feature(Feature::crypto, aes && pmull && sha1 && sha2); + enable_feature(Feature::tme, bits_shift(aa64isar0, 27, 24) == 1); enable_feature(Feature::lse, bits_shift(aa64isar0, 23, 20) >= 1); enable_feature(Feature::crc, bits_shift(aa64isar0, 19, 16) >= 1); diff --git a/crates/std_detect/src/detect/os/freebsd/aarch64.rs b/crates/std_detect/src/detect/os/freebsd/aarch64.rs index e5df9ba4c9..7e086ca057 100644 --- a/crates/std_detect/src/detect/os/freebsd/aarch64.rs +++ b/crates/std_detect/src/detect/os/freebsd/aarch64.rs @@ -17,5 +17,6 @@ mod tests { println!("rdm: {:?}", is_aarch64_feature_detected!("rdm")); println!("rcpc: {:?}", is_aarch64_feature_detected!("rcpc")); println!("dotprod: {:?}", is_aarch64_feature_detected!("dotprod")); + println!("tme: {:?}", is_aarch64_feature_detected!("tme")); } }