From e12b4bd79edea40f58433856893f7449f59a1f8e Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Fri, 10 Feb 2023 23:10:55 +0900 Subject: [PATCH] detect: Support run-time detection of FEAT_LRCPC3 on FreeBSD/OpenBSD/macOS Currently, outline-atomics on aarch64 is only done for FEAT_LSE, so the detection of FEAT_LRCPC3 is currently test-only. --- .github/.cspell/project-dictionary.txt | 1 + src/imp/atomic128/aarch64.rs | 2 +- src/imp/atomic128/detect/aarch64_aa64reg.rs | 43 ++++++++++++++++++++- src/imp/atomic128/detect/aarch64_macos.rs | 8 ++++ src/imp/atomic128/detect/common.rs | 27 +++++++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index ca41752b..ce8c2ef3 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -80,6 +80,7 @@ quadword qword RAII rclass +rcpc reentrancy sbcs seqlock diff --git a/src/imp/atomic128/aarch64.rs b/src/imp/atomic128/aarch64.rs index 7de84666..6fb6d66b 100644 --- a/src/imp/atomic128/aarch64.rs +++ b/src/imp/atomic128/aarch64.rs @@ -60,7 +60,7 @@ include!("macros.rs"); target_os = "android", target_os = "windows", target_os = "freebsd", - target_os = "openbsd" + target_os = "openbsd", )), path = "detect/aarch64_std.rs" )] diff --git a/src/imp/atomic128/detect/aarch64_aa64reg.rs b/src/imp/atomic128/detect/aarch64_aa64reg.rs index c4024cbf..3873c36d 100644 --- a/src/imp/atomic128/detect/aarch64_aa64reg.rs +++ b/src/imp/atomic128/detect/aarch64_aa64reg.rs @@ -17,7 +17,7 @@ // - OpenBSD 7.1+ (through sysctl) // https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8 // -// For now, this module is only used on FreeBSD and OpenBSD. +// For now, this module is only used on FreeBSD/OpenBSD. // On Linux, this module is test only because this approach requires a higher // kernel version than Rust supports, and also does not work with qemu-user // (as of QEMU 7.2) and Valgrind. (Looking into HWCAP_CPUID in auxvec, it appears @@ -36,6 +36,8 @@ include!("common.rs"); struct AA64Reg { aa64isar0: u64, #[cfg(test)] + aa64isar1: u64, + #[cfg(test)] aa64mmfr2: u64, } @@ -44,6 +46,8 @@ fn _detect(info: &mut CpuInfo) { let AA64Reg { aa64isar0, #[cfg(test)] + aa64isar1, + #[cfg(test)] aa64mmfr2, } = imp::aa64reg(); @@ -62,6 +66,11 @@ fn _detect(info: &mut CpuInfo) { #[cfg(test)] { + // ID_AA64ISAR1_EL1, Instruction Set Attribute Register 1 + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/ID-AA64ISAR1-EL1--AArch64-Instruction-Set-Attribute-Register-1?lang=en + if extract(aa64isar1, 23, 20) >= 3 { + info.set(CpuInfo::HAS_RCPC3); + } // ID_AA64MMFR2_EL1, AArch64 Memory Model Feature Register 2 // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/ID-AA64MMFR2-EL1--AArch64-Memory-Model-Feature-Register-2?lang=en if extract(aa64mmfr2, 35, 32) >= 1 { @@ -95,6 +104,16 @@ mod imp { options(pure, nomem, nostack, preserves_flags) ); #[cfg(test)] + let aa64isar1: u64; + #[cfg(test)] + { + asm!( + "mrs {}, ID_AA64ISAR1_EL1", + out(reg) aa64isar1, + options(pure, nomem, nostack, preserves_flags) + ); + } + #[cfg(test)] let aa64mmfr2: u64; #[cfg(test)] { @@ -107,6 +126,8 @@ mod imp { AA64Reg { aa64isar0, #[cfg(test)] + aa64isar1, + #[cfg(test)] aa64mmfr2, } } @@ -154,6 +175,8 @@ mod imp { // https://github.com/openbsd/src/blob/72ccc03bd11da614f31f7ff76e3f6fce99bc1c79/sys/arch/arm64/include/cpu.h#L25-L40 pub(crate) const CPU_ID_AA64ISAR0: c_int = 2; #[cfg(test)] + pub(crate) const CPU_ID_AA64ISAR1: c_int = 3; + #[cfg(test)] pub(crate) const CPU_ID_AA64MMFR2: c_int = 7; } @@ -167,10 +190,14 @@ mod imp { pub(super) fn aa64reg() -> AA64Reg { let aa64isar0 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR0]).unwrap_or(0); #[cfg(test)] + let aa64isar1 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR1]).unwrap_or(0); + #[cfg(test)] let aa64mmfr2 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64MMFR2]).unwrap_or(0); AA64Reg { aa64isar0, #[cfg(test)] + aa64isar1, + #[cfg(test)] aa64mmfr2, } } @@ -221,8 +248,9 @@ mod tests { #[test] fn test_aa64reg() { - let AA64Reg { aa64isar0, aa64mmfr2 } = imp::aa64reg(); + let AA64Reg { aa64isar0, aa64isar1, aa64mmfr2 } = imp::aa64reg(); std::eprintln!("aa64isar0={}", aa64isar0); + std::eprintln!("aa64isar1={}", aa64isar1); std::eprintln!("aa64mmfr2={}", aa64mmfr2); if cfg!(target_os = "openbsd") { let output = Command::new("sysctl").arg("machdep").output().unwrap(); @@ -232,6 +260,10 @@ mod tests { stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64isar0=")).unwrap(), aa64isar0.to_string(), ); + assert_eq!( + stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64isar1=")).unwrap(), + aa64isar1.to_string(), + ); assert_eq!( stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64mmfr2=")).unwrap_or("0"), aa64mmfr2.to_string(), @@ -248,6 +280,9 @@ mod tests { if detect().test(CpuInfo::HAS_LSE2) { assert_eq!(extract(aa64mmfr2, 35, 32), 1); } + if detect().test(CpuInfo::HAS_RCPC3) { + assert_eq!(extract(aa64isar1, 23, 20), 3); + } } // Static assertions for FFI bindings. @@ -290,6 +325,10 @@ mod tests { let [] = [(); (ffi::CPU_ID_AA64ISAR0 - machine_cpu::CPU_ID_AA64ISAR0 as ffi::c_int) as usize]; // libc doesn't have this + // let [] = [(); (ffi::CPU_ID_AA64ISAR1 - libc::CPU_ID_AA64ISAR1) as usize]; + let [] = + [(); (ffi::CPU_ID_AA64ISAR1 - machine_cpu::CPU_ID_AA64ISAR1 as ffi::c_int) as usize]; + // libc doesn't have this // let [] = [(); (ffi::CPU_ID_AA64MMFR2 - libc::CPU_ID_AA64MMFR2) as usize]; let [] = [(); (ffi::CPU_ID_AA64MMFR2 - machine_cpu::CPU_ID_AA64MMFR2 as ffi::c_int) as usize]; diff --git a/src/imp/atomic128/detect/aarch64_macos.rs b/src/imp/atomic128/detect/aarch64_macos.rs index 385cbe19..341dc514 100644 --- a/src/imp/atomic128/detect/aarch64_macos.rs +++ b/src/imp/atomic128/detect/aarch64_macos.rs @@ -96,6 +96,14 @@ fn _detect(info: &mut CpuInfo) { if unsafe { sysctlbyname32(b"hw.optional.arm.FEAT_LSE2\0").unwrap_or(0) != 0 } { info.set(CpuInfo::HAS_LSE2); } + // SAFETY: we passed a valid C string. + if unsafe { sysctlbyname32(b"hw.optional.arm.FEAT_LSE128\0").unwrap_or(0) != 0 } { + info.set(CpuInfo::HAS_LSE128); + } + // SAFETY: we passed a valid C string. + if unsafe { sysctlbyname32(b"hw.optional.arm.FEAT_LRCPC3\0").unwrap_or(0) != 0 } { + info.set(CpuInfo::HAS_RCPC3); + } } } diff --git a/src/imp/atomic128/detect/common.rs b/src/imp/atomic128/detect/common.rs index b207f7e5..015ff733 100644 --- a/src/imp/atomic128/detect/common.rs +++ b/src/imp/atomic128/detect/common.rs @@ -50,6 +50,10 @@ impl CpuInfo { // This is currently only used in tests. #[cfg(test)] const HAS_LSE128: u32 = 3; + /// Whether FEAT_LRCPC3 is available + // This is currently only used in tests. + #[cfg(test)] + const HAS_RCPC3: u32 = 4; } #[cfg(target_arch = "aarch64")] #[inline] @@ -131,26 +135,37 @@ mod tests_aarch64_common { assert!(!x.test(CpuInfo::HAS_LSE)); assert!(!x.test(CpuInfo::HAS_LSE2)); assert!(!x.test(CpuInfo::HAS_LSE128)); + assert!(!x.test(CpuInfo::HAS_RCPC3)); x.set(CpuInfo::INIT); assert!(x.test(CpuInfo::INIT)); assert!(!x.test(CpuInfo::HAS_LSE)); assert!(!x.test(CpuInfo::HAS_LSE2)); assert!(!x.test(CpuInfo::HAS_LSE128)); + assert!(!x.test(CpuInfo::HAS_RCPC3)); x.set(CpuInfo::HAS_LSE); assert!(x.test(CpuInfo::INIT)); assert!(x.test(CpuInfo::HAS_LSE)); assert!(!x.test(CpuInfo::HAS_LSE2)); assert!(!x.test(CpuInfo::HAS_LSE128)); + assert!(!x.test(CpuInfo::HAS_RCPC3)); x.set(CpuInfo::HAS_LSE2); assert!(x.test(CpuInfo::INIT)); assert!(x.test(CpuInfo::HAS_LSE)); assert!(x.test(CpuInfo::HAS_LSE2)); assert!(!x.test(CpuInfo::HAS_LSE128)); + assert!(!x.test(CpuInfo::HAS_RCPC3)); x.set(CpuInfo::HAS_LSE128); assert!(x.test(CpuInfo::INIT)); assert!(x.test(CpuInfo::HAS_LSE)); assert!(x.test(CpuInfo::HAS_LSE2)); assert!(x.test(CpuInfo::HAS_LSE128)); + assert!(!x.test(CpuInfo::HAS_RCPC3)); + x.set(CpuInfo::HAS_RCPC3); + assert!(x.test(CpuInfo::INIT)); + assert!(x.test(CpuInfo::HAS_LSE)); + assert!(x.test(CpuInfo::HAS_LSE2)); + assert!(x.test(CpuInfo::HAS_LSE128)); + assert!(x.test(CpuInfo::HAS_RCPC3)); } // CPU feature detection from reading /proc/cpuinfo (Linux/NetBSD) @@ -311,5 +326,17 @@ mod tests_aarch64_common { // assert!(!std::arch::is_aarch64_feature_detected!("lse128")); // } } + if detect().test(CpuInfo::HAS_RCPC3) { + assert!(detect().test(CpuInfo::HAS_RCPC3)); + } else { + assert!(!detect().test(CpuInfo::HAS_RCPC3)); + // #[cfg(not(any( + // portable_atomic_no_aarch64_target_feature, + // portable_atomic_unstable_aarch64_target_feature + // )))] + // { + // assert!(!std::arch::is_aarch64_feature_detected!("rcpc3")); + // } + } } }