Add force-amo feature (portable_atomic_force_amo cfg)

.github/.cspell/project-dictionary.txt

@@ -4,6 +4,7 @@ alcgr
 algr
 allnoconfig
 aosp
+aqrl
 armasm
 armreg
 Auxinfo
@@ -79,9 +80,11 @@ lqarx
 lrcpc
 lwsync
 machdep
+maxu
 mfence
 mgba
 midr
+minu
 mipsn
 miscompiles
 mmfr
@@ -125,8 +128,10 @@ simavr
 skiboot
 slbgr
 slgr
+sllw
 spinlock
 sreg
+srlw
 sstatus
 stdarch
 stdbool

.github/workflows/ci.yml

@@ -446,6 +446,7 @@ jobs:
       matrix:
         rust:
           - '1.64'
+          - '1.72'
           - stable
           - nightly
     runs-on: ubuntu-latest

Cargo.toml

@@ -58,9 +58,16 @@ require-cas = []
 # https://github.com/taiki-e/portable-atomic#optional-features-unsafe-assume-single-core
 unsafe-assume-single-core = []
 
+# The following are sub-features of the unsafe-assume-single-core feature and if enabled without
+# the unsafe-assume-single-core feature will result in a compile error.
+# There is no explicit "unsafe-" prefix because the user has already opted in to "unsafe" by
+# enabling the unsafe-assume-single-core feature, but misuse of these features is also usually
+# considered unsound.
+
 # For RISC-V targets, generate code for S mode to disable interrupts.
 s-mode = []
-
+# For RISC-V targets, use AMO instructions even if A-extension is disabled.
+force-amo = []
 # For ARM targets, also disable FIQs when disabling interrupts.
 disable-fiq = []
 

build.rs

@@ -22,6 +22,8 @@ fn main() {
     println!("cargo:rustc-cfg=portable_atomic_unsafe_assume_single_core");
     #[cfg(feature = "s-mode")]
     println!("cargo:rustc-cfg=portable_atomic_s_mode");
+    #[cfg(feature = "force-amo")]
+    println!("cargo:rustc-cfg=portable_atomic_force_amo");
     #[cfg(feature = "disable-fiq")]
     println!("cargo:rustc-cfg=portable_atomic_disable_fiq");
 

src/imp/interrupt/README.md

@@ -17,10 +17,11 @@ For some targets, the implementation can be changed by explicitly enabling featu
 - On pre-v6 ARM with the `disable-fiq` feature, this disables interrupts by modifying the I (IRQ mask) bit and F (FIQ mask) bit of the CPSR.
 - On RISC-V (without A-extension), this disables interrupts by modifying the MIE (Machine Interrupt Enable) bit of the `mstatus` register.
 - On RISC-V (without A-extension) with the `s-mode` feature, this disables interrupts by modifying the SIE (Supervisor Interrupt Enable) bit of the `sstatus` register.
+- On RISC-V (without A-extension) with the `force-amo` feature, this uses AMO instructions for RMWs that have corresponding AMO instructions even if A-extension is disabled. For other RMWs, this disables interrupts as usual.
 - On MSP430, this disables interrupts by modifying the GIE (Global Interrupt Enable) bit of the status register (SR).
 - On AVR, this disables interrupts by modifying the I (Global Interrupt Enable) bit of the status register (SREG).
 - On Xtensa, this disables interrupts by modifying the PS special register.
 
-Some operations don't require disabling interrupts (loads and stores on targets except for AVR, but additionally on MSP430 `add`, `sub`, `and`, `or`, `xor`, `not`). However, when the `critical-section` feature is enabled, critical sections are taken for all atomic operations.
+Some operations don't require disabling interrupts (loads and stores on targets except for AVR, but additionally on MSP430 `add,sub,and,or,xor,not`, on RISC-V with the `force-amo` feature `swap,fetch_{add,sub,and,or,xor,not,max,min},add,sub,and,or,xor,not`). However, when the `critical-section` feature is enabled, critical sections are taken for all atomic operations.
 
 Feel free to submit an issue if your target is not supported yet.

src/imp/interrupt/mod.rs

@@ -175,7 +175,13 @@ impl<T> AtomicPtr<T> {
     }
 
     #[inline]
-    pub(crate) fn swap(&self, ptr: *mut T, _order: Ordering) -> *mut T {
+    pub(crate) fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T {
+        let _ = order;
+        #[cfg(portable_atomic_force_amo)]
+        {
+            self.as_native().swap(ptr, order)
+        }
+        #[cfg(not(portable_atomic_force_amo))]
         // SAFETY: any data races are prevented by disabling interrupts (see
         // module-level comments) and the raw pointer is valid because we got it
         // from a reference.
@@ -277,9 +283,12 @@ macro_rules! atomic_int {
             }
         }
     };
-    (load_store_atomic, $atomic_type:ident, $int_type:ident, $align:literal) => {
+    (load_store_atomic $([$kind:ident])?, $atomic_type:ident, $int_type:ident, $align:literal) => {
         atomic_int!(base, $atomic_type, $int_type, $align);
-        atomic_int!(cas, $atomic_type, $int_type);
+        #[cfg(not(portable_atomic_force_amo))]
+        atomic_int!(cas[emulate], $atomic_type, $int_type);
+        #[cfg(portable_atomic_force_amo)]
+        atomic_int!(cas $([$kind])?, $atomic_type, $int_type);
         impl $atomic_type {
             #[inline]
             #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
@@ -360,7 +369,7 @@ macro_rules! atomic_int {
     };
     (load_store_critical_session, $atomic_type:ident, $int_type:ident, $align:literal) => {
         atomic_int!(base, $atomic_type, $int_type, $align);
-        atomic_int!(cas, $atomic_type, $int_type);
+        atomic_int!(cas[emulate], $atomic_type, $int_type);
         impl_default_no_fetch_ops!($atomic_type, $int_type);
         impl_default_bit_opts!($atomic_type, $int_type);
         impl $atomic_type {
@@ -390,7 +399,7 @@ macro_rules! atomic_int {
             }
         }
     };
-    (cas, $atomic_type:ident, $int_type:ident) => {
+    (cas[emulate], $atomic_type:ident, $int_type:ident) => {
         impl $atomic_type {
             #[inline]
             pub(crate) fn swap(&self, val: $int_type, _order: Ordering) -> $int_type {
@@ -544,6 +553,256 @@ macro_rules! atomic_int {
                 })
             }
 
+            #[inline]
+            pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    self.v.get().write(prev.wrapping_neg());
+                    prev
+                })
+            }
+            #[inline]
+            pub(crate) fn neg(&self, order: Ordering) {
+                self.fetch_neg(order);
+            }
+        }
+    };
+    // cfg(portable_atomic_force_amo) 32-bit(RV32)/{32,64}-bit(RV64) RMW
+    (cas, $atomic_type:ident, $int_type:ident) => {
+        impl $atomic_type {
+            #[inline]
+            pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().swap(val, order)
+            }
+
+            #[inline]
+            #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
+            pub(crate) fn compare_exchange(
+                &self,
+                current: $int_type,
+                new: $int_type,
+                success: Ordering,
+                failure: Ordering,
+            ) -> Result<$int_type, $int_type> {
+                crate::utils::assert_compare_exchange_ordering(success, failure);
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    if prev == current {
+                        self.v.get().write(new);
+                        Ok(prev)
+                    } else {
+                        Err(prev)
+                    }
+                })
+            }
+
+            #[inline]
+            #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
+            pub(crate) fn compare_exchange_weak(
+                &self,
+                current: $int_type,
+                new: $int_type,
+                success: Ordering,
+                failure: Ordering,
+            ) -> Result<$int_type, $int_type> {
+                self.compare_exchange(current, new, success, failure)
+            }
+
+            #[inline]
+            pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_add(val, order)
+            }
+            #[inline]
+            pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_sub(val, order)
+            }
+            #[inline]
+            pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_and(val, order)
+            }
+
+            #[inline]
+            pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    self.v.get().write(!(prev & val));
+                    prev
+                })
+            }
+
+            #[inline]
+            pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_or(val, order)
+            }
+            #[inline]
+            pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_xor(val, order)
+            }
+            #[inline]
+            pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_max(val, order)
+            }
+            #[inline]
+            pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_min(val, order)
+            }
+            #[inline]
+            pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
+                self.as_native().fetch_not(order)
+            }
+
+            #[inline]
+            pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    self.v.get().write(prev.wrapping_neg());
+                    prev
+                })
+            }
+            #[inline]
+            pub(crate) fn neg(&self, order: Ordering) {
+                self.fetch_neg(order);
+            }
+        }
+    };
+    // cfg(portable_atomic_force_amo) {8,16}-bit RMW
+    (cas[sub_word], $atomic_type:ident, $int_type:ident) => {
+        impl $atomic_type {
+            #[inline]
+            pub(crate) fn swap(&self, val: $int_type, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe { self.v.get().replace(val) })
+            }
+
+            #[inline]
+            #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
+            pub(crate) fn compare_exchange(
+                &self,
+                current: $int_type,
+                new: $int_type,
+                success: Ordering,
+                failure: Ordering,
+            ) -> Result<$int_type, $int_type> {
+                crate::utils::assert_compare_exchange_ordering(success, failure);
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    if prev == current {
+                        self.v.get().write(new);
+                        Ok(prev)
+                    } else {
+                        Err(prev)
+                    }
+                })
+            }
+
+            #[inline]
+            #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
+            pub(crate) fn compare_exchange_weak(
+                &self,
+                current: $int_type,
+                new: $int_type,
+                success: Ordering,
+                failure: Ordering,
+            ) -> Result<$int_type, $int_type> {
+                self.compare_exchange(current, new, success, failure)
+            }
+
+            #[inline]
+            pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    self.v.get().write(prev.wrapping_add(val));
+                    prev
+                })
+            }
+
+            #[inline]
+            pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    self.v.get().write(prev.wrapping_sub(val));
+                    prev
+                })
+            }
+
+            #[inline]
+            pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_and(val, order)
+            }
+
+            #[inline]
+            pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    self.v.get().write(!(prev & val));
+                    prev
+                })
+            }
+
+            #[inline]
+            pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_or(val, order)
+            }
+            #[inline]
+            pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type {
+                self.as_native().fetch_xor(val, order)
+            }
+
+            #[inline]
+            pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    self.v.get().write(core::cmp::max(prev, val));
+                    prev
+                })
+            }
+
+            #[inline]
+            pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type {
+                // SAFETY: any data races are prevented by disabling interrupts (see
+                // module-level comments) and the raw pointer is valid because we got it
+                // from a reference.
+                with(|| unsafe {
+                    let prev = self.v.get().read();
+                    self.v.get().write(core::cmp::min(prev, val));
+                    prev
+                })
+            }
+
+            #[inline]
+            pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type {
+                self.as_native().fetch_not(order)
+            }
+
             #[inline]
             pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type {
                 // SAFETY: any data races are prevented by disabling interrupts (see
@@ -580,10 +839,10 @@ atomic_int!(load_store_atomic, AtomicIsize, isize, 16);
 #[cfg(target_pointer_width = "128")]
 atomic_int!(load_store_atomic, AtomicUsize, usize, 16);
 
-atomic_int!(load_store_atomic, AtomicI8, i8, 1);
-atomic_int!(load_store_atomic, AtomicU8, u8, 1);
-atomic_int!(load_store_atomic, AtomicI16, i16, 2);
-atomic_int!(load_store_atomic, AtomicU16, u16, 2);
+atomic_int!(load_store_atomic[sub_word], AtomicI8, i8, 1);
+atomic_int!(load_store_atomic[sub_word], AtomicU8, u8, 1);
+atomic_int!(load_store_atomic[sub_word], AtomicI16, i16, 2);
+atomic_int!(load_store_atomic[sub_word], AtomicU16, u16, 2);
 
 #[cfg(not(target_pointer_width = "16"))]
 atomic_int!(load_store_atomic, AtomicI32, i32, 4);