From 4e194051d4ebb2a2a750480f3b1bb5d5fe97f37a Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Fri, 29 Nov 2024 17:10:32 +0800 Subject: [PATCH] Add 128-bit SIMD implementation for LoongArch --- src/control/group/lsx.rs | 137 +++++++++++++++++++++++++++++++++++++++ src/control/group/mod.rs | 8 +++ src/lib.rs | 1 + 3 files changed, 146 insertions(+) create mode 100644 src/control/group/lsx.rs diff --git a/src/control/group/lsx.rs b/src/control/group/lsx.rs new file mode 100644 index 000000000..5f45bc8d3 --- /dev/null +++ b/src/control/group/lsx.rs @@ -0,0 +1,137 @@ +use super::super::{BitMask, Tag}; +use core::mem; +use core::num::NonZeroU16; + +use core::arch::loongarch64::*; +use mem::transmute; + +pub(crate) type BitMaskWord = u16; +pub(crate) type NonZeroBitMaskWord = NonZeroU16; +pub(crate) const BITMASK_STRIDE: usize = 1; +pub(crate) const BITMASK_MASK: BitMaskWord = 0xffff; +pub(crate) const BITMASK_ITER_MASK: BitMaskWord = !0; + +/// Abstraction over a group of control tags which can be scanned in +/// parallel. +/// +/// This implementation uses a 128-bit LSX value. +#[derive(Copy, Clone)] +pub(crate) struct Group(v16i8); + +// FIXME: https://github.com/rust-lang/rust-clippy/issues/3859 +#[allow(clippy::use_self)] +impl Group { + /// Number of bytes in the group. + pub(crate) const WIDTH: usize = mem::size_of::(); + + /// Returns a full group of empty tags, suitable for use as the initial + /// value for an empty hash table. + /// + /// This is guaranteed to be aligned to the group size. + #[inline] + #[allow(clippy::items_after_statements)] + pub(crate) const fn static_empty() -> &'static [Tag; Group::WIDTH] { + #[repr(C)] + struct AlignedTags { + _align: [Group; 0], + tags: [Tag; Group::WIDTH], + } + const ALIGNED_TAGS: AlignedTags = AlignedTags { + _align: [], + tags: [Tag::EMPTY; Group::WIDTH], + }; + &ALIGNED_TAGS.tags + } + + /// Loads a group of tags starting at the given address. + #[inline] + #[allow(clippy::cast_ptr_alignment)] // unaligned load + pub(crate) unsafe fn load(ptr: *const Tag) -> Self { + Group(lsx_vld::<0>(ptr.cast())) + } + + /// Loads a group of tags starting at the given address, which must be + /// aligned to `mem::align_of::()`. + #[inline] + #[allow(clippy::cast_ptr_alignment)] + pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + Group(lsx_vld::<0>(ptr.cast())) + } + + /// Stores the group of tags to the given address, which must be + /// aligned to `mem::align_of::()`. + #[inline] + #[allow(clippy::cast_ptr_alignment)] + pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + lsx_vst::<0>(self.0, ptr.cast()); + } + + /// Returns a `BitMask` indicating all tags in the group which have + /// the given value. + #[inline] + pub(crate) fn match_tag(self, tag: Tag) -> BitMask { + #[allow(clippy::missing_transmute_annotations)] + unsafe { + let cmp = lsx_vseq_b(self.0, lsx_vreplgr2vr_b(tag.0 as i32)); + BitMask(lsx_vpickve2gr_hu::<0>(transmute(lsx_vmskltz_b(cmp))) as u16) + } + } + + /// Returns a `BitMask` indicating all tags in the group which are + /// `EMPTY`. + #[inline] + pub(crate) fn match_empty(self) -> BitMask { + #[allow(clippy::missing_transmute_annotations)] + unsafe { + let cmp = lsx_vseqi_b::<{ Tag::EMPTY.0 as i8 as i32 }>(self.0); + BitMask(lsx_vpickve2gr_hu::<0>(transmute(lsx_vmskltz_b(cmp))) as u16) + } + } + + /// Returns a `BitMask` indicating all tags in the group which are + /// `EMPTY` or `DELETED`. + #[inline] + pub(crate) fn match_empty_or_deleted(self) -> BitMask { + #[allow(clippy::missing_transmute_annotations)] + unsafe { + // A tag is EMPTY or DELETED iff the high bit is set + BitMask(lsx_vpickve2gr_hu::<0>(transmute(lsx_vmskltz_b(self.0))) as u16) + } + } + + /// Returns a `BitMask` indicating all tags in the group which are full. + #[inline] + pub(crate) fn match_full(&self) -> BitMask { + #[allow(clippy::missing_transmute_annotations)] + unsafe { + // A tag is EMPTY or DELETED iff the high bit is set + BitMask(lsx_vpickve2gr_hu::<0>(transmute(lsx_vmskgez_b(self.0))) as u16) + } + } + + /// Performs the following transformation on all tags in the group: + /// - `EMPTY => EMPTY` + /// - `DELETED => EMPTY` + /// - `FULL => DELETED` + #[inline] + pub(crate) fn convert_special_to_empty_and_full_to_deleted(self) -> Self { + // Map high_bit = 1 (EMPTY or DELETED) to 1111_1111 + // and high_bit = 0 (FULL) to 1000_0000 + // + // Here's this logic expanded to concrete values: + // let special = 0 > tag = 1111_1111 (true) or 0000_0000 (false) + // 1111_1111 | 1000_0000 = 1111_1111 + // 0000_0000 | 1000_0000 = 1000_0000 + #[allow(clippy::missing_transmute_annotations)] + unsafe { + let zero = lsx_vreplgr2vr_b(0); + let special = lsx_vslt_b(self.0, zero); + Group(transmute(lsx_vor_v( + transmute(special), + transmute(lsx_vreplgr2vr_b(Tag::DELETED.0 as i32)), + ))) + } + } +} diff --git a/src/control/group/mod.rs b/src/control/group/mod.rs index 614326048..fe2d77483 100644 --- a/src/control/group/mod.rs +++ b/src/control/group/mod.rs @@ -24,6 +24,14 @@ cfg_if! { ))] { mod neon; use neon as imp; + } else if #[cfg(all( + feature = "nightly", + target_arch = "loongarch64", + target_feature = "lsx", + not(miri), + ))] { + mod lsx; + use lsx as imp; } else { mod generic; use generic as imp; diff --git a/src/lib.rs b/src/lib.rs index 5fa2cda4c..60dc4bd23 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,7 @@ feature = "nightly", allow(clippy::incompatible_msrv, internal_features) )] +#![cfg_attr(target_arch = "loongarch64", feature(stdarch_loongarch))] /// Default hasher for [`HashMap`] and [`HashSet`]. #[cfg(feature = "default-hasher")]