From d1653635b4ce086c50bbe56615d7d6db5e2d9e57 Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Sat, 25 May 2024 15:19:32 +0200 Subject: [PATCH 01/13] Make scalar backends generic over inner SIMD backends --- src/parser.rs | 1 + src/util/simd/avx2.rs | 4 +-- src/util/simd/mod.rs | 34 +++++++++++++++--------- src/util/simd/sse2.rs | 4 +-- src/util/simd/traits.rs | 8 +++--- src/util/simd/v128.rs | 8 ++++-- src/util/simd/v256.rs | 58 +++++++++++++++++++++++------------------ src/util/simd/v512.rs | 58 +++++++++++++++++++++++------------------ 8 files changed, 103 insertions(+), 72 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index d219298..85f505b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -29,6 +29,7 @@ use crate::{ arch::{get_nonspace_bits, prefix_xor}, num::{parse_number, ParserNumber}, simd::{i8x32, m8x32, u8x32, u8x64, Mask, Simd}, + //simd::{Mask, Simd}, string::*, unicode::{codepoint_to_utf8, hex_to_u32_nocheck}, }, diff --git a/src/util/simd/avx2.rs b/src/util/simd/avx2.rs index 2bb6e4e..eb4e106 100644 --- a/src/util/simd/avx2.rs +++ b/src/util/simd/avx2.rs @@ -56,9 +56,9 @@ impl Simd for Simd256i { #[repr(transparent)] pub struct Mask256(__m256i); -impl_lanes!(Simd256u, 32); +impl_lanes!([impl Simd256u] 32); -impl_lanes!(Mask256, 32); +impl_lanes!([impl Mask256] 32); impl Mask for Mask256 { type BitMask = u32; diff --git a/src/util/simd/mod.rs b/src/util/simd/mod.rs index a7431a7..c3cfa3d 100644 --- a/src/util/simd/mod.rs +++ b/src/util/simd/mod.rs @@ -1,11 +1,25 @@ pub mod bits; mod traits; +// Link to all the SIMD backends by default for later runtime choice +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + pub(crate) mod avx2; + pub(crate) mod sse2; + } else if #[cfg(target_arch = "aarch64")] { + pub(crate) mod neon; + } +} + +pub(crate) mod v128; +pub(crate) mod v256; +pub(crate) mod v512; + #[doc(hidden)] #[macro_export] macro_rules! impl_lanes { - ($simd: ty, $lane: expr) => { - impl $simd { + ([$($impl_statement:tt)+] $lane: expr) => { + $($impl_statement)+ { pub const fn lanes() -> usize { $lane } @@ -16,14 +30,11 @@ macro_rules! impl_lanes { // pick v128 simd cfg_if::cfg_if! { if #[cfg(target_feature = "sse2")] { - mod sse2; use self::sse2::*; } else if #[cfg(all(target_feature="neon", target_arch="aarch64"))] { - pub(crate) mod neon; use self::neon::*; } else { // TODO: support wasm - mod v128; use self::v128::*; } } @@ -31,10 +42,8 @@ cfg_if::cfg_if! { // pick v256 simd cfg_if::cfg_if! { if #[cfg(target_feature = "avx2")] { - mod avx2; use self::avx2::*; } else { - mod v256; use self::v256::*; } } @@ -42,15 +51,14 @@ cfg_if::cfg_if! { pub use self::traits::{BitMask, Mask, Simd}; // pick v512 simd // TODO: support avx512? -mod v512; use self::v512::*; pub type u8x16 = Simd128u; -pub type u8x32 = Simd256u; -pub type u8x64 = Simd512u; +pub type u8x32 = Simd256u; +pub type u8x64 = Simd512u; -pub type i8x32 = Simd256i; +pub type i8x32 = Simd256i; pub type m8x16 = Mask128; -pub type m8x32 = Mask256; -pub type m8x64 = Mask512; +pub type m8x32 = Mask256; +pub type m8x64 = Mask512; diff --git a/src/util/simd/sse2.rs b/src/util/simd/sse2.rs index f4065a7..a9727e3 100644 --- a/src/util/simd/sse2.rs +++ b/src/util/simd/sse2.rs @@ -56,9 +56,9 @@ impl Simd for Simd128i { #[repr(transparent)] pub struct Mask128(__m128i); -impl_lanes!(Simd128u, 16); +impl_lanes!([impl Simd128u] 16); -impl_lanes!(Mask128, 16); +impl_lanes!([impl Mask128] 16); impl Mask for Mask128 { type BitMask = u16; diff --git a/src/util/simd/traits.rs b/src/util/simd/traits.rs index 793993f..05cf646 100644 --- a/src/util/simd/traits.rs +++ b/src/util/simd/traits.rs @@ -1,6 +1,6 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; -/// Portbal SIMD traits +/// Portable SIMD traits pub trait Simd: Sized { const LANES: usize; @@ -32,8 +32,10 @@ pub trait Simd: Sized { fn le(&self, rhs: &Self) -> Self::Mask; } -/// Portbal SIMD mask traits -pub trait Mask: Sized + BitOr + BitOrAssign + BitAnd { +/// Portable SIMD mask traits +pub trait Mask: + Sized + BitOr + BitOrAssign + BitAnd +{ type Element; type BitMask: BitMask; diff --git a/src/util/simd/v128.rs b/src/util/simd/v128.rs index 40f6171..3fa1ae5 100644 --- a/src/util/simd/v128.rs +++ b/src/util/simd/v128.rs @@ -16,6 +16,8 @@ pub struct Mask128([u8; 16]); impl Simd for Simd128i { const LANES: usize = 16; + + type Element = i8; type Mask = Mask128; unsafe fn loadu(ptr: *const u8) -> Self { @@ -38,8 +40,8 @@ impl Simd for Simd128i { Mask128(mask) } - fn splat(value: u8) -> Self { - Self([value as i8; Self::LANES]) + fn splat(value: i8) -> Self { + Self([value; Self::LANES]) } fn le(&self, rhs: &Self) -> Self::Mask { @@ -61,6 +63,8 @@ impl Simd for Simd128i { impl Simd for Simd128u { const LANES: usize = 16; + + type Element = u8; type Mask = Mask128; unsafe fn loadu(ptr: *const u8) -> Self { diff --git a/src/util/simd/v256.rs b/src/util/simd/v256.rs index 0151564..4d8744b 100644 --- a/src/util/simd/v256.rs +++ b/src/util/simd/v256.rs @@ -3,23 +3,23 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; use super::{bits::combine_u16, Mask, Mask128, Simd, Simd128i, Simd128u}; use crate::impl_lanes; -impl_lanes!(Simd256u, 32); +impl_lanes!([impl Simd256u] 32); -impl_lanes!(Mask256, 32); +impl_lanes!([impl Mask256] 32); #[derive(Debug)] #[repr(transparent)] -pub struct Simd256u((Simd128u, Simd128u)); +pub struct Simd256u((B, B)); #[derive(Debug)] #[repr(transparent)] -pub struct Simd256i((Simd128i, Simd128i)); +pub struct Simd256i((B, B)); #[derive(Debug)] #[repr(transparent)] -pub struct Mask256(pub(crate) (Mask128, Mask128)); +pub struct Mask256(pub(crate) (M, M)); -impl Mask for Mask256 { +impl> Mask for Mask256 { type BitMask = u32; type Element = u8; @@ -38,11 +38,11 @@ impl Mask for Mask256 { #[inline(always)] fn splat(b: bool) -> Self { - Mask256((Mask128::splat(b), Mask128::splat(b))) + Mask256((M::splat(b), M::splat(b))) } } -impl BitOr for Mask256 { +impl BitOr for Mask256 { type Output = Self; #[inline(always)] @@ -53,7 +53,7 @@ impl BitOr for Mask256 { } } -impl BitOrAssign for Mask256 { +impl BitOrAssign for Mask256 { #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { self.0 .0 |= rhs.0 .0; @@ -61,34 +61,38 @@ impl BitOrAssign for Mask256 { } } -impl BitAnd for Mask256 { +impl BitAnd> for Mask256 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: Mask256) -> Self::Output { + fn bitand(self, rhs: Mask256) -> Self::Output { let lo = self.0 .0 & rhs.0 .0; let hi = self.0 .1 & rhs.0 .1; Mask256((lo, hi)) } } -impl Simd for Simd256u { +impl Simd for Simd256u +where + B: Simd, + B::Mask: Mask, +{ const LANES: usize = 32; - type Mask = Mask256; + type Mask = Mask256; type Element = u8; #[inline(always)] unsafe fn loadu(ptr: *const u8) -> Self { - let lo = Simd128u::loadu(ptr); - let hi = Simd128u::loadu(ptr.add(Simd128u::LANES)); + let lo = B::loadu(ptr); + let hi = B::loadu(ptr.add(Simd128u::LANES)); Simd256u((lo, hi)) } #[inline(always)] unsafe fn storeu(&self, ptr: *mut u8) { - Simd128u::storeu(&self.0 .0, ptr); - Simd128u::storeu(&self.0 .1, ptr.add(Simd128u::LANES)); + B::storeu(&self.0 .0, ptr); + B::storeu(&self.0 .1, ptr.add(Simd128u::LANES)); } #[inline(always)] @@ -100,7 +104,7 @@ impl Simd for Simd256u { #[inline(always)] fn splat(elem: u8) -> Self { - Simd256u((Simd128u::splat(elem), Simd128u::splat(elem))) + Simd256u((B::splat(elem), B::splat(elem))) } #[inline(always)] @@ -118,23 +122,27 @@ impl Simd for Simd256u { } } -impl Simd for Simd256i { +impl Simd for Simd256i +where + B: Simd, + B::Mask: Mask, +{ const LANES: usize = 32; - type Mask = Mask256; + type Mask = Mask256; type Element = i8; #[inline(always)] unsafe fn loadu(ptr: *const u8) -> Self { - let lo = Simd128i::loadu(ptr); - let hi = Simd128i::loadu(ptr.add(Simd128i::LANES)); + let lo = B::loadu(ptr); + let hi = B::loadu(ptr.add(Simd128i::LANES)); Simd256i((lo, hi)) } #[inline(always)] unsafe fn storeu(&self, ptr: *mut u8) { - Simd128i::storeu(&self.0 .0, ptr); - Simd128i::storeu(&self.0 .1, ptr.add(Simd128i::LANES)); + B::storeu(&self.0 .0, ptr); + B::storeu(&self.0 .1, ptr.add(Simd128i::LANES)); } #[inline(always)] @@ -146,7 +154,7 @@ impl Simd for Simd256i { #[inline(always)] fn splat(elem: i8) -> Self { - Simd256i((Simd128i::splat(elem), Simd128i::splat(elem))) + Simd256i((B::splat(elem), B::splat(elem))) } #[inline(always)] diff --git a/src/util/simd/v512.rs b/src/util/simd/v512.rs index 0b6ead7..fa33354 100644 --- a/src/util/simd/v512.rs +++ b/src/util/simd/v512.rs @@ -3,23 +3,23 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; use super::{bits::combine_u32, Mask, Mask256, Simd, Simd256i, Simd256u}; use crate::impl_lanes; -impl_lanes!(Simd512u, 64); +impl_lanes!([impl Simd512u] 64); -impl_lanes!(Mask512, 64); +impl_lanes!([impl Mask512] 64); #[derive(Debug)] #[repr(transparent)] -pub struct Simd512u((Simd256u, Simd256u)); +pub struct Simd512u((B, B)); #[derive(Debug)] #[repr(transparent)] -pub struct Simd512i((Simd256i, Simd256i)); +pub struct Simd512i((B, B)); #[derive(Debug)] #[repr(transparent)] -pub struct Mask512((Mask256, Mask256)); +pub struct Mask512((M, M)); -impl Mask for Mask512 { +impl> Mask for Mask512 { type BitMask = u64; type Element = u8; @@ -40,11 +40,11 @@ impl Mask for Mask512 { #[inline(always)] fn splat(b: bool) -> Self { - Mask512((Mask256::splat(b), Mask256::splat(b))) + Mask512((M::splat(b), M::splat(b))) } } -impl BitOr for Mask512 { +impl BitOr for Mask512 { type Output = Self; #[inline(always)] @@ -55,7 +55,7 @@ impl BitOr for Mask512 { } } -impl BitOrAssign for Mask512 { +impl BitOrAssign for Mask512 { #[inline(always)] fn bitor_assign(&mut self, rhs: Self) { self.0 .0 |= rhs.0 .0; @@ -63,33 +63,37 @@ impl BitOrAssign for Mask512 { } } -impl BitAnd for Mask512 { +impl BitAnd> for Mask512 { type Output = Self; #[inline(always)] - fn bitand(self, rhs: Mask512) -> Self::Output { + fn bitand(self, rhs: Mask512) -> Self::Output { let lo = self.0 .0 & rhs.0 .0; let hi = self.0 .1 & rhs.0 .1; Mask512((lo, hi)) } } -impl Simd for Simd512u { +impl Simd for Simd512u +where + B: Simd, + B::Mask: Mask, +{ const LANES: usize = 64; type Element = u8; - type Mask = Mask512; + type Mask = Mask512; #[inline(always)] unsafe fn loadu(ptr: *const u8) -> Self { - let lo = Simd256u::loadu(ptr); - let hi = Simd256u::loadu(ptr.add(Simd256u::LANES)); + let lo = B::loadu(ptr); + let hi = B::loadu(ptr.add(B::LANES)); Simd512u((lo, hi)) } #[inline(always)] unsafe fn storeu(&self, ptr: *mut u8) { - Simd256u::storeu(&self.0 .0, ptr); - Simd256u::storeu(&self.0 .1, ptr.add(Simd256u::LANES)); + B::storeu(&self.0 .0, ptr); + B::storeu(&self.0 .1, ptr.add(B::LANES)); } #[inline(always)] @@ -101,7 +105,7 @@ impl Simd for Simd512u { #[inline(always)] fn splat(ch: u8) -> Self { - Simd512u((Simd256u::splat(ch), Simd256u::splat(ch))) + Simd512u((B::splat(ch), B::splat(ch))) } #[inline(always)] @@ -119,23 +123,27 @@ impl Simd for Simd512u { } } -impl Simd for Simd512i { +impl Simd for Simd512i +where + B: Simd, + B::Mask: Mask, +{ const LANES: usize = 64; type Element = i8; - type Mask = Mask512; + type Mask = Mask512; #[inline(always)] unsafe fn loadu(ptr: *const u8) -> Self { - let lo = Simd256i::loadu(ptr); - let hi = Simd256i::loadu(ptr.add(Simd256i::LANES)); + let lo = B::loadu(ptr); + let hi = B::loadu(ptr.add(B::LANES)); Simd512i((lo, hi)) } #[inline(always)] unsafe fn storeu(&self, ptr: *mut u8) { - Simd256i::storeu(&self.0 .0, ptr); - Simd256i::storeu(&self.0 .1, ptr.add(Simd256i::LANES)); + B::storeu(&self.0 .0, ptr); + B::storeu(&self.0 .1, ptr.add(B::LANES)); } #[inline(always)] @@ -147,7 +155,7 @@ impl Simd for Simd512i { #[inline(always)] fn splat(elem: i8) -> Self { - Simd512i((Simd256i::splat(elem), Simd256i::splat(elem))) + Simd512i((B::splat(elem), B::splat(elem))) } #[inline(always)] From 00b45382c8eb1dafb98dd938d6d94d2715165095 Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Sat, 25 May 2024 16:12:53 +0200 Subject: [PATCH 02/13] Make parser generic over SIMD ops --- src/parser.rs | 57 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 85f505b..18ee972 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,4 +1,5 @@ use std::{ + marker::PhantomData, num::NonZeroU8, ops::Deref, slice::{from_raw_parts, from_raw_parts_mut}, @@ -28,8 +29,8 @@ use crate::{ arc::Arc, arch::{get_nonspace_bits, prefix_xor}, num::{parse_number, ParserNumber}, - simd::{i8x32, m8x32, u8x32, u8x64, Mask, Simd}, - //simd::{Mask, Simd}, + //simd::{i8x32, m8x32, u8x32, u8x64, Mask, Simd}, + simd::{Mask, Simd}, string::*, unicode::{codepoint_to_utf8, hex_to_u32_nocheck}, }, @@ -92,7 +93,11 @@ pub(crate) fn is_whitespace(ch: u8) -> bool { } #[inline(always)] -fn get_string_bits(data: &[u8; 64], prev_instring: &mut u64, prev_escaped: &mut u64) -> u64 { +fn get_string_bits(data: &[u8; 64], prev_instring: &mut u64, prev_escaped: &mut u64) -> u64 +where + u8x64: Simd, + u8x64::Mask: Mask, +{ let v = unsafe { u8x64::from_slice_unaligned_unchecked(data) }; let bs_bits = (v.eq(&u8x64::splat(b'\\'))).bitmask(); @@ -110,7 +115,7 @@ fn get_string_bits(data: &[u8; 64], prev_instring: &mut u64, prev_escaped: &mut } #[inline(always)] -fn skip_container_loop( +fn skip_container_loop( input: &[u8; 64], /* a 64-bytes slice from json */ prev_instring: &mut u64, /* the bitmap of last string */ prev_escaped: &mut u64, @@ -118,9 +123,13 @@ fn skip_container_loop( rbrace_num: &mut usize, left: u8, right: u8, -) -> Option { +) -> Option +where + u8x64: Simd, + u8x64::Mask: Mask, +{ // get the bitmao - let instring = get_string_bits(input, prev_instring, prev_escaped); + let instring = get_string_bits::(input, prev_instring, prev_escaped); // #Safety // the input is 64 bytes, so the v is always valid. let v = unsafe { u8x64::from_slice_unaligned_unchecked(input) }; @@ -142,12 +151,14 @@ fn skip_container_loop( None } -pub(crate) struct Parser { +pub(crate) struct Parser { pub(crate) read: R, error_index: usize, // mark the error position nospace_bits: u64, // SIMD marked nospace bitmap nospace_start: isize, // the start position of nospace_bits pub(crate) shared: Option>, // the shared allocator for `Value` + + _marker: PhantomData<(i8x32, u8x32, u8x64)>, } /// Records the parse status @@ -157,9 +168,15 @@ pub(crate) enum ParseStatus { HasEscaped, } -impl<'de, R> Parser +impl<'de, R, i8x32, u8x32, u8x64> Parser where R: Reader<'de>, + i8x32: Simd, + i8x32::Mask: Mask, + u8x32: Simd, + u8x32::Mask: Mask, + u8x64: Simd, + u8x64::Mask: Mask, { pub fn new(read: R) -> Self { Self { @@ -168,6 +185,8 @@ where nospace_bits: 0, nospace_start: -128, shared: None, + + _marker: PhantomData, } } @@ -974,10 +993,9 @@ where #[inline(always)] fn get_next_token(&mut self, tokens: [u8; N], advance: usize) -> Option { let r = &mut self.read; - const LANS: usize = u8x32::lanes(); - while let Some(chunk) = r.peek_n(LANS) { + while let Some(chunk) = r.peek_n(u8x32::LANES) { let v = unsafe { u8x32::from_slice_unaligned_unchecked(chunk) }; - let mut vor = m8x32::splat(false); + let mut vor = u8x32::Mask::splat(false); for t in tokens.iter().take(N) { vor |= v.eq(&u8x32::splat(*t)); } @@ -988,7 +1006,7 @@ where r.eat(cnt + advance); return Some(ch); } - r.eat(LANS); + r.eat(u8x32::LANES); } while let Some(ch) = r.peek() { @@ -1007,14 +1025,13 @@ where // escaped status. skip_string always start with the quote marks. #[inline(always)] fn skip_string_impl(&mut self) -> Result { - const LANS: usize = u8x32::lanes(); let r = &mut self.read; let mut quote_bits; let mut escaped; let mut prev_escaped = 0; let mut status = ParseStatus::None; - while let Some(chunk) = r.peek_n(LANS) { + while let Some(chunk) = r.peek_n(u8x32::LANES) { let v = unsafe { u8x32::from_slice_unaligned_unchecked(chunk) }; let bs_bits = (v.eq(&u8x32::splat(b'\\'))).bitmask(); quote_bits = (v.eq(&u8x32::splat(b'"'))).bitmask(); @@ -1030,7 +1047,7 @@ where r.eat(quote_bits.trailing_zeros() as usize + 1); return Ok(status); } - r.eat(LANS) + r.eat(u8x32::LANES) } // skip the possible prev escaped quote @@ -1086,10 +1103,8 @@ where // skip_string skips a JSON string with validation. #[inline(always)] fn skip_string(&mut self) -> Result { - const LANS: usize = u8x32::lanes(); - let mut status = ParseStatus::None; - while let Some(chunk) = self.read.peek_n(LANS) { + while let Some(chunk) = self.read.peek_n(u8x32::LANES) { let v = unsafe { u8x32::from_slice_unaligned_unchecked(chunk) }; let v_bs = v.eq(&u8x32::splat(b'\\')); let v_quote = v.eq(&u8x32::splat(b'"')); @@ -1111,7 +1126,7 @@ where _ => unreachable!(), } } else { - self.read.eat(LANS) + self.read.eat(u8x32::LANES) } } @@ -1219,7 +1234,7 @@ where while let Some(chunk) = reader.peek_n(64) { let input = array_ref![chunk, 0, 64]; - if let Some(count) = skip_container_loop( + if let Some(count) = skip_container_loop::( input, &mut prev_instring, &mut prev_escaped, @@ -1239,7 +1254,7 @@ where let n = reader.remain(); remain[..n].copy_from_slice(reader.peek_n(n).unwrap_unchecked()); } - if let Some(count) = skip_container_loop( + if let Some(count) = skip_container_loop::( &remain, &mut prev_instring, &mut prev_escaped, From 3e4afa1c62449cfa9f3f53e3e6ad6d11bed8c101 Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Sat, 25 May 2024 17:44:16 +0200 Subject: [PATCH 03/13] Add runtime dispatch --- Cargo.toml | 3 +- src/lazyvalue/get.rs | 4 +- src/{parser.rs => parser/inner.rs} | 23 +-- src/parser/mod.rs | 20 +++ src/parser/runtime.rs | 265 +++++++++++++++++++++++++++++ src/serde/de.rs | 70 ++++---- src/util/simd/mod.rs | 2 +- src/value/de.rs | 2 +- src/value/node.rs | 4 +- 9 files changed, 340 insertions(+), 53 deletions(-) rename src/{parser.rs => parser/inner.rs} (99%) create mode 100644 src/parser/mod.rs create mode 100644 src/parser/runtime.rs diff --git a/Cargo.toml b/Cargo.toml index ab17c50..60c650a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,7 +79,8 @@ name = "get_from" harness = false [features] -default = [] +default = ["runtime-detection"] +runtime-detection = [] # Use an arbitrary precision number type representation when parsing JSON into `sonic_rs::Value`. # This allows the JSON numbers will be serialized without loss of precision. diff --git a/src/lazyvalue/get.rs b/src/lazyvalue/get.rs index ae52269..e04351b 100644 --- a/src/lazyvalue/get.rs +++ b/src/lazyvalue/get.rs @@ -392,7 +392,7 @@ where let lv = LazyValue::new(json.from_subset(sub), status == ParseStatus::HasEscaped)?; // validate the utf-8 if slice - let index = parser.read.index(); + let index = parser.read().index(); if json.need_utf8_valid() { from_utf8(&slice[..index])?; } @@ -429,7 +429,7 @@ where let nodes = parser.get_many(tree, true)?; // validate the utf-8 if slice - let index = parser.read.index(); + let index = parser.read().index(); if json.need_utf8_valid() { from_utf8(&slice[..index])?; } diff --git a/src/parser.rs b/src/parser/inner.rs similarity index 99% rename from src/parser.rs rename to src/parser/inner.rs index 18ee972..635d7ac 100644 --- a/src/parser.rs +++ b/src/parser/inner.rs @@ -11,7 +11,8 @@ use faststr::FastStr; use serde::de::{self, Expected, Unexpected}; use smallvec::SmallVec; -use super::reader::{Reader, Reference}; +use super::{as_str, DEFAULT_KEY_BUF_CAPACITY}; +use crate::reader::{Reader, Reference}; #[cfg(all(target_feature = "neon", target_arch = "aarch64"))] use crate::util::simd::bits::NeonBits; use crate::{ @@ -38,11 +39,6 @@ use crate::{ JsonType, LazyValue, }; -pub(crate) const DEFAULT_KEY_BUF_CAPACITY: usize = 128; -pub(crate) fn as_str(data: &[u8]) -> &str { - unsafe { from_utf8_unchecked(data) } -} - #[inline(always)] fn get_escaped_branchless_u32(prev_escaped: &mut u32, backslash: u32) -> u32 { const EVEN_BITS: u32 = 0x5555_5555; @@ -152,11 +148,11 @@ where } pub(crate) struct Parser { - pub(crate) read: R, - error_index: usize, // mark the error position - nospace_bits: u64, // SIMD marked nospace bitmap - nospace_start: isize, // the start position of nospace_bits - pub(crate) shared: Option>, // the shared allocator for `Value` + read: R, + error_index: usize, // mark the error position + nospace_bits: u64, // SIMD marked nospace bitmap + nospace_start: isize, // the start position of nospace_bits + shared: Option>, // the shared allocator for `Value` _marker: PhantomData<(i8x32, u8x32, u8x64)>, } @@ -190,6 +186,11 @@ where } } + #[inline(always)] + pub(crate) fn read(&mut self) -> &mut R { + &mut self.read + } + #[inline(always)] fn error_index(&self) -> usize { // when parsing strings , we need record the error position. diff --git a/src/parser/mod.rs b/src/parser/mod.rs new file mode 100644 index 0000000..4a42455 --- /dev/null +++ b/src/parser/mod.rs @@ -0,0 +1,20 @@ +use std::str::from_utf8_unchecked; + +mod inner; + +cfg_if::cfg_if! { + if #[cfg(feature = "runtime-detection")] { + mod runtime; + pub(crate) use self::runtime::Parser; + } else { + use crate::util::simd::{i8x32, u8x32, u8x64}; + pub(crate) type Parser = self::inner::Parser; + } +} + +pub(crate) use self::inner::ParseStatus; + +pub(crate) const DEFAULT_KEY_BUF_CAPACITY: usize = 128; +pub(crate) fn as_str(data: &[u8]) -> &str { + unsafe { from_utf8_unchecked(data) } +} diff --git a/src/parser/runtime.rs b/src/parser/runtime.rs new file mode 100644 index 0000000..4266ef5 --- /dev/null +++ b/src/parser/runtime.rs @@ -0,0 +1,265 @@ +use faststr::FastStr; +use serde::de::Expected; + +use super::{inner, ParseStatus}; +#[cfg(target_arch = "aarch64")] +use crate::util::simd::neon; +#[cfg(target_arch = "x86_64")] +use crate::util::simd::{avx2, sse2}; +use crate::{ + error::{Error, ErrorCode, Result}, + reader::{Reader, Reference}, + util::{ + arc::Arc, + num::ParserNumber, + simd::{v128, v256, v512}, + }, + value::{shared::Shared, visitor::JsonVisitor}, + Index, LazyValue, PointerTree, +}; + +pub enum Parser { + #[cfg(target_arch = "x86_64")] + Sse2( + inner::Parser< + R, + v256::Simd256i, + v256::Simd256u, + v512::Simd512u>, + >, + ), + #[cfg(target_arch = "x86_64")] + Avx2(inner::Parser>), + #[cfg(target_arch = "aarch64")] + Neon( + inner::Parser< + R, + v256::Simd256i, + v256::Simd256u, + v512::Simd512u>, + >, + ), + Scalar( + inner::Parser< + R, + v256::Simd256i, + v256::Simd256u, + v512::Simd512u>, + >, + ), +} + +macro_rules! dispatch { + ($self:ident => $($stuff:tt)*) => {{ + cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + match $self { + Self::Avx2(val) => { + val.$($stuff)* + }, + Self::Sse2(val) => { + val.$($stuff)* + }, + Self::Scalar(val) => { + val.$($stuff)* + } + } + } else if #[cfg(target_arch = "aarch64")] { + match self { + Self::Neon(val) => { + val.$($stuff)* + }, + Self::Scalar(val) => { + val.$($stuff)* + } + } + } else { + let Self::Scalar(val) = self; + val.$($stuff)* + } + } + }}; +} + +impl<'de, R> Parser +where + R: Reader<'de>, +{ + #[inline(always)] + pub fn new(read: R) -> Self { + cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + let has_sse2 = is_x86_feature_detected!("sse2"); + let has_avx2 = is_x86_feature_detected!("avx2"); + + match (has_sse2, has_avx2) { + (false, false) => Self::Scalar(inner::Parser::new(read)), + (_, true) => Self::Avx2(inner::Parser::new(read)), + (true, false) => Self::Sse2(inner::Parser::new(read)), + } + } else if #[cfg(target_arch = "aarch64")] { + if is_aarch64_feature_detected!("neon") { + Self::Neon(inner::Parser::new(read)) + } else { + Self::Scalar(inner::Parser::new(read)) + } + } else { + Self::Scalar(inner::Parser::new(read)) + } + } + } + + #[inline(always)] + pub(crate) fn error(&self, reason: ErrorCode) -> Error { + dispatch!(self => error(reason)) + } + + #[inline(always)] + pub(crate) fn fix_position(&self, err: Error) -> Error { + dispatch!(self => fix_position(err)) + } + + #[inline(always)] + pub(crate) fn get_from_with_iter( + &mut self, + path: P, + ) -> Result<(&'de [u8], ParseStatus)> + where + P::Item: Index, + { + dispatch!(self => get_from_with_iter(path)) + } + + #[inline(always)] + pub(crate) fn get_from_with_iter_checked( + &mut self, + path: P, + ) -> Result<(&'de [u8], ParseStatus)> + where + P::Item: Index, + { + dispatch!(self => get_from_with_iter_checked(path)) + } + + #[inline(always)] + pub(crate) fn get_many( + &mut self, + tree: &PointerTree, + is_safe: bool, + ) -> Result>> { + dispatch!(self => get_many(tree, is_safe)) + } + + #[inline(always)] + pub(crate) fn get_shared_inc_count(&mut self) -> Arc { + dispatch!(self => get_shared_inc_count()) + } + + #[inline(always)] + pub(crate) fn parse_array_elem_lazy( + &mut self, + first: &mut bool, + check: bool, + ) -> Result> { + dispatch!(self => parse_array_elem_lazy(first, check)) + } + + #[inline(always)] + pub(crate) fn parse_array_end(&mut self) -> Result<()> { + dispatch!(self => parse_array_end()) + } + + #[inline(always)] + pub(crate) fn parse_entry_lazy( + &mut self, + strbuf: &mut Vec, + first: &mut bool, + check: bool, + ) -> Result> { + dispatch!(self => parse_entry_lazy(strbuf, first, check)) + } + + #[inline(always)] + pub(crate) fn parse_literal(&mut self, literal: &str) -> Result<()> { + dispatch!(self => parse_literal(literal)) + } + + #[inline(always)] + pub(crate) fn parse_number(&mut self, first: u8) -> Result { + dispatch!(self => parse_number(first)) + } + + #[inline(always)] + pub(crate) fn parse_object_clo(&mut self) -> Result<()> { + dispatch!(self => parse_object_clo()) + } + + #[inline(always)] + pub(crate) fn parse_str_impl<'own>( + &mut self, + buf: &'own mut Vec, + ) -> Result> { + dispatch!(self => parse_str_impl(buf)) + } + + #[inline(always)] + pub(crate) fn parse_string_raw<'own>( + &mut self, + buf: &'own mut Vec, + ) -> Result> { + dispatch!(self => parse_string_raw(buf)) + } + + #[inline(always)] + pub(crate) fn parse_trailing(&mut self) -> Result<()> { + dispatch!(self => parse_trailing()) + } + + #[inline(always)] + pub(crate) fn parse_value(&mut self, visitor: &mut V) -> Result<()> + where + V: JsonVisitor<'de>, + { + dispatch!(self => parse_value(visitor)) + } + + #[inline(always)] + pub(crate) fn parse_value_without_padding(&mut self, visitor: &mut V) -> Result<()> + where + V: JsonVisitor<'de>, + { + dispatch!(self => parse_value_without_padding(visitor)) + } + + #[inline(always)] + pub(crate) fn peek_invalid_type(&mut self, peek: u8, exp: &dyn Expected) -> Error { + dispatch!(self => peek_invalid_type(peek, exp)) + } + + #[inline(always)] + pub(crate) fn skip_number(&mut self, first: u8) -> Result<()> { + dispatch!(self => skip_number(first)) + } + + #[inline(always)] + pub(crate) fn skip_one(&mut self) -> Result<(&'de [u8], ParseStatus)> { + dispatch!(self => skip_one()) + } + + #[inline(always)] + pub(crate) fn skip_space(&mut self) -> Option { + dispatch!(self => skip_space()) + } + + #[inline(always)] + pub(crate) fn skip_space_peek(&mut self) -> Option { + dispatch!(self => skip_space_peek()) + } + + // --- FIELD GETTERS --- + + #[inline(always)] + pub(crate) fn read(&mut self) -> &mut R { + dispatch!(self => read()) + } +} diff --git a/src/serde/de.rs b/src/serde/de.rs index ef26dcd..1b13178 100644 --- a/src/serde/de.rs +++ b/src/serde/de.rs @@ -147,23 +147,23 @@ impl<'de, R: Reader<'de>> Deserializer { } fn scan_integer128(&mut self, buf: &mut String) -> Result<()> { - match self.parser.read.peek() { + match self.parser.read().peek() { Some(b'0') => { buf.push('0'); // There can be only one leading '0'. - if let Some(ch) = self.parser.read.peek() { + if let Some(ch) = self.parser.read().peek() { if ch.is_ascii_digit() { return Err(self.parser.error(ErrorCode::InvalidNumber)); } } - self.parser.read.eat(1); + self.parser.read().eat(1); Ok(()) } Some(c) if c.is_ascii_digit() => { buf.push(c as char); - self.parser.read.eat(1); - while let c @ b'0'..=b'9' = self.parser.read.peek().unwrap_or_default() { - self.parser.read.eat(1); + self.parser.read().eat(1); + while let c @ b'0'..=b'9' = self.parser.read().peek().unwrap_or_default() { + self.parser.read().eat(1); buf.push(c as char); } Ok(()) @@ -190,10 +190,10 @@ impl<'de, R: Reader<'de>> Deserializer { { let shared = self.parser.get_shared_inc_count(); let mut val = Value::new_null(shared.data_ptr()); - let val = if self.parser.read.index() == 0 { + let val = if self.parser.read().index() == 0 { // get n to check trailing characters in later - let n = val.parse_with_padding(self.parser.read.as_u8_slice())?; - self.parser.read.eat(n); + let n = val.parse_with_padding(self.parser.read().as_u8_slice())?; + self.parser.read().eat(n); val } else { // deserialize some json parts into `Value`, not use padding buffer, avoid the memory @@ -226,25 +226,25 @@ impl<'de, R: Reader<'de>> Deserializer { { let raw = match self.parser.skip_space_peek() { Some(c @ b'-' | c @ b'0'..=b'9') => { - let start = self.parser.read.index(); - self.parser.read.eat(1); + let start = self.parser.read().index(); + self.parser.read().eat(1); self.parser.skip_number(c)?; - let end = self.parser.read.index(); - as_str(self.parser.read.slice_unchecked(start, end)) + let end = self.parser.read().index(); + as_str(self.parser.read().slice_unchecked(start, end)) } Some(b'"') => { - self.parser.read.eat(1); - let start = self.parser.read.index(); - match self.parser.read.next() { + self.parser.read().eat(1); + let start = self.parser.read().index(); + match self.parser.read().next() { Some(c @ b'-' | c @ b'0'..=b'9') => { self.parser.skip_number(c)?; } _ => return Err(self.parser.error(ErrorCode::InvalidNumber)), } - let end = self.parser.read.index(); - let raw = as_str(self.parser.read.slice_unchecked(start, end)); + let end = self.parser.read().index(); + let raw = as_str(self.parser.read().slice_unchecked(start, end)); // match the right quote - if self.parser.read.next() != Some(b'"') { + if self.parser.read().next() != Some(b'"') { return Err(self.parser.error(ErrorCode::InvalidNumber)); } raw @@ -367,7 +367,7 @@ impl<'de, 'a, R: Reader<'de>> de::Deserializer<'de> for &'a mut Deserializer match self.parser.skip_space_peek() { Some(b'-') => { buf.push('-'); - self.parser.read.eat(1); + self.parser.read().eat(1); } Some(_) => {} None => { @@ -478,7 +478,7 @@ impl<'de, 'a, R: Reader<'de>> de::Deserializer<'de> for &'a mut Deserializer Reference::Copied(b) => visitor.visit_bytes(b), }, b'[' => { - self.parser.read.backward(1); + self.parser.read().backward(1); self.deserialize_seq(visitor) } _ => Err(self.peek_invalid_type(peek, &visitor)), @@ -510,7 +510,7 @@ impl<'de, 'a, R: Reader<'de>> de::Deserializer<'de> for &'a mut Deserializer { match self.parser.skip_space_peek() { Some(b'n') => { - self.parser.read.eat(1); + self.parser.read().eat(1); tri!(self.parser.parse_literal("ull")); visitor.visit_none() } @@ -697,7 +697,7 @@ impl<'de, 'a, R: Reader<'de>> de::Deserializer<'de> for &'a mut Deserializer { match self.parser.skip_space_peek() { Some(b'{') => { - self.parser.read.eat(1); + self.parser.read().eat(1); let value = { let _ = DepthGuard::guard(self); tri!(visitor.visit_enum(VariantAccess::new(self))) @@ -753,7 +753,7 @@ impl<'de, 'a, R: Reader<'de> + 'a> de::SeqAccess<'de> for SeqAccess<'a, R> { match self.de.parser.skip_space_peek() { Some(b']') => Ok(None), // we will check the ending brace after `visit_seq` Some(b',') if !self.first => { - self.de.parser.read.eat(1); + self.de.parser.read().eat(1); Ok(Some(tri!(seed.deserialize(&mut *self.de)))) } Some(_) => { @@ -761,7 +761,7 @@ impl<'de, 'a, R: Reader<'de> + 'a> de::SeqAccess<'de> for SeqAccess<'a, R> { self.first = false; Ok(Some(tri!(seed.deserialize(&mut *self.de)))) } else { - self.de.parser.read.eat(1); // makes the error position is correct + self.de.parser.read().eat(1); // makes the error position is correct Err(self.de.parser.error(ErrorCode::ExpectedArrayCommaOrEnd)) } } @@ -794,11 +794,11 @@ impl<'de, 'a, R: Reader<'de> + 'a> de::MapAccess<'de> for MapAccess<'a, R> { return Ok(None); } Some(b',') if !self.first => { - self.de.parser.read.eat(1); + self.de.parser.read().eat(1); self.de.parser.skip_space() } Some(b) => { - self.de.parser.read.eat(1); + self.de.parser.read().eat(1); if self.first { self.first = false; Some(b) @@ -976,7 +976,7 @@ macro_rules! deserialize_numeric_key { V: de::Visitor<'de>, { let value = tri!(self.de.deserialize_number(visitor)); - if self.de.parser.read.next() != Some(b'"') { + if self.de.parser.read().next() != Some(b'"') { return Err(self.de.parser.error(ErrorCode::ExpectedQuote)); } @@ -989,14 +989,14 @@ macro_rules! deserialize_numeric_key { where V: de::Visitor<'de>, { - match self.de.parser.read.peek() { + match self.de.parser.read().peek() { Some(b'0'..=b'9' | b'-') => {} _ => return Err(self.de.parser.error(ErrorCode::ExpectedNumericKey)), } let value = tri!(self.de.$delegate(visitor)); - if self.de.parser.read.next() != Some(b'"') { + if self.de.parser.read().next() != Some(b'"') { return Err(self.de.parser.error(ErrorCode::ExpectedQuote)); } @@ -1013,14 +1013,14 @@ where where V: de::Visitor<'de>, { - match self.de.parser.read.peek() { + match self.de.parser.read().peek() { Some(b'0'..=b'9' | b'-') => {} _ => return Err(self.de.parser.error(ErrorCode::ExpectedNumericKey)), } let value = tri!(self.de.deserialize_number(visitor)); - if self.de.parser.read.next() != Some(b'"') { + if self.de.parser.read().next() != Some(b'"') { return Err(self.de.parser.error(ErrorCode::ExpectedQuote)); } @@ -1063,7 +1063,7 @@ where where V: de::Visitor<'de>, { - let mut value = match self.de.parser.read.next() { + let mut value = match self.de.parser.read().next() { Some(b't') => { tri!(self.de.parser.parse_literal("rue")); visitor.visit_bool(true) @@ -1076,7 +1076,7 @@ where Some(peek) => Err(self.de.peek_invalid_type(peek, &visitor)), }; - if self.de.parser.read.next() != Some(b'"') { + if self.de.parser.read().next() != Some(b'"') { value = Err(self.de.parser.error(ErrorCode::ExpectedQuote)); } @@ -1113,7 +1113,7 @@ where where V: de::Visitor<'de>, { - self.de.parser.read.backward(1); + self.de.parser.read().backward(1); self.de.deserialize_enum(name, variants, visitor) } diff --git a/src/util/simd/mod.rs b/src/util/simd/mod.rs index c3cfa3d..72814ad 100644 --- a/src/util/simd/mod.rs +++ b/src/util/simd/mod.rs @@ -31,7 +31,7 @@ macro_rules! impl_lanes { cfg_if::cfg_if! { if #[cfg(target_feature = "sse2")] { use self::sse2::*; - } else if #[cfg(all(target_feature="neon", target_arch="aarch64"))] { + } else if #[cfg(all(target_feature="neon"))] { use self::neon::*; } else { // TODO: support wasm diff --git a/src/value/de.rs b/src/value/de.rs index d02e42e..451081e 100644 --- a/src/value/de.rs +++ b/src/value/de.rs @@ -208,7 +208,7 @@ macro_rules! deserialize_numeric_key { { let mut de = crate::Deserializer::new(crate::reader::Read::new(self.key.as_bytes(), false)); - match de.parser.read.peek() { + match de.parser.read().peek() { Some(b'0'..=b'9' | b'-') => {} _ => return Err(Error::syntax(ErrorCode::ExpectedNumericKey, b"", 0)), } diff --git a/src/value/node.rs b/src/value/node.rs index 5fa5880..fa2db08 100644 --- a/src/value/node.rs +++ b/src/value/node.rs @@ -1371,7 +1371,7 @@ impl Value { self.data = visitor.nodes()[0].data; self.meta = visitor.nodes()[0].meta; self.mark_root(); - Ok(parser.read.index()) + Ok(parser.read().index()) } #[inline(never)] @@ -1379,7 +1379,7 @@ impl Value { &mut self, parser: &mut Parser, ) -> Result<()> { - let remain_len = parser.read.remain(); + let remain_len = parser.read().remain(); let max_len = (remain_len / 2) + 2; let mut buf = TlsBuf::with_capacity(max_len); From f1e458e651510efc81f2be2d06e9c09d1bf249ed Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Sat, 1 Jun 2024 16:48:46 +0200 Subject: [PATCH 04/13] Use default types instead --- src/util/simd/mod.rs | 10 +++++----- src/util/simd/v256.rs | 16 ++++++++-------- src/util/simd/v512.rs | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/util/simd/mod.rs b/src/util/simd/mod.rs index 72814ad..79bc45c 100644 --- a/src/util/simd/mod.rs +++ b/src/util/simd/mod.rs @@ -54,11 +54,11 @@ pub use self::traits::{BitMask, Mask, Simd}; use self::v512::*; pub type u8x16 = Simd128u; -pub type u8x32 = Simd256u; -pub type u8x64 = Simd512u; +pub type u8x32 = Simd256u; +pub type u8x64 = Simd512u; -pub type i8x32 = Simd256i; +pub type i8x32 = Simd256i; pub type m8x16 = Mask128; -pub type m8x32 = Mask256; -pub type m8x64 = Mask512; +pub type m8x32 = Mask256; +pub type m8x64 = Mask512; diff --git a/src/util/simd/v256.rs b/src/util/simd/v256.rs index 4d8744b..fd681d4 100644 --- a/src/util/simd/v256.rs +++ b/src/util/simd/v256.rs @@ -1,6 +1,6 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; -use super::{bits::combine_u16, Mask, Mask128, Simd, Simd128i, Simd128u}; +use super::{bits::combine_u16, Mask, Simd}; use crate::impl_lanes; impl_lanes!([impl Simd256u] 32); @@ -9,15 +9,15 @@ impl_lanes!([impl Mask256] 32); #[derive(Debug)] #[repr(transparent)] -pub struct Simd256u((B, B)); +pub struct Simd256u((B, B)); #[derive(Debug)] #[repr(transparent)] -pub struct Simd256i((B, B)); +pub struct Simd256i((B, B)); #[derive(Debug)] #[repr(transparent)] -pub struct Mask256(pub(crate) (M, M)); +pub struct Mask256(pub(crate) (M, M)); impl> Mask for Mask256 { type BitMask = u32; @@ -85,14 +85,14 @@ where #[inline(always)] unsafe fn loadu(ptr: *const u8) -> Self { let lo = B::loadu(ptr); - let hi = B::loadu(ptr.add(Simd128u::LANES)); + let hi = B::loadu(ptr.add(B::LANES)); Simd256u((lo, hi)) } #[inline(always)] unsafe fn storeu(&self, ptr: *mut u8) { B::storeu(&self.0 .0, ptr); - B::storeu(&self.0 .1, ptr.add(Simd128u::LANES)); + B::storeu(&self.0 .1, ptr.add(B::LANES)); } #[inline(always)] @@ -135,14 +135,14 @@ where #[inline(always)] unsafe fn loadu(ptr: *const u8) -> Self { let lo = B::loadu(ptr); - let hi = B::loadu(ptr.add(Simd128i::LANES)); + let hi = B::loadu(ptr.add(B::LANES)); Simd256i((lo, hi)) } #[inline(always)] unsafe fn storeu(&self, ptr: *mut u8) { B::storeu(&self.0 .0, ptr); - B::storeu(&self.0 .1, ptr.add(Simd128i::LANES)); + B::storeu(&self.0 .1, ptr.add(B::LANES)); } #[inline(always)] diff --git a/src/util/simd/v512.rs b/src/util/simd/v512.rs index fa33354..f5b44be 100644 --- a/src/util/simd/v512.rs +++ b/src/util/simd/v512.rs @@ -1,6 +1,6 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; -use super::{bits::combine_u32, Mask, Mask256, Simd, Simd256i, Simd256u}; +use super::{bits::combine_u32, Mask, Simd}; use crate::impl_lanes; impl_lanes!([impl Simd512u] 64); @@ -9,15 +9,15 @@ impl_lanes!([impl Mask512] 64); #[derive(Debug)] #[repr(transparent)] -pub struct Simd512u((B, B)); +pub struct Simd512u((B, B)); #[derive(Debug)] #[repr(transparent)] -pub struct Simd512i((B, B)); +pub struct Simd512i((B, B)); #[derive(Debug)] #[repr(transparent)] -pub struct Mask512((M, M)); +pub struct Mask512((M, M)); impl> Mask for Mask512 { type BitMask = u64; From 1e433f4510f91979611073ea176732a87c07bc37 Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Mon, 3 Jun 2024 10:48:31 +0200 Subject: [PATCH 05/13] Fix warnings and compilation with arbitrary-precision --- src/lazyvalue/value.rs | 3 ++ src/parser/inner.rs | 86 +++++++++++++++++++++--------------------- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/src/lazyvalue/value.rs b/src/lazyvalue/value.rs index 521f0a8..c5916cf 100644 --- a/src/lazyvalue/value.rs +++ b/src/lazyvalue/value.rs @@ -16,6 +16,9 @@ use crate::{ JsonValueTrait, Result, }; +#[cfg(feature = "arbitrary_precision")] +use crate::RawNumber; + /// LazyValue wrappers a unparsed raw JSON text. It is borrowed from the origin JSON text. /// /// LazyValue can be [`get`](crate::get), [`get_unchecked`](crate::get_unchecked) or diff --git a/src/parser/inner.rs b/src/parser/inner.rs index 635d7ac..13808bc 100644 --- a/src/parser/inner.rs +++ b/src/parser/inner.rs @@ -89,14 +89,14 @@ pub(crate) fn is_whitespace(ch: u8) -> bool { } #[inline(always)] -fn get_string_bits(data: &[u8; 64], prev_instring: &mut u64, prev_escaped: &mut u64) -> u64 +fn get_string_bits(data: &[u8; 64], prev_instring: &mut u64, prev_escaped: &mut u64) -> u64 where - u8x64: Simd, - u8x64::Mask: Mask, + U8x64: Simd, + U8x64::Mask: Mask, { - let v = unsafe { u8x64::from_slice_unaligned_unchecked(data) }; + let v = unsafe { U8x64::from_slice_unaligned_unchecked(data) }; - let bs_bits = (v.eq(&u8x64::splat(b'\\'))).bitmask(); + let bs_bits = (v.eq(&U8x64::splat(b'\\'))).bitmask(); let escaped: u64; if bs_bits != 0 { escaped = get_escaped_branchless_u64(prev_escaped, bs_bits); @@ -104,14 +104,14 @@ where escaped = *prev_escaped; *prev_escaped = 0; } - let quote_bits = (v.eq(&u8x64::splat(b'"'))).bitmask() & !escaped; + let quote_bits = (v.eq(&U8x64::splat(b'"'))).bitmask() & !escaped; let in_string = unsafe { prefix_xor(quote_bits) ^ *prev_instring }; *prev_instring = (in_string as i64 >> 63) as u64; in_string } #[inline(always)] -fn skip_container_loop( +fn skip_container_loop( input: &[u8; 64], /* a 64-bytes slice from json */ prev_instring: &mut u64, /* the bitmap of last string */ prev_escaped: &mut u64, @@ -121,17 +121,17 @@ fn skip_container_loop( right: u8, ) -> Option where - u8x64: Simd, - u8x64::Mask: Mask, + U8x64: Simd, + U8x64::Mask: Mask, { // get the bitmao - let instring = get_string_bits::(input, prev_instring, prev_escaped); + let instring = get_string_bits::(input, prev_instring, prev_escaped); // #Safety // the input is 64 bytes, so the v is always valid. - let v = unsafe { u8x64::from_slice_unaligned_unchecked(input) }; + let v = unsafe { U8x64::from_slice_unaligned_unchecked(input) }; let last_lbrace_num = *lbrace_num; - let mut rbrace = (v.eq(&u8x64::splat(right))).bitmask() & !instring; - let lbrace = (v.eq(&u8x64::splat(left))).bitmask() & !instring; + let mut rbrace = (v.eq(&U8x64::splat(right))).bitmask() & !instring; + let lbrace = (v.eq(&U8x64::splat(left))).bitmask() & !instring; while rbrace != 0 { *rbrace_num += 1; *lbrace_num = last_lbrace_num + (lbrace & (rbrace - 1)).count_ones() as usize; @@ -147,14 +147,14 @@ where None } -pub(crate) struct Parser { +pub(crate) struct Parser { read: R, error_index: usize, // mark the error position nospace_bits: u64, // SIMD marked nospace bitmap nospace_start: isize, // the start position of nospace_bits shared: Option>, // the shared allocator for `Value` - _marker: PhantomData<(i8x32, u8x32, u8x64)>, + _marker: PhantomData<(I8x32, U8x32, U8x64)>, } /// Records the parse status @@ -164,15 +164,15 @@ pub(crate) enum ParseStatus { HasEscaped, } -impl<'de, R, i8x32, u8x32, u8x64> Parser +impl<'de, R, I8x32, U8x32, U8x64> Parser where R: Reader<'de>, - i8x32: Simd, - i8x32::Mask: Mask, - u8x32: Simd, - u8x32::Mask: Mask, - u8x64: Simd, - u8x64::Mask: Mask, + I8x32: Simd, + I8x32::Mask: Mask, + U8x32: Simd, + U8x32::Mask: Mask, + U8x64: Simd, + U8x64::Mask: Mask, { pub fn new(read: R) -> Self { Self { @@ -994,11 +994,11 @@ where #[inline(always)] fn get_next_token(&mut self, tokens: [u8; N], advance: usize) -> Option { let r = &mut self.read; - while let Some(chunk) = r.peek_n(u8x32::LANES) { - let v = unsafe { u8x32::from_slice_unaligned_unchecked(chunk) }; - let mut vor = u8x32::Mask::splat(false); + while let Some(chunk) = r.peek_n(U8x32::LANES) { + let v = unsafe { U8x32::from_slice_unaligned_unchecked(chunk) }; + let mut vor = U8x32::Mask::splat(false); for t in tokens.iter().take(N) { - vor |= v.eq(&u8x32::splat(*t)); + vor |= v.eq(&U8x32::splat(*t)); } let next = vor.bitmask(); if next != 0 { @@ -1007,7 +1007,7 @@ where r.eat(cnt + advance); return Some(ch); } - r.eat(u8x32::LANES); + r.eat(U8x32::LANES); } while let Some(ch) = r.peek() { @@ -1032,10 +1032,10 @@ where let mut prev_escaped = 0; let mut status = ParseStatus::None; - while let Some(chunk) = r.peek_n(u8x32::LANES) { - let v = unsafe { u8x32::from_slice_unaligned_unchecked(chunk) }; - let bs_bits = (v.eq(&u8x32::splat(b'\\'))).bitmask(); - quote_bits = (v.eq(&u8x32::splat(b'"'))).bitmask(); + while let Some(chunk) = r.peek_n(U8x32::LANES) { + let v = unsafe { U8x32::from_slice_unaligned_unchecked(chunk) }; + let bs_bits = (v.eq(&U8x32::splat(b'\\'))).bitmask(); + quote_bits = (v.eq(&U8x32::splat(b'"'))).bitmask(); // maybe has escaped quotes if ((quote_bits.wrapping_sub(1)) & bs_bits) != 0 || prev_escaped != 0 { escaped = get_escaped_branchless_u32(&mut prev_escaped, bs_bits); @@ -1048,7 +1048,7 @@ where r.eat(quote_bits.trailing_zeros() as usize + 1); return Ok(status); } - r.eat(u8x32::LANES) + r.eat(U8x32::LANES) } // skip the possible prev escaped quote @@ -1105,11 +1105,11 @@ where #[inline(always)] fn skip_string(&mut self) -> Result { let mut status = ParseStatus::None; - while let Some(chunk) = self.read.peek_n(u8x32::LANES) { - let v = unsafe { u8x32::from_slice_unaligned_unchecked(chunk) }; - let v_bs = v.eq(&u8x32::splat(b'\\')); - let v_quote = v.eq(&u8x32::splat(b'"')); - let v_cc = v.le(&u8x32::splat(0x1f)); + while let Some(chunk) = self.read.peek_n(U8x32::LANES) { + let v = unsafe { U8x32::from_slice_unaligned_unchecked(chunk) }; + let v_bs = v.eq(&U8x32::splat(b'\\')); + let v_quote = v.eq(&U8x32::splat(b'"')); + let v_cc = v.le(&U8x32::splat(0x1f)); let mask = (v_bs | v_quote | v_cc).bitmask(); // check the mask @@ -1127,7 +1127,7 @@ where _ => unreachable!(), } } else { - self.read.eat(u8x32::LANES) + self.read.eat(U8x32::LANES) } } @@ -1235,7 +1235,7 @@ where while let Some(chunk) = reader.peek_n(64) { let input = array_ref![chunk, 0, 64]; - if let Some(count) = skip_container_loop::( + if let Some(count) = skip_container_loop::( input, &mut prev_instring, &mut prev_escaped, @@ -1255,7 +1255,7 @@ where let n = reader.remain(); remain[..n].copy_from_slice(reader.peek_n(n).unwrap_unchecked()); } - if let Some(count) = skip_container_loop::( + if let Some(count) = skip_container_loop::( &remain, &mut prev_instring, &mut prev_escaped, @@ -1478,9 +1478,9 @@ where // SIMD path for long number while let Some(chunk) = self.read.peek_n(32) { - let v = unsafe { i8x32::from_slice_unaligned_unchecked(chunk) }; - let zero = i8x32::splat(b'0' as i8); - let nine = i8x32::splat(b'9' as i8); + let v = unsafe { I8x32::from_slice_unaligned_unchecked(chunk) }; + let zero = I8x32::splat(b'0' as i8); + let nine = I8x32::splat(b'9' as i8); let nondigits = (zero.gt(&v) | v.gt(&nine)).bitmask(); if nondigits != 0 { let cnt = nondigits.trailing_zeros() as usize; From 7a4dd0e75987b5dd1c0a22c1f2f6ac9c7ec7234f Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Mon, 3 Jun 2024 11:02:04 +0200 Subject: [PATCH 06/13] Introduce new function on bitmask to allow for generic bitmasks --- src/lazyvalue/value.rs | 3 --- src/parser/inner.rs | 43 ++++++++++++++++++++++------------------- src/util/simd/bits.rs | 14 ++++++++++++++ src/util/simd/traits.rs | 6 ++++++ 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/lazyvalue/value.rs b/src/lazyvalue/value.rs index c5916cf..521f0a8 100644 --- a/src/lazyvalue/value.rs +++ b/src/lazyvalue/value.rs @@ -16,9 +16,6 @@ use crate::{ JsonValueTrait, Result, }; -#[cfg(feature = "arbitrary_precision")] -use crate::RawNumber; - /// LazyValue wrappers a unparsed raw JSON text. It is borrowed from the origin JSON text. /// /// LazyValue can be [`get`](crate::get), [`get_unchecked`](crate::get_unchecked) or diff --git a/src/parser/inner.rs b/src/parser/inner.rs index 13808bc..990b7b8 100644 --- a/src/parser/inner.rs +++ b/src/parser/inner.rs @@ -12,7 +12,6 @@ use serde::de::{self, Expected, Unexpected}; use smallvec::SmallVec; use super::{as_str, DEFAULT_KEY_BUF_CAPACITY}; -use crate::reader::{Reader, Reference}; #[cfg(all(target_feature = "neon", target_arch = "aarch64"))] use crate::util::simd::bits::NeonBits; use crate::{ @@ -38,6 +37,10 @@ use crate::{ value::{shared::Shared, visitor::JsonVisitor}, JsonType, LazyValue, }; +use crate::{ + reader::{Reader, Reference}, + util::simd::BitMask, +}; #[inline(always)] fn get_escaped_branchless_u32(prev_escaped: &mut u32, backslash: u32) -> u32 { @@ -92,19 +95,19 @@ pub(crate) fn is_whitespace(ch: u8) -> bool { fn get_string_bits(data: &[u8; 64], prev_instring: &mut u64, prev_escaped: &mut u64) -> u64 where U8x64: Simd, - U8x64::Mask: Mask, + ::BitMask: BitMask, { let v = unsafe { U8x64::from_slice_unaligned_unchecked(data) }; let bs_bits = (v.eq(&U8x64::splat(b'\\'))).bitmask(); let escaped: u64; - if bs_bits != 0 { - escaped = get_escaped_branchless_u64(prev_escaped, bs_bits); + if !bs_bits.all_zero() { + escaped = get_escaped_branchless_u64(prev_escaped, bs_bits.as_primitive()); } else { escaped = *prev_escaped; *prev_escaped = 0; } - let quote_bits = (v.eq(&U8x64::splat(b'"'))).bitmask() & !escaped; + let quote_bits = (v.eq(&U8x64::splat(b'"'))).bitmask().as_primitive() & !escaped; let in_string = unsafe { prefix_xor(quote_bits) ^ *prev_instring }; *prev_instring = (in_string as i64 >> 63) as u64; in_string @@ -122,7 +125,7 @@ fn skip_container_loop( ) -> Option where U8x64: Simd, - U8x64::Mask: Mask, + ::BitMask: BitMask, { // get the bitmao let instring = get_string_bits::(input, prev_instring, prev_escaped); @@ -130,8 +133,8 @@ where // the input is 64 bytes, so the v is always valid. let v = unsafe { U8x64::from_slice_unaligned_unchecked(input) }; let last_lbrace_num = *lbrace_num; - let mut rbrace = (v.eq(&U8x64::splat(right))).bitmask() & !instring; - let lbrace = (v.eq(&U8x64::splat(left))).bitmask() & !instring; + let mut rbrace = (v.eq(&U8x64::splat(right))).bitmask().as_primitive() & !instring; + let lbrace = (v.eq(&U8x64::splat(left))).bitmask().as_primitive() & !instring; while rbrace != 0 { *rbrace_num += 1; *lbrace_num = last_lbrace_num + (lbrace & (rbrace - 1)).count_ones() as usize; @@ -168,11 +171,11 @@ impl<'de, R, I8x32, U8x32, U8x64> Parser where R: Reader<'de>, I8x32: Simd, - I8x32::Mask: Mask, + ::BitMask: BitMask, U8x32: Simd, - U8x32::Mask: Mask, + ::BitMask: BitMask, U8x64: Simd, - U8x64::Mask: Mask, + ::BitMask: BitMask, { pub fn new(read: R) -> Self { Self { @@ -1001,8 +1004,8 @@ where vor |= v.eq(&U8x32::splat(*t)); } let next = vor.bitmask(); - if next != 0 { - let cnt = next.trailing_zeros() as usize; + if !next.all_zero() { + let cnt = next.as_primitive().trailing_zeros() as usize; let ch = chunk[cnt]; r.eat(cnt + advance); return Some(ch); @@ -1034,8 +1037,8 @@ where while let Some(chunk) = r.peek_n(U8x32::LANES) { let v = unsafe { U8x32::from_slice_unaligned_unchecked(chunk) }; - let bs_bits = (v.eq(&U8x32::splat(b'\\'))).bitmask(); - quote_bits = (v.eq(&U8x32::splat(b'"'))).bitmask(); + let bs_bits = (v.eq(&U8x32::splat(b'\\'))).bitmask().as_primitive(); + quote_bits = (v.eq(&U8x32::splat(b'"'))).bitmask().as_primitive(); // maybe has escaped quotes if ((quote_bits.wrapping_sub(1)) & bs_bits) != 0 || prev_escaped != 0 { escaped = get_escaped_branchless_u32(&mut prev_escaped, bs_bits); @@ -1113,8 +1116,8 @@ where let mask = (v_bs | v_quote | v_cc).bitmask(); // check the mask - if mask != 0 { - let cnt = mask.trailing_zeros() as usize; + if !mask.all_zero() { + let cnt = mask.as_primitive().trailing_zeros() as usize; self.read.eat(cnt + 1); match chunk[cnt] { @@ -1482,8 +1485,8 @@ where let zero = I8x32::splat(b'0' as i8); let nine = I8x32::splat(b'9' as i8); let nondigits = (zero.gt(&v) | v.gt(&nine)).bitmask(); - if nondigits != 0 { - let cnt = nondigits.trailing_zeros() as usize; + if !nondigits.all_zero() { + let cnt = nondigits.as_primitive().trailing_zeros() as usize; let ch = chunk[cnt]; if ch == b'.' && !is_float { self.read.eat(cnt + 1); @@ -1492,7 +1495,7 @@ where let traversed = cnt + 2; // check the remaining digits - let nondigts = nondigits.wrapping_shr((traversed) as u32); + let nondigts = nondigits.as_primitive().wrapping_shr((traversed) as u32); if nondigts != 0 { while let Some(ch) = self.read.peek() { if ch == b'e' || ch == b'E' { diff --git a/src/util/simd/bits.rs b/src/util/simd/bits.rs index ac39549..d2732d7 100644 --- a/src/util/simd/bits.rs +++ b/src/util/simd/bits.rs @@ -9,6 +9,13 @@ macro_rules! impl_bits { impl BitMask for $ty { const LEN: usize = std::mem::size_of::<$ty>() * 8; + type Primitive = $ty; + + #[inline] + fn as_primitive(&self) -> $ty { + *self + } + #[inline] fn before(&self, rhs: &Self) -> bool { (self.as_little_endian() & rhs.as_little_endian().wrapping_sub(1)) != 0 @@ -69,6 +76,13 @@ impl NeonBits { impl BitMask for NeonBits { const LEN: usize = 16; + type Primitive = u64; + + #[inline] + fn as_primitive(&self) -> Self::Primitive { + self.0 + } + #[inline] fn first_offset(&self) -> usize { (self.as_little_endian().0.trailing_zeros() as usize) >> 2 diff --git a/src/util/simd/traits.rs b/src/util/simd/traits.rs index 05cf646..ccc98af 100644 --- a/src/util/simd/traits.rs +++ b/src/util/simd/traits.rs @@ -49,6 +49,12 @@ pub trait BitMask { /// Total bits in the bitmask. const LEN: usize; + /// The primitive type this bit mask is represented by + type Primitive; + + /// get the primitive type of this bitmask + fn as_primitive(&self) -> Self::Primitive; + /// get the offset of the first `1` bit. fn first_offset(&self) -> usize; From c8dc14fd8c6b82584d94e3a2cc802dd9c23e4510 Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Mon, 3 Jun 2024 15:12:16 +0200 Subject: [PATCH 07/13] Fix linter warnings --- src/util/simd/v128.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/util/simd/v128.rs b/src/util/simd/v128.rs index 3fa1ae5..3070371 100644 --- a/src/util/simd/v128.rs +++ b/src/util/simd/v128.rs @@ -34,8 +34,8 @@ impl Simd for Simd128i { fn eq(&self, rhs: &Self) -> Self::Mask { let mut mask = [0u8; 16]; - for i in 0..Self::LANES { - mask[i] = if self.0[i] == rhs.0[i] { 1 } else { 0 }; + for (i, item) in mask.iter_mut().enumerate() { + *item = if self.0[i] == rhs.0[i] { 1 } else { 0 }; } Mask128(mask) } @@ -46,16 +46,16 @@ impl Simd for Simd128i { fn le(&self, rhs: &Self) -> Self::Mask { let mut mask = [0u8; 16]; - for i in 0..Self::LANES { - mask[i] = if self.0[i] <= rhs.0[i] { 1 } else { 0 }; + for (i, item) in mask.iter_mut().enumerate() { + *item = if self.0[i] <= rhs.0[i] { 1 } else { 0 }; } Mask128(mask) } fn gt(&self, rhs: &Self) -> Self::Mask { let mut mask = [0u8; 16]; - for i in 0..Self::LANES { - mask[i] = if self.0[i] > rhs.0[i] { 1 } else { 0 }; + for (i, item) in mask.iter_mut().enumerate() { + *item = if self.0[i] > rhs.0[i] { 1 } else { 0 }; } Mask128(mask) } @@ -81,8 +81,8 @@ impl Simd for Simd128u { fn eq(&self, rhs: &Self) -> Self::Mask { let mut mask = [0u8; 16]; - for i in 0..Self::LANES { - mask[i] = if self.0[i] == rhs.0[i] { 1 } else { 0 }; + for (i, item) in mask.iter_mut().enumerate() { + *item = if self.0[i] == rhs.0[i] { 1 } else { 0 }; } Mask128(mask) } @@ -93,16 +93,16 @@ impl Simd for Simd128u { fn le(&self, rhs: &Self) -> Self::Mask { let mut mask = [0u8; 16]; - for i in 0..Self::LANES { - mask[i] = if self.0[i] <= rhs.0[i] { 1 } else { 0 }; + for (i, item) in mask.iter_mut().enumerate() { + *item = if self.0[i] <= rhs.0[i] { 1 } else { 0 }; } Mask128(mask) } fn gt(&self, rhs: &Self) -> Self::Mask { let mut mask = [0u8; 16]; - for i in 0..Self::LANES { - mask[i] = if self.0[i] > rhs.0[i] { 1 } else { 0 }; + for (i, item) in mask.iter_mut().enumerate() { + *item = if self.0[i] > rhs.0[i] { 1 } else { 0 }; } Mask128(mask) } @@ -139,8 +139,8 @@ impl BitAnd for Mask128 { fn bitand(self, rhs: Self) -> Self::Output { let mut result = [0u8; 16]; - for i in 0..16 { - result[i] = self.0[i] & rhs.0[i]; + for (i, item) in result.iter_mut().enumerate() { + *item = self.0[i] & rhs.0[i]; } Mask128(result) } @@ -151,8 +151,8 @@ impl BitOr for Mask128 { fn bitor(self, rhs: Self) -> Self::Output { let mut result = [0u8; 16]; - for i in 0..16 { - result[i] = self.0[i] | rhs.0[i]; + for (i, item) in result.iter_mut().enumerate() { + *item = self.0[i] | rhs.0[i]; } Mask128(result) } From 7a3a864b353e51944a8281d3ee3e830772d48d72 Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Mon, 3 Jun 2024 15:19:54 +0200 Subject: [PATCH 08/13] Replace explicit bitmask requirement with abstraction --- src/util/simd/v256.rs | 13 ++++++++----- src/util/simd/v512.rs | 13 ++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/util/simd/v256.rs b/src/util/simd/v256.rs index fd681d4..4e0f5a6 100644 --- a/src/util/simd/v256.rs +++ b/src/util/simd/v256.rs @@ -1,6 +1,6 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; -use super::{bits::combine_u16, Mask, Simd}; +use super::{bits::combine_u16, BitMask, Mask, Simd}; use crate::impl_lanes; impl_lanes!([impl Simd256u] 32); @@ -19,7 +19,10 @@ pub struct Simd256i((B, B)); #[repr(transparent)] pub struct Mask256(pub(crate) (M, M)); -impl> Mask for Mask256 { +impl Mask for Mask256 +where + ::BitMask: BitMask, +{ type BitMask = u32; type Element = u8; @@ -31,7 +34,7 @@ impl> Mask for Mask256 { let(v0, v1) = self.0; unsafe { super::neon::to_bitmask32(v0.0, v1.0) } } else { - combine_u16(self.0 .0.bitmask(), self.0 .1.bitmask()) + combine_u16(self.0.0.bitmask().as_primitive(), self.0.1.bitmask().as_primitive()) } } } @@ -75,7 +78,7 @@ impl BitAnd> for Mask256 { impl Simd for Simd256u where B: Simd, - B::Mask: Mask, + ::BitMask: BitMask, { const LANES: usize = 32; @@ -125,7 +128,7 @@ where impl Simd for Simd256i where B: Simd, - B::Mask: Mask, + ::BitMask: BitMask, { const LANES: usize = 32; diff --git a/src/util/simd/v512.rs b/src/util/simd/v512.rs index f5b44be..cb7da1b 100644 --- a/src/util/simd/v512.rs +++ b/src/util/simd/v512.rs @@ -1,6 +1,6 @@ use std::ops::{BitAnd, BitOr, BitOrAssign}; -use super::{bits::combine_u32, Mask, Simd}; +use super::{bits::combine_u32, BitMask, Mask, Simd}; use crate::impl_lanes; impl_lanes!([impl Simd512u] 64); @@ -19,7 +19,10 @@ pub struct Simd512i((B, B)); #[repr(transparent)] pub struct Mask512((M, M)); -impl> Mask for Mask512 { +impl Mask for Mask512 +where + ::BitMask: BitMask, +{ type BitMask = u64; type Element = u8; @@ -33,7 +36,7 @@ impl> Mask for Mask512 { let (m2, m3) = v1.0; unsafe { super::neon::to_bitmask64(m0.0, m1.0, m2.0, m3.0) } } else { - combine_u32(self.0 .0.bitmask(), self.0 .1.bitmask()) + combine_u32(self.0.0.bitmask().as_primitive(), self.0.1.bitmask().as_primitive()) } } } @@ -77,7 +80,7 @@ impl BitAnd> for Mask512 { impl Simd for Simd512u where B: Simd, - B::Mask: Mask, + ::BitMask: BitMask, { const LANES: usize = 64; type Element = u8; @@ -126,7 +129,7 @@ where impl Simd for Simd512i where B: Simd, - B::Mask: Mask, + ::BitMask: BitMask, { const LANES: usize = 64; type Element = i8; From c6904d65e63875bdfb61a648f8b9fb6e6fdcbb9e Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Fri, 12 Jul 2024 15:25:47 +0200 Subject: [PATCH 09/13] Fix compile errors --- src/lazyvalue/iterator.rs | 8 ++++---- src/serde/de.rs | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lazyvalue/iterator.rs b/src/lazyvalue/iterator.rs index 3359774..6f2e624 100644 --- a/src/lazyvalue/iterator.rs +++ b/src/lazyvalue/iterator.rs @@ -102,9 +102,9 @@ impl<'de> ObjectInner<'de> { if self.parser.is_none() { let slice = self.json.as_ref(); let slice = unsafe { std::slice::from_raw_parts(slice.as_ptr(), slice.len()) }; - let parser = Parser::new(Read::new(slice, check)); + let mut parser = Parser::new(Read::new(slice, check)); // check invalid utf8 - match parser.read.check_utf8_final() { + match parser.read().check_utf8_final() { Err(err) if check => { self.ending = true; return Some(Err(err)); @@ -152,9 +152,9 @@ impl<'de> ArrayInner<'de> { if self.parser.is_none() { let slice = self.json.as_ref(); let slice = unsafe { std::slice::from_raw_parts(slice.as_ptr(), slice.len()) }; - let parser = Parser::new(Read::new(slice, check)); + let mut parser = Parser::new(Read::new(slice, check)); // check invalid utf8 - match parser.read.check_utf8_final() { + match parser.read().check_utf8_final() { Err(err) if check => { self.ending = true; return Some(Err(err)); diff --git a/src/serde/de.rs b/src/serde/de.rs index 1b13178..ac85fda 100644 --- a/src/serde/de.rs +++ b/src/serde/de.rs @@ -471,7 +471,7 @@ impl<'de, 'a, R: Reader<'de>> de::Deserializer<'de> for &'a mut Deserializer return Err(self.parser.error(ErrorCode::EofWhileParsing)); }; - let start = self.parser.read.index(); + let start = self.parser.read().index(); let value = match peek { b'"' => match tri!(self.parser.parse_string_raw(&mut self.scratch)) { Reference::Borrowed(b) => visitor.visit_borrowed_bytes(b), @@ -485,9 +485,11 @@ impl<'de, 'a, R: Reader<'de>> de::Deserializer<'de> for &'a mut Deserializer }; // check invalid utf8 with allow space here + let index = self.parser.read().index(); self.parser - .read - .validate_utf8((start, self.parser.read.index()))?; + .read() + .validate_utf8((start, index))?; + match value { Ok(value) => Ok(value), Err(err) => Err(self.parser.fix_position(err)), @@ -1152,7 +1154,7 @@ where tri!(de.parser.parse_trailing()); // check invalid utf8 - tri!(de.parser.read.check_utf8_final()); + tri!(de.parser.read().check_utf8_final()); Ok(value) } From d153039fc68cc47ea8376a7026acc8e8dff7211b Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Wed, 31 Jul 2024 19:45:31 +0200 Subject: [PATCH 10/13] Fix compile error --- src/serde/de.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/serde/de.rs b/src/serde/de.rs index c20a366..310ae76 100644 --- a/src/serde/de.rs +++ b/src/serde/de.rs @@ -1124,7 +1124,7 @@ where where V: de::Visitor<'de>, { - self.de.parser.read.backward(1); + self.de.parser.read().backward(1); self.de.deserialize_bytes(visitor) } @@ -1133,7 +1133,7 @@ where where V: de::Visitor<'de>, { - self.de.parser.read.backward(1); + self.de.parser.read().backward(1); self.de.deserialize_bytes(visitor) } From e3fa2cfb1a030d3581c08e6d3627a64497418e13 Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Tue, 20 Aug 2024 16:20:49 +0200 Subject: [PATCH 11/13] Runtime bitmask dispatch for NEON --- src/serde/de.rs | 4 +--- src/util/simd/neon.rs | 15 +++++++++++++++ src/util/simd/v256.rs | 17 +++++++++-------- src/util/simd/v512.rs | 21 +++++++++++---------- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/serde/de.rs b/src/serde/de.rs index c24efe9..6528466 100644 --- a/src/serde/de.rs +++ b/src/serde/de.rs @@ -604,9 +604,7 @@ impl<'de, 'a, R: Reader<'de>> de::Deserializer<'de> for &'a mut Deserializer // check invalid utf8 with allow space here let index = self.parser.read().index(); - self.parser - .read() - .validate_utf8((start, index))?; + self.parser.read().validate_utf8((start, index))?; match value { Ok(value) => Ok(value), diff --git a/src/util/simd/neon.rs b/src/util/simd/neon.rs index c6e5d63..d25226a 100644 --- a/src/util/simd/neon.rs +++ b/src/util/simd/neon.rs @@ -1,10 +1,25 @@ use std::{ arch::aarch64::*, ops::{BitAnd, BitOr, BitOrAssign}, + sync::Once, }; use super::{bits::NeonBits, Mask, Simd}; +#[inline] +pub fn is_supported() -> bool { + unsafe { + static INIT: Once = Once::new(); + static mut SUPPORTED: bool = false; + + INIT.call_once(|| { + SUPPORTED = std::is_aarch64_feature_detected!("neon"); + }); + + SUPPORTED + } +} + #[derive(Debug)] #[repr(transparent)] pub struct Simd128u(uint8x16_t); diff --git a/src/util/simd/v256.rs b/src/util/simd/v256.rs index 4e0f5a6..155e0af 100644 --- a/src/util/simd/v256.rs +++ b/src/util/simd/v256.rs @@ -28,15 +28,16 @@ where #[inline(always)] fn bitmask(self) -> Self::BitMask { - cfg_if::cfg_if! { - if #[cfg(all(target_feature="neon", target_arch="aarch64"))] { - use std::arch::aarch64::uint8x16_t; - let(v0, v1) = self.0; - unsafe { super::neon::to_bitmask32(v0.0, v1.0) } - } else { - combine_u16(self.0.0.bitmask().as_primitive(), self.0.1.bitmask().as_primitive()) - } + #[cfg(target_arch = "aarch64")] + if super::neon::is_supported() { + let (v0, v1) = self.0; + return unsafe { super::neon::to_bitmask32(v0.0, v1.0) }; } + + combine_u16( + self.0 .0.bitmask().as_primitive(), + self.0 .1.bitmask().as_primitive(), + ) } #[inline(always)] diff --git a/src/util/simd/v512.rs b/src/util/simd/v512.rs index cb7da1b..7b048e2 100644 --- a/src/util/simd/v512.rs +++ b/src/util/simd/v512.rs @@ -28,17 +28,18 @@ where #[inline(always)] fn bitmask(self) -> Self::BitMask { - cfg_if::cfg_if! { - if #[cfg(all(target_feature="neon", target_arch="aarch64"))] { - use std::arch::aarch64::uint8x16_t; - let (v0, v1) = self.0; - let (m0, m1) = v0.0; - let (m2, m3) = v1.0; - unsafe { super::neon::to_bitmask64(m0.0, m1.0, m2.0, m3.0) } - } else { - combine_u32(self.0.0.bitmask().as_primitive(), self.0.1.bitmask().as_primitive()) - } + #[cfg(target_arch = "aarch64")] + if super::neon::is_supported() { + let (v0, v1) = self.0; + let (m0, m1) = v0.0; + let (m2, m3) = v1.0; + return unsafe { super::neon::to_bitmask64(m0.0, m1.0, m2.0, m3.0) }; } + + combine_u32( + self.0 .0.bitmask().as_primitive(), + self.0 .1.bitmask().as_primitive(), + ) } #[inline(always)] From cb52681686b6ecf30e3ef79981385291bfb346fd Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Wed, 21 Aug 2024 13:53:13 +0200 Subject: [PATCH 12/13] Fix dispatch, fix macro references --- src/parser/runtime.rs | 6 +++--- src/util/simd/neon.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/parser/runtime.rs b/src/parser/runtime.rs index 4266ef5..9a43a6b 100644 --- a/src/parser/runtime.rs +++ b/src/parser/runtime.rs @@ -65,7 +65,7 @@ macro_rules! dispatch { } } } else if #[cfg(target_arch = "aarch64")] { - match self { + match $self { Self::Neon(val) => { val.$($stuff)* }, @@ -74,7 +74,7 @@ macro_rules! dispatch { } } } else { - let Self::Scalar(val) = self; + let Self::Scalar(val) = $self; val.$($stuff)* } } @@ -98,7 +98,7 @@ where (true, false) => Self::Sse2(inner::Parser::new(read)), } } else if #[cfg(target_arch = "aarch64")] { - if is_aarch64_feature_detected!("neon") { + if std::arch::is_aarch64_feature_detected!("neon") { Self::Neon(inner::Parser::new(read)) } else { Self::Scalar(inner::Parser::new(read)) diff --git a/src/util/simd/neon.rs b/src/util/simd/neon.rs index d25226a..ff8747f 100644 --- a/src/util/simd/neon.rs +++ b/src/util/simd/neon.rs @@ -13,7 +13,7 @@ pub fn is_supported() -> bool { static mut SUPPORTED: bool = false; INIT.call_once(|| { - SUPPORTED = std::is_aarch64_feature_detected!("neon"); + SUPPORTED = std::arch::is_aarch64_feature_detected!("neon"); }); SUPPORTED From dab5e6810e738885e92f96d9552d17f86a5de008 Mon Sep 17 00:00:00 2001 From: Aumetra Weisman Date: Wed, 21 Aug 2024 16:48:41 +0200 Subject: [PATCH 13/13] Cache cpuid result --- src/parser/runtime.rs | 9 +++++---- src/util/simd/avx2.rs | 15 +++++++++++++++ src/util/simd/sse2.rs | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/parser/runtime.rs b/src/parser/runtime.rs index 9a43a6b..189d640 100644 --- a/src/parser/runtime.rs +++ b/src/parser/runtime.rs @@ -89,16 +89,17 @@ where pub fn new(read: R) -> Self { cfg_if::cfg_if! { if #[cfg(target_arch = "x86_64")] { - let has_sse2 = is_x86_feature_detected!("sse2"); - let has_avx2 = is_x86_feature_detected!("avx2"); + use crate::util::simd::{avx2, sse2}; - match (has_sse2, has_avx2) { + match (sse2::is_supported(), avx2::is_supported()) { (false, false) => Self::Scalar(inner::Parser::new(read)), (_, true) => Self::Avx2(inner::Parser::new(read)), (true, false) => Self::Sse2(inner::Parser::new(read)), } } else if #[cfg(target_arch = "aarch64")] { - if std::arch::is_aarch64_feature_detected!("neon") { + use crate::util::simd::neon; + + if neon::is_supported() { Self::Neon(inner::Parser::new(read)) } else { Self::Scalar(inner::Parser::new(read)) diff --git a/src/util/simd/avx2.rs b/src/util/simd/avx2.rs index eb4e106..641455f 100644 --- a/src/util/simd/avx2.rs +++ b/src/util/simd/avx2.rs @@ -2,11 +2,26 @@ use std::{ arch::x86_64::*, mem::transmute, ops::{BitAnd, BitOr, BitOrAssign}, + sync::Once, }; use super::{Mask, Simd}; use crate::impl_lanes; +#[inline] +pub fn is_supported() -> bool { + unsafe { + static INIT: Once = Once::new(); + static mut SUPPORTED: bool = false; + + INIT.call_once(|| { + SUPPORTED = std::arch::is_x86_feature_detected!("avx2"); + }); + + SUPPORTED + } +} + #[derive(Debug)] #[repr(transparent)] pub struct Simd256u(__m256i); diff --git a/src/util/simd/sse2.rs b/src/util/simd/sse2.rs index a9727e3..03d5ddf 100644 --- a/src/util/simd/sse2.rs +++ b/src/util/simd/sse2.rs @@ -1,11 +1,26 @@ use std::{ arch::x86_64::*, ops::{BitAnd, BitOr, BitOrAssign}, + sync::Once, }; use super::{Mask, Simd}; use crate::impl_lanes; +#[inline] +pub fn is_supported() -> bool { + unsafe { + static INIT: Once = Once::new(); + static mut SUPPORTED: bool = false; + + INIT.call_once(|| { + SUPPORTED = std::arch::is_x86_feature_detected!("sse2"); + }); + + SUPPORTED + } +} + #[derive(Debug)] #[repr(transparent)] pub struct Simd128i(__m128i);