simd-json update

Signed-off-by: Heinz N. Gies <[email protected]>

Cargo.toml

@@ -83,8 +83,8 @@ rand = "0.8.5"
 regex = "1.9"
 serde = { version = "1", features = ["derive"] }
 serde_yaml = "0.9"
-simd-json = { version = "0.11", features = ["known-key"] }
-simd-json-derive = "0.11"
+simd-json = { version = "0.12", features = ["known-key"] }
+simd-json-derive = "0.12"
 socket2 = { version = "0.5", features = ["all"] }
 tremor-common = { path = "tremor-common" }
 tremor-config = { path = "tremor-config" }

tremor-api/Cargo.toml

@@ -12,7 +12,7 @@ http-types = "2"
 log = "0.4"
 serde = "1"
 serde_yaml = "0.9"
-simd-json = "0.11"
+simd-json = "0.12"
 tokio = { version = "1.32", features = ["full"] }
 # we don't need sessions or cookies or shitty logging middleware
 tide = { version = "0.16", default-features = false, features = ["h1-server"] }

tremor-cli/Cargo.toml

@@ -36,7 +36,7 @@ serde = "1"
 serde_yaml = "0.9"
 signal-hook = "0.3"
 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
-simd-json = { version = "0.11", features = ["known-key"] }
+simd-json = { version = "0.12", features = ["known-key"] }
 # We need to stay with 0.2 for now as there are reasons that can be named for the need to be able to
 # compile and run on operating systems that are a decade old. (insert apropriate ammount rage)
 snmalloc-rs = { version = "0.3" }

tremor-codec/Cargo.toml

@@ -18,7 +18,7 @@ tokio = { version = "1.32", features = ["full"] }
 async-trait = "0.1"
 error-chain = "0.12"
 futures = "0.3"
-simd-json-derive = "0.11"
+simd-json-derive = "0.12"
 value-trait = "0.6"
 beef = "0.5"
 test-case = "3.1"
@@ -46,7 +46,7 @@ reqwest = { version = "0.11", default-features = false, features = [
 ] }
 csv = "1.2"
 tremor-influx = { version = "0.13.0-rc.16", path = "../tremor-influx" }
-simd-json = "0.11"
+simd-json = "0.12"
 apache-avro = { version = "0.16", features = [
     "snappy",
     "bzip",

tremor-codec/src/codec/json.rs

@@ -23,9 +23,9 @@
 //! The codec can be configured with a mode, either `sorted` or `unsorted`. The default is `unsorted` as it is singnificantly faster, `sorted` json is only needed in testing situations where the key order in maps matters for compairson.
 
 use crate::prelude::*;
-use std::{cmp::max, marker::PhantomData};
+use simd_json::Buffers;
+use std::marker::PhantomData;
 use tremor_value::utils::sorted_serialize;
-use tremor_value::AlignedBuf;
 
 /// Sorting for JSON
 pub trait Sorting: Sync + Send + Copy + Clone + 'static {
@@ -50,8 +50,7 @@ impl Sorting for Unsorted {
 /// JSON codec
 pub struct Json<S: Sorting> {
     _phantom: PhantomData<S>,
-    input_buffer: AlignedBuf,
-    string_buffer: Vec<u8>,
+    buffers: Buffers,
     data_buf: Vec<u8>,
 }
 
@@ -65,8 +64,7 @@ impl<S: Sorting> Default for Json<S> {
     fn default() -> Self {
         Self {
             _phantom: PhantomData,
-            input_buffer: AlignedBuf::with_capacity(1024),
-            string_buffer: vec![0u8; 1024],
+            buffers: Buffers::new(1024),
             data_buf: Vec::new(),
         }
     }
@@ -103,18 +101,9 @@ impl<S: Sorting> Codec for Json<S> {
         _ingest_ns: u64,
         meta: Value<'input>,
     ) -> Result<Option<(Value<'input>, Value<'input>)>> {
-        // The input buffer will be automatically grown if required
-        if self.string_buffer.capacity() < data.len() {
-            let new_len = max(self.string_buffer.capacity(), data.len()) * 2;
-            self.string_buffer.resize(new_len, 0);
-        }
-        tremor_value::parse_to_value_with_buffers(
-            data,
-            &mut self.input_buffer,
-            &mut self.string_buffer,
-        )
-        .map(|v| Some((v, meta)))
-        .map_err(Error::from)
+        tremor_value::parse_to_value_with_buffers(data, &mut self.buffers)
+            .map(|v| Some((v, meta)))
+            .map_err(Error::from)
     }
     async fn encode(&mut self, data: &Value, _meta: &Value) -> Result<Vec<u8>> {
         if S::SORTED {
@@ -140,8 +129,7 @@ mod test {
     #[tokio::test(flavor = "multi_thread")]
     async fn decode() -> Result<()> {
         let mut codec: Json<Unsorted> = Json {
-            input_buffer: AlignedBuf::with_capacity(0),
-            string_buffer: Vec::new(),
+            buffers: Buffers::default(),
             ..Default::default()
         };
         let expected = literal!({ "snot": "badger" });

tremor-common/Cargo.toml

@@ -14,8 +14,8 @@ rand = { version = "0.8", features = ["small_rng"] }
 beef = { version = "0.5", features = ["impl_serde"] }
 serde = "1"
 url = { version = "2", features = ["serde"] }
-simd-json = { version = "0.11", features = ["known-key"] }
-simd-json-derive = "0.11"
+simd-json = { version = "0.12", features = ["known-key"] }
+simd-json-derive = "0.12"
 base64 = "0.21"
 regex = "*"
 lazy_static = "*"

tremor-config/Cargo.toml

@@ -14,4 +14,4 @@ version = "0.13.0-rc.16"
 [dependencies]
 tremor-value = { path = "../tremor-value" }
 serde = "1"
-simd-json = "0.11"
+simd-json = "0.12"

tremor-influx/Cargo.toml

@@ -23,7 +23,7 @@ value-trait = "0.6"
 [dev-dependencies]
 criterion = "0.5"
 pretty_assertions = "1.4"
-simd-json = "0.11"
+simd-json = "0.12"
 snmalloc-rs = "0.3"
 
 [[bench]]

tremor-interceptor/Cargo.toml

@@ -20,7 +20,7 @@ tremor-config = { path = "../tremor-config" }
 tremor-common = { path = "../tremor-common" }
 log = "0.4"
 serde = { version = "1", features = ["derive"] }
-simd-json = "0.11"
+simd-json = "0.12"
 libflate = "2"
 xz2 = "0.1"
 lz4 = "1"

tremor-pipeline/Cargo.toml

@@ -22,8 +22,8 @@ regex = "1"
 rust-bert = { version = "0.21.0", optional = true }
 serde = "1"
 serde_yaml = "0.9"
-simd-json = { version = "0.11", features = ["known-key"] }
-simd-json-derive = "0.11"
+simd-json = { version = "0.12", features = ["known-key"] }
+simd-json-derive = "0.12"
 sled = "0.34"
 tremor-common = { version = "0.13.0-rc.16", path = "../tremor-common" }
 tremor-config = { version = "0.13.0-rc.16", path = "../tremor-config" }

tremor-script/Cargo.toml

@@ -46,8 +46,8 @@ percent-encoding = "2"
 rand = { version = "0.8", features = ["small_rng"] }
 regex = "1"
 serde = { version = "1", features = ["derive"] }
-simd-json = { version = "0.11", features = ["known-key"] }
-simd-json-derive = "0.11"
+simd-json = { version = "0.12", features = ["known-key"] }
+simd-json-derive = "0.12"
 sketches-ddsketch = "0.2"
 strip-ansi-escapes = "0.2"
 termcolor = "1.2"
@@ -79,8 +79,6 @@ test-case = "3"
 erlang-float-testing = []
 # This is required for the language server to prevent unbounded growth of the area
 arena-delete = []
-# This is required for the language server as w3e want to allow the use of it without platfor specific flags
-allow-non-simd = ["simd-json/allow-non-simd"]
 
 [[bench]]
 name = "array_flatten"

tremor-value/Cargo.toml

@@ -16,8 +16,8 @@ base64 = "0.21"
 beef = "0.5"
 halfbrown = "0.2"
 serde = "1.0"
-simd-json = "0.11"
-simd-json-derive = "0.11"
+simd-json = "0.12"
+simd-json-derive = "0.12"
 value-trait = { version = "0.6", features = ["custom-types"] }
 tremor-common = { version = "0.13.0-rc.16", path = "../tremor-common" }
 

tremor-value/src/lib.rs

@@ -49,7 +49,7 @@ pub mod value;
 pub use crate::serde::structurize;
 pub use error::*;
 pub use known_key::{Error as KnownKeyError, KnownKey};
-pub use simd_json::{json, json_typed, AlignedBuf, StaticNode};
+pub use simd_json::{json, json_typed, Buffers, StaticNode};
 pub use value::from::*;
 pub use value::{parse_to_value, parse_to_value_with_buffers, to_value, Object, Value};
 
@@ -102,8 +102,8 @@ impl<'input, 'tape> ValueDeser<'input, 'tape> {
         match self.0.next() {
             Some(Node::Static(s)) => Ok(Value::Static(s)),
             Some(Node::String(s)) => Ok(Value::from(s)),
-            Some(Node::Array(len, _)) => Ok(self.parse_array(len)),
-            Some(Node::Object(len, _)) => Ok(self.parse_map(len)),
+            Some(Node::Array { len, .. }) => Ok(self.parse_array(len)),
+            Some(Node::Object { len, .. }) => Ok(self.parse_map(len)),
             None => Err(simd_json::Error::generic(simd_json::ErrorType::Eof)),
         }
     }

tremor-value/src/value.rs

@@ -25,8 +25,8 @@ use crate::{Error, Result};
 use beef::Cow;
 use halfbrown::HashMap;
 pub use r#static::StaticValue;
-use simd_json::{prelude::*, ObjectHasher};
-use simd_json::{AlignedBuf, Deserializer, Node, StaticNode};
+use simd_json::{prelude::*, Buffers, ObjectHasher};
+use simd_json::{Deserializer, Node, StaticNode};
 use std::{borrow::Borrow, convert::TryInto, fmt};
 use std::{cmp::Ord, hash::Hash};
 use std::{
@@ -64,10 +64,9 @@ pub fn parse_to_value(s: &mut [u8]) -> Result<Value> {
 /// Will return `Err` if `s` is invalid JSON.
 pub fn parse_to_value_with_buffers<'value>(
     s: &'value mut [u8],
-    input_buffer: &mut AlignedBuf,
-    string_buffer: &mut [u8],
+    buffer: &mut Buffers,
 ) -> Result<Value<'value>> {
-    match Deserializer::from_slice_with_buffers(s, input_buffer, string_buffer) {
+    match Deserializer::from_slice_with_buffers(s, buffer) {
         Ok(de) => Ok(ValueDeserializer::from_deserializer(de).parse()),
         Err(e) => Err(Error::SimdJson(e)),
     }
@@ -600,8 +599,8 @@ impl<'de> ValueDeserializer<'de> {
         match unsafe { self.0.next_() } {
             Node::Static(s) => Value::Static(s),
             Node::String(s) => Value::from(s),
-            Node::Array(len, _) => self.parse_array(len),
-            Node::Object(len, _) => self.parse_map(len),
+            Node::Array { len, .. } => self.parse_array(len),
+            Node::Object { len, .. } => self.parse_map(len),
         }
     }