Merge remote-tracking branch 'up/main' into stream-sort-spill

Cargo.toml

@@ -542,6 +542,7 @@ mysql_common = "0.32.4"
 quickcheck = "1.0"
 sqllogictest = "0.21.0"
 sqlparser = "0.50.0"
+threadpool = "1.8"
 
 [workspace.lints.rust]
 async_fn_in_trait = "allow"

src/common/storage/src/merge.rs

@@ -23,18 +23,6 @@ pub struct MutationStatus {
 }
 
 impl MutationStatus {
-    pub fn add_insert_rows(&mut self, insert_rows: u64) {
-        self.insert_rows += insert_rows;
-    }
-
-    pub fn add_deleted_rows(&mut self, deleted_rows: u64) {
-        self.deleted_rows += deleted_rows
-    }
-
-    pub fn add_update_rows(&mut self, update_rows: u64) {
-        self.update_rows += update_rows
-    }
-
     pub fn merge_mutation_status(&mut self, mutation_status: MutationStatus) {
         self.insert_rows += mutation_status.insert_rows;
         self.deleted_rows += mutation_status.deleted_rows;

src/meta/types/src/cluster.rs

@@ -82,6 +82,11 @@ pub struct NodeInfo {
     pub flight_address: String,
     pub discovery_address: String,
     pub binary_version: String,
+
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub cluster_id: String,
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub warehouse_id: String,
 }
 
 impl NodeInfo {
@@ -103,6 +108,8 @@ impl NodeInfo {
             flight_address,
             discovery_address,
             binary_version,
+            cluster_id: "".to_string(),
+            warehouse_id: "".to_string(),
         }
     }
 

src/meta/types/tests/it/cluster.rs

@@ -25,6 +25,8 @@ fn test_node_info_ip_port() -> anyhow::Result<()> {
         flight_address: "1.2.3.4:123".to_string(),
         discovery_address: "4.5.6.7:456".to_string(),
         binary_version: "v0.8-binary-version".to_string(),
+        cluster_id: "".to_string(),
+        warehouse_id: "".to_string(),
     };
 
     let (ip, port) = n.ip_port()?;
@@ -34,3 +36,32 @@ fn test_node_info_ip_port() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn test_serde_node_info() {
+    let mut info = NodeInfo {
+        id: "test_id".to_string(),
+        secret: "test_secret".to_string(),
+        version: 1,
+        cpu_nums: 1,
+        http_address: "7.8.9.10:987".to_string(),
+        flight_address: "1.2.3.4:123".to_string(),
+        discovery_address: "4.5.6.7:456".to_string(),
+        binary_version: "v0.8-binary-version".to_string(),
+        cluster_id: String::new(),
+        warehouse_id: String::new(),
+    };
+
+    let json_str = serde_json::to_string(&info).unwrap();
+    assert_eq!(info, serde_json::from_str::<NodeInfo>(&json_str).unwrap());
+    assert!(!json_str.contains("cluster"));
+    assert!(!json_str.contains("warehouse"));
+
+    info.cluster_id = String::from("test-cluster-id");
+    info.warehouse_id = String::from("test-warehouse-id");
+
+    assert_eq!(
+        info,
+        serde_json::from_slice::<NodeInfo>(&serde_json::to_vec(&info).unwrap()).unwrap()
+    );
+}

src/query/catalog/src/statistics/data_cache_statistics.rs

@@ -19,7 +19,7 @@ use serde::Serialize;
 
 #[derive(Default)]
 pub struct DataCacheMetrics {
-    bytes_from_remote_disk: AtomicUsize,
+    pub bytes_from_remote_disk: AtomicUsize,
     bytes_from_local_disk: AtomicUsize,
     bytes_from_memory: AtomicUsize,
 }

src/query/expression/src/kernels/group_by.rs

@@ -23,16 +23,11 @@ use super::group_by_hash::HashMethodSerializer;
 use super::group_by_hash::HashMethodSingleBinary;
 use crate::types::DataType;
 use crate::DataBlock;
-use crate::HashMethodDictionarySerializer;
 use crate::HashMethodKeysU128;
 use crate::HashMethodKeysU256;
 
 impl DataBlock {
-    pub fn choose_hash_method(
-        chunk: &DataBlock,
-        indices: &[usize],
-        efficiently_memory: bool,
-    ) -> Result<HashMethodKind> {
+    pub fn choose_hash_method(chunk: &DataBlock, indices: &[usize]) -> Result<HashMethodKind> {
         let hash_key_types = indices
             .iter()
             .map(|&offset| {
@@ -42,13 +37,10 @@ impl DataBlock {
             .collect::<Result<Vec<_>>>();
 
         let hash_key_types = hash_key_types?;
-        Self::choose_hash_method_with_types(&hash_key_types, efficiently_memory)
+        Self::choose_hash_method_with_types(&hash_key_types)
     }
 
-    pub fn choose_hash_method_with_types(
-        hash_key_types: &[DataType],
-        efficiently_memory: bool,
-    ) -> Result<HashMethodKind> {
+    pub fn choose_hash_method_with_types(hash_key_types: &[DataType]) -> Result<HashMethodKind> {
         if hash_key_types.len() == 1
             && matches!(
                 hash_key_types[0],
@@ -74,14 +66,8 @@ impl DataBlock {
                 if hash_key_type.is_nullable() {
                     group_key_len += 1;
                 }
-            } else if !efficiently_memory || hash_key_types.len() == 1 {
-                return Ok(HashMethodKind::Serializer(HashMethodSerializer::default()));
             } else {
-                return Ok(HashMethodKind::DictionarySerializer(
-                    HashMethodDictionarySerializer {
-                        dict_keys: hash_key_types.len(),
-                    },
-                ));
+                return Ok(HashMethodKind::Serializer(HashMethodSerializer::default()));
             }
         }
 

src/query/expression/src/kernels/group_by_hash/method.rs

@@ -31,7 +31,6 @@ use crate::types::DecimalDataType;
 use crate::types::NumberDataType;
 use crate::types::StringColumn;
 use crate::Column;
-use crate::HashMethodDictionarySerializer;
 use crate::HashMethodKeysU128;
 use crate::HashMethodKeysU16;
 use crate::HashMethodKeysU256;
@@ -91,7 +90,6 @@ pub trait HashMethod: Clone + Sync + Send + 'static {
 #[derive(Clone, Debug)]
 pub enum HashMethodKind {
     Serializer(HashMethodSerializer),
-    DictionarySerializer(HashMethodDictionarySerializer),
     SingleBinary(HashMethodSingleBinary),
     KeysU8(HashMethodKeysU8),
     KeysU16(HashMethodKeysU16),
@@ -106,7 +104,7 @@ macro_rules! with_hash_method {
     ( | $t:tt | $($tail:tt)* ) => {
         match_template::match_template! {
             $t = [Serializer, SingleBinary, KeysU8, KeysU16,
-            KeysU32, KeysU64, KeysU128, KeysU256, DictionarySerializer],
+            KeysU32, KeysU64, KeysU128, KeysU256],
             $($tail)*
         }
     }
@@ -123,26 +121,6 @@ macro_rules! with_join_hash_method {
     }
 }
 
-#[macro_export]
-macro_rules! with_mappedhash_method {
-    ( | $t:tt | $($tail:tt)* ) => {
-        match_template::match_template! {
-            $t = [
-                Serializer => HashMethodSerializer,
-                SingleBinary => HashMethodSingleBinary,
-                KeysU8 => HashMethodKeysU8,
-                KeysU16 => HashMethodKeysU16,
-                KeysU32 => HashMethodKeysU32,
-                KeysU64 => HashMethodKeysU64,
-                KeysU128 => HashMethodKeysU128,
-                KeysU256 => HashMethodKeysU256,
-                DictionarySerializer => HashMethodDictionarySerializer
-            ],
-            $($tail)*
-        }
-    }
-}
-
 impl HashMethodKind {
     pub fn name(&self) -> String {
         with_hash_method!(|T| match self {
@@ -164,7 +142,6 @@ impl HashMethodKind {
             HashMethodKind::KeysU256(_) => {
                 DataType::Decimal(DecimalDataType::Decimal256(i256::default_decimal_size()))
             }
-            HashMethodKind::DictionarySerializer(_) => DataType::Binary,
         }
     }
 }