refactor: add kv-api::Key::ValueType (databendlabs#14425)

Assign the value type to a key type.

- Part of: databendlabs#14407

src/meta/api/src/id_generator.rs

@@ -99,6 +99,8 @@ impl IdGenerator {
 impl kvapi::Key for IdGenerator {
     const PREFIX: &'static str = PREFIX_ID_GEN;
 
+    type ValueType = ();
+
     fn to_string_key(&self) -> String {
         kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
             .push_raw(&self.resource)

src/meta/api/src/schema_api_impl.rs

@@ -1704,7 +1704,7 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
 
             let db_meta = {
                 let d = data.remove(0);
-                let (k, v) = deserialize_struct_get_response::<DatabaseId, DatabaseMeta>(d)?;
+                let (k, v) = deserialize_struct_get_response::<DatabaseId>(d)?;
                 assert_eq!(key_dbid, k);
 
                 v.ok_or_else(|| {
@@ -1744,7 +1744,7 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
 
             let mut tb_id_list = {
                 let d = data.remove(0);
-                let (k, v) = deserialize_struct_get_response::<TableIdListKey, TableIdList>(d)?;
+                let (k, v) = deserialize_struct_get_response::<TableIdListKey>(d)?;
                 assert_eq!(key_table_id_list, k);
 
                 v.unwrap_or_default()

src/meta/api/src/util.rs

@@ -102,12 +102,13 @@ pub async fn get_u64_value<T: kvapi::Key>(
     }
 }
 
-pub fn deserialize_struct_get_response<K, T>(
+#[allow(clippy::type_complexity)]
+pub fn deserialize_struct_get_response<K>(
     resp: TxnGetResponse,
-) -> Result<(K, Option<SeqV<T>>), MetaError>
+) -> Result<(K, Option<SeqV<K::ValueType>>), MetaError>
 where
     K: kvapi::Key,
-    T: FromToProto,
+    K::ValueType: FromToProto,
 {
     let key = K::from_str_key(&resp.key).map_err(|e| {
         let inv = InvalidReply::new(
@@ -119,7 +120,7 @@ where
 
     if let Some(pb_seqv) = resp.value {
         let seqv = SeqV::from(pb_seqv);
-        let value = deserialize_struct::<T>(&seqv.data)?;
+        let value = deserialize_struct::<K::ValueType>(&seqv.data)?;
         let seqv = SeqV::with_meta(seqv.seq, seqv.meta, value);
         Ok((key, Some(seqv)))
     } else {
@@ -152,13 +153,13 @@ where K: kvapi::Key {
 /// Get value that are encoded with FromToProto.
 ///
 /// It returns seq number and the data.
-pub async fn get_pb_value<K, T>(
+pub async fn get_pb_value<K>(
     kv_api: &(impl kvapi::KVApi<Error = MetaError> + ?Sized),
     k: &K,
-) -> Result<(u64, Option<T>), MetaError>
+) -> Result<(u64, Option<K::ValueType>), MetaError>
 where
     K: kvapi::Key,
-    T: FromToProto,
+    K::ValueType: FromToProto,
 {
     let res = kv_api.get_kv(&k.to_string_key()).await?;
 

src/meta/app/src/background/background_job.rs

@@ -382,13 +382,17 @@ mod kvapi_key_impl {
 
     use crate::background::background_job::BackgroundJobId;
     use crate::background::background_job::BackgroundJobIdent;
+    use crate::background::BackgroundJobInfo;
+
     const PREFIX_BACKGROUND_JOB: &str = "__fd_background_job";
     const PREFIX_BACKGROUND_JOB_BY_ID: &str = "__fd_background_job_by_id";
 
     /// <prefix>/<tenant>/<background_job_ident> -> <id>
     impl kvapi::Key for BackgroundJobIdent {
         const PREFIX: &'static str = PREFIX_BACKGROUND_JOB;
 
+        type ValueType = BackgroundJobId;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -410,6 +414,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for BackgroundJobId {
         const PREFIX: &'static str = PREFIX_BACKGROUND_JOB_BY_ID;
 
+        type ValueType = BackgroundJobInfo;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.id)

src/meta/app/src/background/background_task.rs

@@ -219,13 +219,17 @@ mod kvapi_key_impl {
     use databend_common_meta_kvapi::kvapi;
 
     use crate::background::background_task::BackgroundTaskIdent;
+    use crate::background::BackgroundTaskInfo;
+
     const PREFIX_BACKGROUND: &str = "__fd_background_task_by_name";
 
     // task is named by id, and will not encounter renaming issue.
     /// <prefix>/<tenant>/<background_task_ident> -> info
     impl kvapi::Key for BackgroundTaskIdent {
         const PREFIX: &'static str = PREFIX_BACKGROUND;
 
+        type ValueType = BackgroundTaskInfo;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)

src/meta/app/src/data_mask/mod.rs

@@ -131,11 +131,15 @@ mod kvapi_key_impl {
     use super::PREFIX_DATAMASK;
     use super::PREFIX_DATAMASK_BY_ID;
     use super::PREFIX_DATAMASK_ID_LIST;
+    use crate::data_mask::DatamaskMeta;
+    use crate::data_mask::MaskpolicyTableIdList;
 
     /// __fd_database/<tenant>/<name> -> <data_mask_id>
     impl kvapi::Key for DatamaskNameIdent {
         const PREFIX: &'static str = PREFIX_DATAMASK;
 
+        type ValueType = DatamaskId;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -158,6 +162,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for DatamaskId {
         const PREFIX: &'static str = PREFIX_DATAMASK_BY_ID;
 
+        type ValueType = DatamaskMeta;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.id)
@@ -177,6 +183,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for MaskpolicyTableIdListKey {
         const PREFIX: &'static str = PREFIX_DATAMASK_ID_LIST;
 
+        type ValueType = MaskpolicyTableIdList;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)

src/meta/app/src/schema/catalog.rs

@@ -235,6 +235,7 @@ mod kvapi_key_impl {
     use super::CatalogId;
     use super::CatalogIdToName;
     use super::CatalogNameIdent;
+    use crate::schema::CatalogMeta;
     use crate::schema::PREFIX_CATALOG;
     use crate::schema::PREFIX_CATALOG_BY_ID;
     use crate::schema::PREFIX_CATALOG_ID_TO_NAME;
@@ -243,6 +244,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for CatalogNameIdent {
         const PREFIX: &'static str = PREFIX_CATALOG;
 
+        type ValueType = CatalogId;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -268,6 +271,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for CatalogId {
         const PREFIX: &'static str = PREFIX_CATALOG_BY_ID;
 
+        type ValueType = CatalogMeta;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.catalog_id)
@@ -288,6 +293,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for CatalogIdToName {
         const PREFIX: &'static str = PREFIX_CATALOG_ID_TO_NAME;
 
+        type ValueType = CatalogNameIdent;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.catalog_id)

src/meta/app/src/schema/database.rs

@@ -320,7 +320,9 @@ mod kvapi_key_impl {
 
     use crate::schema::DatabaseId;
     use crate::schema::DatabaseIdToName;
+    use crate::schema::DatabaseMeta;
     use crate::schema::DatabaseNameIdent;
+    use crate::schema::DbIdList;
     use crate::schema::DbIdListKey;
     use crate::schema::PREFIX_DATABASE;
     use crate::schema::PREFIX_DATABASE_BY_ID;
@@ -331,6 +333,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for DatabaseNameIdent {
         const PREFIX: &'static str = PREFIX_DATABASE;
 
+        type ValueType = DatabaseId;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -353,6 +357,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for DatabaseId {
         const PREFIX: &'static str = PREFIX_DATABASE_BY_ID;
 
+        type ValueType = DatabaseMeta;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.db_id)
@@ -373,6 +379,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for DatabaseIdToName {
         const PREFIX: &'static str = PREFIX_DATABASE_ID_TO_NAME;
 
+        type ValueType = DatabaseNameIdent;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.db_id)
@@ -393,6 +401,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for DbIdListKey {
         const PREFIX: &'static str = PREFIX_DB_ID_LIST;
 
+        type ValueType = DbIdList;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)

src/meta/app/src/schema/index.rs

@@ -230,6 +230,7 @@ mod kvapi_key_impl {
 
     use crate::schema::IndexId;
     use crate::schema::IndexIdToName;
+    use crate::schema::IndexMeta;
     use crate::schema::IndexNameIdent;
     use crate::schema::PREFIX_INDEX;
     use crate::schema::PREFIX_INDEX_BY_ID;
@@ -239,6 +240,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for IndexNameIdent {
         const PREFIX: &'static str = PREFIX_INDEX;
 
+        type ValueType = IndexId;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_str(&self.tenant)
@@ -261,6 +264,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for IndexId {
         const PREFIX: &'static str = PREFIX_INDEX_BY_ID;
 
+        type ValueType = IndexMeta;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.index_id)
@@ -281,6 +286,8 @@ mod kvapi_key_impl {
     impl kvapi::Key for IndexIdToName {
         const PREFIX: &'static str = PREFIX_INDEX_ID_TO_NAME;
 
+        type ValueType = IndexNameIdent;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.index_id)

src/meta/app/src/schema/lock.rs

@@ -102,7 +102,7 @@ impl LockKey {
         }
     }
 
-    pub fn gen_key(&self, revision: u64) -> impl Key {
+    pub fn gen_key(&self, revision: u64) -> TableLockKey {
         match self {
             LockKey::Table { table_id } => TableLockKey {
                 table_id: *table_id,
@@ -181,13 +181,16 @@ pub struct TableLockKey {
 mod kvapi_key_impl {
     use databend_common_meta_kvapi::kvapi;
 
+    use crate::schema::LockMeta;
     use crate::schema::TableLockKey;
     use crate::schema::PREFIX_TABLE_LOCK;
 
     /// __fd_table_lock/table_id/revision -> LockMeta
     impl kvapi::Key for TableLockKey {
         const PREFIX: &'static str = PREFIX_TABLE_LOCK;
 
+        type ValueType = LockMeta;
+
         fn to_string_key(&self) -> String {
             kvapi::KeyBuilder::new_prefixed(Self::PREFIX)
                 .push_u64(self.table_id)

src/meta/app/src/schema/mod.rs

@@ -86,7 +86,6 @@ pub use table::SetTableColumnMaskPolicyAction;
 pub use table::SetTableColumnMaskPolicyReply;
 pub use table::SetTableColumnMaskPolicyReq;
 pub use table::TableCopiedFileInfo;
-pub use table::TableCopiedFileLockKey;
 pub use table::TableCopiedFileNameIdent;
 pub use table::TableId;
 pub use table::TableIdList;
@@ -134,7 +133,6 @@ const PREFIX_TABLE_ID_LIST: &str = "__fd_table_id_list";
 const PREFIX_TABLE_COUNT: &str = "__fd_table_count";
 const PREFIX_TABLE_ID_TO_NAME: &str = "__fd_table_id_to_name";
 const PREFIX_TABLE_COPIED_FILES: &str = "__fd_table_copied_files";
-const PREFIX_TABLE_COPIED_FILES_LOCK: &str = "__fd_table_copied_file_lock";
 const PREFIX_INDEX: &str = "__fd_index";
 const PREFIX_INDEX_ID_TO_NAME: &str = "__fd_index_id_to_name";
 const PREFIX_INDEX_BY_ID: &str = "__fd_index_by_id";