Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(digest): Digest wrappers #102

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
5 changes: 5 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ crate-type = ["cdylib"]
name = "data_type"
crate-type = ["cdylib"]

[[example]]
# name = "data_type2"
name = "type"
crate-type = ["cdylib"]

[[example]]
name = "load_unload"
crate-type = ["cdylib"]
Expand Down
10 changes: 9 additions & 1 deletion examples/data_type.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use std::os::raw::c_void;
use valkey_module::alloc::ValkeyAlloc;
use valkey_module::digest::Digest;
use valkey_module::native_types::ValkeyType;
use valkey_module::{raw, valkey_module, Context, NextArg, ValkeyResult, ValkeyString};

Expand All @@ -17,10 +18,10 @@ static MY_VALKEY_TYPE: ValkeyType = ValkeyType::new(
rdb_save: None,
aof_rewrite: None,
free: Some(free),
digest: Some(digest),

// Currently unused by Redis
mem_usage: None,
digest: None,

// Aux data
aux_load: None,
Expand All @@ -44,6 +45,13 @@ unsafe extern "C" fn free(value: *mut c_void) {
drop(Box::from_raw(value.cast::<MyType>()));
}

unsafe extern "C" fn digest(md: *mut raw::RedisModuleDigest, value: *mut c_void) {
let mut dig = Digest::new(md);
let val = &*(value.cast::<MyType>());
dig.add_string_buffer(&val.data.as_bytes());
dig.end_sequence();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so we exercise add_string_buffer and end_sequence via Integration Tests. How can we add tests to also test get_key_name, get_db_id and add_long_long?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for calling this out. I'm thinking that we can follow core and integration test get_key_name, get_db_id and add_long_long

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@KarthikSubbarao - your thoughts?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it makes sense to test all the new functionality that being added, so I agree with your comment, @dmitrypol .

@hahnandrew That makes sense. We can follow the core and have the Digest callback of the example data type use all the new wrapper Digest APIs added.

Here is how the core tests the Module APIs (using the core example module).

Core Unit Tests written in TCL: https://github.com/valkey-io/valkey/blob/2df56d87c0ebe802f38e8922bb2ea1e4ca9cfa76/tests/unit/moduleapi/datatype2.tcl#L183

Core Module: https://github.com/valkey-io/valkey/blob/b0f23df16522e91a769c75646166045ae70e8d4e/tests/modules/datatype2.c#L614

For the get_db_id, notice how they use SELECT to switch data bases. We can do the same and validate that the APIs work as expected

}

fn alloc_set(ctx: &Context, args: Vec<ValkeyString>) -> ValkeyResult {
let mut args = args.into_iter().skip(1);
let key = args.next_arg()?;
Expand Down
133 changes: 133 additions & 0 deletions examples/data_type2.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
use std::collections::HashMap;
use std::os::raw::c_void;
use std::ptr::null;
use raw::KeyType;
use valkey_module::alloc::ValkeyAlloc;
use valkey_module::native_types::ValkeyType;
use valkey_module::digest::Digest;
use valkey_module::key::{ValkeyKey, ValkeyKeyWritable};
use valkey_module::{raw, valkey_module, Context, NextArg, ValkeyResult, ValkeyString};

#[derive(Debug)]
struct MyType {
data: String,
}

#[derive(Debug)]
struct Memblock {
dictionary: HashMap<i32, MyType>,
}

impl Memblock {
fn new() -> Self {
Memblock {
dictionary: HashMap::new(),
}
}

fn insert(&mut self, key: i32, value: MyType) {
self.dictionary.insert(key, value);
}

fn get(&self, key: &i32) -> Option<&MyType> {
self.dictionary.get(key)
}
}

static MY_VALKEY_TYPE: ValkeyType = ValkeyType::new(
"mytype123",
0,
raw::RedisModuleTypeMethods {
version: raw::REDISMODULE_TYPE_METHOD_VERSION as u64,
rdb_load: None,
rdb_save: None,
aof_rewrite: None,
free: Some(free),
digest: Some(digest),

// Currently unused by Redis
mem_usage: None,

// Aux data
aux_load: None,
aux_save: None,
aux_save2: None,
aux_save_triggers: 0,

free_effort: None,
unlink: None,
copy: None,
defrag: None,

copy2: None,
free_effort2: None,
mem_usage2: None,
unlink2: None,
},
);

unsafe extern "C" fn free(value: *mut c_void) {
drop(Box::from_raw(value.cast::<MyType>()));
}

unsafe extern "C" fn digest(md: *mut raw::RedisModuleDigest, value: *mut c_void) {
// let mut dig = Digest::new(md);
// let val = &*(value.cast::<MyType>());

// dig.add_long_long(val.data.parse::<i64>().unwrap());
// let keyname = dig.get_key_name();
// let dbid = dig.get_db_id();
// assert!(!keyname.is_empty());
// assert!(dbid != i32::MIN && dbid != i32::MAX, "dbid should not be i32::MIN or i32::MAX, got: {}", dbid);
// dig.end_sequence();
}

fn alloc_set(ctx: &Context, args: Vec<ValkeyString>) -> ValkeyResult {
let mut args = args.into_iter().skip(1);
let key = args.next_arg()?;
let size = args.next_i64()?;

ctx.log_debug(format!("key: {key}, size: {size}").as_str());

let key = ctx.open_key_writable(&key);

if let Some(value) = key.get_value::<MyType>(&MY_VALKEY_TYPE)? {
value.data = "B".repeat(size as usize);
} else {
let value = MyType {
data: "A".repeat(size as usize),
};

key.set_value(&MY_VALKEY_TYPE, value)?;
}
Ok(size.into())
}

fn alloc_get(ctx: &Context, args: Vec<ValkeyString>) -> ValkeyResult {
let mut args = args.into_iter().skip(1);
let key = args.next_arg()?;

let key = ctx.open_key(&key);

let value = match key.get_value::<MyType>(&MY_VALKEY_TYPE)? {
Some(value) => value.data.as_str().into(),
None => ().into(),
};

Ok(value)
}

//////////////////////////////////////////////////////

valkey_module! {
name: "alloc",
version: 1,
allocator: (ValkeyAlloc, ValkeyAlloc),
data_types: [
MY_VALKEY_TYPE,
],
commands: [
["alloc.set", alloc_set, "write", 1, 1, 1],
["alloc.get", alloc_get, "readonly", 1, 1, 1],
],
}
140 changes: 140 additions & 0 deletions examples/type.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
use std::collections::HashMap;
use std::os::raw::c_void;
use std::ptr::null;
use raw::KeyType;
use valkey_module::alloc::ValkeyAlloc;
use valkey_module::native_types::ValkeyType;
use valkey_module::digest::Digest;
use valkey_module::key::{ValkeyKey, ValkeyKeyWritable};
use valkey_module::{raw, valkey_module, Context, NextArg, ValkeyResult, ValkeyString};

#[derive(Debug)]
struct MyType {
data: String,
}

// #[derive(Debug)]
// struct Memblock {
// dictionary: HashMap<i32, MyType>,
// }

// impl Memblock {
// fn new() -> Self {
// Memblock {
// dictionary: HashMap::new(),
// }
// }

// fn insert(&mut self, key: i32, value: MyType) {
// self.dictionary.insert(key, value);
// }

// fn get(&self, key: &i32) -> Option<&MyType> {
// self.dictionary.get(key)
// }
// }

static MY_VALKEY_TYPE: ValkeyType = ValkeyType::new(
"mytype123",
0,
raw::RedisModuleTypeMethods {
version: raw::REDISMODULE_TYPE_METHOD_VERSION as u64,
rdb_load: None,
rdb_save: None,
aof_rewrite: None,
free: Some(free),
digest: Some(digest),

// Currently unused by Redis
mem_usage: None,

// Aux data
aux_load: None,
aux_save: None,
aux_save2: None,
aux_save_triggers: 0,

free_effort: None,
unlink: None,
copy: None,
defrag: None,

copy2: None,
free_effort2: None,
mem_usage2: None,
unlink2: None,
},
);

unsafe extern "C" fn free(value: *mut c_void) {
drop(Box::from_raw(value.cast::<MyType>()));
}

unsafe extern "C" fn digest(md: *mut raw::RedisModuleDigest, value: *mut c_void) {
let mut dig = Digest::new(md);
let val = &*(value.cast::<MyType>());
dig.add_string_buffer(&val.data.as_bytes());
dig.end_sequence();
}

// unsafe extern "C" fn digest(md: *mut raw::RedisModuleDigest, value: *mut c_void) {
// let mut dig = Digest::new(md);
// let val = &*(value.cast::<MyType>());

// dig.add_long_long(val.data.parse::<i64>().unwrap());
// let keyname = dig.get_key_name();
// let dbid = dig.get_db_id();
// assert!(!keyname.is_empty());
// assert!(dbid != i32::MIN && dbid != i32::MAX, "dbid should not be i32::MIN or i32::MAX, got: {}", dbid);
// dig.end_sequence();
// }

fn alloc_set(ctx: &Context, args: Vec<ValkeyString>) -> ValkeyResult {
let mut args = args.into_iter().skip(1);
let key = args.next_arg()?;
let size = args.next_i64()?;

ctx.log_debug(format!("key: {key}, size: {size}").as_str());

let key = ctx.open_key_writable(&key);

if let Some(value) = key.get_value::<MyType>(&MY_VALKEY_TYPE)? {
value.data = "B".repeat(size as usize);
} else {
let value = MyType {
data: "A".repeat(size as usize),
};

key.set_value(&MY_VALKEY_TYPE, value)?;
}
Ok(size.into())
}

fn alloc_get(ctx: &Context, args: Vec<ValkeyString>) -> ValkeyResult {
let mut args = args.into_iter().skip(1);
let key = args.next_arg()?;

let key = ctx.open_key(&key);

let value = match key.get_value::<MyType>(&MY_VALKEY_TYPE)? {
Some(value) => value.data.as_str().into(),
None => ().into(),
};

Ok(value)
}

//////////////////////////////////////////////////////

valkey_module! {
name: "alloc",
version: 1,
allocator: (ValkeyAlloc, ValkeyAlloc),
data_types: [
MY_VALKEY_TYPE,
],
commands: [
["alloc.set", alloc_set, "write", 1, 1, 1],
["alloc.get", alloc_get, "readonly", 1, 1, 1],
],
}
Loading