Skip to content

Commit

Permalink
fix: avro serialisation of nested records with implicit namespace (#545)
Browse files Browse the repository at this point in the history
  • Loading branch information
andrewinci committed Jul 20, 2023
1 parent 76716df commit fe51935
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 5 deletions.
2 changes: 1 addition & 1 deletion backend/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 18 additions & 4 deletions backend/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,33 @@ openssl-src = { version = "111", features = ["force-engine"] }
serde_json = "1.0"
dirs = "5.0.1"
serde = { version = "1.0.171", features = ["derive"] }
tauri = { version = "1.4", features = ["dialog-open", "dialog-save", "fs-write-file", "os-all", "updater", "window-create", "window-set-focus", "window-set-min-size", "window-set-title"] }
tauri = { version = "1.4", features = [
"dialog-open",
"dialog-save",
"fs-write-file",
"os-all",
"updater",
"window-create",
"window-set-focus",
"window-set-min-size",
"window-set-title",
] }
reqwest = { version = "0.11", features = ["json", "blocking"] }
url = { version = "2", features = ["serde"] }
tokio = { version = "1", features = ["full"] }
futures = { version = "0.3" }
apache-avro = { git = "https://github.com/apache/avro", rev = "8c3ee165432cf557429a984df6acbc075a848ac9" }
# todo: replace the fork with the apache repo when https://github.com/apache/avro/pull/2374 is merged
# apache-avro = { git = "https://github.com/apache/avro", rev = "<todo>" }
apache-avro = { git = "https://github.com/andrewinci/avro" }
log = { version = "0.4" }
env_logger = { version = "0.10.0" }
async-trait = "0.1.71"
num-bigint = "0.4"
rust_decimal = "1.30"
rusqlite = { version = "0.28.0", features = ["bundled", "backup", "hooks"] }
rust-keystore = { git = "https://github.com/andrewinci/rust-keystore", features = ["p12"], tag = "v0.1.2" }
rust-keystore = { git = "https://github.com/andrewinci/rust-keystore", features = [
"p12",
], tag = "v0.1.2" }
r2d2 = "0.8.10"
r2d2_sqlite = "0.21.0"
toml = "0.7"
Expand Down Expand Up @@ -69,6 +83,6 @@ custom-protocol = ["tauri/custom-protocol"]
integration_tests = []

[profile.release]
strip = true
strip = true
lto = true
opt-level = "s"
75 changes: 75 additions & 0 deletions backend/src/core/avro/json_to_avro.rs
Original file line number Diff line number Diff line change
Expand Up @@ -222,12 +222,42 @@ mod tests {
use super::parse_decimal;
use crate::core::avro::avro_schema::AvroSchema;
use crate::core::avro::avro_schema::RecordField;
use crate::core::avro::error::AvroResult;
use crate::core::avro::AvroError;
use crate::core::avro::AvroParser;
use crate::core::avro::ResolvedAvroSchema;
use crate::core::avro::SchemaProvider;
use apache_avro::Schema;

use apache_avro::types::Value as AvroValue;
use async_trait::async_trait;
use serde_json::json;
use serde_json::Value as JsonValue;

struct MockSchemaProvider {}
#[async_trait]
impl SchemaProvider for MockSchemaProvider {
async fn get_schema_by_id(&self, id: i32) -> AvroResult<ResolvedAvroSchema> {
let json_schema = &get_test_avro_schema();
let schema = Schema::parse_str(json_schema).expect("invalid test schema");
Ok(ResolvedAvroSchema::from(id, &schema))
}
async fn get_schema_by_name(&self, _: &str) -> AvroResult<ResolvedAvroSchema> {
let json_schema = &get_test_avro_schema();
let schema = Schema::parse_str(json_schema).expect("invalid test schema");
Ok(ResolvedAvroSchema::from(123, &schema))
}
}

#[tokio::test]
async fn test_parse_nested_records_with_implicit_namespace() {
let mock_provider = MockSchemaProvider {};
let sut = AvroParser::new(mock_provider.into());
let sample_json = &get_test_avro_message();
let res = sut.json_to_avro(sample_json, "sample").await;
assert!(res.is_ok())
}

#[test]
fn test_decimal() {
// happy path
Expand Down Expand Up @@ -311,4 +341,49 @@ mod tests {
schema: schema,
}
}

fn get_test_avro_message() -> String {
r#"{
"outer_field_1": {
"middle_field_1": {
"inner_field_1": 1.7
},
"middle_field_2": {
"inner_field_1": 1.8
}
}
}"#
.to_string()
}
fn get_test_avro_schema() -> String {
r#"{
"name": "record_name",
"namespace": "space",
"type": "record",
"fields": [
{
"name": "outer_field_1",
"type": {
"type": "record",
"name": "middle_record_name",
"namespace": "middle_namespace",
"fields": [
{
"name": "middle_field_1",
"type": {
"type": "record",
"name": "inner_record_name",
"fields": [
{ "name": "inner_field_1", "type": "double" }
]
}
},
{ "name": "middle_field_2", "type": "inner_record_name" }
]
}
}
]
}"#
.to_string()
}
}

0 comments on commit fe51935

Please sign in to comment.