diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eefc64..25f670b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.6.1 - 2023-06-02 +- Added `--fragment` and `--zstd-fragment` CLI arguments to read DBN streams + without metadata +- Added `csv::Decoder::get_ref` that returns reference to the underlying writer +- Added missing Python getter for `InstrumentDefMsg::group` +- Added dataset constants +- Changed `c_char` fields to be exposed to Python as `str` + ## 0.6.0 - 2023-05-26 - Renamed `booklevel` MBP field to `levels` for brevity and consistent naming - Added `--limit NUM` CLI argument to output only the first `NUM` records diff --git a/Cargo.lock b/Cargo.lock index 0708d14..190f9af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", @@ -52,9 +52,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -109,9 +109,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bstr" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" dependencies = [ "memchr", "once_cell", @@ -127,9 +127,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cbindgen" -version = "0.24.3" +version = "0.24.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6358dedf60f4d9b8db43ad187391afe959746101346fe51bb978126bec61dfb" +checksum = "4b922faaf31122819ec80c4047cc684c6979a087366c069611e33649bf98e18d" dependencies = [ "heck", "indexmap", @@ -160,9 +160,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.2.3" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f9152d70e42172fdb87de2efd7327160beee37886027cf86f30a233d5b30b4" +checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" dependencies = [ "clap_builder", "clap_derive", @@ -171,9 +171,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.3" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e067b220911598876eb55d52725ddcc201ffe3f0904018195973bc5b012ea2ca" +checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" dependencies = [ "anstream", "anstyle", @@ -184,21 +184,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "colorchoice" @@ -208,9 +208,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "csv" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" dependencies = [ "csv-core", "itoa", @@ -229,7 +229,7 @@ dependencies = [ [[package]] name = "databento-dbn" -version = "0.6.0" +version = "0.6.1" dependencies = [ "dbn", "pyo3", @@ -238,7 +238,7 @@ dependencies = [ [[package]] name = "dbn" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "async-compression", @@ -255,7 +255,7 @@ dependencies = [ [[package]] name = "dbn-c" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "cbindgen", @@ -265,7 +265,7 @@ dependencies = [ [[package]] name = "dbn-cli" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "assert_cmd", @@ -274,16 +274,17 @@ dependencies = [ "predicates", "serde", "tempfile", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] name = "dbn-macros" -version = "0.6.0" +version = "0.6.1" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -403,9 +404,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -460,15 +461,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.142" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "linux-raw-sys" -version = "0.3.2" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f508063cc7bb32987c71511216bd5a32be15bccb6a80b52df8b9d7f01fc3aa2" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" @@ -482,12 +483,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "memchr" @@ -497,9 +495,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] @@ -547,14 +545,14 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] name = "once_cell" -version = "1.17.1" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" [[package]] name = "parking_lot" @@ -587,9 +585,9 @@ checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "predicates" @@ -634,18 +632,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b1ac5b3731ba34fdaa9785f8d74d17448cd18f30cf19e0c7e7b1fdb5272109" +checksum = "cffef52f74ec3b1a1baf295d9b8fcc3070327aefc39a6d00656b13c1d0b8885c" dependencies = [ "cfg-if", "indoc", @@ -660,9 +658,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cb946f5ac61bb61a5014924910d936ebd2b23b705f7a4a3c40b05c720b079a3" +checksum = "713eccf888fb05f1a96eb78c0dbc51907fee42b3377272dc902eb38985f418d5" dependencies = [ "once_cell", "target-lexicon", @@ -670,9 +668,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4d7c5337821916ea2a1d21d1092e8443cf34879e53a0ac653fbb98f44ff65c" +checksum = "5b2ecbdcfb01cbbf56e179ce969a048fd7305a66d4cdf3303e0da09d69afe4c3" dependencies = [ "libc", "pyo3-build-config", @@ -680,9 +678,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d39c55dab3fc5a4b25bbd1ac10a2da452c4aca13bb450f22818a002e29648d" +checksum = "b78fdc0899f2ea781c463679b20cb08af9247febc8d052de941951024cd8aea0" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -692,9 +690,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97daff08a4c48320587b5224cc98d609e3c27b6d437315bd40b605c98eeb5918" +checksum = "60da7b84f1227c3e2fe7593505de274dcf4c8928b4e0a1c23d551a14e4e80a0f" dependencies = [ "proc-macro2", "quote", @@ -703,9 +701,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -730,9 +728,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -747,15 +745,15 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.29" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "rustix" -version = "0.37.12" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags", "errno", @@ -779,22 +777,22 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -839,9 +837,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -850,9 +848,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.6" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" +checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" [[package]] name = "tempfile" @@ -875,9 +873,9 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "time" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" dependencies = [ "itoa", "serde", @@ -887,42 +885,42 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", "num_cpus", "pin-project-lite", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -936,15 +934,15 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ "indexmap", "toml_datetime", @@ -953,9 +951,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unindent" @@ -1112,9 +1110,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.1" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" dependencies = [ "memchr", ] diff --git a/README.md b/README.md index 5bdd7de..0fba6cf 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Current Crates.io Version](https://img.shields.io/crates/v/dbn.svg)](https://crates.io/crates/dbn) [![pypi-version](https://img.shields.io/pypi/v/databento_dbn)](https://pypi.org/project/databento-dbn) -A library (`dbn`) and CLI tool (`dbn`) for working with Databento Binary +Libraries and a CLI tool for working with Databento Binary Encoding (DBN) files and streams. Python bindings for `dbn` are provided in the `databento-dbn` package. @@ -21,9 +21,9 @@ highly-compressible binary encoding suitable for bulk financial time series data ## Usage See the respective READMEs for usage details: -- [`dbn`](rust/dbn/README.md) -- [`dbn-cli`](rust/dbn-cli/README.md) -- [`databento-dbn`](python/README.md) +- [`dbn`](rust/dbn/README.md): Rust library crate +- [`dbn-cli`](rust/dbn-cli/README.md): CLI crate providing a `dbn` binary +- [`databento-dbn`](python/README.md): Python package # License diff --git a/c/Cargo.toml b/c/Cargo.toml index 6a12c73..7d8f957 100644 --- a/c/Cargo.toml +++ b/c/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dbn-c" authors = ["Databento "] -version = "0.6.0" +version = "0.6.1" edition = "2021" description = "C bindings for working with Databento Binary Encoding (DBN)" license = "Apache-2.0" diff --git a/python/Cargo.toml b/python/Cargo.toml index 6286825..b1383b5 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "databento-dbn" authors = ["Databento "] -version = "0.6.0" +version = "0.6.1" edition = "2021" description = "Python library written in Rust for working with Databento Binary Encoding (DBN)" license = "Apache-2.0" @@ -16,7 +16,7 @@ name = "databento_dbn" # Python modules can't contain dashes # DBN library dbn = { path = "../rust/dbn", features = ["python"] } # Python bindings for Rust -pyo3 = "0.18.3" +pyo3 = "0.19" [build-dependencies] -pyo3-build-config = { version = "0.18.3" } +pyo3-build-config = { version = "0.19" } diff --git a/python/pyproject.toml b/python/pyproject.toml index 8e85525..8904a3b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,10 +1,23 @@ +[tool.poetry] +name = "databento-dbn" +version = "0.6.1" +description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)" +authors = ["Databento "] +license = "Apache-2.0" + +[tool.poetry.dependencies] +python = ">=3.7" + +[tool.poetry.dev-dependencies] +maturin = ">=1.0" + [build-system] -requires = ["maturin>=0.15,<0.16"] +requires = ["maturin>=1.0"] build-backend = "maturin" [project] name = "databento-dbn" -description = "Native Databento bindings based on dbn Rust crate" +description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)" readme = "README.md" license = { file = "../LICENSE" } requires-python = ">=3.7" diff --git a/python/src/dbn_decoder.rs b/python/src/dbn_decoder.rs index 16dcdde..234709d 100644 --- a/python/src/dbn_decoder.rs +++ b/python/src/dbn_decoder.rs @@ -112,6 +112,7 @@ impl DbnDecoder { #[cfg(test)] mod tests { use dbn::{ + datasets::XNAS_ITCH, encode::{dbn::Encoder, EncodeDbn}, enums::{rtype, SType, Schema}, record::{ErrorMsg, OhlcvMsg, RecordHeader}, @@ -130,7 +131,7 @@ mod tests { let mut encoder = Encoder::new( buffer, &MetadataBuilder::new() - .dataset("XNAS.ITCH".to_owned()) + .dataset(XNAS_ITCH.to_owned()) .schema(Some(Schema::Trades)) .stype_in(Some(SType::RawSymbol)) .stype_out(SType::InstrumentId) @@ -169,7 +170,7 @@ mod tests { let mut encoder = Encoder::new( buffer, &MetadataBuilder::new() - .dataset("XNAS.ITCH".to_owned()) + .dataset(XNAS_ITCH.to_owned()) .schema(Some(Schema::Ohlcv1S)) .stype_in(Some(SType::RawSymbol)) .stype_out(SType::InstrumentId) diff --git a/python/src/encode.rs b/python/src/encode.rs index b606911..5965baf 100644 --- a/python/src/encode.rs +++ b/python/src/encode.rs @@ -183,6 +183,7 @@ mod tests { use std::io::{Cursor, Seek, Write}; use std::sync::{Arc, Mutex}; + use dbn::datasets::GLBX_MDP3; use dbn::{ decode::{dbn::Decoder as DbnDecoder, DecodeDbn}, enums::SType, @@ -240,7 +241,7 @@ mod tests { } } - const DATASET: &str = "GLBX.MDP3"; + const DATASET: &str = GLBX_MDP3; const STYPE: SType = SType::InstrumentId; macro_rules! test_writing_dbn_from_python { diff --git a/rust/dbn-cli/Cargo.toml b/rust/dbn-cli/Cargo.toml index 1b37baf..7b212c6 100644 --- a/rust/dbn-cli/Cargo.toml +++ b/rust/dbn-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dbn-cli" authors = ["Databento "] -version = "0.6.0" +version = "0.6.1" edition = "2021" description = "Command-line utility for converting Databento Binary Encoding (DBN) files to text-based formats" default-run = "dbn" @@ -17,7 +17,7 @@ path = "src/main.rs" [dependencies] # Databento common DBN library -dbn = { path = "../dbn", version = "=0.6.0" } +dbn = { path = "../dbn", version = "=0.6.1" } # Error handling anyhow = "1.0.70" @@ -25,6 +25,8 @@ anyhow = "1.0.70" clap = { version = "4.2", features = ["derive"] } # deserialization for CLI args serde = { version = "1.0", features = ["derive"] } +# (de)compression +zstd = "=0.12.3" [dev-dependencies] # CLI integration tests diff --git a/rust/dbn-cli/README.md b/rust/dbn-cli/README.md index 83f96a3..d83b296 100644 --- a/rust/dbn-cli/README.md +++ b/rust/dbn-cli/README.md @@ -8,6 +8,13 @@ This crate provides a CLI tool `dbn` for converting the Databento Binary Encoding (DBN) files to text formats, as well as updating legacy DBZ files to DBN. +## Installation + +To install the latest version, run the following command: +```sh +cargo install dbn-cli +``` + ## Usage `dbn` currently supports CSV and JSON (technically [newline-delimited JSON](http://ndjson.org/)) @@ -16,9 +23,9 @@ By default, `dbn` outputs the result to standard output for ease of use with text-based command-line utilities. Running ```sh -dbn some.dbn --csv | head -n 5 +dbn some.dbn --csv --limit 5 ``` -will print the header row and 4 data rows in `some.dbn` in CSV format to the console. +will print the header row and the first 5 data rows in `some.dbn` in CSV format to the console. Similarly, running ```sh dbn ohlcv-1d.dbn.zst --json | jq '.high' @@ -53,7 +60,7 @@ dbn ohlcv-1d.dbn -o ohclv-1d.json.zst or explicitly ``` -dbn ohlcv-1d.dbn --json --zstd > ohlcv-1d.json.zst +dbn ohlcv-1d.dbn --json --zstd -o ohlcv-1d.json.zst ``` ### Converting DBZ files to DBN diff --git a/rust/dbn-cli/src/lib.rs b/rust/dbn-cli/src/lib.rs index 7320ffd..f1d8f55 100644 --- a/rust/dbn-cli/src/lib.rs +++ b/rust/dbn-cli/src/lib.rs @@ -98,6 +98,22 @@ pub struct Args { help = "Limit the number of records in the output to the specified number" )] pub limit: Option, + #[clap( + long = "fragment", + action = ArgAction::SetTrue, + default_value = "false", + conflicts_with_all = ["is_zstd_fragment", "should_output_metadata", "dbn"], + help = "Interpret the input as an uncompressed DBN fragment, i.e. records without metadata. Only valid with text output encodings" + )] + pub is_fragment: bool, + #[clap( + long = "zstd-fragment", + action = ArgAction::SetTrue, + default_value = "false", + conflicts_with_all = ["should_output_metadata", "dbn"], + help = "Interpret the input as a Zstd-compressed DBN fragment, i.e. records without metadata. Only valid with text output encodings" + )] + pub is_zstd_fragment: bool, } impl Args { diff --git a/rust/dbn-cli/src/main.rs b/rust/dbn-cli/src/main.rs index f0c121b..56b723a 100644 --- a/rust/dbn-cli/src/main.rs +++ b/rust/dbn-cli/src/main.rs @@ -1,9 +1,11 @@ -use std::io; +use std::{fs::File, io}; use clap::Parser; use dbn::{ - decode::{DecodeDbn, DynDecoder}, + decode::{DbnRecordDecoder, DecodeDbn, DynDecoder}, encode::{json, DynEncoder, EncodeDbn}, + enums::SType, + MetadataBuilder, }; use dbn_cli::{infer_encoding_and_compression, output_from_args, Args}; @@ -47,9 +49,68 @@ fn write_dbn(decoder: DynDecoder, args: &Args) -> anyhow::Res } } +fn write_dbn_frag( + mut decoder: DbnRecordDecoder, + args: &Args, +) -> anyhow::Result<()> { + let writer = output_from_args(args)?; + let (encoding, compression) = infer_encoding_and_compression(args)?; + assert!(!args.should_output_metadata); + + let mut encoder = DynEncoder::new( + writer, + encoding, + compression, + // dummy metadata won't be encoded + &MetadataBuilder::new() + .dataset(String::new()) + .schema(None) + .start(0) + .stype_in(None) + .stype_out(SType::InstrumentId) + .build(), + args.should_pretty_print, + args.should_pretty_print, + args.should_pretty_print, + )?; + let mut n = 0; + while let Some(record) = decoder.decode_ref()? { + // Assume no ts_out for safety + unsafe { + encoder.encode_record_ref(record, false)?; + } + n += 1; + if args.limit.map_or(false, |l| n >= l.get()) { + break; + } + } + Ok(()) +} + fn main() -> anyhow::Result<()> { let args = Args::parse(); - if args.input.as_os_str() == "-" { + if args.is_fragment { + if args.input.as_os_str() == "-" { + write_dbn_frag(DbnRecordDecoder::new(io::stdin().lock()), &args) + } else { + write_dbn_frag( + DbnRecordDecoder::new(File::open(args.input.clone())?), + &args, + ) + } + } else if args.is_zstd_fragment { + if args.input.as_os_str() == "-" { + write_dbn_frag( + DbnRecordDecoder::new(zstd::stream::Decoder::with_buffer(io::stdin().lock())?), + &args, + ) + } else { + write_dbn_frag( + DbnRecordDecoder::new(zstd::stream::Decoder::new(File::open(args.input.clone())?)?), + &args, + ) + } + } else if args.input.as_os_str() == "-" { write_dbn(DynDecoder::inferred_with_buffer(io::stdin().lock())?, &args) } else { write_dbn(DynDecoder::from_file(&args.input)?, &args) diff --git a/rust/dbn-cli/tests/integration_tests.rs b/rust/dbn-cli/tests/integration_tests.rs index b30208a..5714779 100644 --- a/rust/dbn-cli/tests/integration_tests.rs +++ b/rust/dbn-cli/tests/integration_tests.rs @@ -356,6 +356,90 @@ fn metadata_conflicts_with_limit() { .stderr(contains("'--metadata' cannot be used with '--limit")); } +#[test] +fn fragment_conflicts_with_metadata() { + cmd() + .args(&[ + &format!("{TEST_DATA_PATH}/test_data.definition.dbn.frag"), + "--fragment", + "--json", + "--metadata", + ]) + .assert() + .failure() + .stderr(contains("'--fragment' cannot be used with '--metadata'")); +} + +#[test] +fn zstd_fragment_conflicts_with_metadata() { + cmd() + .args(&[ + &format!("{TEST_DATA_PATH}/test_data.definition.dbn.frag.zst"), + "--zstd-fragment", + "--json", + "--metadata", + ]) + .assert() + .failure() + .stderr(contains( + "'--zstd-fragment' cannot be used with '--metadata'", + )); +} + +#[test] +fn fragment_conflicts_with_dbn_output() { + cmd() + .args(&[ + &format!("{TEST_DATA_PATH}/test_data.definition.dbn.frag"), + "--fragment", + "--dbn", + ]) + .assert() + .failure() + .stderr(contains("'--fragment' cannot be used with '--dbn'")); +} + +#[test] +fn zstd_fragment_conflicts_with_dbn_output() { + cmd() + .args(&[ + &format!("{TEST_DATA_PATH}/test_data.definition.dbn.frag.zst"), + "--zstd-fragment", + "--dbn", + ]) + .assert() + .failure() + .stderr(contains("'--zstd-fragment' cannot be used with '--dbn'")); +} + +#[test] +fn test_fragment() { + cmd() + .args(&[ + &format!("{TEST_DATA_PATH}/test_data.definition.dbn.frag"), + "--fragment", + "--json", + ]) + .assert() + .success() + .stdout(contains('\n').count(2)) + .stderr(is_empty()); +} + +#[test] +fn test_zstd_fragment() { + cmd() + .args(&[ + &format!("{TEST_DATA_PATH}/test_data.definition.dbn.frag.zst"), + "--zstd-fragment", + "--json", + ]) + .assert() + .success() + .stdout(contains('\n').count(2)) + .stderr(is_empty()); +} + #[test] fn test_limit_updates_metadata() { // Check metadata shows limit = 2 diff --git a/rust/dbn-macros/Cargo.toml b/rust/dbn-macros/Cargo.toml index ab22b9e..4bbfc2d 100644 --- a/rust/dbn-macros/Cargo.toml +++ b/rust/dbn-macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dbn-macros" authors = ["Databento "] -version = "0.6.0" +version = "0.6.1" edition = "2021" description = "Proc macros for dbn crate" license = "Apache-2.0" diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index 68eafe2..c9bc79b 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "dbn" authors = ["Databento "] -version = "0.6.0" +version = "0.6.1" edition = "2021" description = "Library for working with Databento Binary Encoding (DBN)" license = "Apache-2.0" @@ -18,18 +18,18 @@ python = ["pyo3"] trivial_copy = [] [dependencies] -dbn-macros = { version = "=0.6.0", path = "../dbn-macros" } +dbn-macros = { version = "=0.6.1", path = "../dbn-macros" } # error handling anyhow = "1.0" # async (de)compression async-compression = { version = "0.3.15", features = ["tokio", "zstd"], optional = true } # CSV serialization -csv = "1.2" +csv = ">=1.2.2" # Deriving translation between integers and enums num_enum = "0.6" # Python bindings for Rust -pyo3 = { version = "0.18.3", optional = true } +pyo3 = { version = "0.19.0", optional = true } # JSON serialization json-writer = "0.3" # zero-copy DBN decoding diff --git a/rust/dbn/README.md b/rust/dbn/README.md index 91ddfc6..8cd4646 100644 --- a/rust/dbn/README.md +++ b/rust/dbn/README.md @@ -6,6 +6,13 @@ The official library for working with the Databento Binary Encoding (DBN, formerly DBZ). +## Installation + +To add the crate to an existing project, run the following command: +```sh +cargo add dbn +``` + ## Usage To read a DBN file with MBO data and print each row: diff --git a/rust/dbn/src/decode/dbn.rs b/rust/dbn/src/decode/dbn.rs index 9dab6dc..b59b0e0 100644 --- a/rust/dbn/src/decode/dbn.rs +++ b/rust/dbn/src/decode/dbn.rs @@ -506,6 +506,7 @@ mod tests { use super::*; use crate::{ + datasets::XNAS_ITCH, decode::tests::TEST_DATA_PATH, encode::{dbn::Encoder, EncodeDbn}, enums::rtype, @@ -649,7 +650,7 @@ mod tests { let mut encoder = Encoder::new( &mut buffer, &MetadataBuilder::new() - .dataset("XNAS.ITCH".to_owned()) + .dataset(XNAS_ITCH.to_owned()) .schema(Some(Schema::Mbo)) .start(0) .stype_in(Some(SType::InstrumentId)) diff --git a/rust/dbn/src/decode/mod.rs b/rust/dbn/src/decode/mod.rs index d843f2c..c715fd4 100644 --- a/rust/dbn/src/decode/mod.rs +++ b/rust/dbn/src/decode/mod.rs @@ -89,7 +89,7 @@ pub trait DecodeDbn: private::BufferSlice { } /// A decoder implementing [`DecodeDbn`] whose [`Encoding`](crate::enums::Encoding) and -/// [`Compression`](crate::enums::Compression) can be determined at runtime by peeking +/// [`Compression`](crate::enums::Compression) are determined at runtime by peeking /// at the first few bytes. pub struct DynDecoder<'a, R>(DynDecoderImpl<'a, R>) where diff --git a/rust/dbn/src/encode/csv.rs b/rust/dbn/src/encode/csv.rs index b12723d..7f597b8 100644 --- a/rust/dbn/src/encode/csv.rs +++ b/rust/dbn/src/encode/csv.rs @@ -37,6 +37,11 @@ where } } + /// Returns a reference to the underlying writer. + pub fn get_ref(&self) -> &W { + self.writer.get_ref() + } + #[doc(hidden)] pub fn encode_header(&mut self) -> anyhow::Result<()> { R::serialize_header(&mut self.writer)?; diff --git a/rust/dbn/src/encode/dbn.rs b/rust/dbn/src/encode/dbn.rs index 62a49c9..1471746 100644 --- a/rust/dbn/src/encode/dbn.rs +++ b/rust/dbn/src/encode/dbn.rs @@ -374,6 +374,7 @@ mod tests { use super::*; use crate::{ + datasets::{GLBX_MDP3, XNAS_ITCH}, decode::{dbn::MetadataDecoder, FromLittleEndianSlice}, enums::{SType, Schema}, MappingInterval, MetadataBuilder, @@ -383,7 +384,7 @@ mod tests { fn test_encode_decode_metadata_identity() { let metadata = Metadata { version: crate::DBN_VERSION, - dataset: "GLBX.MDP3".to_owned(), + dataset: GLBX_MDP3.to_owned(), schema: Some(Schema::Mbp10), stype_in: Some(SType::RawSymbol), stype_out: SType::InstrumentId, @@ -500,7 +501,7 @@ mod tests { fn test_update_encoded() { let orig_metadata = Metadata { version: crate::DBN_VERSION, - dataset: "GLBX.MDP3".to_owned(), + dataset: GLBX_MDP3.to_owned(), schema: Some(Schema::Mbo), stype_in: Some(SType::Parent), stype_out: SType::RawSymbol, @@ -544,7 +545,7 @@ mod tests { #[test] fn test_encode_decode_nulls() { let metadata = MetadataBuilder::new() - .dataset("XNAS.ITCH".to_owned()) + .dataset(XNAS_ITCH.to_owned()) .schema(Some(Schema::Mbo)) .start(1697240529000000000) .stype_in(Some(SType::RawSymbol)) @@ -562,7 +563,7 @@ mod tests { #[test] fn test_metadata_min_encoded_size() { let metadata = MetadataBuilder::new() - .dataset("XNAS.ITCH".to_owned()) + .dataset(XNAS_ITCH.to_owned()) .schema(Some(Schema::Mbo)) .start(1697240529000000000) .stype_in(Some(SType::RawSymbol)) @@ -850,6 +851,7 @@ mod r#async { use super::*; use crate::{ + datasets::{GLBX_MDP3, XNAS_ITCH}, decode::{dbn::MetadataDecoder, FromLittleEndianSlice}, enums::{SType, Schema}, MappingInterval, MetadataBuilder, @@ -859,7 +861,7 @@ mod r#async { async fn test_encode_decode_metadata_identity() { let metadata = Metadata { version: crate::DBN_VERSION, - dataset: "GLBX.MDP3".to_owned(), + dataset: GLBX_MDP3.to_owned(), schema: Some(Schema::Mbp10), stype_in: Some(SType::RawSymbol), stype_out: SType::InstrumentId, @@ -989,7 +991,7 @@ mod r#async { #[tokio::test] async fn test_encode_decode_nulls() { let metadata = MetadataBuilder::new() - .dataset("XNAS.ITCH".to_owned()) + .dataset(XNAS_ITCH.to_owned()) .schema(Some(Schema::Mbo)) .start(1697240529000000000) .stype_in(Some(SType::RawSymbol)) diff --git a/rust/dbn/src/encode/json.rs b/rust/dbn/src/encode/json.rs index f5d0b11..bc27384 100644 --- a/rust/dbn/src/encode/json.rs +++ b/rust/dbn/src/encode/json.rs @@ -458,6 +458,7 @@ mod tests { use super::*; use crate::{ + datasets::GLBX_MDP3, encode::test_data::{VecStream, BID_ASK, RECORD_HEADER}, enums::{ InstrumentClass, SType, Schema, SecurityUpdateAction, StatType, StatUpdateAction, @@ -853,7 +854,7 @@ mod tests { fn test_metadata_write_json() { let metadata = Metadata { version: 1, - dataset: "GLBX.MDP3".to_owned(), + dataset: GLBX_MDP3.to_owned(), schema: Some(Schema::Ohlcv1H), start: 1662734705128748281, end: NonZeroU64::new(1662734720914876944), diff --git a/rust/dbn/src/lib.rs b/rust/dbn/src/lib.rs index 98d01ed..a27c3d3 100644 --- a/rust/dbn/src/lib.rs +++ b/rust/dbn/src/lib.rs @@ -43,3 +43,11 @@ pub const UNDEF_ORDER_SIZE: u32 = u32::MAX; pub const UNDEF_STAT_QUANTITY: i32 = i32::MAX; /// The sentinel value for an unset or null timestamp. pub const UNDEF_TIMESTAMP: u64 = u64::MAX; + +/// Contains dataset code constants. +pub mod datasets { + /// The dataset code for CME Globex MDP 3.0. + pub const GLBX_MDP3: &str = "GLBX.MDP3"; + /// The dataset code for Nasdaq TotalView ITCH. + pub const XNAS_ITCH: &str = "XNAS.ITCH"; +} diff --git a/rust/dbn/src/metadata.rs b/rust/dbn/src/metadata.rs index f4a2c1b..f452541 100644 --- a/rust/dbn/src/metadata.rs +++ b/rust/dbn/src/metadata.rs @@ -69,7 +69,9 @@ pub struct Metadata { /// /// # Required fields /// - [`dataset`](Metadata::dataset) +/// - [`schema`](Metadata::schema) /// - [`start`](Metadata::start) +/// - [`stype_in`](Metadata::stype_in) /// - [`stype_out`](Metadata::stype_out) #[derive(Debug)] pub struct MetadataBuilder { diff --git a/rust/dbn/src/python.rs b/rust/dbn/src/python.rs index f842c6e..8309634 100644 --- a/rust/dbn/src/python.rs +++ b/rust/dbn/src/python.rs @@ -355,6 +355,18 @@ impl MboMsg { fn size_hint(_: &PyType) -> PyResult { Ok(mem::size_of::()) } + + #[getter] + #[pyo3(name = "action")] + fn py_action(&self) -> char { + self.action as u8 as char + } + + #[getter] + #[pyo3(name = "side")] + fn py_side(&self) -> char { + self.side as u8 as char + } } #[pymethods] @@ -459,6 +471,18 @@ impl TradeMsg { fn size_hint(_: &PyType) -> PyResult { Ok(mem::size_of::()) } + + #[getter] + #[pyo3(name = "action")] + fn py_action(&self) -> char { + self.action as u8 as char + } + + #[getter] + #[pyo3(name = "side")] + fn py_side(&self) -> char { + self.side as u8 as char + } } #[pymethods] @@ -539,6 +563,18 @@ impl Mbp1Msg { fn size_hint(_: &PyType) -> PyResult { Ok(mem::size_of::()) } + + #[getter] + #[pyo3(name = "action")] + fn py_action(&self) -> char { + self.action as u8 as char + } + + #[getter] + #[pyo3(name = "side")] + fn py_side(&self) -> char { + self.side as u8 as char + } } #[pymethods] @@ -631,6 +667,18 @@ impl Mbp10Msg { fn size_hint(_: &PyType) -> PyResult { Ok(mem::size_of::()) } + + #[getter] + #[pyo3(name = "action")] + fn py_action(&self) -> char { + self.action as u8 as char + } + + #[getter] + #[pyo3(name = "side")] + fn py_side(&self) -> char { + self.side as u8 as char + } } #[pymethods] @@ -995,6 +1043,12 @@ impl InstrumentDefMsg { self.raw_symbol().map_err(to_val_err) } + #[getter] + #[pyo3(name = "group")] + fn py_group(&self) -> PyResult<&str> { + self.group().map_err(to_val_err) + } + #[getter] #[pyo3(name = "exchange")] fn py_exchange(&self) -> PyResult<&str> { @@ -1036,6 +1090,30 @@ impl InstrumentDefMsg { fn py_strike_price_currency(&self) -> PyResult<&str> { self.strike_price_currency().map_err(to_val_err) } + + #[getter] + #[pyo3(name = "instrument_class")] + fn py_instrument_class(&self) -> char { + self.instrument_class as u8 as char + } + + #[getter] + #[pyo3(name = "match_algorithm")] + fn py_match_algorithm(&self) -> char { + self.match_algorithm as u8 as char + } + + #[getter] + #[pyo3(name = "security_update_action")] + fn py_security_update_action(&self) -> char { + self.security_update_action as u8 as char + } + + #[getter] + #[pyo3(name = "user_defined_instrument")] + fn py_user_defined_instrument(&self) -> char { + self.user_defined_instrument as u8 as char + } } #[pymethods] @@ -1126,6 +1204,30 @@ impl ImbalanceMsg { fn size_hint(_: &PyType) -> PyResult { Ok(mem::size_of::()) } + + #[getter] + #[pyo3(name = "auction_type")] + fn py_auction_type(&self) -> char { + self.auction_type as u8 as char + } + + #[getter] + #[pyo3(name = "side")] + fn py_side(&self) -> char { + self.side as u8 as char + } + + #[getter] + #[pyo3(name = "unpaired_side")] + fn py_unpaired_side(&self) -> char { + self.unpaired_side as u8 as char + } + + #[getter] + #[pyo3(name = "significant_imbalance")] + fn py_significant_imbalance(&self) -> char { + self.significant_imbalance as u8 as char + } } #[pymethods] diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index c8149f2..e99568a 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -52,24 +52,31 @@ pub struct RecordHeader { #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", - pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn", name = "MBOMsg") + pyo3::pyclass(set_all, dict, module = "databento_dbn", name = "MBOMsg") )] +#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope #[dbn_record(rtype::MBO)] pub struct MboMsg { /// The common header. + #[pyo3(get)] pub hd: RecordHeader, /// The order ID assigned at the venue. + #[pyo3(get)] pub order_id: u64, /// The order price expressed as a signed integer where every 1 unit /// corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001. #[dbn(fixed_price)] + #[pyo3(get)] pub price: i64, /// The order quantity. + #[pyo3(get)] pub size: u32, /// A combination of packet end with matching engine status. See /// [`enums::flags`](crate::enums::flags) for possible values. + #[pyo3(get)] pub flags: u8, /// A channel ID within the venue. + #[pyo3(get)] pub channel_id: u8, /// The event action. Can be **A**dd, **C**ancel, **M**odify, clea**R**, /// **T**rade, or **F**ill. @@ -81,10 +88,13 @@ pub struct MboMsg { /// The capture-server-received timestamp expressed as number of nanoseconds since /// the UNIX epoch. #[dbn(unix_nanos)] + #[pyo3(get)] pub ts_recv: u64, /// The delta of `ts_recv - ts_exchange_send`, max 2 seconds. + #[pyo3(get)] pub ts_in_delta: i32, /// The message sequence number assigned at the venue. + #[pyo3(get)] pub sequence: u32, } @@ -118,17 +128,21 @@ pub struct BidAskPair { #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", - pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn", name = "TradeMsg") + pyo3::pyclass(set_all, dict, module = "databento_dbn") )] +#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope #[dbn_record(rtype::MBP_0)] pub struct TradeMsg { /// The common header. + #[pyo3(get)] pub hd: RecordHeader, /// The order price expressed as a signed integer where every 1 unit /// corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001. #[dbn(fixed_price)] + #[pyo3(get)] pub price: i64, /// The order quantity. + #[pyo3(get)] pub size: u32, /// The event action. Always **T**rade in the trades schema. #[dbn(c_char)] @@ -138,16 +152,21 @@ pub struct TradeMsg { pub side: c_char, /// A combination of packet end with matching engine status. See /// [`enums::flags`](crate::enums::flags) for possible values. + #[pyo3(get)] pub flags: u8, /// The depth of actual book change. + #[pyo3(get)] pub depth: u8, /// The capture-server-received timestamp expressed as number of nanoseconds since /// the UNIX epoch. #[dbn(unix_nanos)] + #[pyo3(get)] pub ts_recv: u64, /// The delta of `ts_recv - ts_exchange_send`, max 2 seconds. + #[pyo3(get)] pub ts_in_delta: i32, /// The message sequence number assigned at the venue. + #[pyo3(get)] pub sequence: u32, } @@ -158,17 +177,21 @@ pub struct TradeMsg { #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", - pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn", name = "MBP1Msg") + pyo3::pyclass(set_all, dict, module = "databento_dbn", name = "MBP1Msg") )] +#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope #[dbn_record(rtype::MBP_1)] pub struct Mbp1Msg { /// The common header. + #[pyo3(get)] pub hd: RecordHeader, /// The order price expressed as a signed integer where every 1 unit /// corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001. #[dbn(fixed_price)] + #[pyo3(get)] pub price: i64, /// The order quantity. + #[pyo3(get)] pub size: u32, /// The event action. Can be **A**dd, **C**ancel, **M**odify, clea**R**, or /// **T**rade. @@ -179,18 +202,24 @@ pub struct Mbp1Msg { pub side: c_char, /// A combination of packet end with matching engine status. See /// [`enums::flags`](crate::enums::flags) for possible values. + #[pyo3(get)] pub flags: u8, /// The depth of actual book change. + #[pyo3(get)] pub depth: u8, /// The capture-server-received timestamp expressed as number of nanoseconds since /// the UNIX epoch. #[dbn(unix_nanos)] + #[pyo3(get)] pub ts_recv: u64, /// The delta of `ts_recv - ts_exchange_send`, max 2 seconds. + #[pyo3(get)] pub ts_in_delta: i32, /// The message sequence number assigned at the venue. + #[pyo3(get)] pub sequence: u32, /// The top of the order book. + #[pyo3(get)] pub levels: [BidAskPair; 1], } @@ -201,17 +230,21 @@ pub struct Mbp1Msg { #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", - pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn", name = "MBP10Msg") + pyo3::pyclass(set_all, dict, module = "databento_dbn", name = "MBP10Msg") )] +#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope #[dbn_record(rtype::MBP_10)] pub struct Mbp10Msg { /// The common header. + #[pyo3(get)] pub hd: RecordHeader, /// The order price expressed as a signed integer where every 1 unit /// corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001. #[dbn(fixed_price)] + #[pyo3(get)] pub price: i64, /// The order quantity. + #[pyo3(get)] pub size: u32, /// The event action. Can be **A**dd, **C**ancel, **M**odify, clea**R**, or /// **T**rade. @@ -222,18 +255,24 @@ pub struct Mbp10Msg { pub side: c_char, /// A combination of packet end with matching engine status. See /// [`enums::flags`](crate::enums::flags) for possible values. + #[pyo3(get)] pub flags: u8, /// The depth of actual book change. + #[pyo3(get)] pub depth: u8, /// The capture-server-received timestamp expressed as number of nanoseconds since /// the UNIX epoch. #[dbn(unix_nanos)] + #[pyo3(get)] pub ts_recv: u64, /// The delta of `ts_recv - ts_exchange_send`, max 2 seconds. + #[pyo3(get)] pub ts_in_delta: i32, /// The message sequence number assigned at the venue. + #[pyo3(get)] pub sequence: u32, /// The top 10 levels of the order book. + #[pyo3(get)] pub levels: [BidAskPair; 10], } @@ -462,7 +501,7 @@ pub struct InstrumentDefMsg { pub strike_price_currency: [c_char; 4], /// The classification of the instrument. #[dbn(c_char)] - #[pyo3(get, set)] + #[pyo3(set)] pub instrument_class: c_char, #[doc(hidden)] pub _reserved4: [u8; 2], @@ -475,7 +514,7 @@ pub struct InstrumentDefMsg { pub _reserved5: [u8; 6], /// The matching algorithm used for the instrument, typically **F**IFO. #[dbn(c_char)] - #[pyo3(get, set)] + #[pyo3(set)] pub match_algorithm: c_char, /// The current trading state of the instrument. #[pyo3(get, set)] @@ -496,7 +535,7 @@ pub struct InstrumentDefMsg { #[pyo3(get, set)] pub underlying_product: u8, /// Indicates if the instrument definition has been added, modified, or deleted. - #[pyo3(get, set)] + #[pyo3(set)] pub security_update_action: SecurityUpdateAction, /// The calendar month reflected in the instrument symbol. #[pyo3(get, set)] @@ -508,7 +547,7 @@ pub struct InstrumentDefMsg { #[pyo3(get, set)] pub maturity_week: u8, /// Indicates if the instrument is user defined: **Y**es or **N**o. - #[pyo3(get, set)] + #[pyo3(set)] pub user_defined_instrument: UserDefinedInstrument, /// The type of `contract_multiplier`. Either `1` for hours, or `2` for days. #[pyo3(get, set)] @@ -530,47 +569,62 @@ pub struct InstrumentDefMsg { #[cfg_attr(feature = "trivial_copy", derive(Copy))] #[cfg_attr( feature = "python", - pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn") + pyo3::pyclass(set_all, dict, module = "databento_dbn") )] +#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope #[dbn_record(rtype::IMBALANCE)] pub struct ImbalanceMsg { /// The common header. + #[pyo3(get)] pub hd: RecordHeader, /// The capture-server-received timestamp expressed as the number of nanoseconds /// since the UNIX epoch. #[dbn(unix_nanos)] + #[pyo3(get)] pub ts_recv: u64, /// The price at which the imbalance shares are calculated, where every 1 unit corresponds to /// 1e-9, i.e. 1/1,000,000,000 or 0.000000001. #[dbn(fixed_price)] + #[pyo3(get)] pub ref_price: i64, /// Reserved for future use. + #[pyo3(get)] pub auction_time: u64, /// The hypothetical auction-clearing price for both cross and continuous orders. #[dbn(fixed_price)] + #[pyo3(get)] pub cont_book_clr_price: i64, /// The hypothetical auction-clearing price for cross orders only. #[dbn(fixed_price)] + #[pyo3(get)] pub auct_interest_clr_price: i64, /// Reserved for future use. #[dbn(fixed_price)] + #[pyo3(get)] pub ssr_filling_price: i64, /// Reserved for future use. #[dbn(fixed_price)] + #[pyo3(get)] pub ind_match_price: i64, /// Reserved for future use. #[dbn(fixed_price)] + #[pyo3(get)] pub upper_collar: i64, /// Reserved for future use. #[dbn(fixed_price)] + #[pyo3(get)] pub lower_collar: i64, /// The quantity of shares that are eligible to be matched at `ref_price`. + #[pyo3(get)] pub paired_qty: u32, /// The quantity of shares that are not paired at `ref_price`. + #[pyo3(get)] pub total_imbalance_qty: u32, /// Reserved for future use. + #[pyo3(get)] pub market_imbalance_qty: u32, /// Reserved for future use. + #[pyo3(get)] pub unpaired_qty: u32, /// Venue-specific character code indicating the auction type. #[dbn(c_char)] @@ -579,10 +633,13 @@ pub struct ImbalanceMsg { #[dbn(c_char)] pub side: c_char, /// Reserved for future use. + #[pyo3(get)] pub auction_status: u8, /// Reserved for future use. + #[pyo3(get)] pub freeze_status: u8, /// Reserved for future use. + #[pyo3(get)] pub num_extensions: u8, /// Reserved for future use. #[dbn(c_char)] @@ -604,6 +661,7 @@ pub struct ImbalanceMsg { feature = "python", pyo3::pyclass(get_all, set_all, dict, module = "databento_dbn") )] +#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope #[dbn_record(rtype::STATISTICS)] pub struct StatMsg { /// The common header. diff --git a/tests/data/test_data.definition.dbn.frag b/tests/data/test_data.definition.dbn.frag new file mode 100644 index 0000000..137e28d Binary files /dev/null and b/tests/data/test_data.definition.dbn.frag differ diff --git a/tests/data/test_data.definition.dbn.frag.zst b/tests/data/test_data.definition.dbn.frag.zst new file mode 100644 index 0000000..89c1e16 Binary files /dev/null and b/tests/data/test_data.definition.dbn.frag.zst differ