From 722723b4880e060d2c2faf70de7b23d4136d4eeb Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 3 Apr 2024 17:39:24 -0500 Subject: [PATCH 01/23] DOC: Add links to example usage --- CHANGELOG.md | 4 +++ README.md | 2 +- python/README.md | 4 +-- rust/dbn-cli/README.md | 2 +- rust/dbn/README.md | 2 +- rust/dbn/src/enums.rs | 58 +++++++++++++++++++++++++++++++----------- rust/dbn/src/lib.rs | 2 +- 7 files changed, 53 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae5fbfd..802156d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.18.0 - TBD +### Enhancements +- Added links to example usage in documentation + ## 0.17.1 - 2024-04-04 ### Bug fixes diff --git a/README.md b/README.md index 7694ede..b2565a5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ DBN is also the default encoding for all Databento APIs, including live data str This repository contains both libraries and a CLI tool for working with DBN files and streams. Python bindings for `dbn` are provided in the `databento_dbn` package. -For more details, read our [introduction to DBN](https://docs.databento.com/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). +For more details, read our [introduction to DBN](https://databento.com/docs/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). ## Features diff --git a/python/README.md b/python/README.md index 1b666dd..2f89db6 100644 --- a/python/README.md +++ b/python/README.md @@ -6,7 +6,7 @@ [![pypi-version](https://img.shields.io/pypi/v/databento_dbn)](https://pypi.org/project/databento-dbn) Python bindings for the `dbn` Rust library, used by the [Databento Python client library](https://github.com/databento/databento-python). -For more information about the encoding, read our [introduction to DBN](https://docs.databento.com/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). +For more information about the encoding, read our [introduction to DBN](https://databento.com/docs/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). Using this library is for advanced users and is not fully documented or supported. @@ -19,7 +19,7 @@ pip install -U databento-dbn ## Usage and documentation -See the [documentation](https://docs.databento.com/getting-started?historical=python&live=python) for the Python client library. +See the [documentation](https://databento.com/docs/getting-started?historical=python&live=python) for the Python client library. ## Building diff --git a/rust/dbn-cli/README.md b/rust/dbn-cli/README.md index c3d66cc..c36330a 100644 --- a/rust/dbn-cli/README.md +++ b/rust/dbn-cli/README.md @@ -8,7 +8,7 @@ This crate provides a CLI tool `dbn` for converting [Databento](https://databent Binary Encoding (DBN) files to text formats, as well as updating legacy DBZ files to DBN. -For more information about DBN, read our [introduction to DBN](https://docs.databento.com/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). +For more information about DBN, read our [introduction to DBN](https://databento.com/docs/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). ## Installation diff --git a/rust/dbn/README.md b/rust/dbn/README.md index 18bf299..e334c3b 100644 --- a/rust/dbn/README.md +++ b/rust/dbn/README.md @@ -6,7 +6,7 @@ [![Current Crates.io Version](https://img.shields.io/crates/v/dbn.svg)](https://crates.io/crates/dbn) The official crate for working with Databento Binary Encoding (DBN). -For more information about DBN, read our [introduction to DBN](https://docs.databento.com/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). +For more information about DBN, read our [introduction to DBN](https://databento.com/docs/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). Check out the [databento crate](https://crates.io/crates/databento) for the official Databento Rust client. diff --git a/rust/dbn/src/enums.rs b/rust/dbn/src/enums.rs index 0e20ea1..7311e02 100644 --- a/rust/dbn/src/enums.rs +++ b/rust/dbn/src/enums.rs @@ -1,6 +1,7 @@ #![allow(deprecated)] // TODO: remove with SType::Smart //! Enums used in Databento APIs. + use std::{ fmt::{self, Display, Formatter}, str::FromStr, @@ -12,8 +13,11 @@ use std::{ use dbn_macros::MockPyo3; use num_enum::{IntoPrimitive, TryFromPrimitive}; -/// A side of the market. The side of the market for resting orders, or the side -/// of the aggressor for trades. +/// A [side](https://databento.com/docs/knowledge-base/new-users/standards-conventions/side) +/// of the market. The side of the market for resting orders, or the side of the +/// aggressor for trades. +/// +/// #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, TryFromPrimitive, IntoPrimitive)] #[cfg_attr( feature = "python", @@ -36,7 +40,12 @@ impl From for char { } } -/// A tick action. +/// A [tick action](https://databento.com/docs/knowledge-base/new-users/standards-conventions/action) +/// used to indicate order life cycle. +/// +/// For example usage see: +/// - [Order actions](https://databento.com/docs/examples/order-book/order-actions) +/// - [Order tracking](https://databento.com/docs/examples/order-book/order-tracking) #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, TryFromPrimitive, IntoPrimitive)] #[cfg_attr( feature = "python", @@ -45,15 +54,15 @@ impl From for char { )] #[repr(u8)] pub enum Action { - /// An existing order was modified. + /// An existing order was modified: price and/or size. Modify = b'M', - /// A trade executed. + /// An aggressing order traded. Does not affect the book. Trade = b'T', - /// An existing order was filled. + /// An existing order was filled. Does not affect the book. Fill = b'F', - /// An order was cancelled. + /// An order was fully or partially cancelled. Cancel = b'C', - /// A new order was added. + /// A new order was added to the book. Add = b'A', /// Reset the book; clear all orders for an instrument. Clear = b'R', @@ -66,6 +75,9 @@ impl From for char { } /// The class of instrument. +/// +/// For example usage see +/// [Getting options with their underlying](https://databento.com/docs/examples/options/options-and-futures). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, TryFromPrimitive, IntoPrimitive)] #[cfg_attr( feature = "python", @@ -145,6 +157,9 @@ impl From for char { } /// Whether the instrument is user-defined. +/// +/// For example usage see +/// [Getting options with their underlying](https://databento.com/docs/examples/options/options-and-futures). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, TryFromPrimitive, IntoPrimitive, Default)] #[cfg_attr( feature = "python", @@ -167,7 +182,8 @@ impl From for char { } } -/// A symbology type. Refer to the [symbology documentation](https://docs.databento.com/api-reference-historical/basics/symbology) +/// A symbology type. Refer to the +/// [symbology documentation](https://databento.com/docs/api-reference-historical/basics/symbology) /// for more information. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TryFromPrimitive)] #[cfg_attr( @@ -244,14 +260,19 @@ impl Display for SType { pub use rtype::RType; -/// Record types, possible values for [`RecordHeader::rtype`][crate::record::RecordHeader::rtype] +/// Record types, possible values for [`RecordHeader::rtype`][crate::RecordHeader::rtype] #[allow(deprecated)] pub mod rtype { use num_enum::TryFromPrimitive; use super::Schema; - /// A type of record, i.e. a struct implementing [`HasRType`](crate::record::HasRType). + /// A [record type](https://databento.com/docs/knowledge-base/new-users/standards-conventions/rtype), + /// i.e. a sentinel for different types implementing [`HasRType`](crate::record::HasRType). + /// + /// Use in [`RecordHeader`](crate::RecordHeader) to indicate the type of record, + /// which is useful when working with DBN streams containing multiple record types + /// or an unknown record type. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TryFromPrimitive)] #[cfg_attr( feature = "python", @@ -496,6 +517,11 @@ pub mod rtype { } /// A data record schema. +/// +/// Each schema has a particular [record](crate::record) type associated with it. +/// +/// See [List of supported market data schemas](https://databento.com/docs/knowledge-base/new-users/market-data-schemas) +/// for an overview of the differences and use cases of each schema. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TryFromPrimitive)] #[cfg_attr( feature = "python", @@ -553,14 +579,16 @@ pub enum Schema { /// Consolidated best bid and offer. #[pyo3(name = "CBBO")] Cbbo = 14, - /// Consolidated best bid and offer subsampled at one-second intervals, in addition to trades. + /// Consolidated best bid and offer subsampled at one-second intervals, in addition + /// to trades. #[pyo3(name = "CBBO_1S")] Cbbo1S = 15, - /// Consolidated best bid and offer subsampled at one-minute intervals, in addition to trades. + /// Consolidated best bid and offer subsampled at one-minute intervals, in addition + /// to trades. #[pyo3(name = "CBBO_1M")] Cbbo1M = 16, - /// All trade events with the consolidated best bid and offer (CBBO) immediately **before** the - /// effect of the trade. + /// All trade events with the consolidated best bid and offer (CBBO) immediately + /// **before** the effect of the trade. #[pyo3(name = "TCBBO")] Tcbbo = 17, /// Best bid and offer subsampled at one-second intervals, in addition to trades. diff --git a/rust/dbn/src/lib.rs b/rust/dbn/src/lib.rs index c5ff123..566202e 100644 --- a/rust/dbn/src/lib.rs +++ b/rust/dbn/src/lib.rs @@ -8,7 +8,7 @@ //! interchange format and for in-memory representation of data. DBN is also the default //! encoding for all Databento APIs, including live data streaming, historical data //! streaming, and batch flat files. For more information about the encoding, read our -//! [introduction to DBN](https://docs.databento.com/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). +//! [introduction to DBN](https://databento.com/docs/knowledge-base/new-users/dbn-encoding/getting-started-with-dbn). //! //! The crate supports reading and writing DBN files and streams, as well as converting //! them to other [`Encoding`]s. It can also be used to update legacy From 16cc4cdb2c96ceba7e3286e9859d6405b49db633 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 10 Apr 2024 11:17:34 -0500 Subject: [PATCH 02/23] MOD: Update for `BorrowedFormatItem` rename --- Cargo.lock | 8 ++++---- rust/dbn/Cargo.toml | 2 +- rust/dbn/src/metadata.rs | 2 +- rust/dbn/src/pretty.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 824b84a..1ba35ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1159,9 +1159,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "ef89ece63debf11bc32d1ed8d078ac870cbeb44da02afb02a9ff135ae7ca0582" dependencies = [ "deranged", "itoa", @@ -1180,9 +1180,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index 1fcc364..de6a17a 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -48,7 +48,7 @@ strum = { version = "0.26", features = ["derive"], optional = true } # Custom error helper thiserror = "1.0" # date and datetime support -time = { version = "0.3", features = ["formatting", "macros"] } +time = { version = ">=0.3.35", features = ["formatting", "macros"] } # async traits tokio = { version = "1", features = ["fs", "io-util"], optional = true } # (de)compression diff --git a/rust/dbn/src/metadata.rs b/rust/dbn/src/metadata.rs index 29b6964..959d327 100644 --- a/rust/dbn/src/metadata.rs +++ b/rust/dbn/src/metadata.rs @@ -398,7 +398,7 @@ pub struct MappingInterval { } /// The date format used for date strings when serializing [`Metadata`]. -pub const DATE_FORMAT: &[time::format_description::FormatItem<'static>] = +pub const DATE_FORMAT: &[time::format_description::BorrowedFormatItem<'static>] = time::macros::format_description!("[year]-[month]-[day]"); #[cfg(feature = "serde")] diff --git a/rust/dbn/src/pretty.rs b/rust/dbn/src/pretty.rs index a21ec70..de2f73e 100644 --- a/rust/dbn/src/pretty.rs +++ b/rust/dbn/src/pretty.rs @@ -3,7 +3,7 @@ use std::fmt; -use time::format_description::FormatItem; +use time::format_description::BorrowedFormatItem; use crate::FIXED_PRICE_SCALE; @@ -68,7 +68,7 @@ pub fn fmt_px(px: i64) -> String { /// Converts a nanosecond UNIX timestamp to a human-readable string in the format /// `[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:9]Z`. pub fn fmt_ts(ts: u64) -> String { - const TS_FORMAT: &[FormatItem<'static>] = time::macros::format_description!( + const TS_FORMAT: &[BorrowedFormatItem<'static>] = time::macros::format_description!( "[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:9]Z" ); if ts == 0 { From bef57a61f893c98f58a215a533cba095a46082f5 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Mon, 15 Apr 2024 10:11:06 -0500 Subject: [PATCH 03/23] REF: Move common DBN dependencies to workspace --- Cargo.toml | 9 +++++++++ c/Cargo.toml | 3 +-- python/Cargo.toml | 12 ++++-------- rust/dbn-cli/Cargo.toml | 13 ++++--------- rust/dbn/Cargo.toml | 24 +++++------------------- 5 files changed, 23 insertions(+), 38 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7968194..2cdbf8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,3 +15,12 @@ version = "0.17.1" documentation = "https://docs.databento.com" repository = "https://github.com/databento/dbn" license = "Apache-2.0" + +[workspace.dependencies] +anyhow = "1.0.80" +pyo3 = "0.20" +pyo3-build-config = "0.20" +rstest = "0.18.2" +serde = { version = "1.0", features = ["derive"] } +time = ">=0.3.35" +zstd = "0.13" diff --git a/c/Cargo.toml b/c/Cargo.toml index 65ebeeb..37c1948 100644 --- a/c/Cargo.toml +++ b/c/Cargo.toml @@ -14,8 +14,7 @@ name = "dbn_c" crate-type = ["staticlib"] [dependencies] -anyhow = "1.0.80" -# DBN library +anyhow = { workspace = true } dbn = { path = "../rust/dbn", features = [] } libc = "0.2.153" diff --git a/python/Cargo.toml b/python/Cargo.toml index 441f248..e3ecc2f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -13,16 +13,12 @@ repository.workspace = true name = "databento_dbn" # Python modules can't contain dashes [dependencies] -# DBN library dbn = { path = "../rust/dbn", features = ["python"] } -# Python bindings for Rust -pyo3 = "0.20" -# Dates and datetimes -time = "0.3.34" +pyo3 = { workspace = true } +time = { workspace = true } [build-dependencies] -pyo3-build-config = { version = "0.20" } +pyo3-build-config = { workspace = true } [dev-dependencies] -# parameterized testing -rstest = "0.18.2" +rstest = { workspace = true } diff --git a/rust/dbn-cli/Cargo.toml b/rust/dbn-cli/Cargo.toml index cd6b4c8..e3b0129 100644 --- a/rust/dbn-cli/Cargo.toml +++ b/rust/dbn-cli/Cargo.toml @@ -16,23 +16,18 @@ name = "dbn" path = "src/main.rs" [dependencies] -# Databento common DBN library dbn = { path = "../dbn", version = "=0.17.1", default-features = false } -# Error handling -anyhow = "1.0" -# CLI argument parsing +anyhow = { workspace = true } clap = { version = "4.5", features = ["derive", "wrap_help"] } -# deserialization for CLI args -serde = { version = "1.0", features = ["derive"] } -# Compression -zstd = "0.13" +serde = { workspace = true, features = ["derive"] } +zstd = { workspace = true } [dev-dependencies] # CLI integration tests assert_cmd = "2.0.14" # assert_cmd companion predicates = "3.1.0" -rstest = "0.18.2" +rstest = { workspace = true } # A library for managing temporary files and directories tempfile = "3.10.0" diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index de6a17a..61cbd53 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -27,39 +27,25 @@ trivial_copy = [] [dependencies] dbn-macros = { version = "=0.17.1", path = "../dbn-macros" } -# async (de)compression async-compression = { version = "0.4.6", features = ["tokio", "zstd"], optional = true } -# CSV serialization csv = "1.3" # Fast integer to string conversion itoa = "1.0" -# Deriving translation between integers and enums num_enum = "0.7" -# Python bindings for Rust -pyo3 = { version = "0.20", optional = true } -# JSON serialization +pyo3 = { workspace = true, optional = true } json-writer = "0.3" -# deserialization -serde = { version = "1.0", features = ["derive"], optional = true } -# zero-copy DBN decoding +serde = { workspace = true, features = ["derive"], optional = true } streaming-iterator = "0.1.9" # extra enum traits for Python strum = { version = "0.26", features = ["derive"], optional = true } -# Custom error helper thiserror = "1.0" -# date and datetime support -time = { version = ">=0.3.35", features = ["formatting", "macros"] } -# async traits +time = { workspace = true, features = ["formatting", "macros"] } tokio = { version = "1", features = ["fs", "io-util"], optional = true } -# (de)compression -zstd = "0.13" +zstd = { workspace = true } [dev-dependencies] -# Parameterized testing -rstest = "0.18.2" -# Enum helpers +rstest = { workspace = true } strum = { version = "0.26", features = ["derive"] } -# Async runtime tokio = { version = "1", features = ["fs", "io-util", "macros", "rt-multi-thread"] } # Checking alignment and padding type-layout = "0.2.0" From 1aad9a8d3c24430a78bf4f526fd689a0b1b77db7 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Tue, 16 Apr 2024 10:56:45 -0500 Subject: [PATCH 04/23] ADD: Add instrument class helper methods --- CHANGELOG.md | 2 ++ rust/dbn/src/enums.rs | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 802156d..2802f35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## 0.18.0 - TBD ### Enhancements - Added links to example usage in documentation +- Added new predicate methods `InstrumentClass::is_option`, `is_future`, and `is_spread` + to make it easier to work with multiple instrument class variants ## 0.17.1 - 2024-04-04 diff --git a/rust/dbn/src/enums.rs b/rust/dbn/src/enums.rs index 7311e02..aebc93e 100644 --- a/rust/dbn/src/enums.rs +++ b/rust/dbn/src/enums.rs @@ -112,6 +112,31 @@ impl From for char { } } +impl InstrumentClass { + /// Returns `true` if the instrument class is a type of option. + /// + /// NOTE: excludes [`Self::MixedSpread`], which *may* include options. + pub fn is_option(&self) -> bool { + matches!(self, Self::Call | Self::Put | Self::OptionSpread) + } + + /// Returns `true` if the instrument class is a type of future. + /// + /// NOTE: excludes [`Self::MixedSpread`], which *may* include futures. + pub fn is_future(&self) -> bool { + matches!(self, Self::Future | Self::FutureSpread) + } + + /// Returns `true` if the instrument class is a type of spread, i.e. composed of two + /// or more instrument legs. + pub fn is_spread(&self) -> bool { + matches!( + self, + Self::FutureSpread | Self::OptionSpread | Self::MixedSpread + ) + } +} + /// The type of matching algorithm used for the instrument at the exchange. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, TryFromPrimitive, IntoPrimitive)] #[cfg_attr( From e5230419c62a6e623b6aacdd42d39d8e628cc8fa Mon Sep 17 00:00:00 2001 From: Carter Green Date: Mon, 15 Apr 2024 11:55:53 -0500 Subject: [PATCH 05/23] MOD: Upgrade to pyo3 0.21 --- CHANGELOG.md | 4 + Cargo.lock | 20 +-- Cargo.toml | 4 +- python/python/databento_dbn/_lib.pyi | 46 ------ python/src/dbn_decoder.rs | 6 +- python/src/encode.rs | 204 ++------------------------- python/src/lib.rs | 7 +- rust/dbn/src/python/enums.rs | 202 +++++++++++++------------- rust/dbn/src/python/metadata.rs | 15 +- rust/dbn/src/python/record.rs | 12 +- 10 files changed, 147 insertions(+), 373 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2802f35..4f6d049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - Added new predicate methods `InstrumentClass::is_option`, `is_future`, and `is_spread` to make it easier to work with multiple instrument class variants +### Breaking changes +- Removed `write_dbn_file` function deprecated in version 0.14.0 from Python interface. + Please use `Transcoder` instead + ## 0.17.1 - 2024-04-04 ### Bug fixes diff --git a/Cargo.lock b/Cargo.lock index 1ba35ab..865352b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -797,9 +797,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.3" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "a7a8b1990bd018761768d5e608a13df8bd1ac5f678456e0f301bb93e5f3ea16b" dependencies = [ "cfg-if", "indoc", @@ -815,9 +815,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.3" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "650dca34d463b6cdbdb02b1d71bfd6eb6b6816afc708faebb3bac1380ff4aef7" dependencies = [ "once_cell", "target-lexicon", @@ -825,9 +825,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "09a7da8fc04a8a2084909b59f29e1b8474decac98b951d77b80b26dc45f046ad" dependencies = [ "libc", "pyo3-build-config", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "4b8a199fce11ebb28e3569387228836ea98110e43a804a530a9fd83ade36d513" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -847,9 +847,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "93fbbfd7eb553d10036513cb122b888dcd362a945a00b06c165f2ab480d4cc3b" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 2cdbf8c..7f94674 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ license = "Apache-2.0" [workspace.dependencies] anyhow = "1.0.80" -pyo3 = "0.20" -pyo3-build-config = "0.20" +pyo3 = "0.21" +pyo3-build-config = "0.21" rstest = "0.18.2" serde = { version = "1.0", features = ["derive"] } time = ">=0.3.35" diff --git a/python/python/databento_dbn/_lib.pyi b/python/python/databento_dbn/_lib.pyi index 99aa6e3..5f98039 100644 --- a/python/python/databento_dbn/_lib.pyi +++ b/python/python/databento_dbn/_lib.pyi @@ -4797,49 +4797,3 @@ def update_encoded_metadata( When the file update fails. """ - -def write_dbn_file( - file: BinaryIO, - compression: str, - dataset: str, - schema: str, - start: int, - stype_in: str, - stype_out: str, - records: Sequence[Record], - end: int | None = None, -) -> None: - """ - Encode the given data in the DBN encoding and writes it to `file`. - - Parameters - ---------- - file : BinaryIO - The file handle to update. - compression : str - The DBN compression format. - dataset : str - The dataset code. - schema : str - The data record schema. - start : int - The UNIX nanosecond timestamp of the query start, or the - first record if the file was split. - stype_in : str - The input symbology type to map from. - stype_out : str - The output symbology type to map to. - records : Sequence[object] - A sequence of DBN record objects. - end : int | None - The UNIX nanosecond timestamp of the query end, or the - last record if the file was split. - - Raises - ------ - ValueError - When any of the enum arguments cannot be converted to their Rust equivalents. - When there's an issue writing the encoded to bytes. - When an expected field is missing from one of the dicts. - - """ diff --git a/python/src/dbn_decoder.rs b/python/src/dbn_decoder.rs index 5ea0ce7..aa14701 100644 --- a/python/src/dbn_decoder.rs +++ b/python/src/dbn_decoder.rs @@ -235,7 +235,7 @@ mod tests { fn test_dbn_decoder() { setup(); Python::with_gil(|py| { - let path = PyString::new( + let path = PyString::new_bound( py, concat!( env!("CARGO_MANIFEST_DIR"), @@ -263,7 +263,7 @@ for record in records[1:]: fn test_dbn_decoder_decoding_error() { setup(); Python::with_gil(|py| { - py.run( + py.run_bound( r#"from _lib import DBNDecoder, Metadata, Schema, SType metadata = Metadata( @@ -299,7 +299,7 @@ except Exception as ex: fn test_dbn_decoder_no_metadata() { setup(); Python::with_gil(|py| { - py.run( + py.run_bound( r#"from _lib import DBNDecoder, OHLCVMsg decoder = DBNDecoder(has_metadata=False) diff --git a/python/src/encode.rs b/python/src/encode.rs index 309974b..151d19b 100644 --- a/python/src/encode.rs +++ b/python/src/encode.rs @@ -3,26 +3,8 @@ use std::{ num::NonZeroU64, }; -use dbn::{ - encode::{ - dbn::{Encoder as DbnEncoder, MetadataEncoder}, - DbnEncodable, DynWriter, EncodeDbn, - }, - enums::{Compression, Schema}, - python::to_val_err, - record::{ - Bbo1MMsg, Bbo1SMsg, Cbbo1MMsg, Cbbo1SMsg, CbboMsg, ImbalanceMsg, InstrumentDefMsg, MboMsg, - Mbp10Msg, Mbp1Msg, OhlcvMsg, StatMsg, TbboMsg, TcbboMsg, TradeMsg, - }, - Metadata, -}; -use pyo3::{ - exceptions::{PyDeprecationWarning, PyTypeError, PyValueError}, - intern, - prelude::*, - types::PyBytes, - PyClass, -}; +use dbn::{encode::dbn::MetadataEncoder, python::to_val_err}; +use pyo3::{exceptions::PyTypeError, intern, prelude::*, types::PyBytes}; /// Updates existing fields that have already been written to the given file. #[pyfunction] @@ -47,72 +29,6 @@ pub fn update_encoded_metadata( .map_err(to_val_err) } -/// Encodes the given data in the DBN encoding and writes it to `file`. -/// -/// `records` is a list of record objects. -/// -/// # Errors -/// This function returns an error if any of the enum arguments cannot be converted to -/// their Rust equivalents. It will also return an error if there's an issue writing -/// the encoded to bytes or an expected field is missing from one of the dicts. -#[pyfunction] -pub fn write_dbn_file( - py: Python<'_>, - file: PyFileLike, - compression: Compression, - metadata: &Metadata, - records: Vec<&PyAny>, -) -> PyResult<()> { - PyErr::warn( - py, - py.get_type::(), - "This function is deprecated. Please switch to using Transcoder", - 0, - )?; - let writer = DynWriter::new(file, compression).map_err(to_val_err)?; - let encoder = DbnEncoder::new(writer, metadata).map_err(to_val_err)?; - match metadata.schema { - Some(Schema::Mbo) => encode_pyrecs::(encoder, &records), - Some(Schema::Mbp1) => encode_pyrecs::(encoder, &records), - Some(Schema::Mbp10) => encode_pyrecs::(encoder, &records), - Some(Schema::Tbbo) => encode_pyrecs::(encoder, &records), - Some(Schema::Trades) => encode_pyrecs::(encoder, &records), - Some(Schema::Ohlcv1S) - | Some(Schema::Ohlcv1M) - | Some(Schema::Ohlcv1H) - | Some(Schema::Ohlcv1D) - | Some(Schema::OhlcvEod) => encode_pyrecs::(encoder, &records), - Some(Schema::Definition) => encode_pyrecs::(encoder, &records), - Some(Schema::Imbalance) => encode_pyrecs::(encoder, &records), - Some(Schema::Statistics) => encode_pyrecs::(encoder, &records), - Some(Schema::Status) | None => Err(PyValueError::new_err( - "Unsupported schema type for writing DBN files", - )), - Some(Schema::Cbbo) => encode_pyrecs::(encoder, &records), - Some(Schema::Cbbo1S) => encode_pyrecs::(encoder, &records), - Some(Schema::Cbbo1M) => encode_pyrecs::(encoder, &records), - Some(Schema::Tcbbo) => encode_pyrecs::(encoder, &records), - Some(Schema::Bbo1S) => encode_pyrecs::(encoder, &records), - Some(Schema::Bbo1M) => encode_pyrecs::(encoder, &records), - } -} - -fn encode_pyrecs( - mut encoder: DbnEncoder>, - records: &[&PyAny], -) -> PyResult<()> { - encoder - .encode_records( - records - .iter() - .map(|obj| obj.extract()) - .collect::>>()? - .iter() - .as_slice(), - ) - .map_err(to_val_err) -} - /// A Python object that implements the Python file interface. pub struct PyFileLike { inner: PyObject, @@ -147,7 +63,7 @@ impl io::Read for PyFileLike { Python::with_gil(|py| { let bytes: Vec = self .inner - .call_method(py, intern!(py, "read"), (buf.len(),), None) + .call_method_bound(py, intern!(py, "read"), (buf.len(),), None) .map_err(py_to_rs_io_err)? .extract(py)?; buf[..bytes.len()].clone_from_slice(&bytes); @@ -159,10 +75,10 @@ impl io::Read for PyFileLike { impl io::Write for PyFileLike { fn write(&mut self, buf: &[u8]) -> Result { Python::with_gil(|py| { - let bytes = PyBytes::new(py, buf).to_object(py); + let bytes = PyBytes::new_bound(py, buf).to_object(py); let number_bytes_written = self .inner - .call_method(py, intern!(py, "write"), (bytes,), None) + .call_method_bound(py, intern!(py, "write"), (bytes,), None) .map_err(py_to_rs_io_err)?; number_bytes_written.extract(py).map_err(py_to_rs_io_err) @@ -172,7 +88,7 @@ impl io::Write for PyFileLike { fn flush(&mut self) -> Result<(), io::Error> { Python::with_gil(|py| { self.inner - .call_method(py, intern!(py, "flush"), (), None) + .call_method_bound(py, intern!(py, "flush"), (), None) .map_err(py_to_rs_io_err)?; Ok(()) @@ -191,7 +107,7 @@ impl io::Seek for PyFileLike { let new_position = self .inner - .call_method(py, intern!(py, "seek"), (offset, whence), None) + .call_method_bound(py, intern!(py, "seek"), (offset, whence), None) .map_err(py_to_rs_io_err)?; new_position.extract(py).map_err(py_to_rs_io_err) @@ -203,7 +119,7 @@ fn py_to_rs_io_err(e: PyErr) -> io::Error { Python::with_gil(|py| { let e_as_object: PyObject = e.into_py(py); - match e_as_object.call_method(py, intern!(py, "__str__"), (), None) { + match e_as_object.call_method_bound(py, intern!(py, "__str__"), (), None) { Ok(repr) => match repr.extract::(py) { Ok(s) => io::Error::new(io::ErrorKind::Other, s), Err(_e) => io::Error::new(io::ErrorKind::Other, "An unknown error has occurred"), @@ -215,23 +131,15 @@ fn py_to_rs_io_err(e: PyErr) -> io::Error { #[cfg(test)] pub mod tests { - use std::{ io::{Cursor, Seek, Write}, sync::{Arc, Mutex}, }; - use dbn::{ - datasets::GLBX_MDP3, - decode::{dbn::Decoder as DbnDecoder, DbnMetadata, DecodeRecord}, - SType, TbboMsg, - }; - use super::*; - const DBN_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../tests/data"); - #[pyclass] + #[derive(Default)] pub struct MockPyFile { buf: Arc>>>, } @@ -267,103 +175,11 @@ pub mod tests { impl MockPyFile { pub fn new() -> Self { - Self { - buf: Arc::new(Mutex::new(Cursor::new(Vec::new()))), - } + Self::default() } pub fn inner(&self) -> Arc>>> { self.buf.clone() } } - - const DATASET: &str = GLBX_MDP3; - const STYPE: SType = SType::InstrumentId; - - macro_rules! test_writing_dbn_from_python { - ($test_name:ident, $record_type:ident, $schema:expr) => { - #[test] - fn $test_name() { - // Required one-time setup - pyo3::prepare_freethreaded_python(); - - // Read in test data - let decoder = DbnDecoder::from_zstd_file(format!( - "{DBN_PATH}/test_data.{}.dbn.zst", - $schema.as_str() - )) - .unwrap(); - let rs_recs = decoder.decode_records::<$record_type>().unwrap(); - let output_buf = Python::with_gil(|py| -> PyResult<_> { - // Convert JSON objects to Python `dict`s - let recs: Vec<_> = rs_recs - .iter() - .map(|rs_rec| rs_rec.clone().into_py(py)) - .collect(); - let mock_file = MockPyFile::new(); - let output_buf = mock_file.inner(); - let mock_file = Py::new(py, mock_file).unwrap().into_py(py); - let metadata = Metadata::builder() - .dataset(DATASET.to_owned()) - .schema(Some($schema)) - .start(0) - .stype_in(Some(STYPE)) - .stype_out(STYPE) - .build(); - // Call target function - write_dbn_file( - py, - mock_file.extract(py).unwrap(), - Compression::ZStd, - &metadata, - recs.iter().map(|r| r.as_ref(py)).collect(), - ) - .unwrap(); - - Ok(output_buf.clone()) - }) - .unwrap(); - let output_buf = output_buf.lock().unwrap().clone().into_inner(); - - assert!(!output_buf.is_empty()); - - dbg!(&output_buf); - dbg!(output_buf.len()); - // Reread output written with `write_dbn_file` and compare to original - // contents - let py_decoder = DbnDecoder::with_zstd(Cursor::new(&output_buf)).unwrap(); - let metadata = py_decoder.metadata().clone(); - assert_eq!(metadata.schema, Some($schema)); - assert_eq!(metadata.dataset, DATASET); - assert_eq!(metadata.stype_in, Some(STYPE)); - assert_eq!(metadata.stype_out, STYPE); - let decoder = DbnDecoder::from_zstd_file(format!( - "{DBN_PATH}/test_data.{}.dbn.zst", - $schema.as_str() - )) - .unwrap(); - - let py_recs = py_decoder.decode_records::<$record_type>().unwrap(); - let exp_recs = decoder.decode_records::<$record_type>().unwrap(); - assert_eq!(py_recs.len(), exp_recs.len()); - for (py_rec, exp_rec) in py_recs.iter().zip(exp_recs.iter()) { - assert_eq!(py_rec, exp_rec); - } - assert_eq!( - py_recs.len(), - if $schema == Schema::Ohlcv1D { 0 } else { 2 } - ); - } - }; - } - - test_writing_dbn_from_python!(test_writing_mbo_from_python, MboMsg, Schema::Mbo); - test_writing_dbn_from_python!(test_writing_mbp1_from_python, Mbp1Msg, Schema::Mbp1); - test_writing_dbn_from_python!(test_writing_mbp10_from_python, Mbp10Msg, Schema::Mbp10); - test_writing_dbn_from_python!(test_writing_ohlcv1d_from_python, OhlcvMsg, Schema::Ohlcv1D); - test_writing_dbn_from_python!(test_writing_ohlcv1h_from_python, OhlcvMsg, Schema::Ohlcv1H); - test_writing_dbn_from_python!(test_writing_ohlcv1m_from_python, OhlcvMsg, Schema::Ohlcv1M); - test_writing_dbn_from_python!(test_writing_ohlcv1s_from_python, OhlcvMsg, Schema::Ohlcv1S); - test_writing_dbn_from_python!(test_writing_tbbo_from_python, TbboMsg, Schema::Tbbo); - test_writing_dbn_from_python!(test_writing_trades_from_python, TradeMsg, Schema::Trades); } diff --git a/python/src/lib.rs b/python/src/lib.rs index 3e95eaf..6399368 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -21,15 +21,14 @@ mod transcoder; /// A Python module wrapping dbn functions #[pymodule] // The name of the function must match `lib.name` in `Cargo.toml` #[pyo3(name = "_lib")] -fn databento_dbn(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - fn checked_add_class(m: &PyModule) -> PyResult<()> { +fn databento_dbn(_py: Python<'_>, m: &Bound) -> PyResult<()> { + fn checked_add_class(m: &Bound) -> PyResult<()> { // ensure a module was specified, otherwise it defaults to builtins assert_eq!(T::MODULE.unwrap(), "databento_dbn"); m.add_class::() } // all functions exposed to Python need to be added here m.add_wrapped(wrap_pyfunction!(encode::update_encoded_metadata))?; - m.add_wrapped(wrap_pyfunction!(encode::write_dbn_file))?; checked_add_class::(m)?; checked_add_class::(m)?; checked_add_class::(m)?; @@ -153,7 +152,7 @@ assert metadata.ts_out is False"# fn test_dbn_decoder_metadata_error() { setup(); Python::with_gil(|py| { - py.run( + py.run_bound( r#"from _lib import DBNDecoder decoder = DBNDecoder() diff --git a/rust/dbn/src/python/enums.rs b/rust/dbn/src/python/enums.rs index b2e5a92..1eee834 100644 --- a/rust/dbn/src/python/enums.rs +++ b/rust/dbn/src/python/enums.rs @@ -1,6 +1,6 @@ use std::str::FromStr; -use pyo3::{prelude::*, pyclass::CompareOp, type_object::PyTypeInfo, types::PyType}; +use pyo3::{prelude::*, pyclass::CompareOp, type_object::PyTypeInfo, types::PyType, Bound}; use crate::{ enums::{Compression, Encoding, SType, Schema, SecurityUpdateAction, UserDefinedInstrument}, @@ -13,11 +13,11 @@ use super::{to_val_err, EnumIterator, PyFieldDesc}; #[pymethods] impl Side { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { - let t = Self::type_object(py); + let t = Self::type_object_bound(py); let c = value.extract::().map_err(to_val_err)?; - return Self::py_from_str(t, c); + return Self::py_from_str(&t, c); }; Self::try_from(i).map_err(to_val_err) } @@ -34,7 +34,7 @@ impl Side { format!("", self.name(), self.value()) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(py, other)) else { return py.NotImplemented(); }; @@ -56,13 +56,13 @@ impl Side { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: char) -> PyResult { + fn py_from_str(_: &Bound, value: char) -> PyResult { Self::try_from(value as u8).map_err(to_val_err) } } @@ -70,11 +70,11 @@ impl Side { #[pymethods] impl Action { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { - let t = Self::type_object(py); + let t = Self::type_object_bound(py); let c = value.extract::().map_err(to_val_err)?; - return Self::py_from_str(t, c); + return Self::py_from_str(&t, c); }; Self::try_from(i).map_err(to_val_err) } @@ -91,7 +91,7 @@ impl Action { format!("", self.name(), self.value()) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(py, other)) else { return py.NotImplemented(); }; @@ -113,13 +113,13 @@ impl Action { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: char) -> PyResult { + fn py_from_str(_: &Bound, value: char) -> PyResult { Self::try_from(value as u8).map_err(to_val_err) } } @@ -127,11 +127,11 @@ impl Action { #[pymethods] impl InstrumentClass { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { - let t = Self::type_object(py); + let t = Self::type_object_bound(py); let c = value.extract::().map_err(to_val_err)?; - return Self::py_from_str(t, c); + return Self::py_from_str(&t, c); }; Self::try_from(i).map_err(to_val_err) } @@ -144,7 +144,7 @@ impl InstrumentClass { format!("{}", *self as u8 as char) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(py, other)) else { return py.NotImplemented(); }; @@ -161,13 +161,13 @@ impl InstrumentClass { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: char) -> PyResult { + fn py_from_str(_: &Bound, value: char) -> PyResult { Self::try_from(value as u8).map_err(to_val_err) } } @@ -175,11 +175,11 @@ impl InstrumentClass { #[pymethods] impl MatchAlgorithm { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { - let t = Self::type_object(py); + let t = Self::type_object_bound(py); let c = value.extract::().map_err(to_val_err)?; - return Self::py_from_str(t, c); + return Self::py_from_str(&t, c); }; Self::try_from(i).map_err(to_val_err) } @@ -192,7 +192,7 @@ impl MatchAlgorithm { format!("{}", *self as u8 as char) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(py, other)) else { return py.NotImplemented(); }; @@ -209,13 +209,13 @@ impl MatchAlgorithm { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: char) -> PyResult { + fn py_from_str(_: &Bound, value: char) -> PyResult { Self::try_from(value as u8).map_err(to_val_err) } } @@ -223,11 +223,11 @@ impl MatchAlgorithm { #[pymethods] impl UserDefinedInstrument { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { - let t = Self::type_object(py); + let t = Self::type_object_bound(py); let c = value.extract::().map_err(to_val_err)?; - return Self::py_from_str(t, c); + return Self::py_from_str(&t, c); }; Self::try_from(i).map_err(to_val_err) } @@ -248,7 +248,7 @@ impl UserDefinedInstrument { ) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(py, other)) else { return py.NotImplemented(); }; @@ -270,13 +270,13 @@ impl UserDefinedInstrument { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: char) -> PyResult { + fn py_from_str(_: &Bound, value: char) -> PyResult { Self::try_from(value as u8).map_err(to_val_err) } } @@ -284,9 +284,9 @@ impl UserDefinedInstrument { #[pymethods] impl SType { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { - let t = Self::type_object(py); - Self::py_from_str(t, value) + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { + let t = Self::type_object_bound(py); + Self::py_from_str(&t, value) } fn __hash__(&self) -> isize { @@ -301,8 +301,8 @@ impl SType { format!("", self.name(), self.value(),) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { - let Ok(other_enum) = Self::py_from_str(Self::type_object(py), other) else { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { + let Ok(other_enum) = Self::py_from_str(&Self::type_object_bound(py), other) else { return py.NotImplemented(); }; match op { @@ -323,14 +323,14 @@ impl SType { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: &PyAny) -> PyResult { - let value_str: &str = value.str().and_then(|s| s.extract())?; + fn py_from_str(_: &Bound, value: &Bound) -> PyResult { + let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.replace('-', "_").to_lowercase(); Self::from_str(&tokenized).map_err(to_val_err) } @@ -339,9 +339,9 @@ impl SType { #[pymethods] impl RType { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { - let t = Self::type_object(py); - Self::py_from_str(t, value).or_else(|_| Self::py_from_int(t, value)) + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { + let t = Self::type_object_bound(py); + Self::py_from_str(&t, value).or_else(|_| Self::py_from_int(&t, value)) } fn __hash__(&self) -> isize { @@ -356,9 +356,9 @@ impl RType { format!("", self.name(), self.value(),) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { - if let Ok(other_enum) = Self::py_from_str(Self::type_object(py), other) - .or_else(|_| Self::py_from_int(Self::type_object(py), other)) + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { + if let Ok(other_enum) = Self::py_from_str(&Self::type_object_bound(py), other) + .or_else(|_| Self::py_from_int(&Self::type_object_bound(py), other)) { match op { CompareOp::Eq => self.eq(&other_enum).into_py(py), @@ -381,31 +381,31 @@ impl RType { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: &PyAny) -> PyResult { - let value_str: &str = value.str().and_then(|s| s.extract())?; + fn py_from_str(_: &Bound, value: &Bound) -> PyResult { + let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.replace('-', "_").to_lowercase(); Self::from_str(&tokenized).map_err(to_val_err) } #[classmethod] #[pyo3(name = "from_int")] - fn py_from_int(_: &PyType, value: &PyAny) -> PyResult { + fn py_from_int(_: &Bound, value: &Bound) -> PyResult { let value: u8 = value.extract()?; Self::try_from(value).map_err(to_val_err) } #[classmethod] #[pyo3(name = "from_schema")] - fn py_from_schema(pytype: &PyType, value: &PyAny) -> PyResult { + fn py_from_schema(pytype: &Bound, value: &Bound) -> PyResult { let schema: Schema = value .extract() - .or_else(|_| Schema::py_from_str(Schema::type_object(pytype.py()), value)) + .or_else(|_| Schema::py_from_str(&Schema::type_object_bound(pytype.py()), value)) .map_err(to_val_err)?; Ok(Self::from(schema)) } @@ -414,9 +414,9 @@ impl RType { #[pymethods] impl Schema { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { - let t = Self::type_object(py); - Self::py_from_str(t, value) + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { + let t = Self::type_object_bound(py); + Self::py_from_str(&t, value) } fn __hash__(&self) -> isize { @@ -431,8 +431,8 @@ impl Schema { format!("", self.name(), self.value(),) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { - let Ok(other_enum) = Self::py_from_str(Self::type_object(py), other) else { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { + let Ok(other_enum) = Self::py_from_str(&Self::type_object_bound(py), other) else { return py.NotImplemented(); }; match op { @@ -453,14 +453,14 @@ impl Schema { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: &PyAny) -> PyResult { - let value_str: &str = value.str().and_then(|s| s.extract())?; + fn py_from_str(_: &Bound, value: &Bound) -> PyResult { + let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.replace('_', "-").to_lowercase(); Self::from_str(&tokenized).map_err(to_val_err) } @@ -469,9 +469,9 @@ impl Schema { #[pymethods] impl Encoding { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { - let t = Self::type_object(py); - Self::py_from_str(t, value) + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { + let t = Self::type_object_bound(py); + Self::py_from_str(&t, value) } fn __hash__(&self) -> isize { @@ -486,8 +486,8 @@ impl Encoding { format!("", self.name(), self.value(),) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { - let Ok(other_enum) = Self::py_from_str(Self::type_object(py), other) else { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { + let Ok(other_enum) = Self::py_from_str(&Self::type_object_bound(py), other) else { return py.NotImplemented(); }; match op { @@ -508,14 +508,14 @@ impl Encoding { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: &PyAny) -> PyResult { - let value_str: &str = value.str().and_then(|s| s.extract())?; + fn py_from_str(_: &Bound, value: &Bound) -> PyResult { + let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.to_lowercase(); Self::from_str(&tokenized).map_err(to_val_err) } @@ -524,9 +524,9 @@ impl Encoding { #[pymethods] impl Compression { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { - let t = Self::type_object(py); - Self::py_from_str(t, value) + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { + let t = Self::type_object_bound(py); + Self::py_from_str(&t, value) } fn __hash__(&self) -> isize { @@ -541,8 +541,8 @@ impl Compression { format!("", self.name(), self.value(),) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { - let Ok(other_enum) = Self::py_from_str(Self::type_object(py), other) else { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { + let Ok(other_enum) = Self::py_from_str(&Self::type_object_bound(py), other) else { return py.NotImplemented(); }; match op { @@ -564,14 +564,14 @@ impl Compression { // No metaclass support with pyo3, so `for c in Compression: ...` isn't possible #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: &PyAny) -> PyResult { - let value_str: &str = value.str().and_then(|s| s.extract())?; + fn py_from_str(_: &Bound, value: &Bound) -> PyResult { + let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.to_lowercase(); Self::from_str(&tokenized).map_err(to_val_err) } @@ -580,11 +580,11 @@ impl Compression { #[pymethods] impl SecurityUpdateAction { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { - let t = Self::type_object(py); + let t = Self::type_object_bound(py); let c = value.extract::().map_err(to_val_err)?; - return Self::py_from_str(t, c); + return Self::py_from_str(&t, c); }; Self::try_from(i).map_err(to_val_err) } @@ -597,7 +597,7 @@ impl SecurityUpdateAction { format!("", self.name(), self.value()) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(py, other)) else { return py.NotImplemented(); }; @@ -619,13 +619,13 @@ impl SecurityUpdateAction { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: char) -> PyResult { + fn py_from_str(_: &Bound, value: char) -> PyResult { Self::try_from(value as u8).map_err(to_val_err) } } @@ -633,7 +633,7 @@ impl SecurityUpdateAction { #[pymethods] impl StatType { #[new] - fn py_new(value: &PyAny) -> PyResult { + fn py_new(value: &Bound) -> PyResult { let i = value.extract::().map_err(to_val_err)?; Self::try_from(i).map_err(to_val_err) } @@ -642,7 +642,7 @@ impl StatType { *self as isize } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(other)) else { return py.NotImplemented(); }; @@ -659,7 +659,7 @@ impl StatType { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } } @@ -667,7 +667,7 @@ impl StatType { #[pymethods] impl StatusAction { #[new] - fn py_new(value: &PyAny) -> PyResult { + fn py_new(value: &Bound) -> PyResult { let i = value.extract::().map_err(to_val_err)?; Self::try_from(i).map_err(to_val_err) } @@ -676,7 +676,7 @@ impl StatusAction { *self as isize } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(other)) else { return py.NotImplemented(); }; @@ -693,7 +693,7 @@ impl StatusAction { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } } @@ -701,7 +701,7 @@ impl StatusAction { #[pymethods] impl StatusReason { #[new] - fn py_new(value: &PyAny) -> PyResult { + fn py_new(value: &Bound) -> PyResult { let i = value.extract::().map_err(to_val_err)?; Self::try_from(i).map_err(to_val_err) } @@ -710,7 +710,7 @@ impl StatusReason { *self as isize } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(other)) else { return py.NotImplemented(); }; @@ -727,7 +727,7 @@ impl StatusReason { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } } @@ -735,7 +735,7 @@ impl StatusReason { #[pymethods] impl TradingEvent { #[new] - fn py_new(value: &PyAny) -> PyResult { + fn py_new(value: &Bound) -> PyResult { let i = value.extract::().map_err(to_val_err)?; Self::try_from(i).map_err(to_val_err) } @@ -744,7 +744,7 @@ impl TradingEvent { *self as isize } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(other)) else { return py.NotImplemented(); }; @@ -761,7 +761,7 @@ impl TradingEvent { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } } @@ -769,11 +769,11 @@ impl TradingEvent { #[pymethods] impl TriState { #[new] - fn py_new(py: Python<'_>, value: &PyAny) -> PyResult { + fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { - let t = Self::type_object(py); + let t = Self::type_object_bound(py); let c = value.extract::().map_err(to_val_err)?; - return Self::py_from_str(t, c); + return Self::py_from_str(&t, c); }; Self::try_from(i).map_err(to_val_err) } @@ -786,7 +786,7 @@ impl TriState { format!("{}", *self as u8 as char) } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::().or_else(|_| Self::py_new(py, other)) else { return py.NotImplemented(); }; @@ -807,13 +807,13 @@ impl TriState { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } #[classmethod] #[pyo3(name = "from_str")] - fn py_from_str(_: &PyType, value: char) -> PyResult { + fn py_from_str(_: &Bound, value: char) -> PyResult { Self::try_from(value as u8).map_err(to_val_err) } } @@ -824,7 +824,7 @@ impl VersionUpgradePolicy { *self as isize } - fn __richcmp__(&self, other: &PyAny, op: CompareOp, py: Python<'_>) -> Py { + fn __richcmp__(&self, other: &Bound, op: CompareOp, py: Python<'_>) -> Py { let Ok(other_enum) = other.extract::() else { return py.NotImplemented(); }; @@ -836,7 +836,7 @@ impl VersionUpgradePolicy { } #[classmethod] - fn variants(_: &PyType, py: Python<'_>) -> EnumIterator { + fn variants(_: &Bound, py: Python<'_>) -> EnumIterator { EnumIterator::new::(py) } } diff --git a/rust/dbn/src/python/metadata.rs b/rust/dbn/src/python/metadata.rs index ebd0b0d..df3f659 100644 --- a/rust/dbn/src/python/metadata.rs +++ b/rust/dbn/src/python/metadata.rs @@ -5,6 +5,7 @@ use pyo3::{ prelude::*, pyclass::CompareOp, types::{PyBytes, PyDate, PyDict, PyType}, + Bound, }; use crate::{ @@ -80,8 +81,8 @@ impl Metadata { #[pyo3(name = "decode")] #[classmethod] fn py_decode( - _cls: &PyType, - data: &PyBytes, + _cls: &Bound, + data: &Bound, upgrade_policy: Option, ) -> PyResult { let upgrade_policy = upgrade_policy.unwrap_or_default(); @@ -99,7 +100,7 @@ impl Metadata { let mut buffer = Vec::new(); let mut encoder = MetadataEncoder::new(&mut buffer); encoder.encode(self).map_err(to_val_err)?; - Ok(PyBytes::new(py, buffer.as_slice()).into()) + Ok(PyBytes::new_bound(py, buffer.as_slice()).into()) } } @@ -112,7 +113,7 @@ impl IntoPy for SymbolMapping { // `ToPyObject` is about copying and is required for `PyDict::set_item` impl ToPyObject for SymbolMapping { fn to_object(&self, py: Python<'_>) -> PyObject { - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); dict.set_item(intern!(py, "raw_symbol"), &self.raw_symbol) .unwrap(); dict.set_item(intern!(py, "intervals"), &self.intervals) @@ -145,10 +146,10 @@ impl<'source> FromPyObject<'source> for MappingInterval { impl ToPyObject for MappingInterval { fn to_object(&self, py: Python<'_>) -> PyObject { - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); dict.set_item( intern!(py, "start_date"), - PyDate::new( + PyDate::new_bound( py, self.start_date.year(), self.start_date.month() as u8, @@ -159,7 +160,7 @@ impl ToPyObject for MappingInterval { .unwrap(); dict.set_item( intern!(py, "end_date"), - PyDate::new( + PyDate::new_bound( py, self.end_date.year(), self.end_date.month() as u8, diff --git a/rust/dbn/src/python/record.rs b/rust/dbn/src/python/record.rs index 1262b15..43c9b6d 100644 --- a/rust/dbn/src/python/record.rs +++ b/rust/dbn/src/python/record.rs @@ -4,7 +4,7 @@ use pyo3::{ intern, prelude::*, pyclass::CompareOp, - types::{timezone_utc, PyDateTime, PyDict}, + types::{timezone_utc_bound, PyDateTime, PyDict}, }; use crate::{ @@ -3022,8 +3022,8 @@ impl>> IntoPy for WithTsOut { } fn get_utc_nanosecond_timestamp(py: Python<'_>, timestamp: u64) -> PyResult { - if let Ok(pandas) = PyModule::import(py, intern!(py, "pandas")) { - let kwargs = PyDict::new(py); + if let Ok(pandas) = PyModule::import_bound(py, intern!(py, "pandas")) { + let kwargs = PyDict::new_bound(py); if kwargs.set_item(intern!(py, "utc"), true).is_ok() && kwargs .set_item(intern!(py, "errors"), intern!(py, "coerce")) @@ -3033,11 +3033,11 @@ fn get_utc_nanosecond_timestamp(py: Python<'_>, timestamp: u64) -> PyResult Date: Fri, 19 Apr 2024 09:30:11 +1000 Subject: [PATCH 06/23] DOC: Standardize flags field description --- python/python/databento_dbn/_lib.pyi | 6 +++--- rust/dbn/src/record.rs | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/python/python/databento_dbn/_lib.pyi b/python/python/databento_dbn/_lib.pyi index 5f98039..dac2214 100644 --- a/python/python/databento_dbn/_lib.pyi +++ b/python/python/databento_dbn/_lib.pyi @@ -1202,7 +1202,7 @@ class _MBOBase: @property def flags(self) -> int: """ - A combination of packet end with matching engine status. + A bit field indicating event end, message characteristics, and data quality. Returns ------- @@ -1616,7 +1616,7 @@ class _MBPBase: @property def flags(self) -> int: """ - A combination of packet end with matching engine status. + A bit field indicating event end, message characteristics, and data quality. Returns ------- @@ -1782,7 +1782,7 @@ class CbboMsg(Record): @property def flags(self) -> int: """ - A combination of packet end with matching engine status. + A bit field indicating event end, message characteristics, and data quality. Returns ------- diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index e1a24d5..b2ae764 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -91,7 +91,7 @@ pub struct MboMsg { #[dbn(encode_order(5))] #[pyo3(get)] pub size: u32, - /// A combination of packet end with matching engine status. See + /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. #[dbn(fmt_binary)] #[pyo3(get)] @@ -224,7 +224,7 @@ pub struct TradeMsg { /// specified by the original source. #[dbn(c_char, encode_order(3))] pub side: c_char, - /// A combination of packet end with matching engine status. See + /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. #[dbn(fmt_binary)] #[pyo3(get)] @@ -281,7 +281,7 @@ pub struct Mbp1Msg { /// **N**one where no side is specified by the original source. #[dbn(c_char, encode_order(3))] pub side: c_char, - /// A combination of packet end with matching engine status. See + /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. #[dbn(fmt_binary)] #[pyo3(get)] @@ -341,7 +341,7 @@ pub struct Mbp10Msg { /// **N**one where no side is specified by the original source. #[dbn(c_char, encode_order(3))] pub side: c_char, - /// A combination of packet end with matching engine status. See + /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. #[dbn(fmt_binary)] #[pyo3(get)] @@ -401,7 +401,7 @@ pub struct CbboMsg { /// **N**one where no side is specified by the original source. #[dbn(c_char, encode_order(3))] pub side: c_char, - /// A combination of packet end with matching engine status. See + /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. #[dbn(fmt_binary)] #[pyo3(get)] From b2236615c5b808703ccb2bb25220a1a0d5cf0bde Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 24 Apr 2024 09:35:37 -0500 Subject: [PATCH 07/23] MOD: Add methods to `AsyncDynReader` --- CHANGELOG.md | 4 + rust/dbn/src/decode.rs | 128 ++++++++++++++++++++++++++------ rust/dbn/src/decode/dbn/sync.rs | 9 +++ rust/dbn/src/decode/stream.rs | 3 +- 4 files changed, 122 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f6d049..2e779d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ - Added links to example usage in documentation - Added new predicate methods `InstrumentClass::is_option`, `is_future`, and `is_spread` to make it easier to work with multiple instrument class variants +- Implemented `DecodeRecord` for `DbnRecordDecoder` +- Added `new_inferred`, `with_buffer`, `inferred_with_buffer`, `from_file`, `get_mut`, + and `get_ref` methods to `AsyncDynReader` for parity with the sync `DynReader` +- Improved documentation enumerating errors returned by functions ### Breaking changes - Removed `write_dbn_file` function deprecated in version 0.14.0 from Python interface. diff --git a/rust/dbn/src/decode.rs b/rust/dbn/src/decode.rs index 3f931a9..fb67176 100644 --- a/rust/dbn/src/decode.rs +++ b/rust/dbn/src/decode.rs @@ -1,5 +1,5 @@ -//! Decoding DBN and Zstd-compressed DBN files and streams. Decoders implement the -//! [`DecodeDbn`] trait. +//! Decoding DBN and Zstd-compressed DBN files and streams. Sync decoders implement +//the ! [`DecodeDbn`] trait. pub mod dbn; // Having any tests in a deprecated module emits many warnings that can't be silenced, see // https://github.com/rust-lang/rust/issues/47238 @@ -33,7 +33,6 @@ use crate::{ enums::{Compression, VersionUpgradePolicy}, record::HasRType, record_ref::RecordRef, - // record_ref::RecordRef, Metadata, }; @@ -328,6 +327,8 @@ where R: io::Read, { /// Creates a new [`DynReader`] from a reader, with the specified `compression`. + /// If `reader` also implements [`BufRead`](io::BufRead), it's better to use + /// [`with_buffer()`](Self::with_buffer). /// /// # Errors /// This function will return an error if it fails to create the zstd decoder. @@ -407,7 +408,7 @@ impl<'a> DynReader<'a, BufReader> { /// /// # Errors /// This function will return an error if the file doesn't exist, it is unable to - /// determine the encoding of the file or it fails to parse the metadata. + /// determine the encoding of the file, or it fails to create the zstd decoder. pub fn from_file(path: impl AsRef) -> crate::Result { let file = File::open(path.as_ref()).map_err(|e| { crate::Error::io( @@ -447,11 +448,13 @@ where } } -mod private { +#[doc(hidden)] +pub mod private { /// An implementation detail for the interaction between [`StreamingIterator`] and - /// implementors of [`DecodeDbn`]. + /// implementors of [`DecodeRecord`]. #[doc(hidden)] pub trait BufferSlice { + /// Returns an immutable slice of the decoder's buffer. fn buffer_slice(&self) -> &[u8]; } } @@ -461,7 +464,8 @@ pub(crate) trait FromLittleEndianSlice { } impl FromLittleEndianSlice for u64 { - /// NOTE: assumes the length of `slice` is at least 8 bytes + /// # Panics + /// Panics if the length of `slice` is less than 8 bytes. fn from_le_slice(slice: &[u8]) -> Self { let (bytes, _) = slice.split_at(mem::size_of::()); Self::from_le_bytes(bytes.try_into().unwrap()) @@ -469,7 +473,8 @@ impl FromLittleEndianSlice for u64 { } impl FromLittleEndianSlice for i32 { - /// NOTE: assumes the length of `slice` is at least 4 bytes + /// # Panics + /// Panics if the length of `slice` is less than 4 bytes. fn from_le_slice(slice: &[u8]) -> Self { let (bytes, _) = slice.split_at(mem::size_of::()); Self::from_le_bytes(bytes.try_into().unwrap()) @@ -477,7 +482,8 @@ impl FromLittleEndianSlice for i32 { } impl FromLittleEndianSlice for u32 { - /// NOTE: assumes the length of `slice` is at least 4 bytes + /// # Panics + /// Panics if the length of `slice` is less than 4 bytes. fn from_le_slice(slice: &[u8]) -> Self { let (bytes, _) = slice.split_at(mem::size_of::()); Self::from_le_bytes(bytes.try_into().unwrap()) @@ -485,7 +491,8 @@ impl FromLittleEndianSlice for u32 { } impl FromLittleEndianSlice for u16 { - /// NOTE: assumes the length of `slice` is at least 2 bytes + /// # Panics + /// Panics if the length of `slice` is less than 2 bytes. fn from_le_slice(slice: &[u8]) -> Self { let (bytes, _) = slice.split_at(mem::size_of::()); Self::from_le_bytes(bytes.try_into().unwrap()) @@ -551,42 +558,121 @@ pub use self::{ #[cfg(feature = "async")] mod r#async { - use std::pin::Pin; + use std::{path::Path, pin::Pin}; use async_compression::tokio::bufread::ZstdDecoder; - use tokio::io::{self, BufReader}; + use tokio::{ + fs::File, + io::{self, BufReader}, + }; use crate::enums::Compression; /// A type for runtime polymorphism on compressed and uncompressed input. + /// The async version of [`DynReader`](super::DynReader). pub struct DynReader(DynReaderImpl) where - R: io::AsyncReadExt + Unpin; + R: io::AsyncBufReadExt + Unpin; enum DynReaderImpl where - R: io::AsyncReadExt + Unpin, + R: io::AsyncBufReadExt + Unpin, { Uncompressed(R), - ZStd(ZstdDecoder>), + ZStd(ZstdDecoder), } - impl DynReader + impl DynReader> where R: io::AsyncReadExt + Unpin, { - /// Creates a new instance of [`DynReader`] with the specified `compression`. + /// Creates a new instance of [`DynReader`] with the specified `compression`. If + /// `reader` also implements [`AsyncBufRead`](tokio::io::AsyncBufRead), it's + /// better to use [`with_buffer()`](Self::with_buffer). pub fn new(reader: R, compression: Compression) -> Self { - Self(match compression { - Compression::None => DynReaderImpl::Uncompressed(reader), - Compression::ZStd => DynReaderImpl::ZStd(ZstdDecoder::new(BufReader::new(reader))), + Self::with_buffer(BufReader::new(reader), compression) + } + + /// Creates a new [`DynReader`] from a reader, inferring the compression. + /// If `reader` also implements [`AsyncBufRead`](tokio::io::AsyncBufRead), it is + /// better to use [`inferred_with_buffer()`](Self::inferred_with_buffer). + /// + /// # Errors + /// This function will return an error if it is unable to read from `reader`. + pub async fn new_inferred(reader: R) -> crate::Result { + Self::inferred_with_buffer(BufReader::new(reader)).await + } + } + + impl DynReader + where + R: io::AsyncBufReadExt + Unpin, + { + /// Creates a new [`DynReader`] from a buffered reader with the specified + /// `compression`. + pub fn with_buffer(reader: R, compression: Compression) -> Self { + match compression { + Compression::None => Self(DynReaderImpl::Uncompressed(reader)), + Compression::ZStd => Self(DynReaderImpl::ZStd(ZstdDecoder::new(reader))), + } + } + + /// Creates a new [`DynReader`] from a buffered reader, inferring the compression. + /// + /// # Errors + /// This function will return an error if it fails to read from `reader`. + pub async fn inferred_with_buffer(mut reader: R) -> crate::Result { + let first_bytes = reader + .fill_buf() + .await + .map_err(|e| crate::Error::io(e, "creating buffer to infer encoding"))?; + Ok(if super::zstd::starts_with_prefix(first_bytes) { + Self(DynReaderImpl::ZStd(ZstdDecoder::new(reader))) + } else { + Self(DynReaderImpl::Uncompressed(reader)) }) } + + /// Returns a mutable reference to the inner reader. + pub fn get_mut(&mut self) -> &mut R { + match &mut self.0 { + DynReaderImpl::Uncompressed(reader) => reader, + DynReaderImpl::ZStd(reader) => reader.get_mut(), + } + } + + /// Returns a reference to the inner reader. + pub fn get_ref(&self) -> &R { + match &self.0 { + DynReaderImpl::Uncompressed(reader) => reader, + DynReaderImpl::ZStd(reader) => reader.get_ref(), + } + } + } + + impl DynReader> { + /// Creates a new [`DynReader`] from the file at `path`. + /// + /// # Errors + /// This function will return an error if the file doesn't exist, it is unable + /// to read from it. + pub async fn from_file(path: impl AsRef) -> crate::Result { + let file = File::open(path.as_ref()).await.map_err(|e| { + crate::Error::io( + e, + format!( + "opening file to decode at path '{}'", + path.as_ref().display() + ), + ) + })?; + DynReader::new_inferred(file).await + } } impl io::AsyncRead for DynReader where - R: io::AsyncRead + io::AsyncReadExt + Unpin, + R: io::AsyncRead + io::AsyncReadExt + io::AsyncBufReadExt + Unpin, { fn poll_read( mut self: std::pin::Pin<&mut Self>, diff --git a/rust/dbn/src/decode/dbn/sync.rs b/rust/dbn/src/decode/dbn/sync.rs index ada4b7e..5e12cfd 100644 --- a/rust/dbn/src/decode/dbn/sync.rs +++ b/rust/dbn/src/decode/dbn/sync.rs @@ -357,6 +357,15 @@ where } } +impl DecodeRecord for RecordDecoder +where + R: io::Read, +{ + fn decode_record(&mut self) -> crate::Result> { + self.decode() + } +} + impl DecodeRecordRef for RecordDecoder where R: io::Read, diff --git a/rust/dbn/src/decode/stream.rs b/rust/dbn/src/decode/stream.rs index 991c566..ac46944 100644 --- a/rust/dbn/src/decode/stream.rs +++ b/rust/dbn/src/decode/stream.rs @@ -30,7 +30,8 @@ where D: DecodeRecord, T: HasRType, { - pub(crate) fn new(decoder: D) -> Self { + /// Creates a new streaming decoder using the given `decoder`. + pub fn new(decoder: D) -> Self { Self { decoder, i: Some(0), From ec5dafd70d05842207d04b811bf60f1436446469 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Mon, 22 Apr 2024 17:32:00 -0500 Subject: [PATCH 08/23] MOD: Improve flags in DBN --- CHANGELOG.md | 4 + rust/dbn/src/encode/csv/serialize.rs | 12 +- rust/dbn/src/encode/csv/sync.rs | 10 +- rust/dbn/src/encode/json/async.rs | 2 +- rust/dbn/src/encode/json/serialize.rs | 18 ++- rust/dbn/src/encode/json/sync.rs | 8 +- rust/dbn/src/flags.rs | 202 ++++++++++++++++++++++++++ rust/dbn/src/lib.rs | 4 +- rust/dbn/src/python.rs | 13 ++ rust/dbn/src/python/record.rs | 28 ++-- rust/dbn/src/record.rs | 26 ++-- rust/dbn/src/record/impl_default.rs | 10 +- rust/dbn/src/record/methods.rs | 6 +- rust/dbn/src/record_ref.rs | 6 +- 14 files changed, 292 insertions(+), 57 deletions(-) create mode 100644 rust/dbn/src/flags.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e779d6..e318101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ - Improved documentation enumerating errors returned by functions ### Breaking changes +- Changed type of `flags` in `MboMsg`, `TradeMsg`, `Mbp1Msg`, `Mbp10Msg`, and `CbboMsg` + from `u8` to a new `FlagSet` type with predicate methods for the various bit flags + as well as setters. The `u8` value can still be obtained by calling the `raw()` method. + Python and encodings are unaffected. - Removed `write_dbn_file` function deprecated in version 0.14.0 from Python interface. Please use `Transcoder` instead diff --git a/rust/dbn/src/encode/csv/serialize.rs b/rust/dbn/src/encode/csv/serialize.rs index 3a38298..43e6f91 100644 --- a/rust/dbn/src/encode/csv/serialize.rs +++ b/rust/dbn/src/encode/csv/serialize.rs @@ -8,7 +8,7 @@ use crate::{ record::{ c_chars_to_str, BidAskPair, ConsolidatedBidAskPair, HasRType, RecordHeader, WithTsOut, }, - UNDEF_PRICE, UNDEF_TIMESTAMP, + FlagSet, UNDEF_PRICE, UNDEF_TIMESTAMP, }; /// Because of the flat nature of CSVs, there are several limitations in the @@ -116,6 +116,16 @@ impl WriteField for [ConsolidatedBidAskPair; N] { Ok(()) } } + +impl WriteField for FlagSet { + fn write_field( + &self, + writer: &mut Writer, + ) -> csv::Result<()> { + self.raw().write_field::(writer) + } +} + macro_rules! impl_write_field_for { ($($ty:ident),+) => { $( diff --git a/rust/dbn/src/encode/csv/sync.rs b/rust/dbn/src/encode/csv/sync.rs index 7f04b45..bc8b3bc 100644 --- a/rust/dbn/src/encode/csv/sync.rs +++ b/rust/dbn/src/encode/csv/sync.rs @@ -431,7 +431,7 @@ mod tests { order_id: 16, price: 5500, size: 3, - flags: 128, + flags: 128.into(), channel_id: 14, action: 'B' as c_char, side: 'B' as c_char, @@ -469,7 +469,7 @@ mod tests { size: 3, action: 'M' as c_char, side: 'A' as c_char, - flags: 128, + flags: 128.into(), depth: 9, ts_recv: 1658441891000000000, ts_in_delta: 22_000, @@ -507,7 +507,7 @@ mod tests { size: 3, action: 'B' as c_char, side: 'A' as c_char, - flags: 128, + flags: 128.into(), depth: 9, ts_recv: 1658441891000000000, ts_in_delta: 22_000, @@ -543,7 +543,7 @@ mod tests { size: 3, action: 'B' as c_char, side: 'B' as c_char, - flags: 128, + flags: 128.into(), depth: 9, ts_recv: 1658441891000000000, ts_in_delta: 22_000, @@ -740,7 +740,7 @@ mod tests { size: 3, action: 'T' as c_char, side: 'A' as c_char, - flags: 128, + flags: 128.into(), depth: 9, ts_recv: 1658441891000000000, ts_in_delta: 22_000, diff --git a/rust/dbn/src/encode/json/async.rs b/rust/dbn/src/encode/json/async.rs index f7ef3ef..c250af4 100644 --- a/rust/dbn/src/encode/json/async.rs +++ b/rust/dbn/src/encode/json/async.rs @@ -195,7 +195,7 @@ mod tests { order_id: 16, price: 5500, size: 3, - flags: 128, + flags: 128.into(), channel_id: 14, action: 'R' as c_char, side: 'N' as c_char, diff --git a/rust/dbn/src/encode/json/serialize.rs b/rust/dbn/src/encode/json/serialize.rs index e875377..e410260 100644 --- a/rust/dbn/src/encode/json/serialize.rs +++ b/rust/dbn/src/encode/json/serialize.rs @@ -4,8 +4,8 @@ use crate::{ json_writer::{JsonObjectWriter, NULL}, pretty::{fmt_px, fmt_ts}, record::{c_chars_to_str, ConsolidatedBidAskPair}, - BidAskPair, HasRType, Metadata, RecordHeader, SecurityUpdateAction, UserDefinedInstrument, - WithTsOut, UNDEF_PRICE, UNDEF_TIMESTAMP, + BidAskPair, FlagSet, HasRType, Metadata, RecordHeader, SecurityUpdateAction, + UserDefinedInstrument, WithTsOut, UNDEF_PRICE, UNDEF_TIMESTAMP, }; /// Serializes `obj` to a JSON string. @@ -216,6 +216,20 @@ impl WriteField for [ConsolidatedBidAskPair; N] { } } +impl WriteField for FlagSet { + fn write_field< + J: crate::json_writer::JsonWriter, + const PRETTY_PX: bool, + const PRETTY_TS: bool, + >( + &self, + writer: &mut JsonObjectWriter, + name: &str, + ) { + writer.value(name, self.raw()) + } +} + impl WriteField for i64 { fn write_field< J: crate::json_writer::JsonWriter, diff --git a/rust/dbn/src/encode/json/sync.rs b/rust/dbn/src/encode/json/sync.rs index 9ab09d0..1b970f9 100644 --- a/rust/dbn/src/encode/json/sync.rs +++ b/rust/dbn/src/encode/json/sync.rs @@ -279,7 +279,7 @@ mod tests { order_id: 16, price: 5500, size: 3, - flags: 128, + flags: 128.into(), channel_id: 14, action: 'R' as c_char, side: 'N' as c_char, @@ -309,7 +309,7 @@ mod tests { size: 3, action: 'B' as c_char, side: 'B' as c_char, - flags: 128, + flags: 128.into(), depth: 9, ts_recv: 1658441891000000000, ts_in_delta: 22_000, @@ -340,7 +340,7 @@ mod tests { size: 3, action: 'T' as c_char, side: 'N' as c_char, - flags: 128, + flags: 128.into(), depth: 9, ts_recv: 1658441891000000000, ts_in_delta: 22_000, @@ -371,7 +371,7 @@ mod tests { size: 3, action: 'C' as c_char, side: 'B' as c_char, - flags: 128, + flags: 128.into(), depth: 9, ts_recv: 1658441891000000000, ts_in_delta: 22_000, diff --git a/rust/dbn/src/flags.rs b/rust/dbn/src/flags.rs new file mode 100644 index 0000000..a769155 --- /dev/null +++ b/rust/dbn/src/flags.rs @@ -0,0 +1,202 @@ +//! Bit set flags used in Databento market data. + +use std::fmt; + +#[cfg(feature = "python")] +use pyo3::prelude::*; + +/// Indicates it's the last record in the event from the venue for a given +/// `instrument_id`. +pub const LAST: u8 = 1 << 7; +/// Indicates a top-of-book record, not an individual order. +pub const TOB: u8 = 1 << 6; +/// Indicates the record was sourced from a replay, such as a snapshot server. +pub const SNAPSHOT: u8 = 1 << 5; +/// Indicates an aggregated price level record, not an individual order. +pub const MBP: u8 = 1 << 4; +/// Indicates the `ts_recv` value is inaccurate due to clock issues or packet +/// reordering. +pub const BAD_TS_RECV: u8 = 1 << 3; +/// Indicates an unrecoverable gap was detected in the channel. +pub const MAYBE_BAD_BOOK: u8 = 1 << 2; + +/// A transparent wrapper around the bit field used in several DBN record types, +/// namely [`MboMsg`](crate::MboMsg) and record types derived from it. +#[repr(transparent)] +#[derive(Copy, Clone, PartialEq, Eq, Default, Hash)] +#[cfg_attr(feature = "python", derive(FromPyObject), pyo3(transparent))] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(transparent) +)] +pub struct FlagSet { + raw: u8, +} + +impl fmt::Debug for FlagSet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut has_written_flag = false; + for (flag, name) in [ + (LAST, stringify!(LAST)), + (TOB, stringify!(TOB)), + (SNAPSHOT, stringify!(SNAPSHOT)), + (MBP, stringify!(MBP)), + (BAD_TS_RECV, stringify!(BAD_TS_RECV)), + (MAYBE_BAD_BOOK, stringify!(MAYBE_BAD_BOOK)), + ] { + if (self.raw() & flag) > 0 { + if has_written_flag { + write!(f, " | {name}")?; + } else { + write!(f, "{name}")?; + has_written_flag = true; + } + } + } + if has_written_flag { + write!(f, " ({})", self.raw()) + } else { + write!(f, "{}", self.raw()) + } + } +} + +impl From for FlagSet { + fn from(raw: u8) -> Self { + Self { raw } + } +} + +impl FlagSet { + /// Returns an empty [`FlagSet`]: one with no flags set. + pub const fn empty() -> Self { + Self { raw: 0 } + } + + /// Creates a new flag set from `raw`. + pub const fn new(raw: u8) -> Self { + Self { raw } + } + + /// Turns all flags off, i.e. to `false`. + pub fn clear(&mut self) -> &mut Self { + self.raw = 0; + self + } + + /// Returns the raw value. + pub const fn raw(&self) -> u8 { + self.raw + } + + /// Sets the flags directly with a raw `u8`. + pub fn set_raw(&mut self, raw: u8) { + self.raw = raw; + } + + /// Returns `true` if any of the flags are on or set to true. + pub const fn any(&self) -> bool { + self.raw > 0 + } + + /// Returns `true` if all flags are unset/false. + pub fn is_empty(&self) -> bool { + self.raw == 0 + } + + /// Returns `true` if it's the last record in the event from the venue for a given + /// `instrument_id`. + pub const fn is_last(&self) -> bool { + (self.raw & LAST) > 0 + } + + /// Sets the `LAST` bit flag to `true` to indicate this is the last record in the + /// event for a given instrument. + pub fn set_last(&mut self) -> Self { + self.raw |= LAST; + *self + } + + /// Returns `true` if it's a top-of-book record, not an individual order. + pub const fn is_tob(&self) -> bool { + (self.raw & TOB) > 0 + } + + /// Sets the `TOB` bit flag to `true` to indicate this is a top-of-book record. + pub fn set_tob(&mut self) -> Self { + self.raw |= TOB; + *self + } + + /// Returns `true` if this record was sourced from a replay, such as a snapshot + /// server. + pub const fn is_snapshot(&self) -> bool { + (self.raw & SNAPSHOT) > 0 + } + + /// Sets the `SNAPSHOT` bit flag to `true` to indicate this record was sourced from + /// a replay. + pub fn set_snapshot(&mut self) -> Self { + self.raw |= SNAPSHOT; + *self + } + + /// Returns `true` if this record is an aggregated price level record, not an + /// individual order. + pub const fn is_mbp(&self) -> bool { + (self.raw & MBP) > 0 + } + + /// Sets the `MBP` bit flag to `true` to indicate this record is an aggregated price + /// level record. + pub fn set_mbp(&mut self) -> Self { + self.raw |= MBP; + *self + } + + /// Returns `true` if this record has an inaccurate `ts_recv` value due to clock + /// issues or packet reordering. + pub const fn is_bad_ts_recv(&self) -> bool { + (self.raw & BAD_TS_RECV) > 0 + } + + /// Sets the `BAD_TS_RECV` bit flag to `true` to indicate this record has an + /// inaccurate `ts_recv` value. + pub fn set_bad_ts_recv(&mut self) -> Self { + self.raw |= BAD_TS_RECV; + *self + } + + /// Returns `true` if this record is from a channel where an unrecoverable gap was + /// detected. + pub const fn is_maybe_bad_book(&self) -> bool { + (self.raw & MAYBE_BAD_BOOK) > 0 + } + + /// Sets the `MAYBE_BAD_BOOK` bit flag to `true` to indicate this record is from a + /// channel where an unrecoverable gap was detected. + pub fn set_maybe_bad_book(&mut self) -> Self { + self.raw |= MAYBE_BAD_BOOK; + *self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use rstest::*; + + #[rstest] + #[case::empty(FlagSet::empty(), "0")] + #[case::one_set(FlagSet::empty().set_mbp(), "MBP (16)")] + #[case::three_set(FlagSet::empty().set_tob().set_snapshot().set_maybe_bad_book(), "TOB | SNAPSHOT | MAYBE_BAD_BOOK (100)")] + #[case::reserved_set( + FlagSet::new(255), + "LAST | TOB | SNAPSHOT | MBP | BAD_TS_RECV | MAYBE_BAD_BOOK (255)" + )] + fn dbg(#[case] target: FlagSet, #[case] exp: &str) { + assert_eq!(format!("{target:?}"), exp); + } +} diff --git a/rust/dbn/src/lib.rs b/rust/dbn/src/lib.rs index 566202e..c9c0780 100644 --- a/rust/dbn/src/lib.rs +++ b/rust/dbn/src/lib.rs @@ -42,6 +42,7 @@ pub mod decode; pub mod encode; pub mod enums; pub mod error; +pub mod flags; mod json_writer; pub mod macros; pub mod metadata; @@ -56,11 +57,12 @@ pub mod symbol_map; pub use crate::{ enums::{ - flags, rtype, Action, Compression, Encoding, InstrumentClass, MatchAlgorithm, RType, SType, + rtype, Action, Compression, Encoding, InstrumentClass, MatchAlgorithm, RType, SType, Schema, SecurityUpdateAction, Side, StatType, StatUpdateAction, StatusAction, StatusReason, TradingEvent, TriState, UserDefinedInstrument, VersionUpgradePolicy, }, error::{Error, Result}, + flags::FlagSet, metadata::{MappingInterval, Metadata, MetadataBuilder, SymbolMapping}, publishers::{Dataset, Publisher, Venue}, record::{ diff --git a/rust/dbn/src/python.rs b/rust/dbn/src/python.rs index 0d4e826..3bfd321 100644 --- a/rust/dbn/src/python.rs +++ b/rust/dbn/src/python.rs @@ -11,6 +11,8 @@ use pyo3::{ }; use strum::IntoEnumIterator; +use crate::FlagSet; + mod enums; mod metadata; mod record; @@ -57,6 +59,12 @@ impl EnumIterator { } } +impl IntoPy for FlagSet { + fn into_py(self, py: Python<'_>) -> PyObject { + self.raw().into_py(py) + } +} + /// Tries to convert `py_date` to a [`time::Date`]. /// /// # Errors @@ -140,6 +148,11 @@ impl PyFieldDesc for [u8; N] { vec![(field_name.to_owned(), format!("S{N}"))] } } +impl PyFieldDesc for FlagSet { + fn field_dtypes(field_name: &str) -> Vec<(String, String)> { + u8::field_dtypes(field_name) + } +} #[cfg(test)] mod tests { diff --git a/rust/dbn/src/python/record.rs b/rust/dbn/src/python/record.rs index 43c9b6d..44cc7de 100644 --- a/rust/dbn/src/python/record.rs +++ b/rust/dbn/src/python/record.rs @@ -10,11 +10,11 @@ use pyo3::{ use crate::{ compat::{ErrorMsgV1, InstrumentDefMsgV1, SymbolMappingMsgV1, SystemMsgV1}, record::{str_to_c_chars, CbboMsg, ConsolidatedBidAskPair}, - rtype, BidAskPair, ErrorMsg, HasRType, ImbalanceMsg, InstrumentDefMsg, MboMsg, Mbp10Msg, - Mbp1Msg, OhlcvMsg, Publisher, Record, RecordHeader, SType, SecurityUpdateAction, StatMsg, - StatUpdateAction, StatusAction, StatusMsg, StatusReason, SymbolMappingMsg, SystemMsg, TradeMsg, - TradingEvent, TriState, UserDefinedInstrument, WithTsOut, FIXED_PRICE_SCALE, UNDEF_ORDER_SIZE, - UNDEF_PRICE, UNDEF_TIMESTAMP, + rtype, BidAskPair, ErrorMsg, FlagSet, HasRType, ImbalanceMsg, InstrumentDefMsg, MboMsg, + Mbp10Msg, Mbp1Msg, OhlcvMsg, Publisher, Record, RecordHeader, SType, SecurityUpdateAction, + StatMsg, StatUpdateAction, StatusAction, StatusMsg, StatusReason, SymbolMappingMsg, SystemMsg, + TradeMsg, TradingEvent, TriState, UserDefinedInstrument, WithTsOut, FIXED_PRICE_SCALE, + UNDEF_ORDER_SIZE, UNDEF_PRICE, UNDEF_TIMESTAMP, }; use super::{to_val_err, PyFieldDesc}; @@ -35,7 +35,7 @@ impl MboMsg { ts_recv: u64, ts_in_delta: i32, sequence: u32, - flags: Option, + flags: Option, ) -> Self { Self { hd: RecordHeader::new::(rtype::MBO, publisher_id, instrument_id, ts_event), @@ -223,10 +223,10 @@ impl CbboMsg { size: u32, action: c_char, side: c_char, - flags: u8, ts_recv: u64, ts_in_delta: i32, sequence: u32, + flags: Option, levels: Option, ) -> Self { Self { @@ -235,12 +235,12 @@ impl CbboMsg { size, action, side, - flags, - _reserved: [0; 1], + flags: flags.unwrap_or_default(), ts_recv, ts_in_delta, sequence, levels: [levels.unwrap_or_default()], + _reserved: Default::default(), } } @@ -436,7 +436,7 @@ impl TradeMsg { ts_recv: u64, ts_in_delta: i32, sequence: u32, - flags: Option, + flags: Option, ) -> Self { Self { hd: RecordHeader::new::(rtype::MBP_0, publisher_id, instrument_id, ts_event), @@ -570,11 +570,11 @@ impl Mbp1Msg { size: u32, action: c_char, side: c_char, - flags: u8, depth: u8, ts_recv: u64, ts_in_delta: i32, sequence: u32, + flags: Option, levels: Option, ) -> Self { Self { @@ -583,7 +583,7 @@ impl Mbp1Msg { size, action, side, - flags, + flags: flags.unwrap_or_default(), depth, ts_recv, ts_in_delta, @@ -710,11 +710,11 @@ impl Mbp10Msg { size: u32, action: c_char, side: c_char, - flags: u8, depth: u8, ts_recv: u64, ts_in_delta: i32, sequence: u32, + flags: Option, levels: Option>, ) -> PyResult { let levels = if let Some(level) = levels { @@ -735,7 +735,7 @@ impl Mbp10Msg { size, action, side, - flags, + flags: flags.unwrap_or_default(), depth, ts_recv, ts_in_delta, diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index b2ae764..4f8d4ed 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -13,14 +13,10 @@ use std::{ffi::CStr, mem, os::raw::c_char, ptr::NonNull, slice}; use dbn_macros::MockPyo3; use crate::{ - enums::{ - rtype::{self, RType}, - Action, InstrumentClass, MatchAlgorithm, SecurityUpdateAction, Side, StatType, - StatUpdateAction, UserDefinedInstrument, - }, + enums::rtype, macros::{dbn_record, CsvSerialize, JsonSerialize, RecordDebug}, - publishers::Publisher, - Error, Result, SYMBOL_CSTR_LEN, + Action, Error, FlagSet, InstrumentClass, MatchAlgorithm, Publisher, RType, Result, + SecurityUpdateAction, Side, StatType, StatUpdateAction, UserDefinedInstrument, SYMBOL_CSTR_LEN, }; pub(crate) use conv::as_u8_slice; #[cfg(feature = "serde")] @@ -93,9 +89,8 @@ pub struct MboMsg { pub size: u32, /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. - #[dbn(fmt_binary)] #[pyo3(get)] - pub flags: u8, + pub flags: FlagSet, /// A channel ID within the venue. #[dbn(encode_order(6))] #[pyo3(get)] @@ -226,9 +221,8 @@ pub struct TradeMsg { pub side: c_char, /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. - #[dbn(fmt_binary)] #[pyo3(get)] - pub flags: u8, + pub flags: FlagSet, /// The depth of actual book change. #[dbn(encode_order(4))] #[pyo3(get)] @@ -283,9 +277,8 @@ pub struct Mbp1Msg { pub side: c_char, /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. - #[dbn(fmt_binary)] #[pyo3(get)] - pub flags: u8, + pub flags: FlagSet, /// The depth of actual book change. #[dbn(encode_order(4))] #[pyo3(get)] @@ -343,9 +336,8 @@ pub struct Mbp10Msg { pub side: c_char, /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. - #[dbn(fmt_binary)] #[pyo3(get)] - pub flags: u8, + pub flags: FlagSet, /// The depth of actual book change. #[dbn(encode_order(4))] #[pyo3(get)] @@ -403,9 +395,8 @@ pub struct CbboMsg { pub side: c_char, /// A bit field indicating event end, message characteristics, and data quality. See /// [`enums::flags`](crate::enums::flags) for possible values. - #[dbn(fmt_binary)] #[pyo3(get)] - pub flags: u8, + pub flags: FlagSet, // Reserved for future usage. #[doc(hidden)] #[cfg_attr(feature = "serde", serde(skip))] @@ -616,6 +607,7 @@ pub struct InstrumentDefMsg { #[pyo3(get, set)] pub strike_price: i64, /// A bitmap of instrument eligibility attributes. + #[dbn(fmt_binary)] #[pyo3(get, set)] pub inst_attrib_value: i32, /// The `instrument_id` of the first underlying instrument. diff --git a/rust/dbn/src/record/impl_default.rs b/rust/dbn/src/record/impl_default.rs index e4291d5..798616b 100644 --- a/rust/dbn/src/record/impl_default.rs +++ b/rust/dbn/src/record/impl_default.rs @@ -23,7 +23,7 @@ impl Default for MboMsg { order_id: 0, price: UNDEF_PRICE, size: UNDEF_ORDER_SIZE, - flags: 0, + flags: FlagSet::default(), channel_id: 0, action: 0, side: 0, @@ -70,7 +70,7 @@ impl Default for TradeMsg { size: UNDEF_ORDER_SIZE, action: 0, side: 0, - flags: 0, + flags: FlagSet::default(), depth: 0, ts_recv: UNDEF_TIMESTAMP, ts_in_delta: 0, @@ -88,7 +88,7 @@ impl Mbp1Msg { size: UNDEF_ORDER_SIZE, action: 0, side: 0, - flags: 0, + flags: FlagSet::default(), depth: 0, ts_recv: UNDEF_TIMESTAMP, ts_in_delta: 0, @@ -107,7 +107,7 @@ impl CbboMsg { size: UNDEF_ORDER_SIZE, action: 0, side: 0, - flags: 0, + flags: FlagSet::default(), _reserved: [0; 1], ts_recv: UNDEF_TIMESTAMP, ts_in_delta: 0, @@ -125,7 +125,7 @@ impl Default for Mbp10Msg { size: UNDEF_ORDER_SIZE, action: 0, side: 0, - flags: 0, + flags: FlagSet::default(), depth: 0, ts_recv: UNDEF_TIMESTAMP, ts_in_delta: 0, diff --git a/rust/dbn/src/record/methods.rs b/rust/dbn/src/record/methods.rs index 990a83c..ffcd09d 100644 --- a/rust/dbn/src/record/methods.rs +++ b/rust/dbn/src/record/methods.rs @@ -979,8 +979,6 @@ impl WithTsOut { #[cfg(test)] mod tests { - use crate::flags; - use super::*; #[test] @@ -1001,7 +999,7 @@ mod tests { 678, 1704468548242628731, ), - flags: flags::LAST | flags::BAD_TS_RECV, + flags: FlagSet::empty().set_last().set_bad_ts_recv(), price: 4_500_500_000_000, side: b'B' as c_char, action: b'A' as c_char, @@ -1011,7 +1009,7 @@ mod tests { format!("{rec:?}"), "MboMsg { hd: RecordHeader { length: 14, rtype: Mbo, publisher_id: OpraPillarXcbo, \ instrument_id: 678, ts_event: 1704468548242628731 }, order_id: 0, \ - price: 4500.500000000, size: 4294967295, flags: 0b10001000, channel_id: 0, \ + price: 4500.500000000, size: 4294967295, flags: LAST | BAD_TS_RECV (136), channel_id: 0, \ action: 'A', side: 'B', ts_recv: 18446744073709551615, ts_in_delta: 0, sequence: 0 }" ); } diff --git a/rust/dbn/src/record_ref.rs b/rust/dbn/src/record_ref.rs index 7632d1e..9d2a147 100644 --- a/rust/dbn/src/record_ref.rs +++ b/rust/dbn/src/record_ref.rs @@ -207,8 +207,8 @@ mod tests { use std::ffi::c_char; use crate::{ - enums::rtype, - record::{ErrorMsg, InstrumentDefMsg, MboMsg, Mbp10Msg, Mbp1Msg, OhlcvMsg, TradeMsg}, + enums::rtype, ErrorMsg, FlagSet, InstrumentDefMsg, MboMsg, Mbp10Msg, Mbp1Msg, OhlcvMsg, + TradeMsg, }; use super::*; @@ -218,7 +218,7 @@ mod tests { order_id: 17, price: 0, size: 32, - flags: 0, + flags: FlagSet::empty(), channel_id: 1, action: 'A' as c_char, side: 'B' as c_char, From 847932e68c0d028b3f72c401ee32e642db48494b Mon Sep 17 00:00:00 2001 From: Carter Green Date: Thu, 25 Apr 2024 15:46:58 -0500 Subject: [PATCH 09/23] MOD: Improve FlagSet in C++ --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e318101..3e1481e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ - Changed type of `flags` in `MboMsg`, `TradeMsg`, `Mbp1Msg`, `Mbp10Msg`, and `CbboMsg` from `u8` to a new `FlagSet` type with predicate methods for the various bit flags as well as setters. The `u8` value can still be obtained by calling the `raw()` method. - Python and encodings are unaffected. + - Improved `Debug` formatting + - Python and encodings are unaffected - Removed `write_dbn_file` function deprecated in version 0.14.0 from Python interface. Please use `Transcoder` instead From 382b9650a253baaca01f6bd336426635936bb0da Mon Sep 17 00:00:00 2001 From: Carter Green Date: Mon, 29 Apr 2024 11:29:24 -0500 Subject: [PATCH 10/23] MOD: Upgrade pyo3 version --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 865352b..51eacf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -797,9 +797,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a8b1990bd018761768d5e608a13df8bd1ac5f678456e0f301bb93e5f3ea16b" +checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" dependencies = [ "cfg-if", "indoc", @@ -815,9 +815,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650dca34d463b6cdbdb02b1d71bfd6eb6b6816afc708faebb3bac1380ff4aef7" +checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" dependencies = [ "once_cell", "target-lexicon", @@ -825,9 +825,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a7da8fc04a8a2084909b59f29e1b8474decac98b951d77b80b26dc45f046ad" +checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" dependencies = [ "libc", "pyo3-build-config", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8a199fce11ebb28e3569387228836ea98110e43a804a530a9fd83ade36d513" +checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -847,9 +847,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fbbfd7eb553d10036513cb122b888dcd362a945a00b06c165f2ab480d4cc3b" +checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 7f94674..16bd3f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ license = "Apache-2.0" [workspace.dependencies] anyhow = "1.0.80" -pyo3 = "0.21" -pyo3-build-config = "0.21" +pyo3 = "0.21.2" +pyo3-build-config = "0.21.2" rstest = "0.18.2" serde = { version = "1.0", features = ["derive"] } time = ">=0.3.35" From 39e734b2fb340f167d8ec4f519b00f06ed023fe7 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Thu, 2 May 2024 10:58:24 -0500 Subject: [PATCH 11/23] MOD: Switch to FallibleStreamingIterator --- CHANGELOG.md | 4 +++ Cargo.lock | 14 ++++---- rust/dbn/Cargo.toml | 2 +- rust/dbn/src/decode/dbz.rs | 56 ++++++++++++------------------ rust/dbn/src/decode/stream.rs | 35 +++++++++---------- rust/dbn/src/encode.rs | 19 ++++++---- rust/dbn/src/encode/csv/sync.rs | 6 ++-- rust/dbn/src/encode/dyn_encoder.rs | 8 ++--- 8 files changed, 69 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e1481e..f78a45b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ - Python and encodings are unaffected - Removed `write_dbn_file` function deprecated in version 0.14.0 from Python interface. Please use `Transcoder` instead +- Switched `DecodeStream` from `streaming_iterator` crate to `fallible_streaming_iterator` + to allow better notification of errors +- Switched `EncodeDbn::encode_stream` from accepting an `impl StreamingIterator` to + accepting an `FallibleStreamingIterator` to allow bubbling up of decoding errors ## 0.17.1 - 2024-04-04 diff --git a/Cargo.lock b/Cargo.lock index 51eacf8..65bbb99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,13 +287,13 @@ dependencies = [ "async-compression", "csv", "dbn-macros", + "fallible-streaming-iterator", "itoa", "json-writer", "num_enum", "pyo3", "rstest", "serde", - "streaming-iterator", "strum", "thiserror", "time", @@ -378,6 +378,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.0.1" @@ -1038,12 +1044,6 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" -[[package]] -name = "streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" - [[package]] name = "strsim" version = "0.11.0" diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index 61cbd53..aacc72c 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -29,13 +29,13 @@ dbn-macros = { version = "=0.17.1", path = "../dbn-macros" } async-compression = { version = "0.4.6", features = ["tokio", "zstd"], optional = true } csv = "1.3" +fallible-streaming-iterator = { version = "0.1.9", features = ["std"] } # Fast integer to string conversion itoa = "1.0" num_enum = "0.7" pyo3 = { workspace = true, optional = true } json-writer = "0.3" serde = { workspace = true, features = ["derive"], optional = true } -streaming-iterator = "0.1.9" # extra enum traits for Python strum = { version = "0.26", features = ["derive"], optional = true } thiserror = "1.0" diff --git a/rust/dbn/src/decode/dbz.rs b/rust/dbn/src/decode/dbz.rs index 060197e..fed71c0 100644 --- a/rust/dbn/src/decode/dbz.rs +++ b/rust/dbn/src/decode/dbz.rs @@ -408,48 +408,36 @@ impl MetadataDecoder { #[cfg(test)] mod tests { - use streaming_iterator::StreamingIterator; + use fallible_streaming_iterator::FallibleStreamingIterator; + use rstest::*; use super::*; use crate::compat::InstrumentDefMsgV1; use crate::decode::tests::TEST_DATA_PATH; use crate::record::{MboMsg, Mbp10Msg, Mbp1Msg, OhlcvMsg, TbboMsg, TradeMsg}; - /// there are crates like rstest that provide pytest-like parameterized tests, however - /// they don't support passing types - macro_rules! test_reading_dbz { - // Rust doesn't allow concatenating identifiers in stable rust, so each test case needs - // to be named explicitly - ($test_name:ident, $record_type:ident, $schema:expr) => { - #[test] - fn $test_name() { - let target = Decoder::from_file(format!( - "{TEST_DATA_PATH}/test_data.{}.dbz", - $schema.as_str() - )) - .unwrap(); - let exp_rec_count = if $schema == Schema::Ohlcv1D { 0 } else { 2 }; - let actual_rec_count = target.decode_stream::<$record_type>().count(); - assert_eq!(exp_rec_count, actual_rec_count); - } - }; + #[rstest] + #[case::mbo(MboMsg::default(), Schema::Mbo, 2)] + #[case::mbp1(Mbp1Msg::default_for_schema(Schema::Mbp1), Schema::Mbp1, 2)] + #[case::mbp10(Mbp10Msg::default(), Schema::Mbp10, 2)] + #[case::ohlcv_1d(OhlcvMsg::default_for_schema(Schema::Ohlcv1D), Schema::Ohlcv1D, 0)] + #[case::ohlcv_1h(OhlcvMsg::default_for_schema(Schema::Ohlcv1H), Schema::Ohlcv1H, 2)] + #[case::ohlcv_1m(OhlcvMsg::default_for_schema(Schema::Ohlcv1M), Schema::Ohlcv1M, 2)] + #[case::ohlcv_1s(OhlcvMsg::default_for_schema(Schema::Ohlcv1S), Schema::Ohlcv1S, 2)] + #[case::tbbo(TbboMsg::default_for_schema(Schema::Tbbo), Schema::Tbbo, 2)] + #[case::trades(TradeMsg::default(), Schema::Trades, 2)] + #[case::definition(InstrumentDefMsgV1::default(), Schema::Definition, 2)] + fn test_decode_stream( + #[case] _rec: R, + #[case] schema: Schema, + #[case] exp_rec_count: usize, + ) { + let target = + Decoder::from_file(format!("{TEST_DATA_PATH}/test_data.{schema}.dbz")).unwrap(); + let actual_rec_count = target.decode_stream::().count().unwrap(); + assert_eq!(exp_rec_count, actual_rec_count); } - test_reading_dbz!(test_reading_mbo, MboMsg, Schema::Mbo); - test_reading_dbz!(test_reading_mbp1, Mbp1Msg, Schema::Mbp1); - test_reading_dbz!(test_reading_mbp10, Mbp10Msg, Schema::Mbp10); - test_reading_dbz!(test_reading_ohlcv1d, OhlcvMsg, Schema::Ohlcv1D); - test_reading_dbz!(test_reading_ohlcv1h, OhlcvMsg, Schema::Ohlcv1H); - test_reading_dbz!(test_reading_ohlcv1m, OhlcvMsg, Schema::Ohlcv1M); - test_reading_dbz!(test_reading_ohlcv1s, OhlcvMsg, Schema::Ohlcv1S); - test_reading_dbz!(test_reading_tbbo, TbboMsg, Schema::Tbbo); - test_reading_dbz!(test_reading_trades, TradeMsg, Schema::Trades); - test_reading_dbz!( - test_reading_definition, - InstrumentDefMsgV1, - Schema::Definition - ); - #[test] fn test_decode_symbol() { let bytes = b"SPX.1.2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; diff --git a/rust/dbn/src/decode/stream.rs b/rust/dbn/src/decode/stream.rs index ac46944..36541dc 100644 --- a/rust/dbn/src/decode/stream.rs +++ b/rust/dbn/src/decode/stream.rs @@ -1,14 +1,14 @@ use std::marker::PhantomData; -use streaming_iterator::StreamingIterator; +use fallible_streaming_iterator::FallibleStreamingIterator; use super::{DecodeRecord, DecodeStream}; -use crate::record::{transmute_record_bytes, HasRType}; +use crate::{record::transmute_record_bytes, Error, HasRType, Result}; /// A consuming iterator wrapping a [`DecodeRecord`]. Lazily decodes the contents of the /// file or other input stream. /// -/// Implements [`streaming_iterator::StreamingIterator`]. +/// Implements [`FallibleStreamingIterator`]. pub struct StreamIterDecoder where D: DecodeRecord, @@ -19,8 +19,6 @@ where /// Number of element sthat have been decoded. Used for [`Iterator::size_hint()`]. /// `None` indicates the end of the stream has been reached. i: Option, - /// Last error encountering when decoding. - last_err: Option, /// Required to associate this type with a specific record type `T`. _marker: PhantomData, } @@ -35,40 +33,39 @@ where Self { decoder, i: Some(0), - last_err: None, _marker: PhantomData, } } - - /// Last error encountering when decoding. - pub fn last_err(&self) -> Option<&crate::Error> { - self.last_err.as_ref() - } } -impl StreamingIterator for StreamIterDecoder +impl FallibleStreamingIterator for StreamIterDecoder where D: DecodeStream, T: HasRType, { + type Error = Error; type Item = T; - fn advance(&mut self) { + fn advance(&mut self) -> Result<()> { if let Some(i) = self.i.as_mut() { match self.decoder.decode_record::() { - Err(err) => { - self.last_err = Some(err); - // set error state sentinel - self.i = None; + Ok(Some(_)) => { + *i += 1; + Ok(()) } Ok(None) => { // set error state sentinel self.i = None; + Ok(()) } - Ok(Some(_)) => { - *i += 1; + Err(err) => { + // set error state sentinel + self.i = None; + Err(err) } } + } else { + Ok(()) } } diff --git a/rust/dbn/src/encode.rs b/rust/dbn/src/encode.rs index ee4e7b9..85c8dc6 100644 --- a/rust/dbn/src/encode.rs +++ b/rust/dbn/src/encode.rs @@ -8,7 +8,7 @@ pub mod json; use std::{fmt, io, num::NonZeroU64}; -use streaming_iterator::StreamingIterator; +use fallible_streaming_iterator::FallibleStreamingIterator; // Re-exports pub use self::{ @@ -105,9 +105,9 @@ pub trait EncodeDbn: EncodeRecord + EncodeRecordRef { /// or there's a serialization error. fn encode_stream( &mut self, - mut stream: impl StreamingIterator, + mut stream: impl FallibleStreamingIterator, ) -> Result<()> { - while let Some(record) = stream.next() { + while let Some(record) = stream.next()? { self.encode_record(record)?; } self.flush()?; @@ -213,9 +213,12 @@ fn zstd_encoder<'a, W: io::Write>(writer: W) -> Result StreamingIterator for VecStream { + impl FallibleStreamingIterator for VecStream { type Item = T; + type Error = Error; - fn advance(&mut self) { + fn advance(&mut self) -> Result<(), Error> { self.idx += 1; + Ok(()) } fn get(&self) -> Option<&Self::Item> { diff --git a/rust/dbn/src/encode/csv/sync.rs b/rust/dbn/src/encode/csv/sync.rs index bc8b3bc..38d215f 100644 --- a/rust/dbn/src/encode/csv/sync.rs +++ b/rust/dbn/src/encode/csv/sync.rs @@ -1,6 +1,6 @@ use std::{io, num::NonZeroU64}; -use streaming_iterator::StreamingIterator; +use fallible_streaming_iterator::FallibleStreamingIterator; use crate::{ decode::{DbnMetadata, DecodeRecordRef}, @@ -290,12 +290,12 @@ where /// or there's a serialization error. fn encode_stream( &mut self, - mut stream: impl StreamingIterator, + mut stream: impl FallibleStreamingIterator, ) -> Result<()> { if !self.has_written_header { self.encode_header::(false)?; } - while let Some(record) = stream.next() { + while let Some(record) = stream.next()? { self.encode_record(record)?; } self.flush()?; diff --git a/rust/dbn/src/encode/dyn_encoder.rs b/rust/dbn/src/encode/dyn_encoder.rs index d314d05..b67f8e6 100644 --- a/rust/dbn/src/encode/dyn_encoder.rs +++ b/rust/dbn/src/encode/dyn_encoder.rs @@ -1,6 +1,6 @@ use std::io; -use streaming_iterator::StreamingIterator; +use fallible_streaming_iterator::FallibleStreamingIterator; use super::{ CsvEncoder, DbnEncodable, DbnEncoder, DynWriter, EncodeDbn, EncodeRecord, EncodeRecordRef, @@ -8,7 +8,7 @@ use super::{ }; use crate::{ decode::{DbnMetadata, DecodeRecordRef}, - Compression, Encoding, Metadata, RecordRef, Result, Schema, + Compression, Encoding, Error, Metadata, RecordRef, Result, Schema, }; /// An encoder whose [`Encoding`] and [`Compression`] can be set at runtime. @@ -267,7 +267,7 @@ where fn encode_stream( &mut self, - stream: impl StreamingIterator, + stream: impl FallibleStreamingIterator, ) -> Result<()> { self.0.encode_stream(stream) } @@ -346,7 +346,7 @@ where fn encode_stream( &mut self, - stream: impl StreamingIterator, + stream: impl FallibleStreamingIterator, ) -> Result<()> { match self { DynEncoderImpl::Dbn(encoder) => encoder.encode_stream(stream), From 50675480e230e8422a0025d86bbcd4df55066fad Mon Sep 17 00:00:00 2001 From: Carter Green Date: Thu, 2 May 2024 11:08:37 -0500 Subject: [PATCH 12/23] FIX: Fix stypes when upgrading SymbolMappingMsgV1 --- CHANGELOG.md | 3 +++ rust/dbn/src/record/impl_default.rs | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f78a45b..7108f6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ to allow better notification of errors - Switched `EncodeDbn::encode_stream` from accepting an `impl StreamingIterator` to accepting an `FallibleStreamingIterator` to allow bubbling up of decoding errors +- Changed default value for `stype_in` and `stype_out` in `SymbolMappingMsg` to + `u8::MAX` to match C++ client and to reflect an unknown value. This also changes the + value of these fields when upgrading a `SymbolMappingMsgV1` to DBNv2 ## 0.17.1 - 2024-04-04 diff --git a/rust/dbn/src/record/impl_default.rs b/rust/dbn/src/record/impl_default.rs index 798616b..3a969d4 100644 --- a/rust/dbn/src/record/impl_default.rs +++ b/rust/dbn/src/record/impl_default.rs @@ -2,8 +2,8 @@ use std::ffi::c_char; use crate::{ compat::{ErrorMsgV1, InstrumentDefMsgV1, SymbolMappingMsgV1, SystemMsgV1, SYMBOL_CSTR_LEN_V1}, - enums::{StatusAction, StatusReason, TradingEvent, TriState}, - SType, Schema, UNDEF_ORDER_SIZE, UNDEF_PRICE, UNDEF_STAT_QUANTITY, UNDEF_TIMESTAMP, + Schema, StatusAction, StatusReason, TradingEvent, TriState, UNDEF_ORDER_SIZE, UNDEF_PRICE, + UNDEF_STAT_QUANTITY, UNDEF_TIMESTAMP, }; use super::*; @@ -379,9 +379,9 @@ impl Default for SymbolMappingMsg { fn default() -> Self { Self { hd: RecordHeader::default::(rtype::SYMBOL_MAPPING), - stype_in: SType::RawSymbol as u8, + stype_in: u8::MAX, stype_in_symbol: [0; SYMBOL_CSTR_LEN], - stype_out: SType::InstrumentId as u8, + stype_out: u8::MAX, stype_out_symbol: [0; SYMBOL_CSTR_LEN], start_ts: UNDEF_TIMESTAMP, end_ts: UNDEF_TIMESTAMP, From 12042b93974c74f311b9a888d33af8412aaa1558 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Thu, 2 May 2024 16:07:44 -0500 Subject: [PATCH 13/23] MOD: Update compiler error output --- .../tests/ui/csv_serialize_conflicting_dbn_attr.stderr | 2 +- .../tests/ui/csv_serialize_invalid_dbn_attr.stderr | 2 +- .../tests/ui/json_serialize_conflicting_dbn_attr.stderr | 5 +---- .../tests/ui/json_serialize_invalid_dbn_attr.stderr | 5 +---- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/rust/dbn-macros/tests/ui/csv_serialize_conflicting_dbn_attr.stderr b/rust/dbn-macros/tests/ui/csv_serialize_conflicting_dbn_attr.stderr index 142d037..6811b92 100644 --- a/rust/dbn-macros/tests/ui/csv_serialize_conflicting_dbn_attr.stderr +++ b/rust/dbn-macros/tests/ui/csv_serialize_conflicting_dbn_attr.stderr @@ -27,7 +27,7 @@ error[E0599]: no function or associated item named `write_header` found for type | = help: items from traits can only be used if the trait is in scope = note: this error originates in the derive macro `CsvSerialize` (in Nightly builds, run with -Z macro-backtrace for more info) -help: the following trait is implemented but not in scope; perhaps add a `use` for it: +help: trait `WriteField` which provides `write_header` is implemented but not in scope; perhaps you want to import it | 1 + use dbn::encode::csv::serialize::WriteField; | diff --git a/rust/dbn-macros/tests/ui/csv_serialize_invalid_dbn_attr.stderr b/rust/dbn-macros/tests/ui/csv_serialize_invalid_dbn_attr.stderr index afbd186..5d8aad3 100644 --- a/rust/dbn-macros/tests/ui/csv_serialize_invalid_dbn_attr.stderr +++ b/rust/dbn-macros/tests/ui/csv_serialize_invalid_dbn_attr.stderr @@ -27,7 +27,7 @@ error[E0599]: no function or associated item named `write_header` found for type | = help: items from traits can only be used if the trait is in scope = note: this error originates in the derive macro `CsvSerialize` (in Nightly builds, run with -Z macro-backtrace for more info) -help: the following trait is implemented but not in scope; perhaps add a `use` for it: +help: trait `WriteField` which provides `write_header` is implemented but not in scope; perhaps you want to import it | 1 + use dbn::encode::csv::serialize::WriteField; | diff --git a/rust/dbn-macros/tests/ui/json_serialize_conflicting_dbn_attr.stderr b/rust/dbn-macros/tests/ui/json_serialize_conflicting_dbn_attr.stderr index b09d017..de4fde3 100644 --- a/rust/dbn-macros/tests/ui/json_serialize_conflicting_dbn_attr.stderr +++ b/rust/dbn-macros/tests/ui/json_serialize_conflicting_dbn_attr.stderr @@ -8,10 +8,7 @@ error[E0433]: failed to resolve: unresolved import --> tests/ui/json_serialize_conflicting_dbn_attr.rs:3:10 | 3 | #[derive(JsonSerialize)] - | ^^^^^^^^^^^^^ - | | - | unresolved import - | help: a similar path exists: `dbn::encode` + | ^^^^^^^^^^^^^ unresolved import | = note: this error originates in the derive macro `JsonSerialize` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/rust/dbn-macros/tests/ui/json_serialize_invalid_dbn_attr.stderr b/rust/dbn-macros/tests/ui/json_serialize_invalid_dbn_attr.stderr index 5672feb..57e9110 100644 --- a/rust/dbn-macros/tests/ui/json_serialize_invalid_dbn_attr.stderr +++ b/rust/dbn-macros/tests/ui/json_serialize_invalid_dbn_attr.stderr @@ -8,10 +8,7 @@ error[E0433]: failed to resolve: unresolved import --> tests/ui/json_serialize_invalid_dbn_attr.rs:3:10 | 3 | #[derive(JsonSerialize)] - | ^^^^^^^^^^^^^ - | | - | unresolved import - | help: a similar path exists: `dbn::encode` + | ^^^^^^^^^^^^^ unresolved import | = note: this error originates in the derive macro `JsonSerialize` (in Nightly builds, run with -Z macro-backtrace for more info) From 468a73bb5879d9c2057f970c4a5df1a3a180060a Mon Sep 17 00:00:00 2001 From: Carter Green Date: Fri, 3 May 2024 16:45:39 -0500 Subject: [PATCH 14/23] DOC: Mention units in display_factor documentation --- rust/dbn/src/record.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index 4f8d4ed..329ae3c 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -554,7 +554,8 @@ pub struct InstrumentDefMsg { #[dbn(fixed_price)] #[pyo3(get, set)] pub min_price_increment: i64, - /// The multiplier to convert the venue’s display price to the conventional price. + /// The multiplier to convert the venue’s display price to the conventional price, + /// in units of 1e-9, i.e. 1/1,000,000,000 or 0.000000001. #[pyo3(get, set)] pub display_factor: i64, /// The last eligible trade time expressed as a number of nanoseconds since the From 3b43027ecfa4b0de660f6f9a1353cef4bc525544 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Fri, 3 May 2024 15:57:35 -0500 Subject: [PATCH 15/23] ADD: Create custom Python exception for Rust code --- CHANGELOG.md | 2 + python/python/databento_dbn/_lib.pyi | 19 ++-- python/src/dbn_decoder.rs | 31 +++--- python/src/encode.rs | 16 ++-- python/src/lib.rs | 7 +- python/src/transcoder.rs | 53 ++++------ rust/dbn/src/python.rs | 26 +++-- rust/dbn/src/python/enums.rs | 74 +++++++------- rust/dbn/src/python/metadata.rs | 13 ++- rust/dbn/src/python/record.rs | 138 +++++++++++++-------------- 10 files changed, 186 insertions(+), 193 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7108f6e..28527ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ - Added `new_inferred`, `with_buffer`, `inferred_with_buffer`, `from_file`, `get_mut`, and `get_ref` methods to `AsyncDynReader` for parity with the sync `DynReader` - Improved documentation enumerating errors returned by functions +- Added new `DBNError` Python exception that's now the primary exception raised by + `databento_dbn` ### Breaking changes - Changed type of `flags` in `MboMsg`, `TradeMsg`, `Mbp1Msg`, `Mbp10Msg`, and `CbboMsg` diff --git a/python/python/databento_dbn/_lib.pyi b/python/python/databento_dbn/_lib.pyi index dac2214..0b429d1 100644 --- a/python/python/databento_dbn/_lib.pyi +++ b/python/python/databento_dbn/_lib.pyi @@ -43,6 +43,11 @@ _DBNRecord = Union[ StatusMsg, ] +class DBNError(Exception): + """ + An exception from databento_dbn Rust code. + """ + class Side(Enum): """ A side of the market. The side of the market for resting orders, or the side @@ -950,7 +955,7 @@ class Metadata(SupportsBytes): Raises ------ - ValueError + DBNError When a Metadata instance cannot be parsed from `data`. """ @@ -965,7 +970,7 @@ class Metadata(SupportsBytes): Raises ------ - ValueError + DBNError When the Metadata object cannot be encoded. """ @@ -4651,7 +4656,7 @@ class DBNDecoder: Raises ------ - ValueError + DBNError When the decoding fails. See Also @@ -4669,7 +4674,7 @@ class DBNDecoder: Raises ------ - ValueError + DBNError When the write to the internal buffer fails. See Also @@ -4753,7 +4758,7 @@ class Transcoder: Raises ------ - ValueError + DBNError When the write to the internal buffer or the output fails. """ @@ -4765,7 +4770,7 @@ class Transcoder: Raises ------ - ValueError + DBNError When the write to the output fails. """ @@ -4793,7 +4798,7 @@ def update_encoded_metadata( Raises ------ - ValueError + DBNError When the file update fails. """ diff --git a/python/src/dbn_decoder.rs b/python/src/dbn_decoder.rs index aa14701..57aac9a 100644 --- a/python/src/dbn_decoder.rs +++ b/python/src/dbn_decoder.rs @@ -4,8 +4,8 @@ use pyo3::prelude::*; use dbn::{ decode::dbn::{MetadataDecoder, RecordDecoder}, - python::to_val_err, - rtype_ts_out_dispatch, HasRType, Record, VersionUpgradePolicy, + python::to_py_err, + rtype_ts_out_dispatch, HasRType, VersionUpgradePolicy, }; #[pyclass(module = "databento_dbn", name = "DBNDecoder")] @@ -36,7 +36,7 @@ impl DbnDecoder { } fn write(&mut self, bytes: &[u8]) -> PyResult<()> { - self.buffer.write_all(bytes).map_err(to_val_err) + self.buffer.write_all(bytes).map_err(to_py_err) } fn buffer(&self) -> &[u8] { @@ -63,7 +63,7 @@ impl DbnDecoder { { return Ok(Vec::new()); } - return Err(to_val_err(err)); + return Err(PyErr::from(err)); } } } @@ -73,10 +73,9 @@ impl DbnDecoder { self.input_version, self.upgrade_policy, self.ts_out, - ) - .map_err(to_val_err)?; + )?; Python::with_gil(|py| -> PyResult<()> { - while let Some(rec) = decoder.decode_ref().map_err(to_val_err)? { + while let Some(rec) = decoder.decode_ref()? { // Bug in clippy generates an error here. trivial_copy feature isn't enabled, // but clippy thinks these records are `Copy` fn push_rec>>( @@ -89,14 +88,7 @@ impl DbnDecoder { // Safety: It's safe to cast to `WithTsOut` because we're passing in the `ts_out` // from the metadata header. - if unsafe { rtype_ts_out_dispatch!(rec, self.ts_out, push_rec, py, &mut recs) } - .is_err() - { - return Err(to_val_err(format!( - "Invalid rtype {} found in record", - rec.header().rtype, - ))); - } + unsafe { rtype_ts_out_dispatch!(rec, self.ts_out, push_rec, py, &mut recs) }?; // keep track of position after last _successful_ decoding to // ensure buffer is left in correct state in the case where one // or more successful decodings is followed by a partial one, i.e. @@ -264,7 +256,7 @@ for record in records[1:]: setup(); Python::with_gil(|py| { py.run_bound( - r#"from _lib import DBNDecoder, Metadata, Schema, SType + r#"from _lib import DBNDecoder, DBNError, Metadata, Schema, SType metadata = Metadata( dataset="GLBX.MDP3", @@ -286,8 +278,11 @@ try: records = decoder.decode() # If this code is called, the test will fail assert False -except Exception as ex: - assert "Invalid rtype" in str(ex) +except DBNError as ex: + assert "couldn't convert" in str(ex) + assert "RType" in str(ex) +except Exception: + assert False "#, None, None, diff --git a/python/src/encode.rs b/python/src/encode.rs index 151d19b..157fe41 100644 --- a/python/src/encode.rs +++ b/python/src/encode.rs @@ -3,7 +3,7 @@ use std::{ num::NonZeroU64, }; -use dbn::{encode::dbn::MetadataEncoder, python::to_val_err}; +use dbn::encode::dbn::MetadataEncoder; use pyo3::{exceptions::PyTypeError, intern, prelude::*, types::PyBytes}; /// Updates existing fields that have already been written to the given file. @@ -19,14 +19,12 @@ pub fn update_encoded_metadata( let mut buf = [0; 4]; file.read_exact(&mut buf)?; let version = buf[3]; - MetadataEncoder::new(file) - .update_encoded( - version, - start, - end.and_then(NonZeroU64::new), - limit.and_then(NonZeroU64::new), - ) - .map_err(to_val_err) + Ok(MetadataEncoder::new(file).update_encoded( + version, + start, + end.and_then(NonZeroU64::new), + limit.and_then(NonZeroU64::new), + )?) } /// A Python object that implements the Python file interface. diff --git a/python/src/lib.rs b/python/src/lib.rs index 6399368..d2f0b53 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -5,7 +5,7 @@ use pyo3::{prelude::*, wrap_pyfunction, PyClass}; use dbn::{ compat::{ErrorMsgV1, InstrumentDefMsgV1, SymbolMappingMsgV1, SystemMsgV1}, flags, - python::EnumIterator, + python::{DBNError, EnumIterator}, Action, BidAskPair, CbboMsg, Compression, ConsolidatedBidAskPair, Encoding, ErrorMsg, ImbalanceMsg, InstrumentClass, InstrumentDefMsg, MatchAlgorithm, MboMsg, Mbp10Msg, Mbp1Msg, Metadata, OhlcvMsg, RType, RecordHeader, SType, Schema, SecurityUpdateAction, Side, StatMsg, @@ -29,10 +29,11 @@ fn databento_dbn(_py: Python<'_>, m: &Bound) -> PyResult<()> { } // all functions exposed to Python need to be added here m.add_wrapped(wrap_pyfunction!(encode::update_encoded_metadata))?; + m.add("DBNError", m.py().get_type_bound::())?; + checked_add_class::(m)?; + checked_add_class::(m)?; checked_add_class::(m)?; checked_add_class::(m)?; - checked_add_class::(m)?; - checked_add_class::(m)?; // Records checked_add_class::(m)?; checked_add_class::(m)?; diff --git a/python/src/transcoder.rs b/python/src/transcoder.rs index 2dc04c5..0cc8106 100644 --- a/python/src/transcoder.rs +++ b/python/src/transcoder.rs @@ -12,7 +12,7 @@ use dbn::{ CsvEncoder, DbnMetadataEncoder, DbnRecordEncoder, DynWriter, EncodeRecordRef, EncodeRecordTextExt, JsonEncoder, }, - python::{py_to_time_date, to_val_err}, + python::{py_to_time_date, to_py_err}, Compression, Encoding, PitSymbolMap, RType, Record, RecordRef, Schema, SymbolIndex, TsSymbolMap, VersionUpgradePolicy, }; @@ -51,9 +51,7 @@ impl Transcoder { } let start_date = py_to_time_date(start_date)?; let end_date = py_to_time_date(end_date)?; - symbol_map - .insert(iid, start_date, end_date, Arc::new(symbol)) - .map_err(to_val_err)?; + symbol_map.insert(iid, start_date, end_date, Arc::new(symbol))?; } } Some(symbol_map) @@ -141,7 +139,7 @@ struct Inner { impl Transcode for Inner { fn write(&mut self, bytes: &[u8]) -> PyResult<()> { - self.buffer.write_all(bytes).map_err(to_val_err)?; + self.buffer.write_all(bytes).map_err(to_py_err)?; self.encode() } @@ -177,7 +175,7 @@ impl Inner { } Ok(Self { buffer: io::Cursor::default(), - output: DynWriter::new(BufWriter::new(file), compression).map_err(to_val_err)?, + output: DynWriter::new(BufWriter::new(file), compression)?, use_pretty_px: pretty_px.unwrap_or(true), use_pretty_ts: pretty_ts.unwrap_or(true), map_symbols: map_symbols.unwrap_or(true), @@ -215,14 +213,12 @@ impl Inner { self.input_version, self.upgrade_policy, self.ts_out, - ) - .map_err(to_val_err)?; + )?; let mut encoder = DbnRecordEncoder::new(&mut self.output); loop { match decoder.decode_record_ref() { Ok(Some(rec)) => { - unsafe { encoder.encode_record_ref_ts_out(rec, self.ts_out) } - .map_err(to_val_err)?; + unsafe { encoder.encode_record_ref_ts_out(rec, self.ts_out) }?; // keep track of position after last _successful_ decoding to // ensure buffer is left in correct state in the case where one // or more successful decodings is followed by a partial one, i.e. @@ -234,7 +230,7 @@ impl Inner { } Err(err) => { self.buffer.set_position(orig_position); - return Err(to_val_err(err)); + return Err(PyErr::from(err)); } } } @@ -248,15 +244,13 @@ impl Inner { self.input_version, self.upgrade_policy, self.ts_out, - ) - .map_err(to_val_err)?; + )?; let mut encoder = CsvEncoder::builder(&mut self.output) .use_pretty_px(self.use_pretty_px) .use_pretty_ts(self.use_pretty_ts) .write_header(false) - .build() - .map_err(to_val_err)?; + .build()?; loop { match decoder.decode_record_ref() { Ok(Some(rec)) => { @@ -275,8 +269,7 @@ impl Inner { unsafe { encoder.encode_ref_ts_out_with_sym(rec, self.ts_out, symbol) } } else { unsafe { encoder.encode_record_ref_ts_out(rec, self.ts_out) } - } - .map_err(to_val_err)?; + }?; } // keep track of position after last _successful_ decoding to // ensure buffer is left in correct state in the case where one @@ -289,7 +282,7 @@ impl Inner { } Err(err) => { self.buffer.set_position(orig_position); - return Err(to_val_err(err)); + return Err(PyErr::from(err)); } } } @@ -303,8 +296,7 @@ impl Inner { self.input_version, self.upgrade_policy, self.ts_out, - ) - .map_err(to_val_err)?; + )?; let mut encoder = JsonEncoder::builder(&mut self.output) .use_pretty_px(self.use_pretty_px) @@ -319,8 +311,7 @@ impl Inner { unsafe { encoder.encode_ref_ts_out_with_sym(rec, self.ts_out, symbol) } } else { unsafe { encoder.encode_record_ref_ts_out(rec, self.ts_out) } - } - .map_err(to_val_err)?; + }?; // keep track of position after last _successful_ decoding to // ensure buffer is left in correct state in the case where one // or more successful decodings is followed by a partial one, i.e. @@ -332,7 +323,7 @@ impl Inner { } Err(err) => { self.buffer.set_position(orig_position); - return Err(to_val_err(err)); + return Err(PyErr::from(err)); } } } @@ -353,9 +344,7 @@ impl Inner { metadata.upgrade(self.upgrade_policy); // Setup live symbol mapping if OUTPUT_ENC == Encoding::Dbn as u8 { - DbnMetadataEncoder::new(&mut self.output) - .encode(&metadata) - .map_err(to_val_err)?; + DbnMetadataEncoder::new(&mut self.output).encode(&metadata)?; // CSV or JSON } else if self.map_symbols { if metadata.schema.is_some() { @@ -363,10 +352,8 @@ impl Inner { // only read from metadata mappings if symbol_map is unpopulated, // i.e. no `symbol_map` was passed in if self.symbol_map.is_empty() { - self.symbol_map = metadata - .symbol_map() - .map(SymbolMap::Historical) - .map_err(to_val_err)?; + self.symbol_map = + metadata.symbol_map().map(SymbolMap::Historical)?; } } else { // live @@ -381,7 +368,7 @@ impl Inner { { return Ok(false); } - return Err(to_val_err(err)); + return Err(PyErr::from(err)); } } // decoding metadata and the header are both done once at the beginning @@ -393,9 +380,7 @@ impl Inner { }; let mut encoder = CsvEncoder::new(&mut self.output, self.use_pretty_px, self.use_pretty_ts); - encoder - .encode_header_for_schema(schema, self.ts_out, self.map_symbols) - .map_err(to_val_err)?; + encoder.encode_header_for_schema(schema, self.ts_out, self.map_symbols)?; } } Ok(true) diff --git a/rust/dbn/src/python.rs b/rust/dbn/src/python.rs index 3bfd321..5205a7c 100644 --- a/rust/dbn/src/python.rs +++ b/rust/dbn/src/python.rs @@ -5,22 +5,36 @@ use std::fmt; use pyo3::{ - exceptions::PyValueError, + create_exception, + exceptions::PyException, prelude::*, types::{PyDate, PyDateAccess}, }; use strum::IntoEnumIterator; -use crate::FlagSet; +use crate::{Error, FlagSet}; mod enums; mod metadata; mod record; +create_exception!( + databento_dbn, + DBNError, + PyException, + "An exception from databento_dbn Rust code." +); + /// A helper function for converting any type that implements `Debug` to a Python /// `ValueError`. -pub fn to_val_err(e: impl fmt::Debug) -> PyErr { - PyValueError::new_err(format!("{e:?}")) +pub fn to_py_err(e: impl fmt::Display) -> PyErr { + DBNError::new_err(format!("{e}")) +} + +impl From for PyErr { + fn from(err: Error) -> Self { + DBNError::new_err(format!("{err}")) + } } /// Python iterator over the variants of an enum. @@ -71,9 +85,9 @@ impl IntoPy for FlagSet { /// This function returns an error if input has an invalid month. pub fn py_to_time_date(py_date: &PyDate) -> PyResult { let month = - time::Month::try_from(py_date.get_month()).map_err(|e| to_val_err(e.to_string()))?; + time::Month::try_from(py_date.get_month()).map_err(|e| DBNError::new_err(e.to_string()))?; time::Date::from_calendar_date(py_date.get_year(), month, py_date.get_day()) - .map_err(|e| to_val_err(e.to_string())) + .map_err(|e| DBNError::new_err(e.to_string())) } /// A trait for records that provide descriptions of their fields. diff --git a/rust/dbn/src/python/enums.rs b/rust/dbn/src/python/enums.rs index 1eee834..1ca8a83 100644 --- a/rust/dbn/src/python/enums.rs +++ b/rust/dbn/src/python/enums.rs @@ -8,7 +8,7 @@ use crate::{ TradingEvent, TriState, VersionUpgradePolicy, }; -use super::{to_val_err, EnumIterator, PyFieldDesc}; +use super::{to_py_err, EnumIterator, PyFieldDesc}; #[pymethods] impl Side { @@ -16,10 +16,10 @@ impl Side { fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { let t = Self::type_object_bound(py); - let c = value.extract::().map_err(to_val_err)?; + let c = value.extract::().map_err(to_py_err)?; return Self::py_from_str(&t, c); }; - Self::try_from(i).map_err(to_val_err) + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -63,7 +63,7 @@ impl Side { #[classmethod] #[pyo3(name = "from_str")] fn py_from_str(_: &Bound, value: char) -> PyResult { - Self::try_from(value as u8).map_err(to_val_err) + Self::try_from(value as u8).map_err(to_py_err) } } @@ -73,10 +73,10 @@ impl Action { fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { let t = Self::type_object_bound(py); - let c = value.extract::().map_err(to_val_err)?; + let c = value.extract::().map_err(to_py_err)?; return Self::py_from_str(&t, c); }; - Self::try_from(i).map_err(to_val_err) + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -120,7 +120,7 @@ impl Action { #[classmethod] #[pyo3(name = "from_str")] fn py_from_str(_: &Bound, value: char) -> PyResult { - Self::try_from(value as u8).map_err(to_val_err) + Self::try_from(value as u8).map_err(to_py_err) } } @@ -130,10 +130,10 @@ impl InstrumentClass { fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { let t = Self::type_object_bound(py); - let c = value.extract::().map_err(to_val_err)?; + let c = value.extract::().map_err(to_py_err)?; return Self::py_from_str(&t, c); }; - Self::try_from(i).map_err(to_val_err) + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -168,7 +168,7 @@ impl InstrumentClass { #[classmethod] #[pyo3(name = "from_str")] fn py_from_str(_: &Bound, value: char) -> PyResult { - Self::try_from(value as u8).map_err(to_val_err) + Self::try_from(value as u8).map_err(to_py_err) } } @@ -178,10 +178,10 @@ impl MatchAlgorithm { fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { let t = Self::type_object_bound(py); - let c = value.extract::().map_err(to_val_err)?; + let c = value.extract::().map_err(to_py_err)?; return Self::py_from_str(&t, c); }; - Self::try_from(i).map_err(to_val_err) + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -216,7 +216,7 @@ impl MatchAlgorithm { #[classmethod] #[pyo3(name = "from_str")] fn py_from_str(_: &Bound, value: char) -> PyResult { - Self::try_from(value as u8).map_err(to_val_err) + Self::try_from(value as u8).map_err(to_py_err) } } @@ -226,10 +226,10 @@ impl UserDefinedInstrument { fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { let t = Self::type_object_bound(py); - let c = value.extract::().map_err(to_val_err)?; + let c = value.extract::().map_err(to_py_err)?; return Self::py_from_str(&t, c); }; - Self::try_from(i).map_err(to_val_err) + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -277,7 +277,7 @@ impl UserDefinedInstrument { #[classmethod] #[pyo3(name = "from_str")] fn py_from_str(_: &Bound, value: char) -> PyResult { - Self::try_from(value as u8).map_err(to_val_err) + Self::try_from(value as u8).map_err(to_py_err) } } @@ -332,7 +332,7 @@ impl SType { fn py_from_str(_: &Bound, value: &Bound) -> PyResult { let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.replace('-', "_").to_lowercase(); - Self::from_str(&tokenized).map_err(to_val_err) + Ok(Self::from_str(&tokenized)?) } } @@ -390,14 +390,14 @@ impl RType { fn py_from_str(_: &Bound, value: &Bound) -> PyResult { let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.replace('-', "_").to_lowercase(); - Self::from_str(&tokenized).map_err(to_val_err) + Ok(Self::from_str(&tokenized)?) } #[classmethod] #[pyo3(name = "from_int")] fn py_from_int(_: &Bound, value: &Bound) -> PyResult { let value: u8 = value.extract()?; - Self::try_from(value).map_err(to_val_err) + Self::try_from(value).map_err(to_py_err) } #[classmethod] @@ -406,7 +406,7 @@ impl RType { let schema: Schema = value .extract() .or_else(|_| Schema::py_from_str(&Schema::type_object_bound(pytype.py()), value)) - .map_err(to_val_err)?; + .map_err(to_py_err)?; Ok(Self::from(schema)) } } @@ -462,7 +462,7 @@ impl Schema { fn py_from_str(_: &Bound, value: &Bound) -> PyResult { let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.replace('_', "-").to_lowercase(); - Self::from_str(&tokenized).map_err(to_val_err) + Ok(Self::from_str(&tokenized)?) } } @@ -517,7 +517,7 @@ impl Encoding { fn py_from_str(_: &Bound, value: &Bound) -> PyResult { let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.to_lowercase(); - Self::from_str(&tokenized).map_err(to_val_err) + Ok(Self::from_str(&tokenized)?) } } @@ -573,7 +573,7 @@ impl Compression { fn py_from_str(_: &Bound, value: &Bound) -> PyResult { let value_str: String = value.str().and_then(|s| s.extract())?; let tokenized = value_str.to_lowercase(); - Self::from_str(&tokenized).map_err(to_val_err) + Ok(Self::from_str(&tokenized)?) } } @@ -583,10 +583,10 @@ impl SecurityUpdateAction { fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { let t = Self::type_object_bound(py); - let c = value.extract::().map_err(to_val_err)?; + let c = value.extract::().map_err(to_py_err)?; return Self::py_from_str(&t, c); }; - Self::try_from(i).map_err(to_val_err) + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -626,7 +626,7 @@ impl SecurityUpdateAction { #[classmethod] #[pyo3(name = "from_str")] fn py_from_str(_: &Bound, value: char) -> PyResult { - Self::try_from(value as u8).map_err(to_val_err) + Self::try_from(value as u8).map_err(to_py_err) } } @@ -634,8 +634,8 @@ impl SecurityUpdateAction { impl StatType { #[new] fn py_new(value: &Bound) -> PyResult { - let i = value.extract::().map_err(to_val_err)?; - Self::try_from(i).map_err(to_val_err) + let i = value.extract::().map_err(to_py_err)?; + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -668,8 +668,8 @@ impl StatType { impl StatusAction { #[new] fn py_new(value: &Bound) -> PyResult { - let i = value.extract::().map_err(to_val_err)?; - Self::try_from(i).map_err(to_val_err) + let i = value.extract::().map_err(to_py_err)?; + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -702,8 +702,8 @@ impl StatusAction { impl StatusReason { #[new] fn py_new(value: &Bound) -> PyResult { - let i = value.extract::().map_err(to_val_err)?; - Self::try_from(i).map_err(to_val_err) + let i = value.extract::().map_err(to_py_err)?; + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -736,8 +736,8 @@ impl StatusReason { impl TradingEvent { #[new] fn py_new(value: &Bound) -> PyResult { - let i = value.extract::().map_err(to_val_err)?; - Self::try_from(i).map_err(to_val_err) + let i = value.extract::().map_err(to_py_err)?; + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -772,10 +772,10 @@ impl TriState { fn py_new(py: Python<'_>, value: &Bound) -> PyResult { let Ok(i) = value.extract::() else { let t = Self::type_object_bound(py); - let c = value.extract::().map_err(to_val_err)?; + let c = value.extract::().map_err(to_py_err)?; return Self::py_from_str(&t, c); }; - Self::try_from(i).map_err(to_val_err) + Self::try_from(i).map_err(to_py_err) } fn __hash__(&self) -> isize { @@ -814,7 +814,7 @@ impl TriState { #[classmethod] #[pyo3(name = "from_str")] fn py_from_str(_: &Bound, value: char) -> PyResult { - Self::try_from(value as u8).map_err(to_val_err) + Self::try_from(value as u8).map_err(to_py_err) } } diff --git a/rust/dbn/src/python/metadata.rs b/rust/dbn/src/python/metadata.rs index df3f659..b544c44 100644 --- a/rust/dbn/src/python/metadata.rs +++ b/rust/dbn/src/python/metadata.rs @@ -15,7 +15,7 @@ use crate::{ MappingInterval, Metadata, SymbolMapping, VersionUpgradePolicy, }; -use super::{py_to_time_date, to_val_err}; +use super::{py_to_time_date, to_py_err}; #[pymethods] impl Metadata { @@ -87,8 +87,7 @@ impl Metadata { ) -> PyResult { let upgrade_policy = upgrade_policy.unwrap_or_default(); let reader = io::BufReader::new(data.as_bytes()); - let mut metadata = DynDecoder::inferred_with_buffer(reader, upgrade_policy) - .map_err(to_val_err)? + let mut metadata = DynDecoder::inferred_with_buffer(reader, upgrade_policy)? .metadata() .clone(); metadata.upgrade(upgrade_policy); @@ -99,7 +98,7 @@ impl Metadata { fn py_encode(&self, py: Python<'_>) -> PyResult> { let mut buffer = Vec::new(); let mut encoder = MetadataEncoder::new(&mut buffer); - encoder.encode(self).map_err(to_val_err)?; + encoder.encode(self)?; Ok(PyBytes::new_bound(py, buffer.as_slice()).into()) } } @@ -126,15 +125,15 @@ impl<'source> FromPyObject<'source> for MappingInterval { fn extract(ob: &'source PyAny) -> PyResult { let start_date = ob .getattr(intern!(ob.py(), "start_date")) - .map_err(|_| to_val_err("Missing start_date".to_owned())) + .map_err(|_| to_py_err("Missing start_date".to_owned())) .and_then(extract_date)?; let end_date = ob .getattr(intern!(ob.py(), "end_date")) - .map_err(|_| to_val_err("Missing end_date".to_owned())) + .map_err(|_| to_py_err("Missing end_date".to_owned())) .and_then(extract_date)?; let symbol = ob .getattr(intern!(ob.py(), "symbol")) - .map_err(|_| to_val_err("Missing symbol".to_owned())) + .map_err(|_| to_py_err("Missing symbol".to_owned())) .and_then(|d| d.extract::())?; Ok(Self { start_date, diff --git a/rust/dbn/src/python/record.rs b/rust/dbn/src/python/record.rs index 44cc7de..67ae212 100644 --- a/rust/dbn/src/python/record.rs +++ b/rust/dbn/src/python/record.rs @@ -17,7 +17,7 @@ use crate::{ UNDEF_ORDER_SIZE, UNDEF_PRICE, UNDEF_TIMESTAMP, }; -use super::{to_val_err, PyFieldDesc}; +use super::{to_py_err, PyFieldDesc}; #[pymethods] impl MboMsg { @@ -720,7 +720,7 @@ impl Mbp10Msg { let levels = if let Some(level) = levels { let mut arr: [BidAskPair; 10] = Default::default(); if level.len() > 10 { - return Err(to_val_err("Only 10 levels are allowed")); + return Err(to_py_err("Only 10 levels are allowed")); } for (i, level) in level.into_iter().enumerate() { arr[i] = level; @@ -1223,21 +1223,18 @@ impl InstrumentDefMsg { maturity_year: maturity_year.unwrap_or(u16::MAX), decay_start_date: decay_start_date.unwrap_or(u16::MAX), channel_id: channel_id.unwrap_or(u16::MAX), - currency: str_to_c_chars(currency.unwrap_or_default()).map_err(to_val_err)?, - settl_currency: str_to_c_chars(settl_currency.unwrap_or_default()) - .map_err(to_val_err)?, - secsubtype: str_to_c_chars(secsubtype.unwrap_or_default()).map_err(to_val_err)?, - raw_symbol: str_to_c_chars(raw_symbol).map_err(to_val_err)?, - group: str_to_c_chars(group).map_err(to_val_err)?, - exchange: str_to_c_chars(exchange).map_err(to_val_err)?, - asset: str_to_c_chars(asset.unwrap_or_default()).map_err(to_val_err)?, - cfi: str_to_c_chars(cfi.unwrap_or_default()).map_err(to_val_err)?, - security_type: str_to_c_chars(security_type.unwrap_or_default()).map_err(to_val_err)?, - unit_of_measure: str_to_c_chars(unit_of_measure.unwrap_or_default()) - .map_err(to_val_err)?, - underlying: str_to_c_chars(underlying.unwrap_or_default()).map_err(to_val_err)?, - strike_price_currency: str_to_c_chars(strike_price_currency.unwrap_or_default()) - .map_err(to_val_err)?, + currency: str_to_c_chars(currency.unwrap_or_default())?, + settl_currency: str_to_c_chars(settl_currency.unwrap_or_default())?, + secsubtype: str_to_c_chars(secsubtype.unwrap_or_default())?, + raw_symbol: str_to_c_chars(raw_symbol)?, + group: str_to_c_chars(group)?, + exchange: str_to_c_chars(exchange)?, + asset: str_to_c_chars(asset.unwrap_or_default())?, + cfi: str_to_c_chars(cfi.unwrap_or_default())?, + security_type: str_to_c_chars(security_type.unwrap_or_default())?, + unit_of_measure: str_to_c_chars(unit_of_measure.unwrap_or_default())?, + underlying: str_to_c_chars(underlying.unwrap_or_default())?, + strike_price_currency: str_to_c_chars(strike_price_currency.unwrap_or_default())?, instrument_class, strike_price: strike_price.unwrap_or(UNDEF_PRICE), match_algorithm, @@ -1401,73 +1398,73 @@ impl InstrumentDefMsg { #[getter] #[pyo3(name = "currency")] fn py_currency(&self) -> PyResult<&str> { - self.currency().map_err(to_val_err) + Ok(self.currency()?) } #[getter] #[pyo3(name = "settl_currency")] fn py_settl_currency(&self) -> PyResult<&str> { - self.settl_currency().map_err(to_val_err) + Ok(self.settl_currency()?) } #[getter] #[pyo3(name = "secsubtype")] fn py_secsubtype(&self) -> PyResult<&str> { - self.secsubtype().map_err(to_val_err) + Ok(self.secsubtype()?) } #[getter] #[pyo3(name = "raw_symbol")] fn py_raw_symbol(&self) -> PyResult<&str> { - self.raw_symbol().map_err(to_val_err) + Ok(self.raw_symbol()?) } #[getter] #[pyo3(name = "group")] fn py_group(&self) -> PyResult<&str> { - self.group().map_err(to_val_err) + Ok(self.group()?) } #[getter] #[pyo3(name = "exchange")] fn py_exchange(&self) -> PyResult<&str> { - self.exchange().map_err(to_val_err) + Ok(self.exchange()?) } #[getter] #[pyo3(name = "asset")] fn py_asset(&self) -> PyResult<&str> { - self.asset().map_err(to_val_err) + Ok(self.asset()?) } #[getter] #[pyo3(name = "cfi")] fn py_cfi(&self) -> PyResult<&str> { - self.cfi().map_err(to_val_err) + Ok(self.cfi()?) } #[getter] #[pyo3(name = "security_type")] fn py_security_type(&self) -> PyResult<&str> { - self.security_type().map_err(to_val_err) + Ok(self.security_type()?) } #[getter] #[pyo3(name = "unit_of_measure")] fn py_unit_of_measure(&self) -> PyResult<&str> { - self.unit_of_measure().map_err(to_val_err) + Ok(self.unit_of_measure()?) } #[getter] #[pyo3(name = "underlying")] fn py_underlying(&self) -> PyResult<&str> { - self.underlying().map_err(to_val_err) + Ok(self.underlying()?) } #[getter] #[pyo3(name = "strike_price_currency")] fn py_strike_price_currency(&self) -> PyResult<&str> { - self.strike_price_currency().map_err(to_val_err) + Ok(self.strike_price_currency()?) } #[getter] @@ -1631,21 +1628,18 @@ impl InstrumentDefMsgV1 { maturity_year: maturity_year.unwrap_or(u16::MAX), decay_start_date: decay_start_date.unwrap_or(u16::MAX), channel_id: channel_id.unwrap_or(u16::MAX), - currency: str_to_c_chars(currency.unwrap_or_default()).map_err(to_val_err)?, - settl_currency: str_to_c_chars(settl_currency.unwrap_or_default()) - .map_err(to_val_err)?, - secsubtype: str_to_c_chars(secsubtype.unwrap_or_default()).map_err(to_val_err)?, - raw_symbol: str_to_c_chars(raw_symbol).map_err(to_val_err)?, - group: str_to_c_chars(group).map_err(to_val_err)?, - exchange: str_to_c_chars(exchange).map_err(to_val_err)?, - asset: str_to_c_chars(asset.unwrap_or_default()).map_err(to_val_err)?, - cfi: str_to_c_chars(cfi.unwrap_or_default()).map_err(to_val_err)?, - security_type: str_to_c_chars(security_type.unwrap_or_default()).map_err(to_val_err)?, - unit_of_measure: str_to_c_chars(unit_of_measure.unwrap_or_default()) - .map_err(to_val_err)?, - underlying: str_to_c_chars(underlying.unwrap_or_default()).map_err(to_val_err)?, - strike_price_currency: str_to_c_chars(strike_price_currency.unwrap_or_default()) - .map_err(to_val_err)?, + currency: str_to_c_chars(currency.unwrap_or_default())?, + settl_currency: str_to_c_chars(settl_currency.unwrap_or_default())?, + secsubtype: str_to_c_chars(secsubtype.unwrap_or_default())?, + raw_symbol: str_to_c_chars(raw_symbol)?, + group: str_to_c_chars(group)?, + exchange: str_to_c_chars(exchange)?, + asset: str_to_c_chars(asset.unwrap_or_default())?, + cfi: str_to_c_chars(cfi.unwrap_or_default())?, + security_type: str_to_c_chars(security_type.unwrap_or_default())?, + unit_of_measure: str_to_c_chars(unit_of_measure.unwrap_or_default())?, + underlying: str_to_c_chars(underlying.unwrap_or_default())?, + strike_price_currency: str_to_c_chars(strike_price_currency.unwrap_or_default())?, instrument_class, strike_price: strike_price.unwrap_or(UNDEF_PRICE), match_algorithm, @@ -1813,73 +1807,73 @@ impl InstrumentDefMsgV1 { #[getter] #[pyo3(name = "currency")] fn py_currency(&self) -> PyResult<&str> { - self.currency().map_err(to_val_err) + Ok(self.currency()?) } #[getter] #[pyo3(name = "settl_currency")] fn py_settl_currency(&self) -> PyResult<&str> { - self.settl_currency().map_err(to_val_err) + Ok(self.settl_currency()?) } #[getter] #[pyo3(name = "secsubtype")] fn py_secsubtype(&self) -> PyResult<&str> { - self.secsubtype().map_err(to_val_err) + Ok(self.secsubtype()?) } #[getter] #[pyo3(name = "raw_symbol")] fn py_raw_symbol(&self) -> PyResult<&str> { - self.raw_symbol().map_err(to_val_err) + Ok(self.raw_symbol()?) } #[getter] #[pyo3(name = "group")] fn py_group(&self) -> PyResult<&str> { - self.group().map_err(to_val_err) + Ok(self.group()?) } #[getter] #[pyo3(name = "exchange")] fn py_exchange(&self) -> PyResult<&str> { - self.exchange().map_err(to_val_err) + Ok(self.exchange()?) } #[getter] #[pyo3(name = "asset")] fn py_asset(&self) -> PyResult<&str> { - self.asset().map_err(to_val_err) + Ok(self.asset()?) } #[getter] #[pyo3(name = "cfi")] fn py_cfi(&self) -> PyResult<&str> { - self.cfi().map_err(to_val_err) + Ok(self.cfi()?) } #[getter] #[pyo3(name = "security_type")] fn py_security_type(&self) -> PyResult<&str> { - self.security_type().map_err(to_val_err) + Ok(self.security_type()?) } #[getter] #[pyo3(name = "unit_of_measure")] fn py_unit_of_measure(&self) -> PyResult<&str> { - self.unit_of_measure().map_err(to_val_err) + Ok(self.unit_of_measure()?) } #[getter] #[pyo3(name = "underlying")] fn py_underlying(&self) -> PyResult<&str> { - self.underlying().map_err(to_val_err) + Ok(self.underlying()?) } #[getter] #[pyo3(name = "strike_price_currency")] fn py_strike_price_currency(&self) -> PyResult<&str> { - self.strike_price_currency().map_err(to_val_err) + Ok(self.strike_price_currency()?) } #[getter] @@ -2307,7 +2301,7 @@ impl ErrorMsg { #[getter] #[pyo3(name = "err")] fn py_err(&self) -> PyResult<&str> { - self.err().map_err(to_val_err) + Ok(self.err()?) } #[classattr] @@ -2403,7 +2397,7 @@ impl ErrorMsgV1 { #[getter] #[pyo3(name = "err")] fn py_err(&self) -> PyResult<&str> { - self.err().map_err(to_val_err) + Ok(self.err()?) } #[classattr] @@ -2459,9 +2453,9 @@ impl SymbolMappingMsg { ts_event, ), stype_in: stype_in as u8, - stype_in_symbol: str_to_c_chars(stype_in_symbol).map_err(to_val_err)?, + stype_in_symbol: str_to_c_chars(stype_in_symbol)?, stype_out: stype_out as u8, - stype_out_symbol: str_to_c_chars(stype_out_symbol).map_err(to_val_err)?, + stype_out_symbol: str_to_c_chars(stype_out_symbol)?, start_ts, end_ts, }) @@ -2534,25 +2528,25 @@ impl SymbolMappingMsg { #[getter] #[pyo3(name = "stype_in")] fn py_stype_in(&self) -> PyResult { - self.stype_in().map_err(to_val_err) + Ok(self.stype_in()?) } #[getter] #[pyo3(name = "stype_in_symbol")] fn py_stype_in_symbol(&self) -> PyResult<&str> { - self.stype_in_symbol().map_err(to_val_err) + Ok(self.stype_in_symbol()?) } #[getter] #[pyo3(name = "stype_out")] fn py_stype_out(&self) -> PyResult { - self.stype_out().map_err(to_val_err) + Ok(self.stype_out()?) } #[getter] #[pyo3(name = "stype_out_symbol")] fn py_stype_out_symbol(&self) -> PyResult<&str> { - self.stype_out_symbol().map_err(to_val_err) + Ok(self.stype_out_symbol()?) } #[classattr] @@ -2605,8 +2599,8 @@ impl SymbolMappingMsgV1 { instrument_id, ts_event, ), - stype_in_symbol: str_to_c_chars(stype_in_symbol).map_err(to_val_err)?, - stype_out_symbol: str_to_c_chars(stype_out_symbol).map_err(to_val_err)?, + stype_in_symbol: str_to_c_chars(stype_in_symbol)?, + stype_out_symbol: str_to_c_chars(stype_out_symbol)?, start_ts, end_ts, _dummy: Default::default(), @@ -2680,13 +2674,13 @@ impl SymbolMappingMsgV1 { #[getter] #[pyo3(name = "stype_in_symbol")] fn py_stype_in_symbol(&self) -> PyResult<&str> { - self.stype_in_symbol().map_err(to_val_err) + Ok(self.stype_in_symbol()?) } #[getter] #[pyo3(name = "stype_out_symbol")] fn py_stype_out_symbol(&self) -> PyResult<&str> { - self.stype_out_symbol().map_err(to_val_err) + Ok(self.stype_out_symbol()?) } #[classattr] @@ -2724,7 +2718,7 @@ impl SymbolMappingMsgV1 { impl SystemMsg { #[new] fn py_new(ts_event: u64, msg: &str) -> PyResult { - SystemMsg::new(ts_event, msg).map_err(to_val_err) + Ok(SystemMsg::new(ts_event, msg)?) } fn __bytes__(&self) -> &[u8] { @@ -2782,7 +2776,7 @@ impl SystemMsg { #[getter] #[pyo3(name = "msg")] fn py_msg(&self) -> PyResult<&str> { - self.msg().map_err(to_val_err) + Ok(self.msg()?) } #[pyo3(name = "is_heartbeat")] @@ -2825,7 +2819,7 @@ impl SystemMsg { impl SystemMsgV1 { #[new] fn py_new(ts_event: u64, msg: &str) -> PyResult { - Self::new(ts_event, msg).map_err(to_val_err) + Ok(Self::new(ts_event, msg)?) } fn __bytes__(&self) -> &[u8] { @@ -2883,7 +2877,7 @@ impl SystemMsgV1 { #[getter] #[pyo3(name = "msg")] fn py_msg(&self) -> PyResult<&str> { - self.msg().map_err(to_val_err) + Ok(self.msg()?) } #[pyo3(name = "is_heartbeat")] From 35078585e0a3e311848beb37d6bbc0731ea77c93 Mon Sep 17 00:00:00 2001 From: Zach Banks Date: Tue, 14 May 2024 18:23:54 -0400 Subject: [PATCH 16/23] DOC: Update release dates in changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28527ac..18dba40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog -## 0.18.0 - TBD +## 0.18.0 - 2024-05-14 + ### Enhancements - Added links to example usage in documentation - Added new predicate methods `InstrumentClass::is_option`, `is_future`, and `is_spread` From 4b5a05e1807d82cf4168dacb942a44be142779aa Mon Sep 17 00:00:00 2001 From: Nick Macholl Date: Tue, 14 May 2024 15:41:12 -0700 Subject: [PATCH 17/23] VER: Release 0.18.0 --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- python/pyproject.toml | 4 ++-- rust/dbn-cli/Cargo.toml | 2 +- rust/dbn/Cargo.toml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65bbb99..58de0f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,7 +271,7 @@ dependencies = [ [[package]] name = "databento-dbn" -version = "0.17.1" +version = "0.18.0" dependencies = [ "dbn", "pyo3", @@ -282,7 +282,7 @@ dependencies = [ [[package]] name = "dbn" -version = "0.17.1" +version = "0.18.0" dependencies = [ "async-compression", "csv", @@ -304,7 +304,7 @@ dependencies = [ [[package]] name = "dbn-c" -version = "0.17.1" +version = "0.18.0" dependencies = [ "anyhow", "cbindgen", @@ -314,7 +314,7 @@ dependencies = [ [[package]] name = "dbn-cli" -version = "0.17.1" +version = "0.18.0" dependencies = [ "anyhow", "assert_cmd", @@ -329,7 +329,7 @@ dependencies = [ [[package]] name = "dbn-macros" -version = "0.17.1" +version = "0.18.0" dependencies = [ "csv", "dbn", diff --git a/Cargo.toml b/Cargo.toml index 16bd3f4..ca1905a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ resolver = "2" [workspace.package] authors = ["Databento "] edition = "2021" -version = "0.17.1" +version = "0.18.0" documentation = "https://docs.databento.com" repository = "https://github.com/databento/dbn" license = "Apache-2.0" diff --git a/python/pyproject.toml b/python/pyproject.toml index 86e2b00..c3105d8 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databento-dbn" -version = "0.17.1" +version = "0.18.0" description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)" authors = ["Databento "] license = "Apache-2.0" @@ -17,7 +17,7 @@ build-backend = "maturin" [project] name = "databento-dbn" -version = "0.17.1" +version = "0.18.0" authors = [ { name = "Databento", email = "support@databento.com" } ] diff --git a/rust/dbn-cli/Cargo.toml b/rust/dbn-cli/Cargo.toml index e3b0129..cb8e9ea 100644 --- a/rust/dbn-cli/Cargo.toml +++ b/rust/dbn-cli/Cargo.toml @@ -16,7 +16,7 @@ name = "dbn" path = "src/main.rs" [dependencies] -dbn = { path = "../dbn", version = "=0.17.1", default-features = false } +dbn = { path = "../dbn", version = "=0.18.0", default-features = false } anyhow = { workspace = true } clap = { version = "4.5", features = ["derive", "wrap_help"] } diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index aacc72c..bc9be4b 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -25,7 +25,7 @@ serde = ["dep:serde", "time/parsing", "time/serde"] trivial_copy = [] [dependencies] -dbn-macros = { version = "=0.17.1", path = "../dbn-macros" } +dbn-macros = { version = "=0.18.0", path = "../dbn-macros" } async-compression = { version = "0.4.6", features = ["tokio", "zstd"], optional = true } csv = "1.3" From fd94e72cd77a3e7f0c140822d57fb6a85c07bc64 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 15 May 2024 09:45:54 -0500 Subject: [PATCH 18/23] BLD: Use stable Rust for DBN build --- .github/workflows/build.yaml | 31 ++++++++++++------ .github/workflows/release.yaml | 57 +++++++++++++++++++++------------- Cargo.lock | 10 +++--- Cargo.toml | 2 +- python/pyproject.toml | 4 +-- rust/dbn-cli/Cargo.toml | 2 +- rust/dbn/Cargo.toml | 2 +- 7 files changed, 66 insertions(+), 42 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3715857..158f72f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,6 +20,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt, clippy + # Cargo setup - name: Set up Cargo cache uses: actions/cache@v4 @@ -43,9 +50,6 @@ jobs: target: x86_64 args: --release --out dist --manifest-path python/Cargo.toml --interpreter python${{ matrix.python-version }} - - name: Install clippy and rustfmt - run: rustup component add clippy rustfmt - shell: bash - name: Format run: scripts/format.sh shell: bash @@ -71,6 +75,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt, clippy + # Cargo setup - name: Set up Cargo cache uses: actions/cache@v4 @@ -94,9 +105,6 @@ jobs: manylinux: auto args: --release --out dist --manifest-path python/Cargo.toml --interpreter python${{ matrix.python-version }} - - name: Install clippy and rustfmt - run: rustup component add clippy rustfmt - shell: bash - name: Format run: scripts/format.sh - name: Build @@ -118,6 +126,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt, clippy + # Cargo setup - name: Set up Cargo cache uses: actions/cache@v4 @@ -133,7 +148,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: x64 - name: Build wheels - x86_64 uses: messense/maturin-action@v1 @@ -146,9 +160,6 @@ jobs: with: args: --release --target universal2-apple-darwin --out dist --manifest-path python/Cargo.toml --interpreter python${{ matrix.python-version }} - - name: Install clippy and rustfmt - run: rustup component add clippy rustfmt - shell: bash - name: Format run: scripts/format.sh - name: Build diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 73f805c..68ff7dc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -22,28 +22,6 @@ jobs: with: fetch-depth: 2 - # Cargo setup - - name: Set up Cargo cache - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-x86_64-cargo-${{ hashFiles('Cargo.lock') }} - - # Python setup - - name: Set up Python environment - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - # Tag the commit with the library version - - name: Create git tag - uses: salsify/action-detect-and-tag-new-version@v2 - with: - version-command: scripts/get_version.sh - # Set release output variables - name: Set output id: vars @@ -75,6 +53,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt, clippy + # Cargo setup - name: Set up Cargo cache uses: actions/cache@v4 @@ -120,6 +105,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt, clippy + # Cargo setup - name: Set up Cargo cache uses: actions/cache@v4 @@ -161,6 +153,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt, clippy + # Cargo setup - name: Set up Cargo cache uses: actions/cache@v4 @@ -202,6 +201,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt, clippy + # Cargo setup - name: Set up Cargo cache uses: actions/cache@v4 @@ -279,6 +285,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + components: rustfmt, clippy + # Cargo setup - name: Set up Cargo cache uses: actions/cache@v4 diff --git a/Cargo.lock b/Cargo.lock index 58de0f1..65bbb99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,7 +271,7 @@ dependencies = [ [[package]] name = "databento-dbn" -version = "0.18.0" +version = "0.17.1" dependencies = [ "dbn", "pyo3", @@ -282,7 +282,7 @@ dependencies = [ [[package]] name = "dbn" -version = "0.18.0" +version = "0.17.1" dependencies = [ "async-compression", "csv", @@ -304,7 +304,7 @@ dependencies = [ [[package]] name = "dbn-c" -version = "0.18.0" +version = "0.17.1" dependencies = [ "anyhow", "cbindgen", @@ -314,7 +314,7 @@ dependencies = [ [[package]] name = "dbn-cli" -version = "0.18.0" +version = "0.17.1" dependencies = [ "anyhow", "assert_cmd", @@ -329,7 +329,7 @@ dependencies = [ [[package]] name = "dbn-macros" -version = "0.18.0" +version = "0.17.1" dependencies = [ "csv", "dbn", diff --git a/Cargo.toml b/Cargo.toml index ca1905a..16bd3f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ resolver = "2" [workspace.package] authors = ["Databento "] edition = "2021" -version = "0.18.0" +version = "0.17.1" documentation = "https://docs.databento.com" repository = "https://github.com/databento/dbn" license = "Apache-2.0" diff --git a/python/pyproject.toml b/python/pyproject.toml index c3105d8..86e2b00 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databento-dbn" -version = "0.18.0" +version = "0.17.1" description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)" authors = ["Databento "] license = "Apache-2.0" @@ -17,7 +17,7 @@ build-backend = "maturin" [project] name = "databento-dbn" -version = "0.18.0" +version = "0.17.1" authors = [ { name = "Databento", email = "support@databento.com" } ] diff --git a/rust/dbn-cli/Cargo.toml b/rust/dbn-cli/Cargo.toml index cb8e9ea..e3b0129 100644 --- a/rust/dbn-cli/Cargo.toml +++ b/rust/dbn-cli/Cargo.toml @@ -16,7 +16,7 @@ name = "dbn" path = "src/main.rs" [dependencies] -dbn = { path = "../dbn", version = "=0.18.0", default-features = false } +dbn = { path = "../dbn", version = "=0.17.1", default-features = false } anyhow = { workspace = true } clap = { version = "4.5", features = ["derive", "wrap_help"] } diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index bc9be4b..aacc72c 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -25,7 +25,7 @@ serde = ["dep:serde", "time/parsing", "time/serde"] trivial_copy = [] [dependencies] -dbn-macros = { version = "=0.18.0", path = "../dbn-macros" } +dbn-macros = { version = "=0.17.1", path = "../dbn-macros" } async-compression = { version = "0.4.6", features = ["tokio", "zstd"], optional = true } csv = "1.3" From 5caf4dcd243b2c9a2999c2935eec6ad34403eb09 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Thu, 9 May 2024 11:17:02 -0500 Subject: [PATCH 19/23] OPT: Optimize async file decoding --- CHANGELOG.md | 1 + Cargo.lock | 4 ++-- rust/dbn/Cargo.toml | 2 +- rust/dbn/src/decode.rs | 8 +++++++- rust/dbn/src/decode/dbn/async.rs | 6 +++--- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18dba40..8f3ad11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Improved documentation enumerating errors returned by functions - Added new `DBNError` Python exception that's now the primary exception raised by `databento_dbn` +- Improved async performance of decoding DBN files ### Breaking changes - Changed type of `flags` in `MboMsg`, `TradeMsg`, `Mbp1Msg`, `Mbp10Msg`, and `CbboMsg` diff --git a/Cargo.lock b/Cargo.lock index 65bbb99..837d9cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.6" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" +checksum = "9c90a406b4495d129f00461241616194cb8a032c8d1c53c657f0961d5f8e0498" dependencies = [ "futures-core", "memchr", diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index aacc72c..b61d011 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -27,7 +27,7 @@ trivial_copy = [] [dependencies] dbn-macros = { version = "=0.17.1", path = "../dbn-macros" } -async-compression = { version = "0.4.6", features = ["tokio", "zstd"], optional = true } +async-compression = { version = "0.4.10", features = ["tokio", "zstd"], optional = true } csv = "1.3" fallible-streaming-iterator = { version = "0.1.9", features = ["std"] } # Fast integer to string conversion diff --git a/rust/dbn/src/decode.rs b/rust/dbn/src/decode.rs index fb67176..5f7ccad 100644 --- a/rust/dbn/src/decode.rs +++ b/rust/dbn/src/decode.rs @@ -566,6 +566,8 @@ mod r#async { io::{self, BufReader}, }; + pub(crate) const ZSTD_FILE_BUFFER_CAPACITY: usize = 1 << 20; + use crate::enums::Compression; /// A type for runtime polymorphism on compressed and uncompressed input. @@ -666,7 +668,11 @@ mod r#async { ), ) })?; - DynReader::new_inferred(file).await + DynReader::inferred_with_buffer(BufReader::with_capacity( + ZSTD_FILE_BUFFER_CAPACITY, + file, + )) + .await } } diff --git a/rust/dbn/src/decode/dbn/async.rs b/rust/dbn/src/decode/dbn/async.rs index cab67ec..b4ef474 100644 --- a/rust/dbn/src/decode/dbn/async.rs +++ b/rust/dbn/src/decode/dbn/async.rs @@ -8,7 +8,7 @@ use tokio::{ use crate::{ compat, - decode::{FromLittleEndianSlice, VersionUpgradePolicy}, + decode::{r#async::ZSTD_FILE_BUFFER_CAPACITY, FromLittleEndianSlice, VersionUpgradePolicy}, HasRType, Metadata, Record, RecordHeader, RecordRef, Result, DBN_VERSION, METADATA_FIXED_LEN, }; @@ -192,7 +192,7 @@ impl Decoder> { format!("opening DBN file at path '{}'", path.as_ref().display()), ) })?; - Self::new(BufReader::new(file)).await + Self::new(BufReader::with_capacity(ZSTD_FILE_BUFFER_CAPACITY, file)).await } } @@ -217,7 +217,7 @@ impl Decoder>> { ), ) })?; - Self::with_zstd(file).await + Self::with_zstd_buffer(BufReader::with_capacity(ZSTD_FILE_BUFFER_CAPACITY, file)).await } } From a7987876d335139ed23fdd81aaa28f02bb62cd9d Mon Sep 17 00:00:00 2001 From: Carter Green Date: Fri, 17 May 2024 14:36:24 -0500 Subject: [PATCH 20/23] ADD: Add missing ts_in_delta method for StatMsg --- CHANGELOG.md | 2 ++ rust/dbn/src/record/methods.rs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f3ad11..f8bab45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - Added new `DBNError` Python exception that's now the primary exception raised by `databento_dbn` - Improved async performance of decoding DBN files +- Added `StatMsg::ts_in_delta()` method that returns a `time::Duration` for consistency + with other records with a `ts_in_delta` field ### Breaking changes - Changed type of `flags` in `MboMsg`, `TradeMsg`, `Mbp1Msg`, `Mbp10Msg`, and `CbboMsg` diff --git a/rust/dbn/src/record/methods.rs b/rust/dbn/src/record/methods.rs index ffcd09d..8d77b88 100644 --- a/rust/dbn/src/record/methods.rs +++ b/rust/dbn/src/record/methods.rs @@ -669,6 +669,11 @@ impl StatMsg { Error::conversion::(format!("{:04X}", self.update_action)) }) } + + /// Parses the raw `ts_in_delta`—the delta of `ts_recv - ts_exchange_send`—into a duration. + pub fn ts_in_delta(&self) -> time::Duration { + time::Duration::new(0, self.ts_in_delta) + } } impl ErrorMsgV1 { From ace310cebd28ae3f950eb5645c74043bfda76934 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Tue, 9 Apr 2024 14:59:30 -0500 Subject: [PATCH 21/23] MOD: Modify display of definition fields --- CHANGELOG.md | 11 ++++++++--- rust/dbn/src/compat.rs | 2 ++ rust/dbn/src/encode/json/sync.rs | 4 ++-- rust/dbn/src/python.rs | 5 +++-- rust/dbn/src/record.rs | 1 + 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8bab45..b07ce16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,8 +32,13 @@ `u8::MAX` to match C++ client and to reflect an unknown value. This also changes the value of these fields when upgrading a `SymbolMappingMsgV1` to DBNv2 -## 0.17.1 - 2024-04-04 +### Breaking changes +- Changed text serialization of `display_factor` to be affected by `pretty_px`. + While it's not a price, it uses the same fixed-price decimal format as other prices +- Changed text serialization of `unit_of_measure_qty` in `InstrumentDefMsgV1` to be + affected by `pretty_px` to match behavior of `InstrumentDefMsgV2` +## 0.17.1 - 2024-04-04 ### Bug fixes - Added missing Python type stub for `StatusMsg` @@ -97,8 +102,8 @@ `DBNDecoder` to `Upgrade` so by default the primary record types can always be used - Changed fields of previously-hidden `StatusMsg` record type - Updated text serialization order of status schema to match other schemas -- Changed text serialization `unit_of_measure_qty` to be affected by `pretty_px`. While - it's not a price, it uses the same fixed-price decimal format as other prices +- Changed text serialization of `unit_of_measure_qty` to be affected by `pretty_px`. + While it's not a price, it uses the same fixed-price decimal format as other prices - Made `StatType` and `VersionUpgradePolicy` non-exhaustive to allow future additions without breaking changes - Renamed `_dummy` field in `ImbalanceMsg` and `StatMsg` to `_reserved` diff --git a/rust/dbn/src/compat.rs b/rust/dbn/src/compat.rs index dccf964..a0a6c23 100644 --- a/rust/dbn/src/compat.rs +++ b/rust/dbn/src/compat.rs @@ -126,6 +126,7 @@ pub struct InstrumentDefMsgV1 { #[pyo3(get, set)] pub min_price_increment: i64, /// The multiplier to convert the venue’s display price to the conventional price. + #[dbn(fixed_price)] #[pyo3(get, set)] pub display_factor: i64, /// The last eligible trade time expressed as a number of nanoseconds since the @@ -158,6 +159,7 @@ pub struct InstrumentDefMsgV1 { #[pyo3(get, set)] pub trading_reference_price: i64, /// The contract size for each instrument, in combination with `unit_of_measure`. + #[dbn(fixed_price)] #[pyo3(get, set)] pub unit_of_measure_qty: i64, /// The value currently under development by the venue. Converted to units of 1e-9, i.e. diff --git a/rust/dbn/src/encode/json/sync.rs b/rust/dbn/src/encode/json/sync.rs index 1b970f9..cfb11c6 100644 --- a/rust/dbn/src/encode/json/sync.rs +++ b/rust/dbn/src/encode/json/sync.rs @@ -448,7 +448,7 @@ mod tests { hd: RECORD_HEADER, ts_recv: 1658441891000000000, min_price_increment: 100, - display_factor: 1000, + display_factor: 1_000_000_000, expiration: 1698450000000000000, activation: 1697350000000000000, high_limit_price: 1_000_000, @@ -519,7 +519,7 @@ mod tests { r#""ts_recv":"2022-07-21T22:18:11.000000000Z""#, r#""hd":{"ts_event":"2022-07-21T22:17:31.000000000Z","rtype":4,"publisher_id":1,"instrument_id":323}"#, concat!( - r#""raw_symbol":"ESZ4 C4100","security_update_action":"A","instrument_class":"C","min_price_increment":"0.000000100","display_factor":"1000","expiration":"2023-10-27T23:40:00.000000000Z","activation":"2023-10-15T06:06:40.000000000Z","#, + r#""raw_symbol":"ESZ4 C4100","security_update_action":"A","instrument_class":"C","min_price_increment":"0.000000100","display_factor":"1.000000000","expiration":"2023-10-27T23:40:00.000000000Z","activation":"2023-10-15T06:06:40.000000000Z","#, r#""high_limit_price":"0.001000000","low_limit_price":"-0.001000000","max_price_variation":"0.000000000","trading_reference_price":"0.000500000","unit_of_measure_qty":"5.000000000","#, r#""min_price_increment_amount":"0.000000005","price_ratio":"0.000000010","inst_attrib_value":10,"underlying_id":256785,"raw_instrument_id":323,"market_depth_implied":0,"#, r#""market_depth":13,"market_segment_id":0,"max_trade_vol":10000,"min_lot_size":1,"min_lot_size_block":1000,"min_lot_size_round_lot":100,"min_trade_vol":1,"#, diff --git a/rust/dbn/src/python.rs b/rust/dbn/src/python.rs index 5205a7c..2a490a4 100644 --- a/rust/dbn/src/python.rs +++ b/rust/dbn/src/python.rs @@ -335,8 +335,8 @@ mod tests { #[test] fn test_cbbo_fields() { let mut exp_price = vec!["price".to_owned()]; - exp_price.push(format!("bid_px_00")); - exp_price.push(format!("ask_px_00")); + exp_price.push("bid_px_00".to_owned()); + exp_price.push("ask_px_00".to_owned()); assert_eq!(CbboMsg::price_fields(""), exp_price); assert_eq!( CbboMsg::hidden_fields(""), @@ -448,6 +448,7 @@ mod tests { InstrumentDefMsg::price_fields(""), vec![ "min_price_increment".to_owned(), + "display_factor".to_owned(), "high_limit_price".to_owned(), "low_limit_price".to_owned(), "max_price_variation".to_owned(), diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index 329ae3c..ea53984 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -556,6 +556,7 @@ pub struct InstrumentDefMsg { pub min_price_increment: i64, /// The multiplier to convert the venue’s display price to the conventional price, /// in units of 1e-9, i.e. 1/1,000,000,000 or 0.000000001. + #[dbn(fixed_price)] #[pyo3(get, set)] pub display_factor: i64, /// The last eligible trade time expressed as a number of nanoseconds since the From 89a492fcffde6e96b4f154b4f391eee1bdee74fa Mon Sep 17 00:00:00 2001 From: Carter Green Date: Mon, 20 May 2024 11:35:57 -0500 Subject: [PATCH 22/23] MOD: Rename `CbboMsg` to `CBBOMsg` in Python --- CHANGELOG.md | 1 + python/python/databento_dbn/_lib.pyi | 6 +++--- rust/dbn/src/record.rs | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b07ce16..543e46d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - Changed default value for `stype_in` and `stype_out` in `SymbolMappingMsg` to `u8::MAX` to match C++ client and to reflect an unknown value. This also changes the value of these fields when upgrading a `SymbolMappingMsgV1` to DBNv2 +- Renamed `CbboMsg` to `CBBOMsg` in Python for consistency with other schemas ### Breaking changes - Changed text serialization of `display_factor` to be affected by `pretty_px`. diff --git a/python/python/databento_dbn/_lib.pyi b/python/python/databento_dbn/_lib.pyi index 0b429d1..92276a4 100644 --- a/python/python/databento_dbn/_lib.pyi +++ b/python/python/databento_dbn/_lib.pyi @@ -26,7 +26,7 @@ _DBNRecord = Union[ Metadata, MBOMsg, MBP1Msg, - CbboMsg, + CBBOMsg, MBP10Msg, OHLCVMsg, TradeMsg, @@ -1714,7 +1714,7 @@ class MBP1Msg(Record, _MBPBase): """ -class CbboMsg(Record): +class CBBOMsg(Record): """ Consolidated best bid and offer implementation. """ @@ -1863,7 +1863,7 @@ class CbboMsg(Record): Notes ----- - CbboMsg contains 1 level of ConsolidatedBidAskPair. + CBBOMsg contains 1 level of ConsolidatedBidAskPair. """ diff --git a/rust/dbn/src/record.rs b/rust/dbn/src/record.rs index ea53984..fa42596 100644 --- a/rust/dbn/src/record.rs +++ b/rust/dbn/src/record.rs @@ -366,7 +366,7 @@ pub struct Mbp10Msg { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( feature = "python", - pyo3::pyclass(set_all, dict, module = "databento_dbn", name = "CbboMsg"), + pyo3::pyclass(set_all, dict, module = "databento_dbn", name = "CBBOMsg"), derive(crate::macros::PyFieldDesc) )] #[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope From e9226da9360e79ed461b94aed8393337b5a2237f Mon Sep 17 00:00:00 2001 From: Carter Green Date: Mon, 20 May 2024 09:25:38 -0500 Subject: [PATCH 23/23] VER: Release 0.18.0 --- CHANGELOG.md | 2 +- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- python/pyproject.toml | 4 ++-- rust/dbn-cli/Cargo.toml | 2 +- rust/dbn/Cargo.toml | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 543e46d..0cecf8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.18.0 - 2024-05-14 +## 0.18.0 - 2024-05-21 ### Enhancements - Added links to example usage in documentation diff --git a/Cargo.lock b/Cargo.lock index 837d9cf..d4b4920 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,7 +271,7 @@ dependencies = [ [[package]] name = "databento-dbn" -version = "0.17.1" +version = "0.18.0" dependencies = [ "dbn", "pyo3", @@ -282,7 +282,7 @@ dependencies = [ [[package]] name = "dbn" -version = "0.17.1" +version = "0.18.0" dependencies = [ "async-compression", "csv", @@ -304,7 +304,7 @@ dependencies = [ [[package]] name = "dbn-c" -version = "0.17.1" +version = "0.18.0" dependencies = [ "anyhow", "cbindgen", @@ -314,7 +314,7 @@ dependencies = [ [[package]] name = "dbn-cli" -version = "0.17.1" +version = "0.18.0" dependencies = [ "anyhow", "assert_cmd", @@ -329,7 +329,7 @@ dependencies = [ [[package]] name = "dbn-macros" -version = "0.17.1" +version = "0.18.0" dependencies = [ "csv", "dbn", diff --git a/Cargo.toml b/Cargo.toml index 16bd3f4..ca1905a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ resolver = "2" [workspace.package] authors = ["Databento "] edition = "2021" -version = "0.17.1" +version = "0.18.0" documentation = "https://docs.databento.com" repository = "https://github.com/databento/dbn" license = "Apache-2.0" diff --git a/python/pyproject.toml b/python/pyproject.toml index 86e2b00..c3105d8 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databento-dbn" -version = "0.17.1" +version = "0.18.0" description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)" authors = ["Databento "] license = "Apache-2.0" @@ -17,7 +17,7 @@ build-backend = "maturin" [project] name = "databento-dbn" -version = "0.17.1" +version = "0.18.0" authors = [ { name = "Databento", email = "support@databento.com" } ] diff --git a/rust/dbn-cli/Cargo.toml b/rust/dbn-cli/Cargo.toml index e3b0129..cb8e9ea 100644 --- a/rust/dbn-cli/Cargo.toml +++ b/rust/dbn-cli/Cargo.toml @@ -16,7 +16,7 @@ name = "dbn" path = "src/main.rs" [dependencies] -dbn = { path = "../dbn", version = "=0.17.1", default-features = false } +dbn = { path = "../dbn", version = "=0.18.0", default-features = false } anyhow = { workspace = true } clap = { version = "4.5", features = ["derive", "wrap_help"] } diff --git a/rust/dbn/Cargo.toml b/rust/dbn/Cargo.toml index b61d011..2ad40eb 100644 --- a/rust/dbn/Cargo.toml +++ b/rust/dbn/Cargo.toml @@ -25,7 +25,7 @@ serde = ["dep:serde", "time/parsing", "time/serde"] trivial_copy = [] [dependencies] -dbn-macros = { version = "=0.17.1", path = "../dbn-macros" } +dbn-macros = { version = "=0.18.0", path = "../dbn-macros" } async-compression = { version = "0.4.10", features = ["tokio", "zstd"], optional = true } csv = "1.3"