diff --git a/atelier/src/data/README.md b/atelier/src/data/README.md new file mode 100644 index 0000000..4c41869 --- /dev/null +++ b/atelier/src/data/README.md @@ -0,0 +1,2 @@ +# Data Module + diff --git a/atelier/src/data/admin.rs b/atelier/src/data/admin.rs new file mode 100644 index 0000000..c2f9e46 --- /dev/null +++ b/atelier/src/data/admin.rs @@ -0,0 +1,9 @@ +/// Admin types and implementations + +#[derive(Debug, Clone, PartialEq, PartialOrd)] +pub struct User { + pub first_name: String, + pub last_name: String, + pub email: String, +} + diff --git a/atelier/src/data/hasher.rs b/atelier/src/data/hasher.rs new file mode 100644 index 0000000..8f5ca7b --- /dev/null +++ b/atelier/src/data/hasher.rs @@ -0,0 +1,23 @@ +/// Hashing generating functions +/// +/// in order to create order_id, and, user_id hashed values with +/// unicity and atomicity operative properties. + +use crate::data::market::Order; +use std::hash::{DefaultHasher, Hash, Hasher}; +use crate::data::admin; + +impl Hash for admin::User { + fn hash(&self, state: &mut H) { + self.first_name.hash(state); + self.last_name.hash(state); + self.email.hash(state); + } +} + +fn hash_user(user: &admin::User) -> u64 { + let mut hasher = DefaultHasher::new(); + user.hash(&mut hasher); + hasher.finish() +} + diff --git a/atelier/src/data/mod.rs b/atelier/src/data/mod.rs index 60e5a3f..918d38e 100644 --- a/atelier/src/data/mod.rs +++ b/atelier/src/data/mod.rs @@ -1 +1,9 @@ +/// Administrative types like users, symbols, etc. +pub mod admin; + +/// Market related types, like Order, Level, Orderbook. pub mod market; + +/// Hashing types and functions. +pub mod hasher; + diff --git a/atelier/src/engine/README.md b/atelier/src/engine/README.md index 78b2cff..f0893a6 100644 --- a/atelier/src/engine/README.md +++ b/atelier/src/engine/README.md @@ -1,14 +1,30 @@ # Engine Module -Is the one in charge from the definition of communication and processing channels, to the creation and management of the queues, the routers for async order management to the logic of the matching engine. It is comprehended by the following: +Within this module exists the definition of the validation processes for income events, routers for async order management, the logic of the matching engine. This module is organized in the following files: -1. `channels.rs`: Communication and Processing channels. -2. `queues.rs`: Order Queues. -3. `management.rs`: Order Management Interface. -4. `routers.rs`: Order Routing Algorithms. -5. `matching.rs`: Matching Engine. +1. `validation.rs`: +2. `routing.rs`: Order Routing Algorithms. +3. `matching.rs`: Matching Engine. -## Communication and Processing Channels `channels.rs` +## System Design Taxonomy + +Concurrent Publisher/Subscriber Pattern that uses unbounded and multi-producer/multi-consumer channels, with atomic events as messages, A routing algorithm to gather and inject the event's data into an execution queue to be consumed by an execution engine that ultimately alters the state represented in the Limit Order Book data structure. + +## General Description + +- Inbound Events Pipeline (connectors, channels, producers): + +An External Data Source produces data, then a connector acting as the Internal API endpoint fetches that data, either through a querie using a REST connector, or through a received data package using a WebSocket connector. Then data goes through a validation/pre-processing layer (to make it compliant with the event-message schema). After validation, data is pre-processed for compliance with the event-message schema and passed into a `Publisher` so it can publish the event into the corresponding channel. + +- Internal Routing: + +Since there are different channels to take events from, an Order Routing Strategy will decide the order of the channels and the amount of events to be extracted from each of them, to then be sent to the Global Executable Tasks Queue. + +- Execution: + +From the Executable Tasks Queue, it will consume tasks acording to the Execution Strategy, and if successfully executed, it will effectively change the state of the Order Book. + +## Communication Channels `channels.rs` *(pending)* diff --git a/atelier/src/engine/mod.rs b/atelier/src/engine/mod.rs index 4d147cd..4b18b0c 100644 --- a/atelier/src/engine/mod.rs +++ b/atelier/src/engine/mod.rs @@ -1,14 +1,8 @@ -/// Communication and Processing Channels -pub mod channels; - -/// Queues for the order management -pub mod queues; - -/// Order Management Interface -pub mod management; +/// Routers for the order flow +pub mod validation; /// Routers for the order flow -pub mod routers; +pub mod routing; /// Matching engine logic pub mod matching; diff --git a/atelier/src/engine/queues.rs b/atelier/src/engine/queues.rs deleted file mode 100644 index ff7bd09..0000000 --- a/atelier/src/engine/queues.rs +++ /dev/null @@ -1 +0,0 @@ -// placeholder diff --git a/atelier/src/engine/routers.rs b/atelier/src/engine/routing.rs similarity index 100% rename from atelier/src/engine/routers.rs rename to atelier/src/engine/routing.rs diff --git a/atelier/src/engine/management.rs b/atelier/src/engine/validation.rs similarity index 100% rename from atelier/src/engine/management.rs rename to atelier/src/engine/validation.rs diff --git a/atelier/src/events/README.md b/atelier/src/events/README.md index ac8561e..6370cdb 100644 --- a/atelier/src/events/README.md +++ b/atelier/src/events/README.md @@ -1,28 +1,98 @@ -# Events +# Events Module -The current version supports 4 types of Market Events, as specified in the `MarketEventType` enum: +## Folder structure + +``` +Events/ +├── channels.rs +├── mod.rs +├── publishers.rs +├── subscribers.rs +└── templates.rs +``` + +1. `channels.rs`: Where events are published to, and/or, subscribed from. +2. `publishers.rs`: The publishers of the events. +3. `subscribers.rs` The subscribers of the events. +4. `templates.rs`: Definition of templates for the events and other uses. + +## Market Event Types + +These are defined within the `src/events/templates.rs` script, as variants of the `MarketEventType` enum, which is being created dynamically with a macro: 1. `CancelLimitOrder`: To cancel a single and specified limit order. 2. `NewMarketOrder`: To create a single new market order. -3. `ModifyLimitOrder`: To modify a single and identified limit order (only the amount). +3. `ModifyLimitOrder`: To modify only the amount for a single and id-identified, limit order. 4. `NewLimitOrder`: To create a single new limit order. -Each of the events has a generation `random__template` process that serves as a pseudo-random -generator of instances of events for testing purposes. +## Market Event + +### Main structure + +The struct that is being carried as the container of the actual market event, defined in `src/events/templates.rs`, it has the +following structure: + +``` +MarketEvent { + event_info: EventInfo, + event_content: EventContent, +} +``` +### Event's information (kind of a meta-data) type structure + +``` +EventInfo { + event_received_ts, + event_type, + user_Id, +} +``` + +### Event's content type symbolic structure + +``` +EventContent { + OrderCreation, + OrderCancelation, + OrderModification, +} +``` + +The necessary contents to be included in the `EventContent` given each of the `MarketEventType` are defined like this: + +| EventInfo::event_type | EventContent::event_object | +|------------------------------------|--------------------------------| +| MarketEventType::CancelLimitOrder | order_id | +| MarketEventType::NewMarketOrder | market::Order | +| MarketEventType::ModifyLimitOrder | order_id, order_amount | +| MarketEventType::NewLimitOrder | market::Order + +With both the `order_id` and `order_amount` being the same type as the one in the `market::Order` struct. + + +# Channels + +Within the `src/engine/channels.rs`, for each `MarketEventType`, there will be one particular channel: + +1. A `CancelLimitOrder` event goes into the `cancel_lo_channel`. +2. A NewMarketOrder event goes into the `new_mo_channel`. +3. A `ModifyLimitOrder` event goes into the `modify_lo_channel`. +4. A `NewLimitOrder` event goes into the `new_lo_channel`. + +Each channel will be unbounded and supports multiple-producers/multiple-consumers logic. + +# Publishers -For each type of the above market events, there will be one channel: +A market participant (exchange client) is the publisher of the MarketEvents, this interaction will be represented by the +pseudo-random generation of such `MarketEvents` by a collection of `generator` instances. Any of these can generate MarketEvents +of any type, and thus, to publish in any of the corresponding channel. -1. A `CancelLimitOrder` event goes into the `cancel_lo_channel` -2. A NewMarketOrder event goes into the `new_mo_channel` -3. A `ModifyLimitOrder` event goes into the `modify_lo_channel` -4. A `NewLimitOrder` event goes into the `new_lo_channel` +There can be more than 1 `Publisher` at/during any given point/period of time. -There is an `event_stream` process that can produce any number of random instances for one or all of the -different market events (as specified by `event_template`), to then use the stream for for testing purposes -of one or all of the created `channel` instances. +# Subscribers -Each channel will be unbounded, and also will support. - -- Multiple producers: Can receive multiple market events (of the same type). -- Multi consumers: Can be queried by multiple execution processes (spawned by the matching engine). +Every `MarketEvent` published by a `Publisher` will pass through an efficient `OrderValidity` process, then will be injected into a channel, +from which eventually it will be picked by an instance of an `OrderRouting` process (as a subscriber), which ultimately is in charge of +applying the `Routing Algorithm` to send the `MarketEvent` to the `ExecutionQueue` and have the `MatchingEngine` execute it and update +the state of the `LimitOrderBook`. diff --git a/atelier/src/engine/channels.rs b/atelier/src/events/channels.rs similarity index 52% rename from atelier/src/engine/channels.rs rename to atelier/src/events/channels.rs index bfe8379..fa67625 100644 --- a/atelier/src/engine/channels.rs +++ b/atelier/src/events/channels.rs @@ -1,4 +1,6 @@ -use crate::events::templates::MarketEventType; +/// Channels + +use crate::events::message::MarketEventType; use crossbeam::channel::unbounded; pub fn create_channel() -> ( @@ -10,6 +12,11 @@ pub fn create_channel() -> ( } fn main() { - let (cancel_lo_queue_s, cancel_lo_queue_r) = create_channel(); - let (new_mo_queue_s, cancel_mo_queue_r) = create_channel(); + + let (cancel_lo_s, cancel_lo_r) = create_channel(); + let (new_mo_s, new_mo_r) = create_channel(); + let (modify_lo_s, modify_lo_r) = create_channel(); + let (new_mo_s, new_mo_r) = create_channel(); + } + diff --git a/atelier/src/events/message.rs b/atelier/src/events/message.rs new file mode 100644 index 0000000..6b072a7 --- /dev/null +++ b/atelier/src/events/message.rs @@ -0,0 +1,103 @@ +/// Market event generator module + +use crate::data::market; +use crate::generators; +use crate::messages::errors; + +use rand::seq::SliceRandom; +use rand::thread_rng; +use std::time::{SystemTime, UNIX_EPOCH}; + +// ---------------------------------------------------------------------- EventType -- // +// ---------------------------------------------------------------------- --------- -- // + +#[macro_export] +macro_rules! enum_create { + ($enum_name:ident, $($variant:ident),+) => { + + #[derive(Debug, Clone, PartialEq, PartialOrd)] + + pub enum $enum_name { + $($variant),+ + } + + impl $enum_name { + + fn variants() -> Vec { + vec![$(Self::$variant),+] + } + + fn random_variants(n_choice:usize) -> Vec { + let mut rng = thread_rng(); + Self::variants() + .choose_multiple(&mut rng, n_choice) + .cloned() + .collect() + } + } + } +} + +// -- Instantiate a MarketEvent Type Enum -- // + +enum_create!( + MarketEventType, + CancelLimitOrder, + NewMarketOrder, + ModifyLimitOrder, + NewLimitOrder +); + +// ------------------------------------------------------- Market Event Info Struct -- // +// ------------------------------------------------------- ------------------------ -- // + +#[derive(Debug)] +pub struct EventInfo { + pub event_received_ts: u128, + pub event_type: MarketEventType, + pub user_id: u32, +} + +impl EventInfo { + pub fn new( + event_received_ts: u128, + event_type: MarketEventType, + user_id: u32, + ) -> Self { + EventInfo { + event_received_ts, + event_type, + user_id, + } + } +} + +// ---------------------------------------------------- Market Event Content Struct -- // +// ---------------------------------------------------- --------------------------- -- // + +// Different market events can have different structures for the event_object. +#[derive(Debug)] +pub enum EventContent { + OrderCreation(market::Order), + OrderCancellation(u32), + OrderModification(u32, f64), +} + +// ------------------------------------------------------------ Market Event Struct -- // +// ------------------------------------------------------------ ------------------- -- // + +#[derive(Debug)] +pub struct MarketEvent { + pub event_info: EventInfo, + pub event_content: EventContent, +} + +impl MarketEvent { + pub fn new(event_info: EventInfo, event_content: EventContent) -> Self { + MarketEvent { + event_info, + event_content, + } + } +} + diff --git a/atelier/src/events/mod.rs b/atelier/src/events/mod.rs index f11a69b..7e73386 100644 --- a/atelier/src/events/mod.rs +++ b/atelier/src/events/mod.rs @@ -1,2 +1,14 @@ -pub mod streams; +/// channels where events are published to, and/or, subscribed from +pub mod channels; + +/// message struct and formation +pub mod message; + +/// publisher of the events. +pub mod publishers; + +/// subscriber of the events. +pub mod subscribers; + +/// Definition of the base templates for the events and other uses. pub mod templates; diff --git a/atelier/src/events/publishers.rs b/atelier/src/events/publishers.rs new file mode 100644 index 0000000..b074a08 --- /dev/null +++ b/atelier/src/events/publishers.rs @@ -0,0 +1,3 @@ +// Publishers + + diff --git a/atelier/src/events/streams.rs b/atelier/src/events/streams.rs deleted file mode 100644 index ff7bd09..0000000 --- a/atelier/src/events/streams.rs +++ /dev/null @@ -1 +0,0 @@ -// placeholder diff --git a/atelier/src/events/subscribers.rs b/atelier/src/events/subscribers.rs new file mode 100644 index 0000000..d782b45 --- /dev/null +++ b/atelier/src/events/subscribers.rs @@ -0,0 +1 @@ +// Subscribers diff --git a/atelier/src/events/templates.rs b/atelier/src/events/templates.rs index 7fddf09..44621a2 100644 --- a/atelier/src/events/templates.rs +++ b/atelier/src/events/templates.rs @@ -1,12 +1,13 @@ -/// Market event generator module use crate::data::market; use crate::generators; use crate::messages::errors; +use crate::events::message; use rand::seq::SliceRandom; use rand::thread_rng; use std::time::{SystemTime, UNIX_EPOCH}; +<<<<<<< HEAD // ---------------------------------------------------------------------- EventType -- // // ---------------------------------------------------------------------- --------- -- // @@ -99,15 +100,20 @@ impl MarketEvent { } } +======= +>>>>>>> 6851723 (Progress on Events logic) // ------------------------------------------------ Template for Cancel Limit Order -- // // ------------------------------------------------ ------------------------------- -- // /// To create a pseudo-random Cancel Limit Order event -pub fn random_cancel_lo_template() -> Result { - let current_ts = SystemTime::now() +pub fn random_cancel_lo_template() -> Result { + + // -- random event info -- // + let random_received_ts = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_nanos() as u128; +<<<<<<< HEAD // -- random event data -- // @@ -118,27 +124,27 @@ pub fn random_cancel_lo_template() -> Result { let i_event_data = EventData::new(i_event_created_ts, i_event_type, i_user_id); +======= + let random_event_type = message::MarketEventType::CancelLimitOrder; + let random_user_id = 123; + +>>>>>>> 6851723 (Progress on Events logic) // -- random event content -- // - - let i_order_id = 345; - let i_order_ts = current_ts; - let i_order_type = market::OrderType::Limit; - let i_order_side = market::Side::random(); - let i_order_price = 70_000.00; - let i_order_amount = 10.01; - - let i_order = market::Order::new( - i_order_id, - i_order_ts, - i_order_type, - i_order_side, - i_order_price, - i_order_amount, + let random_order_id: u32 = 123; + + let i_event_info = message::EventInfo::new( + random_received_ts, + random_event_type, + random_user_id, ); - let i_event_content = EventContent::new(i_order); - - let r_market_event = MarketEvent::new(i_event_data, i_event_content); + let i_event_content = message::EventContent::OrderCancellation(random_order_id); + + // -- market event formation -- // + let r_market_event = message::MarketEvent::new( + i_event_info, + i_event_content + ); // returns the message {event data, event content} Ok(r_market_event) @@ -147,7 +153,7 @@ pub fn random_cancel_lo_template() -> Result { // -------------------------------------------------- Template for New Market Order -- // // -------------------------------------------------- ----------------------------- -- // -pub fn random_new_mo_template() -> Result { +pub fn random_new_mo_template() -> Result { let current_ts = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -157,10 +163,18 @@ pub fn random_new_mo_template() -> Result { let i_event_created_ts = current_ts; let i_event_executed_ts = current_ts + 1; - let i_event_type = MarketEventType::NewMarketOrder; + let i_event_type = message::MarketEventType::NewMarketOrder; let i_user_id = 654; +<<<<<<< HEAD let i_event_data = EventData::new(i_event_created_ts, i_event_type, i_user_id); +======= + let i_event_data = message::EventInfo::new( + i_event_created_ts, + i_event_type, + i_user_id, + ); +>>>>>>> 6851723 (Progress on Events logic) // -- random event content -- // @@ -180,9 +194,9 @@ pub fn random_new_mo_template() -> Result { i_order_amount, ); - let i_event_content = EventContent::new(i_order); + let i_event_content = message::EventContent::new(i_order); - let r_market_event = MarketEvent::new(i_event_data, i_event_content); + let r_market_event = message::MarketEvent::new(i_event_data, i_event_content); // returns the message {event data, event content} Ok(r_market_event) @@ -191,7 +205,12 @@ pub fn random_new_mo_template() -> Result { // ------------------------------------------------ Template for Modify Limit Order -- // // ------------------------------------------------ ------------------------------- -- // +<<<<<<< HEAD pub fn random_modify_lo_template() -> Result { +======= +pub fn random_modify_lo_template() -> Result { + +>>>>>>> 6851723 (Progress on Events logic) let current_ts = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -200,11 +219,23 @@ pub fn random_modify_lo_template() -> Result { // -- random event data -- // let i_event_created_ts = current_ts; +<<<<<<< HEAD let i_event_type = MarketEventType::ModifyLimitOrder; let i_user_id = 654; let i_event_data = EventData::new(i_event_created_ts, i_event_type, i_user_id); +======= + let i_event_type = message::MarketEventType::ModifyLimitOrder; + + let i_user_id = 654; + + let i_event_data = message::EventInfo::new( + i_event_created_ts, + i_event_type, + i_user_id, + ); +>>>>>>> 6851723 (Progress on Events logic) // -- random event content -- // @@ -226,8 +257,8 @@ pub fn random_modify_lo_template() -> Result { i_order_amount, ); - let i_event_content = EventContent::new(i_order); - let r_market_event = MarketEvent::new(i_event_data, i_event_content); + let i_event_content = message::EventContent::new(i_order); + let r_market_event = message::MarketEvent::new(i_event_data, i_event_content); // returns the message {event data, event content} Ok(r_market_event) @@ -236,7 +267,12 @@ pub fn random_modify_lo_template() -> Result { // --------------------------------------------------- Template for New Limit Order -- // // --------------------------------------------------- ---------------------------- -- // +<<<<<<< HEAD pub fn random_new_lo_template() -> Result { +======= +pub fn random_new_lo_template() -> Result { + +>>>>>>> 6851723 (Progress on Events logic) let current_ts = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -245,12 +281,25 @@ pub fn random_new_lo_template() -> Result { // -- random event data -- // let i_event_created_ts = current_ts; +<<<<<<< HEAD let i_event_type = MarketEventType::NewLimitOrder; // TODO: Hash value for user_id + Create a list of users let i_user_id = 654; let i_event_data = EventData::new(i_event_created_ts, i_event_type, i_user_id); +======= + let i_event_type = message::MarketEventType::NewLimitOrder; + + // TODO: Hash value for user_id + Create a list of users + let i_user_id = 654; + + let i_event_data = message::EventInfo::new( + i_event_created_ts, + i_event_type, + i_user_id, + ); +>>>>>>> 6851723 (Progress on Events logic) // -- random event content -- // @@ -274,8 +323,8 @@ pub fn random_new_lo_template() -> Result { i_order_amount, ); - let i_event_content = EventContent::new(i_order); - let r_market_event = MarketEvent::new(i_event_data, i_event_content); + let i_event_content = message::EventContent::new(i_order); + let r_market_event = message::MarketEvent::new(i_event_data, i_event_content); Ok(r_market_event) } diff --git a/atelier/src/generators/README.md b/atelier/src/generators/README.md new file mode 100644 index 0000000..b7b80b8 --- /dev/null +++ b/atelier/src/generators/README.md @@ -0,0 +1 @@ +# Generators Module diff --git a/atelier/src/lib.rs b/atelier/src/lib.rs index 804d7ae..7b7c27a 100644 --- a/atelier/src/lib.rs +++ b/atelier/src/lib.rs @@ -10,17 +10,18 @@ /// Core data types and data structures. pub mod data; +/// The Order Management and Matching Engine logics. +pub mod engine; + +/// Market event generator. +pub mod events; + /// Stochastic data generation with probabilistic models. pub mod generators; -/// Metrics calculation about various aspects and processes. -pub mod metrics; - /// Messages structures for: Errors, Success, Events, and Logs. pub mod messages; -/// Market event generator. -pub mod events; +/// Metrics calculation about various aspects and processes. +pub mod metrics; -/// The Order Management and Matching Engine logics. -pub mod engine; diff --git a/atelier/src/messages/README.md b/atelier/src/messages/README.md new file mode 100644 index 0000000..896736b --- /dev/null +++ b/atelier/src/messages/README.md @@ -0,0 +1 @@ +# Messages Module diff --git a/atelier/src/metrics/README.md b/atelier/src/metrics/README.md new file mode 100644 index 0000000..1346d13 --- /dev/null +++ b/atelier/src/metrics/README.md @@ -0,0 +1 @@ +# Metrics Module diff --git a/atelier/src/subscriptors/mod.rs b/atelier/src/subscriptors/mod.rs deleted file mode 100644 index e69de29..0000000