From d6e51f6d918b8117347b004d25c549b60e5b0913 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 9 Sep 2024 15:16:05 +0200 Subject: [PATCH 01/18] Initial commit --- crates/autopilot/src/domain/settlement/mod.rs | 21 +++++++++++++-- .../src/domain/settlement/trade/math.rs | 14 +++++++--- .../src/domain/settlement/trade/mod.rs | 11 ++++++-- crates/autopilot/src/infra/persistence/mod.rs | 27 +++++++------------ crates/database/src/jit_orders.rs | 1 + crates/database/src/order_execution.rs | 1 + crates/database/src/orders.rs | 2 ++ crates/model/src/order.rs | 1 + crates/orderbook/src/database/orders.rs | 2 ++ crates/shared/src/db_order_conversions.rs | 1 + 10 files changed, 56 insertions(+), 25 deletions(-) diff --git a/crates/autopilot/src/domain/settlement/mod.rs b/crates/autopilot/src/domain/settlement/mod.rs index 626d553f6b..18187ec9d3 100644 --- a/crates/autopilot/src/domain/settlement/mod.rs +++ b/crates/autopilot/src/domain/settlement/mod.rs @@ -84,10 +84,27 @@ impl Settlement { } /// Per order fees breakdown. Contains all orders from the settlement - pub fn fee_breakdown(&self) -> HashMap> { + pub fn fee_breakdown(&self) -> HashMap { self.trades .iter() - .map(|trade| (*trade.uid(), trade.fee_breakdown(&self.auction).ok())) + .map(|trade| { + let fee_breakdown = trade.fee_breakdown(&self.auction).unwrap_or_else(|err| { + tracing::warn!( + ?err, + trade = %trade.uid(), + "possible incomplete fee breakdown calculation", + ); + trade::FeeBreakdown { + total: eth::Asset { + // TODO surplus token + token: trade.sell_token(), + amount: num::zero(), + }, + protocol: vec![], + } + }); + (*trade.uid(), fee_breakdown) + }) .collect() } diff --git a/crates/autopilot/src/domain/settlement/trade/math.rs b/crates/autopilot/src/domain/settlement/trade/math.rs index 52bf4df308..7a8105814c 100644 --- a/crates/autopilot/src/domain/settlement/trade/math.rs +++ b/crates/autopilot/src/domain/settlement/trade/math.rs @@ -113,9 +113,9 @@ impl Trade { pub fn fee_in_ether(&self, prices: &auction::Prices) -> Result { let total_fee = self.fee_in_sell_token()?; let price = prices - .get(&self.sell.token) - .ok_or(Error::MissingPrice(self.sell.token))?; - Ok(price.in_eth(total_fee.into())) + .get(&total_fee.token) + .ok_or(Error::MissingPrice(total_fee.token))?; + Ok(price.in_eth(total_fee.amount)) } /// Converts given surplus fee into sell token fee. @@ -134,9 +134,15 @@ impl Trade { /// Total fee (protocol fee + network fee). Equal to a surplus difference /// before and after applying the fees. - pub fn fee_in_sell_token(&self) -> Result { + /// + /// Denominated in SELL token + pub fn fee_in_sell_token(&self) -> Result { let fee = self.fee()?; self.fee_into_sell_token(fee.amount) + .map(|amount| eth::Asset { + token: self.sell.token, + amount: amount.into(), + }) } /// Total fee (protocol fee + network fee). Equal to a surplus difference diff --git a/crates/autopilot/src/domain/settlement/trade/mod.rs b/crates/autopilot/src/domain/settlement/trade/mod.rs index 0e41ad4236..31c43d6687 100644 --- a/crates/autopilot/src/domain/settlement/trade/mod.rs +++ b/crates/autopilot/src/domain/settlement/trade/mod.rs @@ -75,6 +75,13 @@ impl Trade { Ok(FeeBreakdown { total, protocol }) } + pub fn sell_token(&self) -> eth::TokenAddress { + match self { + Self::Fulfillment(trade) => trade.sell.token, + Self::Jit(trade) => trade.sell.token, + } + } + pub fn new(trade: transaction::EncodedTrade, auction: &super::Auction, created: u32) -> Self { if auction.orders.contains_key(&trade.uid) { Trade::Fulfillment(Fulfillment { @@ -150,8 +157,8 @@ pub struct Jit { #[derive(Debug, Clone)] pub struct FeeBreakdown { /// Total fee the trade was charged (network fee + protocol fee) - // TODO: express in surplus token - pub total: eth::SellTokenAmount, + // TODO surplus token + pub total: eth::Asset, /// Breakdown of protocol fees. pub protocol: Vec, } diff --git a/crates/autopilot/src/infra/persistence/mod.rs b/crates/autopilot/src/infra/persistence/mod.rs index 5b365d2282..1d11c73f7d 100644 --- a/crates/autopilot/src/infra/persistence/mod.rs +++ b/crates/autopilot/src/infra/persistence/mod.rs @@ -645,28 +645,21 @@ impl Persistence { .await; for (order, order_fee) in fee_breakdown { - let total_fee = order_fee - .as_ref() - .map(|fee| u256_to_big_decimal(&fee.total.0)) - .unwrap_or_default(); - let executed_protocol_fees = order_fee - .map(|fee| { - fee.protocol - .into_iter() - .map(|executed| Asset { - token: ByteArray(executed.fee.token.0 .0), - amount: u256_to_big_decimal(&executed.fee.amount.0), - }) - .collect::>() - }) - .unwrap_or_default(); database::order_execution::save( &mut ex, &ByteArray(order.0), auction_id, block_number, - &total_fee, - &executed_protocol_fees, + &u256_to_big_decimal(&order_fee.total.amount.0), + &ByteArray(order_fee.total.token.0 .0), + &order_fee + .protocol + .into_iter() + .map(|executed| Asset { + token: ByteArray(executed.fee.token.0 .0), + amount: u256_to_big_decimal(&executed.fee.amount.0), + }) + .collect::>(), ) .await?; } diff --git a/crates/database/src/jit_orders.rs b/crates/database/src/jit_orders.rs index 12f79a0e6d..ae8590c6d9 100644 --- a/crates/database/src/jit_orders.rs +++ b/crates/database/src/jit_orders.rs @@ -32,6 +32,7 @@ NULL AS ethflow_data, NULL AS onchain_user, NULL AS onchain_placement_error, COALESCE((SELECT SUM(surplus_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_surplus_fee, +COALESCE((SELECT SUM(surplus_fee_token) FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_surplus_fee_token, -- TODO surplus token NULL AS full_app_data "#; diff --git a/crates/database/src/order_execution.rs b/crates/database/src/order_execution.rs index 11eb9f5ba1..85a009d95c 100644 --- a/crates/database/src/order_execution.rs +++ b/crates/database/src/order_execution.rs @@ -19,6 +19,7 @@ pub async fn save( auction: AuctionId, block_number: i64, executed_fee: &BigDecimal, + _executed_fee_token: &Address, // todo executed_protocol_fees: &[Asset], ) -> Result<(), sqlx::Error> { let (protocol_fee_tokens, protocol_fee_amounts): (Vec<_>, Vec<_>) = executed_protocol_fees diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index cc90aeeb58..91615b4fbd 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -486,6 +486,7 @@ pub struct FullOrder { pub onchain_user: Option
, pub onchain_placement_error: Option, pub executed_surplus_fee: BigDecimal, + pub executed_surplus_fee_token: Address, pub full_app_data: Option>, } @@ -549,6 +550,7 @@ array(Select (p.target, p.value, p.data) from interactions p where p.order_uid = (SELECT onchain_o.sender from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_user, (SELECT onchain_o.placement_error from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_placement_error, COALESCE((SELECT SUM(surplus_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_surplus_fee, +COALESCE((SELECT SUM(surplus_fee_token) FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_surplus_fee_token, -- TODO surplus token (SELECT full_app_data FROM app_data ad WHERE o.app_data = ad.contract_app_data LIMIT 1) as full_app_data "#; diff --git a/crates/model/src/order.rs b/crates/model/src/order.rs index 078df93bb6..650d64db9e 100644 --- a/crates/model/src/order.rs +++ b/crates/model/src/order.rs @@ -701,6 +701,7 @@ pub struct OrderMetadata { pub executed_fee_amount: U256, #[serde_as(as = "HexOrDecimalU256")] pub executed_surplus_fee: U256, + pub executed_surplus_fee_token: H160, pub invalidated: bool, pub status: OrderStatus, #[serde(flatten)] diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 2a2060262f..ffaea93aa8 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -516,6 +516,7 @@ fn full_order_into_model_order(order: FullOrder) -> Result { .context("executed fee amount is not a valid u256")?, executed_surplus_fee: big_decimal_to_u256(&order.executed_surplus_fee) .context("executed surplus fee is not a valid u256")?, + executed_surplus_fee_token: H160(order.executed_surplus_fee_token.0), invalidated: order.invalidated, status, is_liquidity_order: class == OrderClass::Liquidity, @@ -638,6 +639,7 @@ mod tests { onchain_user: None, onchain_placement_error: None, executed_surplus_fee: Default::default(), + executed_surplus_fee_token: ByteArray([1; 20]), // TODO surplus token full_app_data: Default::default(), }; diff --git a/crates/shared/src/db_order_conversions.rs b/crates/shared/src/db_order_conversions.rs index 5128c1e190..bfdb42eb57 100644 --- a/crates/shared/src/db_order_conversions.rs +++ b/crates/shared/src/db_order_conversions.rs @@ -92,6 +92,7 @@ pub fn full_order_into_model_order(order: database::orders::FullOrder) -> Result .context("executed fee amount is not a valid u256")?, executed_surplus_fee: big_decimal_to_u256(&order.executed_surplus_fee) .context("executed surplus fee is not a valid u256")?, + executed_surplus_fee_token: H160(order.executed_surplus_fee_token.0), invalidated: order.invalidated, status, is_liquidity_order: class == OrderClass::Liquidity, From bb686b0b1a835462c14dc660a3c78d3804786ef7 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 9 Sep 2024 16:11:01 +0200 Subject: [PATCH 02/18] db migration --- crates/database/src/order_execution.rs | 9 +++++---- database/sql/V071__surplus_fee_token.sql | 13 +++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 database/sql/V071__surplus_fee_token.sql diff --git a/crates/database/src/order_execution.rs b/crates/database/src/order_execution.rs index 85a009d95c..d2d3b317f4 100644 --- a/crates/database/src/order_execution.rs +++ b/crates/database/src/order_execution.rs @@ -19,7 +19,7 @@ pub async fn save( auction: AuctionId, block_number: i64, executed_fee: &BigDecimal, - _executed_fee_token: &Address, // todo + executed_fee_token: &Address, executed_protocol_fees: &[Asset], ) -> Result<(), sqlx::Error> { let (protocol_fee_tokens, protocol_fee_amounts): (Vec<_>, Vec<_>) = executed_protocol_fees @@ -28,16 +28,17 @@ pub async fn save( .unzip(); const QUERY: &str = r#" -INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, block_number, protocol_fee_tokens, protocol_fee_amounts) -VALUES ($1, $2, $3, $4, $5, $6, $7) +INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, surplus_fee_token, block_number, protocol_fee_tokens, protocol_fee_amounts) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (order_uid, auction_id) -DO UPDATE SET reward = $3, surplus_fee = $4, block_number = $5, protocol_fee_tokens = $6, protocol_fee_amounts = $7 +DO UPDATE SET reward = $3, surplus_fee = $4, surplus_fee_token = $5, block_number = $6, protocol_fee_tokens = $7, protocol_fee_amounts = $8 ;"#; sqlx::query(QUERY) .bind(order) .bind(auction) .bind(0.) // reward is deprecated but saved for historical analysis .bind(Some(executed_fee)) + .bind(executed_fee_token) .bind(block_number) .bind(protocol_fee_tokens) .bind(protocol_fee_amounts) diff --git a/database/sql/V071__surplus_fee_token.sql b/database/sql/V071__surplus_fee_token.sql new file mode 100644 index 0000000000..fb96a5721c --- /dev/null +++ b/database/sql/V071__surplus_fee_token.sql @@ -0,0 +1,13 @@ +-- Add a new column to the order_execution table to store the surplus fee token +ALTER TABLE order_execution +ADD COLUMN surplus_fee_token bytea NOT NULL DEFAULT '\x0000000000000000000000000000000000000000'; + +-- Now populate existing rows with the sell token taken from the 'trades' table, JOIN-ing on the order uid. +UPDATE order_execution oe +SET surplus_fee_token = COALESCE(t.sell_token, '\x0000000000000000000000000000000000000000') +FROM trades t +WHERE oe.order_uid = t.order_uid +OR NOT EXISTS (SELECT 1 FROM trades WHERE order_uid = oe.order_uid); + +-- Drop the default value for the surplus_fee_token column +ALTER TABLE order_execution ALTER COLUMN surplus_fee_token DROP DEFAULT; From 33a9381e16641ac7d75fe67228126abae679d92d Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 9 Sep 2024 16:20:53 +0200 Subject: [PATCH 03/18] fix unit tests --- crates/database/src/order_execution.rs | 15 ++++++++++++--- crates/database/src/orders.rs | 6 +++++- crates/model/src/order.rs | 2 ++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/database/src/order_execution.rs b/crates/database/src/order_execution.rs index d2d3b317f4..a2c56472a9 100644 --- a/crates/database/src/order_execution.rs +++ b/crates/database/src/order_execution.rs @@ -126,6 +126,7 @@ mod tests { 1, 0, &Default::default(), + &Default::default(), protocol_fees.as_slice(), ) .await @@ -133,9 +134,17 @@ mod tests { // save entry without protocol fees (simulate case when we are still not // calculating them) - save(&mut db, &Default::default(), 2, 0, &Default::default(), &[]) - .await - .unwrap(); + save( + &mut db, + &Default::default(), + 2, + 0, + &Default::default(), + &Default::default(), + &[], + ) + .await + .unwrap(); let keys: Vec<(AuctionId, OrderUid)> = vec![ (1, Default::default()), diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 91615b4fbd..bf362469af 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -1769,7 +1769,7 @@ mod tests { assert_eq!(order.executed_surplus_fee, fee); let fee: BigDecimal = 1.into(); - crate::order_execution::save(&mut db, &order_uid, 1, 0, &fee, &[]) + crate::order_execution::save(&mut db, &order_uid, 1, 0, &fee, &Default::default(), &[]) .await .unwrap(); @@ -1892,6 +1892,7 @@ mod tests { 1, 1, &BigDecimal::from(1), + &Default::default(), Default::default(), ) .await @@ -1902,6 +1903,7 @@ mod tests { 2, 2, &BigDecimal::from(2), + &Default::default(), Default::default(), ) .await @@ -1912,6 +1914,7 @@ mod tests { 3, 0, &BigDecimal::from(4), + &Default::default(), Default::default(), ) .await @@ -1922,6 +1925,7 @@ mod tests { 2, 3, &BigDecimal::from(4), + &Default::default(), Default::default(), ) .await diff --git a/crates/model/src/order.rs b/crates/model/src/order.rs index 650d64db9e..2129f8e2ae 100644 --- a/crates/model/src/order.rs +++ b/crates/model/src/order.rs @@ -1062,6 +1062,7 @@ mod tests { "appData": "0x6000000000000000000000000000000000000000000000000000000000000007", "feeAmount": "115792089237316195423570985008687907853269984665640564039457584007913129639935", "executedSurplusFee": "1", + "executedSurplusFeeToken": "0x000000000000000000000000000000000000000a", "fullFeeAmount": "115792089237316195423570985008687907853269984665640564039457584007913129639935", "solverFee": "115792089237316195423570985008687907853269984665640564039457584007913129639935", "kind": "buy", @@ -1093,6 +1094,7 @@ mod tests { executed_sell_amount_before_fees: 4.into(), executed_fee_amount: 1.into(), executed_surplus_fee: 1.into(), + executed_surplus_fee_token: H160::from_low_u64_be(10), invalidated: true, status: OrderStatus::Open, settlement_contract: H160::from_low_u64_be(2), From 6768adce2f494fcd35f108a9c2c56da9e831106c Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 9 Sep 2024 16:25:33 +0200 Subject: [PATCH 04/18] update readme --- database/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database/README.md b/database/README.md index 20f6469c43..a03082cf4d 100644 --- a/database/README.md +++ b/database/README.md @@ -175,7 +175,8 @@ Protocol fee tokens/amounts are stored in the same order as fee policies in fee_ order\_uid | bytea | not null | which order this trade execution is related to auction\_id | bigint | not null | in which auction this trade was initiated reward | double | not null | revert adjusted solver rewards, deprecated in favor of [CIP-20](https://snapshot.org/#/cow.eth/proposal/0x2d3f9bd1ea72dca84b03e97dda3efc1f4a42a772c54bd2037e8b62e7d09a491f) - surplus\_fee | numeric | nullable | dynamic fee computed by the protocol that should get taken from the surplus of a trade, this value only applies and is set for fill-or-kill limit orders. + surplus\_fee | numeric | nullable | total fee taken for execution of the trade + surplus\_fee\_token | bytea | not null | token in which the surplus fee is taken block\_number | bigint | not null | block in which the order was executed protocol\_fee\_tokens | bytea[] | not null | tokens in which the protocol fees are taken protocol\_fee\_amounts | numeric[] | not null | amounts of protocol fees taken, aligned protocol\_fee\_tokens array From 5f3d670cf6855899e7b5d1a1dcb10e4f15e769bd Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 9 Sep 2024 16:33:56 +0200 Subject: [PATCH 05/18] openapi --- crates/orderbook/openapi.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/orderbook/openapi.yml b/crates/orderbook/openapi.yml index e7eb381ef9..38433c1e8c 100644 --- a/crates/orderbook/openapi.yml +++ b/crates/orderbook/openapi.yml @@ -886,6 +886,11 @@ components: allOf: - $ref: "#/components/schemas/BigUint" nullable: true + executedSurplusFeeToken: + description: Token of the surplus fee. + allOf: + - $ref: "#/components/schemas/Address" + nullable: true fullAppData: description: | Full `appData`, which the contract-level `appData` is a hash of. See `OrderCreation` From ad453eab89289a4bf6632d6abc34ad05b69cbc37 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Tue, 10 Sep 2024 15:44:24 +0200 Subject: [PATCH 06/18] fix bugs --- crates/database/src/jit_orders.rs | 2 +- crates/database/src/orders.rs | 2 +- database/sql/V071__surplus_fee_token.sql | 18 +++++++++++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/database/src/jit_orders.rs b/crates/database/src/jit_orders.rs index ae8590c6d9..a2d0cf9af1 100644 --- a/crates/database/src/jit_orders.rs +++ b/crates/database/src/jit_orders.rs @@ -32,7 +32,7 @@ NULL AS ethflow_data, NULL AS onchain_user, NULL AS onchain_placement_error, COALESCE((SELECT SUM(surplus_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_surplus_fee, -COALESCE((SELECT SUM(surplus_fee_token) FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_surplus_fee_token, -- TODO surplus token +COALESCE((SELECT surplus_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_surplus_fee_token, -- TODO surplus token NULL AS full_app_data "#; diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index bf362469af..596cad8ee4 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -550,7 +550,7 @@ array(Select (p.target, p.value, p.data) from interactions p where p.order_uid = (SELECT onchain_o.sender from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_user, (SELECT onchain_o.placement_error from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_placement_error, COALESCE((SELECT SUM(surplus_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_surplus_fee, -COALESCE((SELECT SUM(surplus_fee_token) FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_surplus_fee_token, -- TODO surplus token +COALESCE((SELECT surplus_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_surplus_fee_token, -- TODO surplus token (SELECT full_app_data FROM app_data ad WHERE o.app_data = ad.contract_app_data LIMIT 1) as full_app_data "#; diff --git a/database/sql/V071__surplus_fee_token.sql b/database/sql/V071__surplus_fee_token.sql index fb96a5721c..46d6ea418e 100644 --- a/database/sql/V071__surplus_fee_token.sql +++ b/database/sql/V071__surplus_fee_token.sql @@ -2,12 +2,20 @@ ALTER TABLE order_execution ADD COLUMN surplus_fee_token bytea NOT NULL DEFAULT '\x0000000000000000000000000000000000000000'; --- Now populate existing rows with the sell token taken from the 'trades' table, JOIN-ing on the order uid. +-- Now populate existing rows with the sell token taken from the 'orders' table, or if it doesn't exist, try from 'jit_orders'. UPDATE order_execution oe -SET surplus_fee_token = COALESCE(t.sell_token, '\x0000000000000000000000000000000000000000') -FROM trades t -WHERE oe.order_uid = t.order_uid -OR NOT EXISTS (SELECT 1 FROM trades WHERE order_uid = oe.order_uid); +SET surplus_fee_token = COALESCE( + sub.sell_token, '\x0000000000000000000000000000000000000000' +) +FROM ( + SELECT o.uid, o.sell_token + FROM orders o + UNION + SELECT j.uid, j.sell_token + FROM jit_orders j + WHERE NOT EXISTS (SELECT 1 FROM orders WHERE orders.uid = j.uid) +) AS sub +WHERE oe.order_uid = sub.uid; -- Drop the default value for the surplus_fee_token column ALTER TABLE order_execution ALTER COLUMN surplus_fee_token DROP DEFAULT; From 281b94e36a6121e1f552ba4bc3f39038c2b6abcc Mon Sep 17 00:00:00 2001 From: sunce86 Date: Tue, 10 Sep 2024 15:58:27 +0200 Subject: [PATCH 07/18] update docs --- crates/orderbook/openapi.yml | 4 ++-- database/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/orderbook/openapi.yml b/crates/orderbook/openapi.yml index 38433c1e8c..e0df651a96 100644 --- a/crates/orderbook/openapi.yml +++ b/crates/orderbook/openapi.yml @@ -885,12 +885,12 @@ components: description: Surplus fee that the limit order was executed with. allOf: - $ref: "#/components/schemas/BigUint" - nullable: true + nullable: false executedSurplusFeeToken: description: Token of the surplus fee. allOf: - $ref: "#/components/schemas/Address" - nullable: true + nullable: false fullAppData: description: | Full `appData`, which the contract-level `appData` is a hash of. See `OrderCreation` diff --git a/database/README.md b/database/README.md index a03082cf4d..8eb0b8c34d 100644 --- a/database/README.md +++ b/database/README.md @@ -175,7 +175,7 @@ Protocol fee tokens/amounts are stored in the same order as fee policies in fee_ order\_uid | bytea | not null | which order this trade execution is related to auction\_id | bigint | not null | in which auction this trade was initiated reward | double | not null | revert adjusted solver rewards, deprecated in favor of [CIP-20](https://snapshot.org/#/cow.eth/proposal/0x2d3f9bd1ea72dca84b03e97dda3efc1f4a42a772c54bd2037e8b62e7d09a491f) - surplus\_fee | numeric | nullable | total fee taken for execution of the trade + surplus\_fee | numeric | not null | total fee taken for execution of the trade surplus\_fee\_token | bytea | not null | token in which the surplus fee is taken block\_number | bigint | not null | block in which the order was executed protocol\_fee\_tokens | bytea[] | not null | tokens in which the protocol fees are taken From 5fe2851b3906cfc9bae791b821e2a8261074f656 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Wed, 11 Sep 2024 14:49:56 +0200 Subject: [PATCH 08/18] rename surplus to total --- crates/autopilot/src/domain/settlement/mod.rs | 2 +- crates/autopilot/src/infra/persistence/mod.rs | 6 ++- crates/database/src/jit_orders.rs | 4 +- crates/database/src/order_execution.rs | 27 +++++++------ crates/database/src/orders.rs | 29 +++++++++----- crates/e2e/tests/e2e/ethflow.rs | 2 +- crates/e2e/tests/e2e/partial_fill.rs | 2 +- .../e2e/tests/e2e/partially_fillable_pool.rs | 2 +- crates/e2e/tests/e2e/protocol_fee.rs | 39 +++++++++---------- crates/model/src/order.rs | 12 +++--- crates/orderbook/openapi.yml | 8 ++-- crates/orderbook/src/database/orders.rs | 8 ++-- crates/shared/src/db_order_conversions.rs | 6 +-- ...ee_token.sql => V071__total_fee_token.sql} | 9 +++-- 14 files changed, 86 insertions(+), 70 deletions(-) rename database/sql/{V071__surplus_fee_token.sql => V071__total_fee_token.sql} (65%) diff --git a/crates/autopilot/src/domain/settlement/mod.rs b/crates/autopilot/src/domain/settlement/mod.rs index 18187ec9d3..6932520628 100644 --- a/crates/autopilot/src/domain/settlement/mod.rs +++ b/crates/autopilot/src/domain/settlement/mod.rs @@ -356,7 +356,7 @@ mod tests { trade.surplus_in_ether(&auction.prices).unwrap().0, eth::U256::from(52937525819789126u128) ); - // fee read from "executedSurplusFee" https://api.cow.fi/mainnet/api/v1/orders/0x10dab31217bb6cc2ace0fe601c15d342f7626a1ee5ef0495449800e73156998740a50cf069e992aa4536211b23f286ef88752187ffffffff + // fee read from "executedTotalFee" https://api.cow.fi/mainnet/api/v1/orders/0x10dab31217bb6cc2ace0fe601c15d342f7626a1ee5ef0495449800e73156998740a50cf069e992aa4536211b23f286ef88752187ffffffff assert_eq!( trade.fee_in_ether(&auction.prices).unwrap().0, eth::U256::from(6890975030480504u128) diff --git a/crates/autopilot/src/infra/persistence/mod.rs b/crates/autopilot/src/infra/persistence/mod.rs index 1d11c73f7d..f70bce9020 100644 --- a/crates/autopilot/src/infra/persistence/mod.rs +++ b/crates/autopilot/src/infra/persistence/mod.rs @@ -650,8 +650,10 @@ impl Persistence { &ByteArray(order.0), auction_id, block_number, - &u256_to_big_decimal(&order_fee.total.amount.0), - &ByteArray(order_fee.total.token.0 .0), + Asset { + token: ByteArray(order_fee.total.token.0 .0), + amount: u256_to_big_decimal(&order_fee.total.amount.0), + }, &order_fee .protocol .into_iter() diff --git a/crates/database/src/jit_orders.rs b/crates/database/src/jit_orders.rs index a2d0cf9af1..f6fb17c398 100644 --- a/crates/database/src/jit_orders.rs +++ b/crates/database/src/jit_orders.rs @@ -31,8 +31,8 @@ ARRAY[]::record[] AS post_interactions, NULL AS ethflow_data, NULL AS onchain_user, NULL AS onchain_placement_error, -COALESCE((SELECT SUM(surplus_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_surplus_fee, -COALESCE((SELECT surplus_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_surplus_fee_token, -- TODO surplus token +COALESCE((SELECT SUM(total_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_total_fee, +COALESCE((SELECT total_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_total_fee_token, -- TODO surplus token NULL AS full_app_data "#; diff --git a/crates/database/src/order_execution.rs b/crates/database/src/order_execution.rs index a2c56472a9..04a17dd855 100644 --- a/crates/database/src/order_execution.rs +++ b/crates/database/src/order_execution.rs @@ -18,27 +18,26 @@ pub async fn save( order: &OrderUid, auction: AuctionId, block_number: i64, - executed_fee: &BigDecimal, - executed_fee_token: &Address, - executed_protocol_fees: &[Asset], + total_fee: Asset, + protocol_fees: &[Asset], ) -> Result<(), sqlx::Error> { - let (protocol_fee_tokens, protocol_fee_amounts): (Vec<_>, Vec<_>) = executed_protocol_fees + let (protocol_fee_tokens, protocol_fee_amounts): (Vec<_>, Vec<_>) = protocol_fees .iter() .map(|entry| (entry.token, entry.amount.clone())) .unzip(); const QUERY: &str = r#" -INSERT INTO order_execution (order_uid, auction_id, reward, surplus_fee, surplus_fee_token, block_number, protocol_fee_tokens, protocol_fee_amounts) +INSERT INTO order_execution (order_uid, auction_id, reward, total_fee, total_fee_token, block_number, protocol_fee_tokens, protocol_fee_amounts) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (order_uid, auction_id) -DO UPDATE SET reward = $3, surplus_fee = $4, surplus_fee_token = $5, block_number = $6, protocol_fee_tokens = $7, protocol_fee_amounts = $8 +DO UPDATE SET reward = $3, total_fee = $4, total_fee_token = $5, block_number = $6, protocol_fee_tokens = $7, protocol_fee_amounts = $8 ;"#; sqlx::query(QUERY) .bind(order) .bind(auction) .bind(0.) // reward is deprecated but saved for historical analysis - .bind(Some(executed_fee)) - .bind(executed_fee_token) + .bind(Some(total_fee.amount)) + .bind(total_fee.token) .bind(block_number) .bind(protocol_fee_tokens) .bind(protocol_fee_amounts) @@ -125,8 +124,10 @@ mod tests { &Default::default(), 1, 0, - &Default::default(), - &Default::default(), + Asset { + amount: BigDecimal::from(3), + token: Default::default(), + }, protocol_fees.as_slice(), ) .await @@ -139,8 +140,10 @@ mod tests { &Default::default(), 2, 0, - &Default::default(), - &Default::default(), + Asset { + amount: BigDecimal::from(3), + token: Default::default(), + }, &[], ) .await diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 596cad8ee4..8be6cca438 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -485,8 +485,8 @@ pub struct FullOrder { pub ethflow_data: Option<(Option, i64)>, pub onchain_user: Option
, pub onchain_placement_error: Option, - pub executed_surplus_fee: BigDecimal, - pub executed_surplus_fee_token: Address, + pub executed_total_fee: BigDecimal, + pub executed_total_fee_token: Address, pub full_app_data: Option>, } @@ -549,8 +549,8 @@ array(Select (p.target, p.value, p.data) from interactions p where p.order_uid = where eth_o.uid = o.uid limit 1) as ethflow_data, (SELECT onchain_o.sender from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_user, (SELECT onchain_o.placement_error from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_placement_error, -COALESCE((SELECT SUM(surplus_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_surplus_fee, -COALESCE((SELECT surplus_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_surplus_fee_token, -- TODO surplus token +COALESCE((SELECT SUM(total_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_total_fee, +COALESCE((SELECT total_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_total_fee_token, -- TODO surplus token (SELECT full_app_data FROM app_data ad WHERE o.app_data = ad.contract_app_data LIMIT 1) as full_app_data "#; @@ -778,6 +778,7 @@ mod tests { events::{Event, EventIndex, Invalidation, PreSignature, Settlement, Trade}, onchain_broadcasted_orders::{insert_onchain_order, OnchainOrderPlacement}, onchain_invalidations::insert_onchain_invalidation, + order_execution::Asset, PgTransaction, }, bigdecimal::num_bigint::{BigInt, ToBigInt}, @@ -1766,18 +1767,28 @@ mod tests { .await .unwrap() .unwrap(); - assert_eq!(order.executed_surplus_fee, fee); + assert_eq!(order.executed_total_fee, fee); let fee: BigDecimal = 1.into(); - crate::order_execution::save(&mut db, &order_uid, 1, 0, &fee, &Default::default(), &[]) - .await - .unwrap(); + crate::order_execution::save( + &mut db, + &order_uid, + 1, + 0, + Asset { + amount: fee, + token: Default::default(), + }, + &[], + ) + .await + .unwrap(); let order = single_full_order(&mut db, &order_uid) .await .unwrap() .unwrap(); - assert_eq!(order.executed_surplus_fee, fee); + assert_eq!(order.executed_total_fee, fee); } #[tokio::test] diff --git a/crates/e2e/tests/e2e/ethflow.rs b/crates/e2e/tests/e2e/ethflow.rs index d6f1dd50ae..55247e1bb4 100644 --- a/crates/e2e/tests/e2e/ethflow.rs +++ b/crates/e2e/tests/e2e/ethflow.rs @@ -118,7 +118,7 @@ async fn eth_flow_tx(web3: Web3) { .get_order(ðflow_order.uid(onchain.contracts()).await) .await .unwrap(); - order.metadata.executed_surplus_fee > U256::zero() + order.metadata.executed_total_fee > U256::zero() }; wait_for_condition(TIMEOUT, fee_charged).await.unwrap(); diff --git a/crates/e2e/tests/e2e/partial_fill.rs b/crates/e2e/tests/e2e/partial_fill.rs index a1491e03ab..fd6766141a 100644 --- a/crates/e2e/tests/e2e/partial_fill.rs +++ b/crates/e2e/tests/e2e/partial_fill.rs @@ -92,7 +92,7 @@ async fn test(web3: Web3) { let settlement_event_processed = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); - order.metadata.executed_surplus_fee > U256::zero() + order.metadata.executed_total_fee > U256::zero() }; wait_for_condition(TIMEOUT, settlement_event_processed) .await diff --git a/crates/e2e/tests/e2e/partially_fillable_pool.rs b/crates/e2e/tests/e2e/partially_fillable_pool.rs index 777f27b0c9..2f308d6d70 100644 --- a/crates/e2e/tests/e2e/partially_fillable_pool.rs +++ b/crates/e2e/tests/e2e/partially_fillable_pool.rs @@ -119,7 +119,7 @@ async fn test(web3: Web3) { let metadata_updated = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); - !order.metadata.executed_surplus_fee.is_zero() + !order.metadata.executed_total_fee.is_zero() && order.metadata.executed_buy_amount != Default::default() && order.metadata.executed_sell_amount != Default::default() }; diff --git a/crates/e2e/tests/e2e/protocol_fee.rs b/crates/e2e/tests/e2e/protocol_fee.rs index 8dc5f2095a..fa391a73e7 100644 --- a/crates/e2e/tests/e2e/protocol_fee.rs +++ b/crates/e2e/tests/e2e/protocol_fee.rs @@ -277,7 +277,7 @@ async fn combined_protocol_fees(web3: Web3) { services .get_order(uid) .await - .is_ok_and(|order| !order.metadata.executed_surplus_fee.is_zero()) + .is_ok_and(|order| !order.metadata.executed_total_fee.is_zero()) }), ) .await @@ -293,39 +293,38 @@ async fn combined_protocol_fees(web3: Web3) { .get_order(&market_price_improvement_uid) .await .unwrap(); - let market_executed_surplus_fee_in_buy_token = - surplus_fee_in_buy_token(&market_price_improvement_order, &market_quote_after.quote); + let market_executed_total_fee_in_buy_token = + total_fee_in_buy_token(&market_price_improvement_order, &market_quote_after.quote); let market_quote_diff = market_quote_after .quote .buy_amount .saturating_sub(market_quote_before.quote.buy_amount); // see `market_price_improvement_policy.factor`, which is 0.3 - assert!(market_executed_surplus_fee_in_buy_token >= market_quote_diff * 3 / 10); + assert!(market_executed_total_fee_in_buy_token >= market_quote_diff * 3 / 10); let partner_fee_order = services.get_order(&partner_fee_order_uid).await.unwrap(); - let partner_fee_executed_surplus_fee_in_buy_token = - surplus_fee_in_buy_token(&partner_fee_order, &partner_fee_quote_after.quote); + let partner_fee_executed_total_fee_in_buy_token = + total_fee_in_buy_token(&partner_fee_order, &partner_fee_quote_after.quote); assert!( // see `--fee-policy-max-partner-fee` autopilot config argument, which is 0.02 - partner_fee_executed_surplus_fee_in_buy_token - >= partner_fee_quote.quote.buy_amount * 2 / 100 + partner_fee_executed_total_fee_in_buy_token >= partner_fee_quote.quote.buy_amount * 2 / 100 ); let limit_quote_diff = partner_fee_quote_after .quote .buy_amount .saturating_sub(partner_fee_order.data.buy_amount); // see `limit_surplus_policy.factor`, which is 0.3 - assert!(partner_fee_executed_surplus_fee_in_buy_token >= limit_quote_diff * 3 / 10); + assert!(partner_fee_executed_total_fee_in_buy_token >= limit_quote_diff * 3 / 10); let limit_surplus_order = services.get_order(&limit_surplus_order_uid).await.unwrap(); - let limit_executed_surplus_fee_in_buy_token = - surplus_fee_in_buy_token(&limit_surplus_order, &limit_quote_after.quote); + let limit_executed_total_fee_in_buy_token = + total_fee_in_buy_token(&limit_surplus_order, &limit_quote_after.quote); let limit_quote_diff = limit_quote_after .quote .buy_amount .saturating_sub(limit_surplus_order.data.buy_amount); // see `limit_surplus_policy.factor`, which is 0.3 - assert!(limit_executed_surplus_fee_in_buy_token >= limit_quote_diff * 3 / 10); + assert!(limit_executed_total_fee_in_buy_token >= limit_quote_diff * 3 / 10); let [market_order_token_balance, limit_order_token_balance, partner_fee_order_token_balance] = futures::future::try_join_all( @@ -345,15 +344,15 @@ async fn combined_protocol_fees(web3: Web3) { .try_into() .expect("Expected exactly four elements"); assert_approximately_eq!( - market_executed_surplus_fee_in_buy_token, + market_executed_total_fee_in_buy_token, market_order_token_balance ); assert_approximately_eq!( - limit_executed_surplus_fee_in_buy_token, + limit_executed_total_fee_in_buy_token, limit_order_token_balance ); assert_approximately_eq!( - partner_fee_executed_surplus_fee_in_buy_token, + partner_fee_executed_total_fee_in_buy_token, partner_fee_order_token_balance ); } @@ -386,8 +385,8 @@ async fn get_quote( services.submit_quote("e_request).await } -fn surplus_fee_in_buy_token(order: &Order, quote: &OrderQuote) -> U256 { - order.metadata.executed_surplus_fee * quote.buy_amount / quote.sell_amount +fn total_fee_in_buy_token(order: &Order, quote: &OrderQuote) -> U256 { + order.metadata.executed_total_fee * quote.buy_amount / quote.sell_amount } fn sell_order_from_quote(quote: &OrderQuoteResponse) -> OrderCreation { @@ -524,13 +523,13 @@ async fn volume_fee_buy_order_test(web3: Web3) { let metadata_updated = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); - !order.metadata.executed_surplus_fee.is_zero() + !order.metadata.executed_total_fee.is_zero() }; wait_for_condition(TIMEOUT, metadata_updated).await.unwrap(); let order = services.get_order(&uid).await.unwrap(); let fee_in_buy_token = quote.fee_amount * quote.buy_amount / quote.sell_amount; - assert!(order.metadata.executed_surplus_fee >= fee_in_buy_token + quote.sell_amount / 10); + assert!(order.metadata.executed_total_fee >= fee_in_buy_token + quote.sell_amount / 10); // Check settlement contract balance let balance_after = token_gno @@ -538,5 +537,5 @@ async fn volume_fee_buy_order_test(web3: Web3) { .call() .await .unwrap(); - assert_eq!(order.metadata.executed_surplus_fee, balance_after); + assert_eq!(order.metadata.executed_total_fee, balance_after); } diff --git a/crates/model/src/order.rs b/crates/model/src/order.rs index 2129f8e2ae..30dfb59141 100644 --- a/crates/model/src/order.rs +++ b/crates/model/src/order.rs @@ -700,8 +700,8 @@ pub struct OrderMetadata { #[serde_as(as = "HexOrDecimalU256")] pub executed_fee_amount: U256, #[serde_as(as = "HexOrDecimalU256")] - pub executed_surplus_fee: U256, - pub executed_surplus_fee_token: H160, + pub executed_total_fee: U256, + pub executed_total_fee_token: H160, pub invalidated: bool, pub status: OrderStatus, #[serde(flatten)] @@ -1061,8 +1061,8 @@ mod tests { "validTo": 4294967295u32, "appData": "0x6000000000000000000000000000000000000000000000000000000000000007", "feeAmount": "115792089237316195423570985008687907853269984665640564039457584007913129639935", - "executedSurplusFee": "1", - "executedSurplusFeeToken": "0x000000000000000000000000000000000000000a", + "executedTotalFee": "1", + "executedTotalFeeToken": "0x000000000000000000000000000000000000000a", "fullFeeAmount": "115792089237316195423570985008687907853269984665640564039457584007913129639935", "solverFee": "115792089237316195423570985008687907853269984665640564039457584007913129639935", "kind": "buy", @@ -1093,8 +1093,8 @@ mod tests { executed_sell_amount: BigUint::from_bytes_be(&[5]), executed_sell_amount_before_fees: 4.into(), executed_fee_amount: 1.into(), - executed_surplus_fee: 1.into(), - executed_surplus_fee_token: H160::from_low_u64_be(10), + executed_total_fee: 1.into(), + executed_total_fee_token: H160::from_low_u64_be(10), invalidated: true, status: OrderStatus::Open, settlement_contract: H160::from_low_u64_be(2), diff --git a/crates/orderbook/openapi.yml b/crates/orderbook/openapi.yml index e0df651a96..9d44645dea 100644 --- a/crates/orderbook/openapi.yml +++ b/crates/orderbook/openapi.yml @@ -881,13 +881,13 @@ components: can be found in this object. allOf: - $ref: "#/components/schemas/OnchainOrderData" - executedSurplusFee: - description: Surplus fee that the limit order was executed with. + executedTotalFee: + description: Total fee charged for execution of the order. Contains network fee and protocol fees. allOf: - $ref: "#/components/schemas/BigUint" nullable: false - executedSurplusFeeToken: - description: Token of the surplus fee. + executedTotalFeeToken: + description: Token of the total fee. allOf: - $ref: "#/components/schemas/Address" nullable: false diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index ffaea93aa8..8b0e77f258 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -514,9 +514,9 @@ fn full_order_into_model_order(order: FullOrder) -> Result { )?, executed_fee_amount: big_decimal_to_u256(&order.sum_fee) .context("executed fee amount is not a valid u256")?, - executed_surplus_fee: big_decimal_to_u256(&order.executed_surplus_fee) + executed_total_fee: big_decimal_to_u256(&order.executed_total_fee) .context("executed surplus fee is not a valid u256")?, - executed_surplus_fee_token: H160(order.executed_surplus_fee_token.0), + executed_total_fee_token: H160(order.executed_total_fee_token.0), invalidated: order.invalidated, status, is_liquidity_order: class == OrderClass::Liquidity, @@ -638,8 +638,8 @@ mod tests { ethflow_data: None, onchain_user: None, onchain_placement_error: None, - executed_surplus_fee: Default::default(), - executed_surplus_fee_token: ByteArray([1; 20]), // TODO surplus token + executed_total_fee: Default::default(), + executed_total_fee_token: ByteArray([1; 20]), // TODO surplus token full_app_data: Default::default(), }; diff --git a/crates/shared/src/db_order_conversions.rs b/crates/shared/src/db_order_conversions.rs index bfdb42eb57..f832376cf3 100644 --- a/crates/shared/src/db_order_conversions.rs +++ b/crates/shared/src/db_order_conversions.rs @@ -90,9 +90,9 @@ pub fn full_order_into_model_order(order: database::orders::FullOrder) -> Result )?, executed_fee_amount: big_decimal_to_u256(&order.sum_fee) .context("executed fee amount is not a valid u256")?, - executed_surplus_fee: big_decimal_to_u256(&order.executed_surplus_fee) - .context("executed surplus fee is not a valid u256")?, - executed_surplus_fee_token: H160(order.executed_surplus_fee_token.0), + executed_total_fee: big_decimal_to_u256(&order.executed_total_fee) + .context("executed total fee is not a valid u256")?, + executed_total_fee_token: H160(order.executed_total_fee_token.0), invalidated: order.invalidated, status, is_liquidity_order: class == OrderClass::Liquidity, diff --git a/database/sql/V071__surplus_fee_token.sql b/database/sql/V071__total_fee_token.sql similarity index 65% rename from database/sql/V071__surplus_fee_token.sql rename to database/sql/V071__total_fee_token.sql index 46d6ea418e..18f1ba90f0 100644 --- a/database/sql/V071__surplus_fee_token.sql +++ b/database/sql/V071__total_fee_token.sql @@ -1,10 +1,11 @@ -- Add a new column to the order_execution table to store the surplus fee token ALTER TABLE order_execution -ADD COLUMN surplus_fee_token bytea NOT NULL DEFAULT '\x0000000000000000000000000000000000000000'; +ADD COLUMN total_fee_token bytea NOT NULL DEFAULT '\x0000000000000000000000000000000000000000', +RENAME COLUMN surplus_fee TO total_fee; -- Now populate existing rows with the sell token taken from the 'orders' table, or if it doesn't exist, try from 'jit_orders'. UPDATE order_execution oe -SET surplus_fee_token = COALESCE( +SET total_fee_token = COALESCE( sub.sell_token, '\x0000000000000000000000000000000000000000' ) FROM ( @@ -17,5 +18,5 @@ FROM ( ) AS sub WHERE oe.order_uid = sub.uid; --- Drop the default value for the surplus_fee_token column -ALTER TABLE order_execution ALTER COLUMN surplus_fee_token DROP DEFAULT; +-- Drop the default value for the total_fee_token column +ALTER TABLE order_execution ALTER COLUMN total_fee_token DROP DEFAULT; From 3df7ce2560c8a54d70e01fdd70d8466feb3a423d Mon Sep 17 00:00:00 2001 From: sunce86 Date: Wed, 11 Sep 2024 14:57:27 +0200 Subject: [PATCH 09/18] fix tests --- crates/database/src/orders.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 8be6cca438..3898f6d367 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -1776,7 +1776,7 @@ mod tests { 1, 0, Asset { - amount: fee, + amount: fee.clone(), token: Default::default(), }, &[], @@ -1902,8 +1902,10 @@ mod tests { &ByteArray([1u8; 56]), 1, 1, - &BigDecimal::from(1), - &Default::default(), + Asset { + amount: BigDecimal::from(1), + token: Default::default(), + }, Default::default(), ) .await @@ -1913,8 +1915,10 @@ mod tests { &ByteArray([1u8; 56]), 2, 2, - &BigDecimal::from(2), - &Default::default(), + Asset { + amount: BigDecimal::from(2), + token: Default::default(), + }, Default::default(), ) .await @@ -1924,8 +1928,10 @@ mod tests { &ByteArray([1u8; 56]), 3, 0, - &BigDecimal::from(4), - &Default::default(), + Asset { + amount: BigDecimal::from(4), + token: Default::default(), + }, Default::default(), ) .await @@ -1935,8 +1941,10 @@ mod tests { &ByteArray([3u8; 56]), 2, 3, - &BigDecimal::from(4), - &Default::default(), + Asset { + amount: BigDecimal::from(4), + token: Default::default(), + }, Default::default(), ) .await From d78ec96e405f8058562d4beb1bd50a4266f0e886 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Wed, 11 Sep 2024 15:07:38 +0200 Subject: [PATCH 10/18] fix migration --- database/sql/V071__total_fee_token.sql | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/database/sql/V071__total_fee_token.sql b/database/sql/V071__total_fee_token.sql index 18f1ba90f0..4510a07300 100644 --- a/database/sql/V071__total_fee_token.sql +++ b/database/sql/V071__total_fee_token.sql @@ -1,8 +1,10 @@ --- Add a new column to the order_execution table to store the surplus fee token -ALTER TABLE order_execution -ADD COLUMN total_fee_token bytea NOT NULL DEFAULT '\x0000000000000000000000000000000000000000', +ALTER TABLE order_execution RENAME COLUMN surplus_fee TO total_fee; +-- Add a new column to the order_execution table to store the total fee token +ALTER TABLE order_execution +ADD COLUMN total_fee_token bytea NOT NULL DEFAULT '\x0000000000000000000000000000000000000000'; + -- Now populate existing rows with the sell token taken from the 'orders' table, or if it doesn't exist, try from 'jit_orders'. UPDATE order_execution oe SET total_fee_token = COALESCE( From 50dc2d303c7e0068a5f4bfeacf4c5074bca9fddf Mon Sep 17 00:00:00 2001 From: sunce86 Date: Thu, 12 Sep 2024 09:57:28 +0200 Subject: [PATCH 11/18] fix comments --- crates/database/src/order_execution.rs | 3 +-- crates/orderbook/openapi.yml | 2 +- database/README.md | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/database/src/order_execution.rs b/crates/database/src/order_execution.rs index 04a17dd855..4874dcc7a5 100644 --- a/crates/database/src/order_execution.rs +++ b/crates/database/src/order_execution.rs @@ -133,8 +133,7 @@ mod tests { .await .unwrap(); - // save entry without protocol fees (simulate case when we are still not - // calculating them) + // save entry for an order without protocol fees save( &mut db, &Default::default(), diff --git a/crates/orderbook/openapi.yml b/crates/orderbook/openapi.yml index 9d44645dea..e6f9157b26 100644 --- a/crates/orderbook/openapi.yml +++ b/crates/orderbook/openapi.yml @@ -887,7 +887,7 @@ components: - $ref: "#/components/schemas/BigUint" nullable: false executedTotalFeeToken: - description: Token of the total fee. + description: Token the total fee was captured in. allOf: - $ref: "#/components/schemas/Address" nullable: false diff --git a/database/README.md b/database/README.md index 8eb0b8c34d..afa944a904 100644 --- a/database/README.md +++ b/database/README.md @@ -175,8 +175,8 @@ Protocol fee tokens/amounts are stored in the same order as fee policies in fee_ order\_uid | bytea | not null | which order this trade execution is related to auction\_id | bigint | not null | in which auction this trade was initiated reward | double | not null | revert adjusted solver rewards, deprecated in favor of [CIP-20](https://snapshot.org/#/cow.eth/proposal/0x2d3f9bd1ea72dca84b03e97dda3efc1f4a42a772c54bd2037e8b62e7d09a491f) - surplus\_fee | numeric | not null | total fee taken for execution of the trade - surplus\_fee\_token | bytea | not null | token in which the surplus fee is taken + total\_fee | numeric | not null | total fee taken for execution of the trade + total\_fee\_token | bytea | not null | token in which the surplus fee is taken block\_number | bigint | not null | block in which the order was executed protocol\_fee\_tokens | bytea[] | not null | tokens in which the protocol fees are taken protocol\_fee\_amounts | numeric[] | not null | amounts of protocol fees taken, aligned protocol\_fee\_tokens array From a6ac8aac88a516446ad8931df607e4b699908c46 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 13 Sep 2024 10:53:17 +0200 Subject: [PATCH 12/18] rename --- .../sql/{V071__total_fee_token.sql => V072__total_fee_token.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename database/sql/{V071__total_fee_token.sql => V072__total_fee_token.sql} (100%) diff --git a/database/sql/V071__total_fee_token.sql b/database/sql/V072__total_fee_token.sql similarity index 100% rename from database/sql/V071__total_fee_token.sql rename to database/sql/V072__total_fee_token.sql From 3994e5310137ce37309f6b53e9f051bf46214436 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 13 Sep 2024 13:35:13 +0200 Subject: [PATCH 13/18] renaming and soft landing for api --- crates/autopilot/src/domain/settlement/mod.rs | 2 +- crates/database/src/jit_orders.rs | 4 +- crates/database/src/orders.rs | 12 ++--- crates/e2e/tests/e2e/ethflow.rs | 2 +- crates/e2e/tests/e2e/partial_fill.rs | 2 +- .../e2e/tests/e2e/partially_fillable_pool.rs | 2 +- crates/e2e/tests/e2e/protocol_fee.rs | 44 ++++++++----------- crates/model/src/order.rs | 16 ++++--- crates/orderbook/openapi.yml | 7 ++- crates/orderbook/src/database/orders.rs | 12 ++--- crates/shared/src/db_order_conversions.rs | 8 ++-- 11 files changed, 59 insertions(+), 52 deletions(-) diff --git a/crates/autopilot/src/domain/settlement/mod.rs b/crates/autopilot/src/domain/settlement/mod.rs index fa975c7c13..a572770023 100644 --- a/crates/autopilot/src/domain/settlement/mod.rs +++ b/crates/autopilot/src/domain/settlement/mod.rs @@ -329,7 +329,7 @@ mod tests { trade.surplus_in_ether(&auction.prices).unwrap().0, eth::U256::from(52937525819789126u128) ); - // fee read from "executedTotalFee" https://api.cow.fi/mainnet/api/v1/orders/0x10dab31217bb6cc2ace0fe601c15d342f7626a1ee5ef0495449800e73156998740a50cf069e992aa4536211b23f286ef88752187ffffffff + // fee read from "executedFee" https://api.cow.fi/mainnet/api/v1/orders/0x10dab31217bb6cc2ace0fe601c15d342f7626a1ee5ef0495449800e73156998740a50cf069e992aa4536211b23f286ef88752187ffffffff // but not equal to 6890975030480504 anymore, since after this tx we switched to // convert the fee from surplus token directly to ether assert_eq!( diff --git a/crates/database/src/jit_orders.rs b/crates/database/src/jit_orders.rs index f6fb17c398..6725527b61 100644 --- a/crates/database/src/jit_orders.rs +++ b/crates/database/src/jit_orders.rs @@ -31,8 +31,8 @@ ARRAY[]::record[] AS post_interactions, NULL AS ethflow_data, NULL AS onchain_user, NULL AS onchain_placement_error, -COALESCE((SELECT SUM(total_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_total_fee, -COALESCE((SELECT total_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_total_fee_token, -- TODO surplus token +COALESCE((SELECT SUM(total_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_fee, +COALESCE((SELECT total_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_fee_token, -- TODO surplus token NULL AS full_app_data "#; diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 1a1b7fd907..b61eb71263 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -485,8 +485,8 @@ pub struct FullOrder { pub ethflow_data: Option<(Option, i64)>, pub onchain_user: Option
, pub onchain_placement_error: Option, - pub executed_total_fee: BigDecimal, - pub executed_total_fee_token: Address, + pub executed_fee: BigDecimal, + pub executed_fee_token: Address, pub full_app_data: Option>, } @@ -561,8 +561,8 @@ array(Select (p.target, p.value, p.data) from interactions p where p.order_uid = where eth_o.uid = o.uid limit 1) as ethflow_data, (SELECT onchain_o.sender from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_user, (SELECT onchain_o.placement_error from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_placement_error, -COALESCE((SELECT SUM(total_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_total_fee, -COALESCE((SELECT total_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_total_fee_token, -- TODO surplus token +COALESCE((SELECT SUM(total_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_fee, +COALESCE((SELECT total_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_fee_token, -- TODO surplus token (SELECT full_app_data FROM app_data ad WHERE o.app_data = ad.contract_app_data LIMIT 1) as full_app_data "#; @@ -1888,7 +1888,7 @@ mod tests { .await .unwrap() .unwrap(); - assert_eq!(order.executed_total_fee, fee); + assert_eq!(order.executed_fee, fee); let fee: BigDecimal = 1.into(); crate::order_execution::save( @@ -1909,7 +1909,7 @@ mod tests { .await .unwrap() .unwrap(); - assert_eq!(order.executed_total_fee, fee); + assert_eq!(order.executed_fee, fee); } #[tokio::test] diff --git a/crates/e2e/tests/e2e/ethflow.rs b/crates/e2e/tests/e2e/ethflow.rs index 9cb5a5b63a..81c201f4de 100644 --- a/crates/e2e/tests/e2e/ethflow.rs +++ b/crates/e2e/tests/e2e/ethflow.rs @@ -112,7 +112,7 @@ async fn eth_flow_tx(web3: Web3) { .get_order(ðflow_order.uid(onchain.contracts()).await) .await .unwrap(); - order.metadata.executed_total_fee > U256::zero() + order.metadata.executed_fee > U256::zero() }; wait_for_condition(TIMEOUT, fee_charged).await.unwrap(); diff --git a/crates/e2e/tests/e2e/partial_fill.rs b/crates/e2e/tests/e2e/partial_fill.rs index fd6766141a..17d2e221ef 100644 --- a/crates/e2e/tests/e2e/partial_fill.rs +++ b/crates/e2e/tests/e2e/partial_fill.rs @@ -92,7 +92,7 @@ async fn test(web3: Web3) { let settlement_event_processed = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); - order.metadata.executed_total_fee > U256::zero() + order.metadata.executed_fee > U256::zero() }; wait_for_condition(TIMEOUT, settlement_event_processed) .await diff --git a/crates/e2e/tests/e2e/partially_fillable_pool.rs b/crates/e2e/tests/e2e/partially_fillable_pool.rs index 1267b699ba..a656d228ff 100644 --- a/crates/e2e/tests/e2e/partially_fillable_pool.rs +++ b/crates/e2e/tests/e2e/partially_fillable_pool.rs @@ -116,7 +116,7 @@ async fn test(web3: Web3) { let metadata_updated = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); - !order.metadata.executed_total_fee.is_zero() + !order.metadata.executed_fee.is_zero() && order.metadata.executed_buy_amount != Default::default() && order.metadata.executed_sell_amount != Default::default() }; diff --git a/crates/e2e/tests/e2e/protocol_fee.rs b/crates/e2e/tests/e2e/protocol_fee.rs index 8c24784182..8a1049001f 100644 --- a/crates/e2e/tests/e2e/protocol_fee.rs +++ b/crates/e2e/tests/e2e/protocol_fee.rs @@ -277,7 +277,7 @@ async fn combined_protocol_fees(web3: Web3) { services .get_order(uid) .await - .is_ok_and(|order| !order.metadata.executed_total_fee.is_zero()) + .is_ok_and(|order| !order.metadata.executed_fee.is_zero()) }), ) .await @@ -293,38 +293,38 @@ async fn combined_protocol_fees(web3: Web3) { .get_order(&market_price_improvement_uid) .await .unwrap(); - let market_executed_total_fee_in_buy_token = - total_fee_in_buy_token(&market_price_improvement_order, &market_quote_after.quote); + let market_executed_fee_in_buy_token = + fee_in_buy_token(&market_price_improvement_order, &market_quote_after.quote); let market_quote_diff = market_quote_after .quote .buy_amount .saturating_sub(market_quote_before.quote.buy_amount); // see `market_price_improvement_policy.factor`, which is 0.3 - assert!(market_executed_total_fee_in_buy_token >= market_quote_diff * 3 / 10); + assert!(market_executed_fee_in_buy_token >= market_quote_diff * 3 / 10); let partner_fee_order = services.get_order(&partner_fee_order_uid).await.unwrap(); - let partner_fee_executed_total_fee_in_buy_token = - total_fee_in_buy_token(&partner_fee_order, &partner_fee_quote_after.quote); + let partner_fee_executed_fee_in_buy_token = + fee_in_buy_token(&partner_fee_order, &partner_fee_quote_after.quote); assert!( // see `--fee-policy-max-partner-fee` autopilot config argument, which is 0.02 - partner_fee_executed_total_fee_in_buy_token >= partner_fee_quote.quote.buy_amount * 2 / 100 + partner_fee_executed_fee_in_buy_token >= partner_fee_quote.quote.buy_amount * 2 / 100 ); let limit_quote_diff = partner_fee_quote_after .quote .buy_amount .saturating_sub(partner_fee_order.data.buy_amount); // see `limit_surplus_policy.factor`, which is 0.3 - assert!(partner_fee_executed_total_fee_in_buy_token >= limit_quote_diff * 3 / 10); + assert!(partner_fee_executed_fee_in_buy_token >= limit_quote_diff * 3 / 10); let limit_surplus_order = services.get_order(&limit_surplus_order_uid).await.unwrap(); - let limit_executed_total_fee_in_buy_token = - total_fee_in_buy_token(&limit_surplus_order, &limit_quote_after.quote); + let limit_executed_fee_in_buy_token = + fee_in_buy_token(&limit_surplus_order, &limit_quote_after.quote); let limit_quote_diff = limit_quote_after .quote .buy_amount .saturating_sub(limit_surplus_order.data.buy_amount); // see `limit_surplus_policy.factor`, which is 0.3 - assert!(limit_executed_total_fee_in_buy_token >= limit_quote_diff * 3 / 10); + assert!(limit_executed_fee_in_buy_token >= limit_quote_diff * 3 / 10); let [market_order_token_balance, limit_order_token_balance, partner_fee_order_token_balance] = futures::future::try_join_all( @@ -343,16 +343,10 @@ async fn combined_protocol_fees(web3: Web3) { .unwrap() .try_into() .expect("Expected exactly four elements"); + assert_approximately_eq!(market_executed_fee_in_buy_token, market_order_token_balance); + assert_approximately_eq!(limit_executed_fee_in_buy_token, limit_order_token_balance); assert_approximately_eq!( - market_executed_total_fee_in_buy_token, - market_order_token_balance - ); - assert_approximately_eq!( - limit_executed_total_fee_in_buy_token, - limit_order_token_balance - ); - assert_approximately_eq!( - partner_fee_executed_total_fee_in_buy_token, + partner_fee_executed_fee_in_buy_token, partner_fee_order_token_balance ); } @@ -385,8 +379,8 @@ async fn get_quote( services.submit_quote("e_request).await } -fn total_fee_in_buy_token(order: &Order, quote: &OrderQuote) -> U256 { - order.metadata.executed_total_fee * quote.buy_amount / quote.sell_amount +fn fee_in_buy_token(order: &Order, quote: &OrderQuote) -> U256 { + order.metadata.executed_fee * quote.buy_amount / quote.sell_amount } fn sell_order_from_quote(quote: &OrderQuoteResponse) -> OrderCreation { @@ -515,13 +509,13 @@ async fn volume_fee_buy_order_test(web3: Web3) { let metadata_updated = || async { onchain.mint_block().await; let order = services.get_order(&uid).await.unwrap(); - !order.metadata.executed_total_fee.is_zero() + !order.metadata.executed_fee.is_zero() }; wait_for_condition(TIMEOUT, metadata_updated).await.unwrap(); let order = services.get_order(&uid).await.unwrap(); let fee_in_buy_token = quote.fee_amount * quote.buy_amount / quote.sell_amount; - assert!(order.metadata.executed_total_fee >= fee_in_buy_token + quote.sell_amount / 10); + assert!(order.metadata.executed_fee >= fee_in_buy_token + quote.sell_amount / 10); // Check settlement contract balance let balance_after = token_gno @@ -529,5 +523,5 @@ async fn volume_fee_buy_order_test(web3: Web3) { .call() .await .unwrap(); - assert_eq!(order.metadata.executed_total_fee, balance_after); + assert_eq!(order.metadata.executed_fee, balance_after); } diff --git a/crates/model/src/order.rs b/crates/model/src/order.rs index 30dfb59141..e9f06854d8 100644 --- a/crates/model/src/order.rs +++ b/crates/model/src/order.rs @@ -700,8 +700,10 @@ pub struct OrderMetadata { #[serde_as(as = "HexOrDecimalU256")] pub executed_fee_amount: U256, #[serde_as(as = "HexOrDecimalU256")] - pub executed_total_fee: U256, - pub executed_total_fee_token: H160, + pub executed_surplus_fee: U256, + #[serde_as(as = "HexOrDecimalU256")] + pub executed_fee: U256, + pub executed_fee_token: H160, pub invalidated: bool, pub status: OrderStatus, #[serde(flatten)] @@ -1061,8 +1063,9 @@ mod tests { "validTo": 4294967295u32, "appData": "0x6000000000000000000000000000000000000000000000000000000000000007", "feeAmount": "115792089237316195423570985008687907853269984665640564039457584007913129639935", - "executedTotalFee": "1", - "executedTotalFeeToken": "0x000000000000000000000000000000000000000a", + "executedSurplusFee": "1", + "executedFee": "1", + "executedFeeToken": "0x000000000000000000000000000000000000000a", "fullFeeAmount": "115792089237316195423570985008687907853269984665640564039457584007913129639935", "solverFee": "115792089237316195423570985008687907853269984665640564039457584007913129639935", "kind": "buy", @@ -1093,8 +1096,9 @@ mod tests { executed_sell_amount: BigUint::from_bytes_be(&[5]), executed_sell_amount_before_fees: 4.into(), executed_fee_amount: 1.into(), - executed_total_fee: 1.into(), - executed_total_fee_token: H160::from_low_u64_be(10), + executed_surplus_fee: 1.into(), + executed_fee: 1.into(), + executed_fee_token: H160::from_low_u64_be(10), invalidated: true, status: OrderStatus::Open, settlement_contract: H160::from_low_u64_be(2), diff --git a/crates/orderbook/openapi.yml b/crates/orderbook/openapi.yml index e98f937eac..58dd420f5a 100644 --- a/crates/orderbook/openapi.yml +++ b/crates/orderbook/openapi.yml @@ -881,7 +881,12 @@ components: can be found in this object. allOf: - $ref: "#/components/schemas/OnchainOrderData" - executedTotalFee: + executedSurplusFee: + description: Surplus fee that the limit order was executed with. + allOf: + - $ref: "#/components/schemas/BigUint" + nullable: false + executedFee: description: Total fee charged for execution of the order. Contains network fee and protocol fees. allOf: - $ref: "#/components/schemas/BigUint" diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 607ddab3af..0152def257 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -582,9 +582,11 @@ fn full_order_into_model_order(order: FullOrder) -> Result { )?, executed_fee_amount: big_decimal_to_u256(&order.sum_fee) .context("executed fee amount is not a valid u256")?, - executed_total_fee: big_decimal_to_u256(&order.executed_total_fee) - .context("executed surplus fee is not a valid u256")?, - executed_total_fee_token: H160(order.executed_total_fee_token.0), + executed_surplus_fee: big_decimal_to_u256(&order.executed_fee) + .context("executed fee is not a valid u256")?, + executed_fee: big_decimal_to_u256(&order.executed_fee) + .context("executed fee is not a valid u256")?, + executed_fee_token: H160(order.executed_fee_token.0), invalidated: order.invalidated, status, is_liquidity_order: class == OrderClass::Liquidity, @@ -707,8 +709,8 @@ mod tests { ethflow_data: None, onchain_user: None, onchain_placement_error: None, - executed_total_fee: Default::default(), - executed_total_fee_token: ByteArray([1; 20]), // TODO surplus token + executed_fee: Default::default(), + executed_fee_token: ByteArray([1; 20]), // TODO surplus token full_app_data: Default::default(), }; diff --git a/crates/shared/src/db_order_conversions.rs b/crates/shared/src/db_order_conversions.rs index f832376cf3..80cd71f171 100644 --- a/crates/shared/src/db_order_conversions.rs +++ b/crates/shared/src/db_order_conversions.rs @@ -90,9 +90,11 @@ pub fn full_order_into_model_order(order: database::orders::FullOrder) -> Result )?, executed_fee_amount: big_decimal_to_u256(&order.sum_fee) .context("executed fee amount is not a valid u256")?, - executed_total_fee: big_decimal_to_u256(&order.executed_total_fee) - .context("executed total fee is not a valid u256")?, - executed_total_fee_token: H160(order.executed_total_fee_token.0), + executed_surplus_fee: big_decimal_to_u256(&order.executed_fee) + .context("executed fee is not a valid u256")?, + executed_fee: big_decimal_to_u256(&order.executed_fee) + .context("executed fee is not a valid u256")?, + executed_fee_token: H160(order.executed_fee_token.0), invalidated: order.invalidated, status, is_liquidity_order: class == OrderClass::Liquidity, From 8ad4105b6d3361f1d31dc1171d6be5d1fda54294 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Fri, 13 Sep 2024 13:59:11 +0200 Subject: [PATCH 14/18] remove Some --- crates/database/src/order_execution.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/database/src/order_execution.rs b/crates/database/src/order_execution.rs index ece087cada..541e05e1a2 100644 --- a/crates/database/src/order_execution.rs +++ b/crates/database/src/order_execution.rs @@ -36,7 +36,7 @@ DO UPDATE SET reward = $3, total_fee = $4, total_fee_token = $5, block_number = .bind(order) .bind(auction) .bind(0.) // reward is deprecated but saved for historical analysis - .bind(Some(total_fee.amount)) + .bind(total_fee.amount) .bind(total_fee.token) .bind(block_number) .bind(protocol_fee_tokens) From 52b194d97fa3ae013c896502bbd5452cfa291e16 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 23 Dec 2024 15:47:08 +0100 Subject: [PATCH 15/18] small fixes --- crates/autopilot/src/domain/settlement/mod.rs | 2 +- database/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/autopilot/src/domain/settlement/mod.rs b/crates/autopilot/src/domain/settlement/mod.rs index 517fd8f44a..f4faf5e8e7 100644 --- a/crates/autopilot/src/domain/settlement/mod.rs +++ b/crates/autopilot/src/domain/settlement/mod.rs @@ -827,7 +827,7 @@ mod tests { let jit_trade = super::trade::Trade::new(transaction.trades[1].clone(), &auction, 0); assert_eq!(jit_trade.fee_in_ether(&auction.prices).unwrap().0, 0.into()); assert_eq!(jit_trade.score(&auction).unwrap().0, 0.into()); - assert_eq!(jit_trade.fee_breakdown(&auction).unwrap().total.0, 0.into()); + assert_eq!(jit_trade.fee_breakdown(&auction).unwrap().total.amount.0, 0.into()); assert!(jit_trade .fee_breakdown(&auction) .unwrap() diff --git a/database/README.md b/database/README.md index 4b5960633c..e35ec83de8 100644 --- a/database/README.md +++ b/database/README.md @@ -211,7 +211,7 @@ Protocol fee tokens/amounts are stored in the same order as fee policies in fee_ auction\_id | bigint | not null | in which auction this trade was initiated reward | double | not null | revert adjusted solver rewards, deprecated in favor of [CIP-20](https://snapshot.org/#/cow.eth/proposal/0x2d3f9bd1ea72dca84b03e97dda3efc1f4a42a772c54bd2037e8b62e7d09a491f) total\_fee | numeric | not null | total fee taken for execution of the trade - total\_fee\_token | bytea | not null | token in which the surplus fee is taken + total\_fee\_token | bytea | not null | token in which the total fee is taken block\_number | bigint | not null | block in which the order was executed protocol\_fee\_tokens | bytea[] | not null | tokens in which the protocol fees are taken protocol\_fee\_amounts | numeric[] | not null | amounts of protocol fees taken, aligned protocol\_fee\_tokens array From 079492b5d1f81f4a662c72f40b46ce6cf75c6093 Mon Sep 17 00:00:00 2001 From: sunce86 Date: Mon, 23 Dec 2024 17:56:00 +0100 Subject: [PATCH 16/18] renaming to executed fee --- crates/autopilot/src/domain/settlement/mod.rs | 5 ++++- crates/autopilot/src/domain/settlement/trade/math.rs | 2 -- crates/database/src/jit_orders.rs | 4 ++-- crates/database/src/order_execution.rs | 10 +++++----- crates/database/src/orders.rs | 4 ++-- database/README.md | 4 ++-- database/sql/V076__total_fee_token.sql | 12 ++++++------ 7 files changed, 21 insertions(+), 20 deletions(-) diff --git a/crates/autopilot/src/domain/settlement/mod.rs b/crates/autopilot/src/domain/settlement/mod.rs index f4faf5e8e7..2a47bd4c75 100644 --- a/crates/autopilot/src/domain/settlement/mod.rs +++ b/crates/autopilot/src/domain/settlement/mod.rs @@ -827,7 +827,10 @@ mod tests { let jit_trade = super::trade::Trade::new(transaction.trades[1].clone(), &auction, 0); assert_eq!(jit_trade.fee_in_ether(&auction.prices).unwrap().0, 0.into()); assert_eq!(jit_trade.score(&auction).unwrap().0, 0.into()); - assert_eq!(jit_trade.fee_breakdown(&auction).unwrap().total.amount.0, 0.into()); + assert_eq!( + jit_trade.fee_breakdown(&auction).unwrap().total.amount.0, + 0.into() + ); assert!(jit_trade .fee_breakdown(&auction) .unwrap() diff --git a/crates/autopilot/src/domain/settlement/trade/math.rs b/crates/autopilot/src/domain/settlement/trade/math.rs index fcca0b5275..5ff42f01d5 100644 --- a/crates/autopilot/src/domain/settlement/trade/math.rs +++ b/crates/autopilot/src/domain/settlement/trade/math.rs @@ -134,8 +134,6 @@ impl Trade { /// Total fee (protocol fee + network fee). Equal to a surplus difference /// before and after applying the fees. - /// - /// Denominated in SELL token pub fn fee_in_sell_token(&self) -> Result { let fee = self.fee()?; self.fee_into_sell_token(fee.amount) diff --git a/crates/database/src/jit_orders.rs b/crates/database/src/jit_orders.rs index 6725527b61..c10de8fe42 100644 --- a/crates/database/src/jit_orders.rs +++ b/crates/database/src/jit_orders.rs @@ -31,8 +31,8 @@ ARRAY[]::record[] AS post_interactions, NULL AS ethflow_data, NULL AS onchain_user, NULL AS onchain_placement_error, -COALESCE((SELECT SUM(total_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_fee, -COALESCE((SELECT total_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_fee_token, -- TODO surplus token +COALESCE((SELECT SUM(executed_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_fee, +COALESCE((SELECT executed_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_fee_token, -- TODO surplus token NULL AS full_app_data "#; diff --git a/crates/database/src/order_execution.rs b/crates/database/src/order_execution.rs index 541e05e1a2..21059b272f 100644 --- a/crates/database/src/order_execution.rs +++ b/crates/database/src/order_execution.rs @@ -18,7 +18,7 @@ pub async fn save( order: &OrderUid, auction: AuctionId, block_number: i64, - total_fee: Asset, + executed_fee: Asset, protocol_fees: &[Asset], ) -> Result<(), sqlx::Error> { let (protocol_fee_tokens, protocol_fee_amounts): (Vec<_>, Vec<_>) = protocol_fees @@ -27,17 +27,17 @@ pub async fn save( .unzip(); const QUERY: &str = r#" -INSERT INTO order_execution (order_uid, auction_id, reward, total_fee, total_fee_token, block_number, protocol_fee_tokens, protocol_fee_amounts) +INSERT INTO order_execution (order_uid, auction_id, reward, executed_fee, executed_fee_token, block_number, protocol_fee_tokens, protocol_fee_amounts) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (order_uid, auction_id) -DO UPDATE SET reward = $3, total_fee = $4, total_fee_token = $5, block_number = $6, protocol_fee_tokens = $7, protocol_fee_amounts = $8 +DO UPDATE SET reward = $3, executed_fee = $4, executed_fee_token = $5, block_number = $6, protocol_fee_tokens = $7, protocol_fee_amounts = $8 ;"#; sqlx::query(QUERY) .bind(order) .bind(auction) .bind(0.) // reward is deprecated but saved for historical analysis - .bind(total_fee.amount) - .bind(total_fee.token) + .bind(executed_fee.amount) + .bind(executed_fee.token) .bind(block_number) .bind(protocol_fee_tokens) .bind(protocol_fee_amounts) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index d30c5f1912..efb67b32bd 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -571,8 +571,8 @@ array(Select (p.target, p.value, p.data) from interactions p where p.order_uid = where eth_o.uid = o.uid limit 1) as ethflow_data, (SELECT onchain_o.sender from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_user, (SELECT onchain_o.placement_error from onchain_placed_orders onchain_o where onchain_o.uid = o.uid limit 1) as onchain_placement_error, -COALESCE((SELECT SUM(total_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_fee, -COALESCE((SELECT total_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_fee_token, -- TODO surplus token +COALESCE((SELECT SUM(executed_fee) FROM order_execution oe WHERE oe.order_uid = o.uid), 0) as executed_fee, +COALESCE((SELECT executed_fee_token FROM order_execution oe WHERE oe.order_uid = o.uid), o.sell_token) as executed_fee_token, -- TODO surplus token (SELECT full_app_data FROM app_data ad WHERE o.app_data = ad.contract_app_data LIMIT 1) as full_app_data "#; diff --git a/database/README.md b/database/README.md index e35ec83de8..8b8980b95c 100644 --- a/database/README.md +++ b/database/README.md @@ -210,8 +210,8 @@ Protocol fee tokens/amounts are stored in the same order as fee policies in fee_ order\_uid | bytea | not null | which order this trade execution is related to auction\_id | bigint | not null | in which auction this trade was initiated reward | double | not null | revert adjusted solver rewards, deprecated in favor of [CIP-20](https://snapshot.org/#/cow.eth/proposal/0x2d3f9bd1ea72dca84b03e97dda3efc1f4a42a772c54bd2037e8b62e7d09a491f) - total\_fee | numeric | not null | total fee taken for execution of the trade - total\_fee\_token | bytea | not null | token in which the total fee is taken + executed\_fee | numeric | not null | fee taken for execution of the trade + executed\_fee\_token | bytea | not null | token in which the executed fee is taken block\_number | bigint | not null | block in which the order was executed protocol\_fee\_tokens | bytea[] | not null | tokens in which the protocol fees are taken protocol\_fee\_amounts | numeric[] | not null | amounts of protocol fees taken, aligned protocol\_fee\_tokens array diff --git a/database/sql/V076__total_fee_token.sql b/database/sql/V076__total_fee_token.sql index 4510a07300..0c767efff9 100644 --- a/database/sql/V076__total_fee_token.sql +++ b/database/sql/V076__total_fee_token.sql @@ -1,13 +1,13 @@ ALTER TABLE order_execution -RENAME COLUMN surplus_fee TO total_fee; +RENAME COLUMN surplus_fee TO executed_fee; --- Add a new column to the order_execution table to store the total fee token +-- Add a new column to the order_execution table to store the executed fee token ALTER TABLE order_execution -ADD COLUMN total_fee_token bytea NOT NULL DEFAULT '\x0000000000000000000000000000000000000000'; +ADD COLUMN executed_fee_token bytea NOT NULL DEFAULT '\x0000000000000000000000000000000000000000'; -- Now populate existing rows with the sell token taken from the 'orders' table, or if it doesn't exist, try from 'jit_orders'. UPDATE order_execution oe -SET total_fee_token = COALESCE( +SET executed_fee_token = COALESCE( sub.sell_token, '\x0000000000000000000000000000000000000000' ) FROM ( @@ -20,5 +20,5 @@ FROM ( ) AS sub WHERE oe.order_uid = sub.uid; --- Drop the default value for the total_fee_token column -ALTER TABLE order_execution ALTER COLUMN total_fee_token DROP DEFAULT; +-- Drop the default value for the executed_fee_token column +ALTER TABLE order_execution ALTER COLUMN executed_fee_token DROP DEFAULT; From a8a503800760786399c71ac88d557181994105ff Mon Sep 17 00:00:00 2001 From: sunce86 Date: Tue, 24 Dec 2024 16:22:13 +0100 Subject: [PATCH 17/18] fix e2e test --- crates/e2e/tests/e2e/limit_orders.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/e2e/tests/e2e/limit_orders.rs b/crates/e2e/tests/e2e/limit_orders.rs index 94eb7e9e21..4f7ed21714 100644 --- a/crates/e2e/tests/e2e/limit_orders.rs +++ b/crates/e2e/tests/e2e/limit_orders.rs @@ -445,9 +445,9 @@ async fn two_limit_orders_multiple_winners_test(web3: Web3) { assert_eq!(competition.transaction_hashes.len(), 2); // Verify that settlement::Observed properly handled events let order_a_settled = services.get_order(&uid_a).await.unwrap(); - assert!(order_a_settled.metadata.executed_surplus_fee > 0.into()); + assert!(order_a_settled.metadata.executed_fee > 0.into()); let order_b_settled = services.get_order(&uid_b).await.unwrap(); - assert!(order_b_settled.metadata.executed_surplus_fee > 0.into()); + assert!(order_b_settled.metadata.executed_fee > 0.into()); } async fn too_many_limit_orders_test(web3: Web3) { From 28ec783beefb3e2c968c1a6fb0a6530cfb5976df Mon Sep 17 00:00:00 2001 From: sunce86 Date: Tue, 24 Dec 2024 16:28:21 +0100 Subject: [PATCH 18/18] fix openapi --- crates/orderbook/openapi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/orderbook/openapi.yml b/crates/orderbook/openapi.yml index 1649d7ede9..aa4c2da3ac 100644 --- a/crates/orderbook/openapi.yml +++ b/crates/orderbook/openapi.yml @@ -957,8 +957,8 @@ components: allOf: - $ref: "#/components/schemas/BigUint" nullable: false - executedTotalFeeToken: - description: Token the total fee was captured in. + executedFeeToken: + description: Token the executed fee was captured in. allOf: - $ref: "#/components/schemas/Address" nullable: false