Withdrawals (#135)

* Add withdrawals to rewards Signed-off-by: cyc60 <[email protected]> * Update python version, fix tests Signed-off-by: cyc60 <[email protected]> * Fix precommit error Signed-off-by: cyc60 <[email protected]> * Remove get withdrawals backoff Signed-off-by: cyc60 <[email protected]> * Withdrawals: Use genesis slot and epoch Signed-off-by: cyc60 <[email protected]> * Withdrawals: Check max slot Signed-off-by: cyc60 <[email protected]> * Withdrawals: Calculate genesis block based on epoch Signed-off-by: cyc60 <[email protected]> --------- Signed-off-by: cyc60 <[email protected]>
stakewise · Mar 24, 2023 · effbdf8 · effbdf8
1 parent 2b129fe
commit effbdf8
Show file tree

Hide file tree

Showing 17 changed files with 1,622 additions and 754 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -30,7 +30,7 @@ jobs:
       - name: Set up python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.10.5
+          python-version: 3.10.10
 
       # Install poetry
       - name: Load cached Poetry installation
@@ -44,6 +44,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.3.2
 
       # Install dependencies
       - name: Install dependencies

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -10,7 +10,7 @@ repos:
       - id: flake8
 
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # `python-base` sets up all our shared environment variables
-FROM python:3.10.5-slim as python-base
+FROM python:3.10.10-slim as python-base
 
     # python
 ENV PYTHONUNBUFFERED=1 \
@@ -13,7 +13,7 @@ ENV PYTHONUNBUFFERED=1 \
     \
     # poetry
     # https://python-poetry.org/docs/configuration/#using-environment-variables
-    POETRY_VERSION=1.1.10 \
+    POETRY_VERSION=1.3.2 \
     # make poetry install to this location
     POETRY_HOME="/opt/poetry" \
     # make poetry create the virtual environment in the project's root

diff --git a/deploy/gnosis/.env.example b/deploy/gnosis/.env.example
@@ -26,6 +26,10 @@ ETHEREUM_SUBGRAPH_URLS=https://api.thegraph.com/subgraphs/name/stakewise/ethereu
 # NB! You must use a different private key for every network
 ORACLE_PRIVATE_KEY=0x<private_key>
 
+# ETH1 (execution) client endpoint
+# Change if running an external ETH1 node
+ETH1_ENDPOINT=http://eth1-node:8545
+
 # ETH2 (consensus) client endpoint
 # Change if running an external ETH2 node
 ETH2_ENDPOINT=http://eth2-node:5052

diff --git a/deploy/goerli/.env.example b/deploy/goerli/.env.example
@@ -27,6 +27,10 @@ UNISWAP_V3_SUBGRAPH_URLS=https://api.thegraph.com/subgraphs/name/stakewise/unisw
 # NB! You must use a different private key for every network
 ORACLE_PRIVATE_KEY=0x<private_key>
 
+# ETH1 (execution) client endpoint
+# Change if running an external ETH1 node
+ETH1_ENDPOINT=http://eth1-node:8545
+
 # ETH2 (consensus) client endpoint
 # Change if running an external ETH2 node
 ETH2_ENDPOINT=http://eth2-node:5052

diff --git a/deploy/harbour_goerli/.env.example b/deploy/harbour_goerli/.env.example
@@ -26,6 +26,10 @@ ETHEREUM_SUBGRAPH_URLS=https://api.thegraph.com/subgraphs/name/stakewise/ethereu
 # NB! You must use a different private key for every network
 ORACLE_PRIVATE_KEY=0x<private_key>
 
+# ETH1 (execution) client endpoint
+# Change if running an external ETH1 node
+ETH1_ENDPOINT=http://eth1-node:8545
+
 # ETH2 (consensus) client endpoint
 # Change if running an external ETH2 node
 ETH2_ENDPOINT=http://eth2-node:5052

diff --git a/deploy/harbour_mainnet/.env.example b/deploy/harbour_mainnet/.env.example
@@ -26,6 +26,10 @@ ETHEREUM_SUBGRAPH_URLS=https://api.thegraph.com/subgraphs/name/stakewise/ethereu
 # NB! You must use a different private key for every network
 ORACLE_PRIVATE_KEY=0x<private_key>
 
+# ETH1 (execution) client endpoint
+# Change if running an external ETH1 node
+ETH1_ENDPOINT=http://eth1-node:8545
+
 # ETH2 (consensus) client endpoint
 # Change if running an external ETH2 node
 ETH2_ENDPOINT=http://eth2-node:5052

diff --git a/deploy/mainnet/.env.example b/deploy/mainnet/.env.example
@@ -28,6 +28,10 @@ UNISWAP_V3_SUBGRAPH_URLS=https://api.thegraph.com/subgraphs/name/stakewise/unisw
 # NB! You must use a different private key for every network
 ORACLE_PRIVATE_KEY=0x<private_key>
 
+# ETH1 (execution) client endpoint
+# Change if running an external ETH1 node
+ETH1_ENDPOINT=http://eth1-node:8545
+
 # ETH2 (consensus) client endpoint
 # Change if running an external ETH2 node
 ETH2_ENDPOINT=http://eth2-node:5052

diff --git a/oracle/networks.py b/oracle/networks.py
@@ -34,6 +34,7 @@
             default="https://graph.stakewise.io/subgraphs/name/stakewise/uniswap-v3,https://api.thegraph.com/subgraphs/name/stakewise/uniswap-v3-mainnet",
             cast=Csv(),
         ),
+        ETH1_ENDPOINT=config("ETH1_ENDPOINT", default=""),
         ETH2_ENDPOINT=config("ETH2_ENDPOINT", default=""),
         VALIDATORS_FETCH_CHUNK_SIZE=config(
             "VALIDATORS_FETCH_CHUNK_SIZE",
@@ -72,6 +73,7 @@
         ORACLE_STAKEWISE_OPERATOR=Web3.toChecksumAddress(
             "0x5fc60576b92c5ce5c341c43e3b2866eb9e0cddd1"
         ),
+        WITHDRAWALS_GENESIS_EPOCH=194048,
         AWS_BUCKET_NAME=config("AWS_BUCKET_NAME", default="oracle-votes-mainnet"),
         AWS_REGION=config("AWS_REGION", default="eu-central-1"),
         AWS_ACCESS_KEY_ID=config("AWS_ACCESS_KEY_ID", default=""),
@@ -112,6 +114,7 @@
             default="",
             cast=Csv(),
         ),
+        ETH1_ENDPOINT=config("ETH1_ENDPOINT", default=""),
         ETH2_ENDPOINT=config("ETH2_ENDPOINT", default=""),
         VALIDATORS_FETCH_CHUNK_SIZE=config(
             "VALIDATORS_FETCH_CHUNK_SIZE",
@@ -148,6 +151,7 @@
         ),
         ORACLE_PRIVATE_KEY=config("ORACLE_PRIVATE_KEY", default=""),
         ORACLE_STAKEWISE_OPERATOR=EMPTY_ADDR_HEX,
+        WITHDRAWALS_GENESIS_EPOCH=194048,
         AWS_BUCKET_NAME=config(
             "AWS_BUCKET_NAME",
             default="oracle-votes-harbour-mainnet",
@@ -187,6 +191,7 @@
             default="https://api.thegraph.com/subgraphs/name/stakewise/uniswap-v3-goerli",
             cast=Csv(),
         ),
+        ETH1_ENDPOINT=config("ETH1_ENDPOINT", default=""),
         ETH2_ENDPOINT=config("ETH2_ENDPOINT", default=""),
         VALIDATORS_FETCH_CHUNK_SIZE=config(
             "VALIDATORS_FETCH_CHUNK_SIZE",
@@ -223,6 +228,7 @@
         ),
         ORACLE_PRIVATE_KEY=config("ORACLE_PRIVATE_KEY", default=""),
         ORACLE_STAKEWISE_OPERATOR=EMPTY_ADDR_HEX,
+        WITHDRAWALS_GENESIS_EPOCH=162304,
         AWS_BUCKET_NAME=config("AWS_BUCKET_NAME", default="oracle-votes-goerli"),
         AWS_REGION=config("AWS_REGION", default="eu-central-1"),
         AWS_ACCESS_KEY_ID=config("AWS_ACCESS_KEY_ID", default=""),
@@ -259,6 +265,7 @@
             default="",
             cast=Csv(),
         ),
+        ETH1_ENDPOINT=config("ETH1_ENDPOINT", default=""),
         ETH2_ENDPOINT=config("ETH2_ENDPOINT", default=""),
         VALIDATORS_FETCH_CHUNK_SIZE=config(
             "VALIDATORS_FETCH_CHUNK_SIZE",
@@ -295,6 +302,7 @@
         ),
         ORACLE_PRIVATE_KEY=config("ORACLE_PRIVATE_KEY", default=""),
         ORACLE_STAKEWISE_OPERATOR=EMPTY_ADDR_HEX,
+        WITHDRAWALS_GENESIS_EPOCH=162304,
         AWS_BUCKET_NAME=config(
             "AWS_BUCKET_NAME",
             default="oracle-votes-perm-goerli",
@@ -334,6 +342,7 @@
             default="",
             cast=Csv(),
         ),
+        ETH1_ENDPOINT=config("ETH1_ENDPOINT", default=""),
         ETH2_ENDPOINT=config("ETH2_ENDPOINT", default=""),
         VALIDATORS_FETCH_CHUNK_SIZE=config(
             "VALIDATORS_FETCH_CHUNK_SIZE",
@@ -370,6 +379,7 @@
         ),
         ORACLE_PRIVATE_KEY=config("ORACLE_PRIVATE_KEY", default=""),
         ORACLE_STAKEWISE_OPERATOR=EMPTY_ADDR_HEX,
+        WITHDRAWALS_GENESIS_EPOCH=0,
         AWS_BUCKET_NAME=config("AWS_BUCKET_NAME", default="oracle-votes-gnosis"),
         AWS_REGION=config("AWS_REGION", default="eu-north-1"),
         AWS_ACCESS_KEY_ID=config("AWS_ACCESS_KEY_ID", default=""),

diff --git a/oracle/oracle/common/eth1.py b/oracle/oracle/common/eth1.py
@@ -2,6 +2,8 @@
 import logging
 from typing import Dict, TypedDict
 
+from web3 import Web3
+from web3.middleware import geth_poa_middleware
 from web3.types import BlockNumber, Timestamp, Wei
 
 from oracle.oracle.common.clients import execute_single_gql_query, execute_sw_gql_query
@@ -14,7 +16,7 @@
 from oracle.oracle.distributor.common.types import DistributorVotingParameters
 from oracle.oracle.rewards.types import RewardsVotingParameters
 from oracle.oracle.validators.types import ValidatorVotingParameters
-from oracle.settings import CONFIRMATION_BLOCKS, NETWORKS
+from oracle.settings import CONFIRMATION_BLOCKS, NETWORK_CONFIG, NETWORKS
 
 logger = logging.getLogger(__name__)
 
@@ -30,6 +32,28 @@ class VotingParameters(TypedDict):
     validator: ValidatorVotingParameters
 
 
+def get_web3_client() -> Web3:
+    """Returns instance of the Web3 client."""
+    endpoint = NETWORK_CONFIG["ETH1_ENDPOINT"]
+
+    # Prefer WS over HTTP
+    if endpoint.startswith("ws"):
+        w3 = Web3(Web3.WebsocketProvider(endpoint, websocket_timeout=60))
+        logger.warning(f"Web3 websocket endpoint={endpoint}")
+    elif endpoint.startswith("http"):
+        w3 = Web3(Web3.HTTPProvider(endpoint))
+        logger.warning(f"Web3 HTTP endpoint={endpoint}")
+    else:
+        w3 = Web3(Web3.IPCProvider(endpoint))
+        logger.warning(f"Web3 HTTP endpoint={endpoint}")
+
+    if NETWORK_CONFIG["IS_POA"]:
+        w3.middleware_onion.inject(geth_poa_middleware, layer=0)
+        logger.warning("Injected POA middleware")
+
+    return w3
+
+
 async def get_finalized_block(network: str) -> Block:
     """Gets the finalized block number and its timestamp."""
     results = await asyncio.gather(

diff --git a/oracle/oracle/rewards/controller.py b/oracle/oracle/rewards/controller.py
@@ -1,5 +1,7 @@
 import asyncio
+import concurrent.futures
 import logging
+from concurrent.futures import as_completed
 from datetime import datetime, timezone
 from typing import Union
 
@@ -10,7 +12,13 @@
 from web3.types import Timestamp, Wei
 
 from oracle.networks import GNOSIS_CHAIN
-from oracle.oracle.rewards.types import RewardsVotingParameters, RewardVote
+from oracle.oracle.common.eth1 import get_web3_client
+from oracle.oracle.rewards.eth1 import get_withdrawals
+from oracle.oracle.rewards.types import (
+    RegisteredValidatorsPublicKeys,
+    RewardsVotingParameters,
+    RewardVote,
+)
 from oracle.oracle.utils import save
 from oracle.oracle.vote import submit_vote
 from oracle.settings import (
@@ -25,6 +33,7 @@
 from .eth2 import (
     PENDING_STATUSES,
     ValidatorStatus,
+    get_execution_block,
     get_finality_checkpoints,
     get_validators,
 )
@@ -52,6 +61,7 @@ def __init__(
             self.slots_per_epoch * NETWORK_CONFIG["SECONDS_PER_SLOT"]
         )
         self.deposit_token_symbol = NETWORK_CONFIG["DEPOSIT_TOKEN_SYMBOL"]
+        self.withdrawals_genesis_epoch = NETWORK_CONFIG["WITHDRAWALS_GENESIS_EPOCH"]
         self.last_vote_total_rewards = None
 
     @save
@@ -100,28 +110,22 @@ async def process(
 
         state_id = str(update_epoch * self.slots_per_epoch)
         total_rewards: Wei = voting_params["total_fees"]
-        activated_validators = 0
-        chunk_size = NETWORK_CONFIG["VALIDATORS_FETCH_CHUNK_SIZE"]
-
-        # fetch balances in chunks
-        for i in range(0, len(public_keys), chunk_size):
-            validators = await get_validators(
-                session=self.aiohttp_session,
-                public_keys=public_keys[i : i + chunk_size],
-                state_id=state_id,
+        validator_indexes, balance_rewards = await self.calculate_balance_rewards(
+            public_keys, state_id
+        )
+        total_rewards += balance_rewards
+        activated_validators = len(validator_indexes)
+
+        if (
+            self.withdrawals_genesis_epoch
+            and update_epoch >= self.withdrawals_genesis_epoch
+        ):
+            withdrawals_rewards = await self.calculate_withdrawal_rewards(
+                validator_indexes=validator_indexes,
+                to_block=current_block_number,
+                current_slot=int(state_id),
             )
-            for validator in validators:
-                if ValidatorStatus(validator["status"]) in PENDING_STATUSES:
-                    continue
-
-                activated_validators += 1
-                validator_reward = (
-                    Web3.toWei(validator["balance"], "gwei") - self.deposit_amount
-                )
-                if NETWORK == GNOSIS_CHAIN:
-                    # apply mGNO <-> GNO exchange rate
-                    validator_reward = Wei(int(validator_reward * WAD // MGNO_RATE))
-                total_rewards += validator_reward
+            total_rewards += withdrawals_rewards
 
         pretty_total_rewards = self.format_ether(total_rewards)
         logger.info(
@@ -172,6 +176,71 @@ async def process(
 
         self.last_vote_total_rewards = total_rewards
 
+    async def calculate_balance_rewards(
+        self, public_keys: RegisteredValidatorsPublicKeys, state_id: str
+    ) -> tuple[set[int], Wei]:
+        validator_indexes = set()
+        rewards = 0
+        chunk_size = NETWORK_CONFIG["VALIDATORS_FETCH_CHUNK_SIZE"]
+        # fetch balances in chunks
+        for i in range(0, len(public_keys), chunk_size):
+            validators = await get_validators(
+                session=self.aiohttp_session,
+                public_keys=public_keys[i : i + chunk_size],
+                state_id=state_id,
+            )
+            for validator in validators:
+                if ValidatorStatus(validator["status"]) in PENDING_STATUSES:
+                    continue
+
+                validator_indexes.add(int(validator["index"]))
+                validator_reward = (
+                    Web3.toWei(validator["balance"], "gwei") - self.deposit_amount
+                )
+                if NETWORK == GNOSIS_CHAIN:
+                    # apply mGNO <-> GNO exchange rate
+                    validator_reward = Wei(int(validator_reward * WAD // MGNO_RATE))
+                rewards += validator_reward
+
+        return validator_indexes, Wei(rewards)
+
+    async def calculate_withdrawal_rewards(
+        self, validator_indexes: set[int], to_block: BlockNumber, current_slot: int
+    ) -> Wei:
+        withdrawals_amount = 0
+        from_block = await self.get_withdrawals_from_block(current_slot)
+        if not from_block or from_block >= to_block:
+            return Wei(0)
+
+        execution_client = get_web3_client()
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+            futures = [
+                executor.submit(get_withdrawals, execution_client, block_number)
+                for block_number in range(from_block, to_block)
+            ]
+            for future in as_completed(futures):
+                withdrawals = future.result()
+                for withdrawal in withdrawals:
+                    if withdrawal["validator_index"] in validator_indexes:
+                        withdrawals_amount += withdrawal["amount"]
+
+        withdrawals_amount = Web3.toWei(withdrawals_amount, "gwei")
+        if NETWORK == GNOSIS_CHAIN:
+            # apply mGNO <-> GNO exchange rate
+            withdrawals_amount = Wei(int(withdrawals_amount * WAD // MGNO_RATE))
+        return withdrawals_amount
+
+    async def get_withdrawals_from_block(self, current_slot: int) -> BlockNumber | None:
+        slot_number = self.withdrawals_genesis_epoch * self.slots_per_epoch
+        while slot_number <= current_slot:
+            from_block = await get_execution_block(
+                session=self.aiohttp_session, slot_number=slot_number
+            )
+            if from_block:
+                return from_block
+            slot_number += 1
+
     def format_ether(self, value: Union[str, int, Wei]) -> str:
         """Converts Wei value."""
         _value = int(value)