From 88777a05366bb0c9d3f1de04f3f5d6794269b007 Mon Sep 17 00:00:00 2001 From: Romakl Date: Mon, 19 Feb 2024 17:50:51 +0200 Subject: [PATCH] chore(exchanges): working with okx data --- exchanges/constants/urls.py | 2 +- exchanges/exchange_manager.py | 17 +++----- exchanges/fetchers/binance_fetcher.py | 37 +++++++++-------- exchanges/fetchers/future_fetcher.py | 28 +++++++++++++ exchanges/fetchers/option_fetcher.py | 58 ++++++++++++++++++++++++++- exchanges/handlers/future.py | 12 ++---- exchanges/handlers/merge.py | 7 ++-- exchanges/main.py | 4 +- exchanges/processing.py | 32 +++++++++------ 9 files changed, 138 insertions(+), 59 deletions(-) diff --git a/exchanges/constants/urls.py b/exchanges/constants/urls.py index fb5292d..1375fb4 100644 --- a/exchanges/constants/urls.py +++ b/exchanges/constants/urls.py @@ -1,4 +1,4 @@ -BINANCE_API_OPTIONS_URL = "https://eapi.binance.com/eapi/v1/exchangeInfo" +BINANCE_API_OPTIONS_URL = "https://eapi.binance.com" BINANCE_API_FUTURES_URL = "https://fapi.binance.com/fapi/v1/premiumIndex" ByBit_API_URL = "https://api.bybit.com/v2/public/symbols" OKX_API_URL = "https://www.okex.com/api/spot/v3/instruments" diff --git a/exchanges/exchange_manager.py b/exchanges/exchange_manager.py index 3ba535f..29eaf82 100644 --- a/exchanges/exchange_manager.py +++ b/exchanges/exchange_manager.py @@ -1,3 +1,5 @@ +import json + import ccxt import logging @@ -22,24 +24,17 @@ def __init__(self, exchange_id, pairs_to_load, market_types): self.futures_data = pd.DataFrame() def fetch_binance_symbols(self): - logger.info("Fetching Binance option and future symbols.") - ( - binance_option_symbols, - binance_future_symbols, - ) = self.binance_fetcher.fetch_symbols() - return binance_option_symbols, binance_future_symbols + binance_option_symbols = self.binance_fetcher.fetch_symbols() + return binance_option_symbols def load_specific_pairs(self) -> pd.DataFrame: try: if self.exchange_id == "binance": - ( - binance_option_symbols, - binance_future_symbols, - ) = self.fetch_binance_symbols() + binance_option_symbols = self.fetch_binance_symbols() data = { "BTC/USD:BTC": { "option": binance_option_symbols, - "future": binance_future_symbols, + "future": None, } } return self.handle_market_type(data) diff --git a/exchanges/fetchers/binance_fetcher.py b/exchanges/fetchers/binance_fetcher.py index adb0509..b8366a3 100644 --- a/exchanges/fetchers/binance_fetcher.py +++ b/exchanges/fetchers/binance_fetcher.py @@ -1,3 +1,5 @@ +import pandas as pd + from exchanges.constants.urls import BINANCE_API_OPTIONS_URL, BINANCE_API_FUTURES_URL @@ -29,14 +31,21 @@ def get_response(url): @staticmethod def fetch_options_symbols(): - data = BinanceFetcher.get_response(BINANCE_API_OPTIONS_URL) - if data: - return [ - symbol_info.get("symbol") - for symbol_info in data.get("optionSymbols", []) - if "BTC" in symbol_info.get("symbol", "") - ] - return [] + data = BinanceFetcher.get_response(BINANCE_API_OPTIONS_URL+"/eapi/v1/exchangeInfo")["optionSymbols"] + data_df = pd.DataFrame(data) + # all symbols with BTC- + symbols = data_df["symbol"].loc[data_df["symbol"].str.contains("BTC-")] + return symbols + + @staticmethod + def fetch_mark_price(): + data = BinanceFetcher.get_response(BINANCE_API_OPTIONS_URL+"/eapi/v1/mark") + data_df = pd.DataFrame(data) + # get all where BTC- is in the symbol + return data_df.loc[data_df["symbol"].str.contains("BTC-")] + + + @staticmethod def fetch_futures_symbols(): @@ -45,14 +54,4 @@ def fetch_futures_symbols(): return [ res.get("symbol") for res in data if "BTCUSDT_" in res.get("symbol", "") ] - return [] - - @staticmethod - def fetch_symbols(): - options = BinanceFetcher.fetch_options_symbols() - futures = BinanceFetcher.fetch_futures_symbols() - if options or futures: - logger.info("Successfully fetched symbols from Binance") - else: - logger.error("Failed to fetch symbols from Binance") - return options, futures + return [] \ No newline at end of file diff --git a/exchanges/fetchers/future_fetcher.py b/exchanges/fetchers/future_fetcher.py index ce69ed9..c816447 100644 --- a/exchanges/fetchers/future_fetcher.py +++ b/exchanges/fetchers/future_fetcher.py @@ -9,6 +9,34 @@ class FutureFetcher: def __init__(self, exchange): self.exchange = exchange + def fetch_future_market_data( + self, market_symbols: list[str], exchange_name: str + ) -> pd.DataFrame: + """ + Fetches market data for a given list of market symbols from a specified exchange and processes it using pandas. + Args: + market_symbols: A list of symbols in the format recognized by the exchange. + exchange_name: String representing the exchange name ('deribit', 'okx', 'binance'). + Returns: + pd.DataFrame: DataFrame with processed market data for each option contract. + """ + try: + all_tickers = self.exchange.fetch_tickers(market_symbols) + tickers_df = pd.DataFrame(all_tickers).transpose() + # if exchange_name == "Deribit": + # return self.process_deribit_data(tickers_df) + # elif exchange_name == "OKX": + # return self.process_okx_data(tickers_df) + # elif exchange_name == "Binance": + # return self.process_binance_data(tickers_df) + # else: + # logging.error(f"Unsupported exchange: {exchange_name}") + # return pd.DataFrame() + return tickers_df + except Exception as e: + logging.error(f"Error fetching tickers from {exchange_name}: {e}") + return pd.DataFrame() + def fetch_market_data_okx(self, market_symbols: list[str]) -> pd.DataFrame: print("Fetching data from OKX for futures") data_list = [] diff --git a/exchanges/fetchers/option_fetcher.py b/exchanges/fetchers/option_fetcher.py index a9ead8a..1e404d1 100644 --- a/exchanges/fetchers/option_fetcher.py +++ b/exchanges/fetchers/option_fetcher.py @@ -1,8 +1,8 @@ -import json import logging import numpy as np import pandas as pd +import requests class OptionFetcher: @@ -23,6 +23,9 @@ def fetch_market_data( try: all_tickers = self.exchange.fetch_tickers(market_symbols) tickers_df = pd.DataFrame(all_tickers).transpose() + tickers_df.to_json( + f"{exchange_name}_raw_data_options.json", orient="records", indent=4 + ) if exchange_name == "Deribit": return self.process_deribit_data(tickers_df) elif exchange_name == "OKX": @@ -86,13 +89,51 @@ def process_deribit_data(self, df: pd.DataFrame) -> pd.DataFrame: ] def process_okx_data(self, df: pd.DataFrame) -> pd.DataFrame: + # Fetch mark prices from OKX API + response = requests.get( + "https://www.okx.com/api/v5/public/mark-price?instType=OPTION" + ) + mark_prices = response.json()["data"] + mark_prices_df = pd.DataFrame(mark_prices) + + # Convert 'instId' in mark_prices_df to 'symbol' format + mark_prices_df["symbol"] = mark_prices_df["instId"].apply( + self.convert_inst_id_to_symbol + ) + # Continue with the rest of the process, assuming the rest of your method is correct + df["underlying_price"] = self.exchange.fetch_ticker("BTC/USDT")["last"] + df["bid"] *= df["underlying_price"] + df["ask"] *= df["underlying_price"] df["datetime"] = pd.to_datetime(df["timestamp"], unit="ms") df["expiry"] = df["symbol"].apply(self.date_parser) df["YTM"] = (df["expiry"] - df["datetime"]) / np.timedelta64(1, "Y") - return df[["symbol", "bid", "ask", "datetime", "expiry", "YTM"]] + + # Merge the mark prices into the df based on the new 'symbol' + df = df.merge(mark_prices_df[["symbol", "markPx"]], on="symbol", how="left") + + # Rename the 'markPx' column to 'mark_price' for clarity (optional) + df.rename(columns={"markPx": "mark_price"}, inplace=True) + df["mark_price"] = pd.to_numeric(df["mark_price"], errors="coerce").fillna(0.0) * df[ + "underlying_price" + ] + + # Select and return the desired columns, including the new 'mark_price' + return df[ + [ + "symbol", + "bid", + "ask", + "mark_price", + "underlying_price", + "datetime", + "expiry", + "YTM", + ] + ] def process_binance_data(self, df: pd.DataFrame) -> pd.DataFrame: # Assuming 'info' contains the required details + print(df) df["bid"] = df["info"].apply(lambda x: float(x.get("bidPrice", 0))) df["ask"] = df["info"].apply(lambda x: float(x.get("askPrice", 0))) df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms") @@ -114,3 +155,16 @@ def date_parser(symbol: str) -> pd.Timestamp: except ValueError: continue return pd.NaT + + @staticmethod + def convert_inst_id_to_symbol(inst_id: str) -> str: + # Split the instId into its components + parts = inst_id.split("-") + currency = f"{parts[0]}/{parts[1]}" # e.g., BTC/USD + date = parts[2][:2] + parts[2][2:4] + parts[2][4:] # Reformat date + strike_price = parts[3] + option_type = parts[4] + + # Reassemble into the symbol format + symbol = f"{currency}:{parts[0]}-{date}-{strike_price}-{option_type}" + return symbol diff --git a/exchanges/handlers/future.py b/exchanges/handlers/future.py index 5a021bd..9978930 100644 --- a/exchanges/handlers/future.py +++ b/exchanges/handlers/future.py @@ -15,14 +15,8 @@ def __init__(self, exchange: str, market_types: List[str]): self.data_fetcher = FutureFetcher(exchange) def handle(self, market_symbols: List[str]) -> pd.DataFrame: - if str(self.exchange) == "Binance": - market_data = self.data_fetcher.fetch_market_data_binance(market_symbols) - elif str(self.exchange) == "OKX": - market_data = self.data_fetcher.fetch_market_data_okx(market_symbols) - elif str(self.exchange) == "Deribit": - market_data = self.data_fetcher.fetch_market_data_deribit(market_symbols) - else: - logger.error(f"Exchange not supported: {self.exchange}") - return pd.DataFrame() + market_data = self.data_fetcher.fetch_future_market_data( + market_symbols, str(self.exchange) + ) return market_data diff --git a/exchanges/handlers/merge.py b/exchanges/handlers/merge.py index 36fb81c..41d0f56 100644 --- a/exchanges/handlers/merge.py +++ b/exchanges/handlers/merge.py @@ -25,6 +25,9 @@ def handle( ) if future_market: futures_data = self.future_market_handler.handle(future_market) + futures_data.to_json( + f"{self.exchange}_futures_data.json", orient="records", indent=4 + ) merged_data = pd.concat([options_data, futures_data], ignore_index=True) else: merged_data = options_data @@ -36,9 +39,7 @@ def handle( spreads = self.processing.calculate_spreads(valid_quotes) remove_large_spreads = self.processing.remove_large_spreads(spreads) - spreads.to_json( - f"{self.exchange}_spreads.json", orient="records", indent=4 - ) + spreads.to_json(f"{self.exchange}_spreads.json", orient="records", indent=4) remove_large_spreads.to_json( f"{self.exchange}_remove_large_spreads.json", orient="records", indent=4 ) diff --git a/exchanges/main.py b/exchanges/main.py index f599d5f..ddbee2d 100644 --- a/exchanges/main.py +++ b/exchanges/main.py @@ -18,11 +18,11 @@ def main(): pairs_to_load=["BTC/USD:BTC"], market_types=["option", "future"] ) okx = OKXManager( - pairs_to_load=["BTC/USD:BTC"], market_types=["option", "future"] + pairs_to_load=["BTC/USD:BTC"], market_types=["option"] ) results = pd.DataFrame() - for manager in [deribit]: + for manager in [binance]: results = pd.concat( [results, manager.load_specific_pairs()], ignore_index=True ) diff --git a/exchanges/processing.py b/exchanges/processing.py index 96e1c58..4a7496e 100644 --- a/exchanges/processing.py +++ b/exchanges/processing.py @@ -12,9 +12,9 @@ def calculate_implied_interest_rates(df): # Calculate implied interest rates df = df.copy() - df["rimp"] = ( - np.log(df["mark_price"]) - np.log(df["underlying_price"]) - ) / df["YTM"] + df["rimp"] = (np.log(df["mark_price"]) - np.log(df["underlying_price"])) / df[ + "YTM" + ] # Rimp = (ln F − ln S)/T return df @@ -84,6 +84,19 @@ def remove_large_spreads(df): df = df[df["spread"] <= df["global_max_spread"]] return df + @staticmethod + def calculate_mid_price(df): + df["mid_price"] = (df["bid"] + df["ask"]) / 2 + return df + + @staticmethod + def select_options_within_range(df): + df = df.copy() + df["Kmin"] = df["Fimp"] / RANGE_MULT + df["Kmax"] = df["Fimp"] * RANGE_MULT + df = df[(df["strike"] >= df["Kmin"]) & (df["strike"] <= df["Kmax"])] + return df + # Calculate the implied forward price of the strike that has minimum ab- # solute mid-price difference between call and put options, for near and # next-term options: Fimp = K + F × (C − P ) where F is the forward price, @@ -97,15 +110,10 @@ def implied_forward_price(self, df): df["Fimp"] = df["strike"] + df["F"] * (df["call"] - df["put"]) return df + # Set the largest strike that is less than the implied forward Fimp as ATM + # strike KAT M for near and next-term options. @staticmethod - def calculate_mid_price(df): - df["mid_price"] = (df["bid"] + df["ask"]) / 2 - return df - - @staticmethod - def select_options_within_range(df): + def atm_strike(df): df = df.copy() - df["Kmin"] = df["Fimp"] / RANGE_MULT - df["Kmax"] = df["Fimp"] * RANGE_MULT - df = df[(df["strike"] >= df["Kmin"]) & (df["strike"] <= df["Kmax"])] + df["KATM"] = df[df["strike"] < df["Fimp"]].groupby("expiry")["strike"].max() return df