From 7bc21e8746a9bb2576194ba5bb0c010a9591d8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josep=20Batall=C3=A9?= Date: Mon, 22 Aug 2022 19:37:13 +0200 Subject: [PATCH] feat: include version. Fix currency missing afecting Clicktrade --- .gitattributes | 1 + README.md | 1 + VERSION | 1 + api/app.py | 7 ++ .../entities/brokers/clicktrade.py | 32 +++-- .../finance_reader/handlers/broker_read.py | 63 +--------- backend/finance_reader/utils/yahoo.py | 117 +++++++----------- backend/wallet_processor/entities/broker.py | 6 - backend/wallet_processor/entities/crypto.py | 1 - frontend/src/App.vue | 13 +- .../Dashboard/Layout/ContentFooter.vue | 23 ++-- .../Dashboard/Views/Broker/Balance.vue | 6 +- .../Dashboard/Views/Broker/Orders.vue | 4 +- .../Dashboard/Views/Dashboard/Overview.vue | 2 +- 14 files changed, 110 insertions(+), 167 deletions(-) create mode 100644 .gitattributes create mode 100644 VERSION diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ce44599 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.vue linguist-detectable=false \ No newline at end of file diff --git a/README.md b/README.md index 7f3dfc8..a9cd193 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ docker-compose run migrate 7. Add stock to a watchlist ## Development +Technology stack: Python, Vue, Redis and Timescaledb. ### Structure . diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..8a9ecc2 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.1 \ No newline at end of file diff --git a/api/app.py b/api/app.py index 0dc1cfa..eadfd4d 100644 --- a/api/app.py +++ b/api/app.py @@ -13,6 +13,9 @@ # import config and models from main path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) +with open("../VERSION") as f: + VERSION = f.read() + from config import settings from models.sql import create_db_connection @@ -81,6 +84,10 @@ def initialize_app(flask_app): blueprint = Blueprint('api', __name__, url_prefix='/api') + @app.route("/api/version") + def version(): + return VERSION + if settings.DEMO_MODE: log.info("Setting DEMO MODE!") diff --git a/backend/finance_reader/entities/brokers/clicktrade.py b/backend/finance_reader/entities/brokers/clicktrade.py index 60cef5a..a80d10d 100644 --- a/backend/finance_reader/entities/brokers/clicktrade.py +++ b/backend/finance_reader/entities/brokers/clicktrade.py @@ -106,10 +106,10 @@ def get_bookings(self, trade_id): def check_if_tradeable(self, isin): url = "https://fssoclicktrader.clicktrade.es/openapi/ref/v1/instruments/?$top=201&$skip=0&includeNonTradable=true&AssetTypes=Stock%2CEtf%2CEtc%2CEtn%2CFund%2CRights%2CCompanyWarrant%2CStockIndex&keywords={}&OrderBy=Popularity&ClientKey={}" - response4 = self._client.get(url.format(isin, self.client_key)) - if response4.status_code != 200: + response = self._client.get(url.format(isin, self.client_key)) + if response.status_code != 200: self._logger.error("Error, unable to search isin {}!!!".format(isin)) - return response4.json() + return response.json()['Data'] def read_transactions(self, start_date): url = "https://fssoclicktrader.clicktrade.es/openapi/cs/v1/reports/trades/{}?fromDate={}&toDate={}" @@ -142,7 +142,7 @@ def read_transactions(self, start_date): fee = 0 currency_rate = 0 - currency = 0 + currency = None exchange_fee = 0 for q in detail.get('Data', []): if q['BkAmountType'] == "Exchange Fee": @@ -158,14 +158,30 @@ def read_transactions(self, start_date): products = self.check_if_tradeable(isin) status = Ticker.Status.ACTIVE - if not len(products['Data']): + if not len(products): status = Ticker.Status.INACTIVE + if len(products) > 1: + self._logger.warning(f"Check multiple tradeable products for {isin}") + products = [p for p in products if p['Identifier'] == d.get('Uic')] + if not len(products): + self._logger.warning(f"No products found for {isin}") + continue + product = products[0] + + new_symbol, new_exchange_mic = product['Symbol'].upper().split(":") + if new_symbol != symbol: + self._logger.warning(f"Different symbol: {new_symbol} vs {symbol}. Saving {new_symbol}") + + if d['InstrumentDescription'] != product['Description']: + self._logger.warning(f"Different description: {d['InstrumentDescription']} vs {product['Description']}") + ticker = Ticker() ticker.isin = isin - ticker.ticker = symbol - ticker.name = d['InstrumentDescription'] + ticker.ticker = new_symbol + ticker.name = product['Description'] ticker.active = status + ticker.currency = product['CurrencyCode'] ticker.exchange = exchange_mic t = Transaction() @@ -179,7 +195,7 @@ def read_transactions(self, start_date): t.fee = fee t.exchange_fee = exchange_fee t.currency_rate = currency_rate - t.currency = currency if currency else None + t.currency = currency if currency else ticker.currency to_insert.append(t) diff --git a/backend/finance_reader/handlers/broker_read.py b/backend/finance_reader/handlers/broker_read.py index a7f394e..b99389e 100644 --- a/backend/finance_reader/handlers/broker_read.py +++ b/backend/finance_reader/handlers/broker_read.py @@ -9,8 +9,7 @@ class BrokerReader: def __init__(self): - self.exchange_mic = {} - self._get_iso_codes() + pass @staticmethod def _validate_data(data): @@ -52,8 +51,7 @@ def process(self, data): if not entity_account: return - # start_date = self._get_start_date(account_id) or "01/01/2017" - start_date = "01/01/2017" + start_date = self._get_start_date(account_id) or "01/01/2017" logger.info(f"Reading transactions from {start_date}...") transactions = broker.read_transactions(start_date) logger.info("Found {} transactions in {}".format(len(transactions), broker_name)) @@ -98,60 +96,9 @@ def _parse_read(self, account_id, entity_account, trans): StockTransaction.bulk_insert(transactions) - def _get_iso_codes(self): - import csv - with open("finance_reader/utils/ISO10383_MIC_NewFormat.csv") as f: - reader = csv.reader(f, delimiter=',') - for idx, row in enumerate(reader): - if idx == 0: - continue - self.exchange_mic[row[0]] = { - "mic": row[1], - "name": row[3], - "acronym": row[7], - "deleted": True if row[11] == 'DELETED' else False - } - - def _search_yahoo_ticker(self, ticker): - # y = YahooClient() - logger.info(f"Searching {ticker.ticker}") - if ticker.ticker == 'SHIP': - print("HCEKC") -# yahoo_symbol = y.get_yahoo_symbol(ticker.ticker) - import requests - client = requests.Session() - client.headers.update({ - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.8,es;q=0.6,ca;q=0.4', - 'Accept-Encoding': 'deflate' - }) - r = client.get(f"https://query2.finance.yahoo.com/v1/finance/search?q={ticker.ticker}") - d = r.json()['quotes'] - items = [c for c in d if c['typeDisp'] == 'Equity'] - yahoo_symbol = None - for item in items: - if item['symbol'] == ticker.ticker: - if item['exchDisp'] == ticker.exchange or item['exchDisp'] == self.exchange_mic.get(ticker.exchange, {}).get('acronym'): - yahoo_symbol = item['symbol'] - break - else: - logger.warning(f"Check ticker exchange {ticker.ticker} - {ticker.exchange}") - - if ticker.exchange and (item['exchange'] in ticker.exchange or item['exchange'] in self.exchange_mic.get(ticker.exchange, {}).get('acronym')): - logger.warning(f"Check ticker exchange {ticker.ticker} - {ticker.exchange}") - if ticker.ticker in item['symbol']: - yahoo_symbol = item['symbol'] - break - - if not yahoo_symbol: - logger.error("Unable to detect the Yahoo symbol") - return None - return yahoo_symbol - - def _create_new_ticker(self, tickers, t): + @staticmethod + def _create_new_ticker(tickers, t): logger.info(f"Creating new ticker {t.ticker.ticker} - {t.ticker.isin}!") - # TODO: search in yahoo if t.ticker.active == Ticker.Status.ACTIVE: # check if some ticker with this ticker already exists, and set to status INACTIVE @@ -165,7 +112,7 @@ def _create_new_ticker(self, tickers, t): except: pass - yahoo_ticker = self._search_yahoo_ticker(t.ticker) + yahoo_ticker = YahooClient().get_ticker(t.ticker) logger.info(f"Ticker {t.ticker.ticker} - Yahoo ticker: {yahoo_ticker}!") ticker = Ticker(ticker=t.ticker.ticker, isin=t.ticker.isin, diff --git a/backend/finance_reader/utils/yahoo.py b/backend/finance_reader/utils/yahoo.py index 15cfe03..cec4062 100644 --- a/backend/finance_reader/utils/yahoo.py +++ b/backend/finance_reader/utils/yahoo.py @@ -1,10 +1,6 @@ import requests -import json -from datetime import datetime -from urllib.parse import urlparse, parse_qs +import csv import requests.utils -import binascii -import pickle import logging @@ -23,74 +19,47 @@ def __init__(self): 'Accept-Encoding': 'deflate' } ) - try: - with open("finance_reader/utils/cookies.pickle") as f: - self.client.cookies = pickle.loads(binascii.a2b_base64(f.read())) - except: - self.generate_cookies() - def get_ticker_info(self, symbol): - endpoint = "https://query1.finance.yahoo.com/v7/finance/quote?lang=en-US®ion=US&corsDomain=finance.yahoo.com" - flds = ("symbol", "shortName", "currency", "fullExchangeName", "exchange") - - url = f"{endpoint}&fields={','.join(flds)}&symbols={symbol}" - r = self.client.get(url) - api_json = json.loads(r.text) - - return api_json['quoteResponse']['result'][0]['shortName'], api_json['quoteResponse']['result'][0]['currency'] - - def generate_cookies(self): - logger.info("Generating cookies!") - url = "https://finance.yahoo.com" - r = self.client.get(url) - - o = urlparse(r.url) - params = parse_qs(o.query) - session_id = params['sessionId'][0] - url = f"https://consent.yahoo.com/v2/collectConsent?sessionId={session_id}" - - o = urlparse(r.history[1].url) - params = parse_qs(o.query) - csrf_token = params['gcrumb'][0] - data = { - "csrfToken": csrf_token, - "sessionId": session_id, - "originalDoneUrl": "https://finance.yahoo.com/?guccounter=1", - "namespace": "yahoo", - "agree": "agree" - } - - self.client.post(url, data=data) - - logger.info("Saving cookies to pickle file") - cookies = requests.utils.dict_from_cookiejar(self.client.cookies) - parsed_cookies = binascii.b2a_base64(pickle.dumps(self.client.cookies)).decode() - - with open("finance_reader/utils/cookies.pickle", 'w') as f: - f.write(parsed_cookies) - - def get_yahoo_symbol(self, symbol): - url = "http://d.yimg.com/autoc.finance.yahoo.com/autoc?query={}®ion=1&lang=en".format(symbol) - url = "https://finance.yahoo.com/_finance_doubledown/api/resource/searchassist;searchTerm={}?device=console&returnMeta=true".format(symbol) - - r = self.client.get(url) - if r.status_code != 200: - logger.error(f"Status code: {r.status_code}. Error: {r.text}") - return None - - try: - result = r.json() - except: - logger.error("Unable to read response as json") + self.exchange_mic = {} + self._get_iso_codes() + + def _get_iso_codes(self): + with open("finance_reader/utils/ISO10383_MIC_NewFormat.csv") as f: + reader = csv.reader(f, delimiter=',') + for idx, row in enumerate(reader): + if idx == 0: + continue + self.exchange_mic[row[0]] = { + "mic": row[1], + "name": row[3], + "acronym": row[7], + "deleted": True if row[11] == 'DELETED' else False + } + + def get_ticker(self, ticker): + r = self.client.get(f"https://query2.finance.yahoo.com/v1/finance/search?q={ticker.ticker}") + d = r.json()['quotes'] + items = [c for c in d if c['typeDisp'] == 'Equity'] + yahoo_symbol = None + for item in items: + if item['symbol'] == ticker.ticker: + if item['exchDisp'] == ticker.exchange or item['exchDisp'] == self.exchange_mic.get(ticker.exchange, + {}).get('acronym'): + yahoo_symbol = item['symbol'] + break + else: + logger.warning(f"Check ticker exchange {ticker.ticker} - {ticker.exchange}") + + if ticker.exchange and ( + item['exchange'] in ticker.exchange or item['exchange'] in self.exchange_mic.get(ticker.exchange, + {}).get( + 'acronym')): + logger.warning(f"Check ticker exchange {ticker.ticker} - {ticker.exchange}") + if ticker.ticker in item['symbol']: + yahoo_symbol = item['symbol'] + break + + if not yahoo_symbol: + logger.error("Unable to detect the Yahoo symbol") return None - - if len(result['data']['items']) == 1: - return result['data']['items'][0]['symbol'] - - # search by name - for x in result['data']['items']: - # if x['name'] - return x['symbol'] - # if x['symbol'] == symbol: - - return None + return yahoo_symbol diff --git a/backend/wallet_processor/entities/broker.py b/backend/wallet_processor/entities/broker.py index 572577f..6366dca 100644 --- a/backend/wallet_processor/entities/broker.py +++ b/backend/wallet_processor/entities/broker.py @@ -86,9 +86,6 @@ def calc_wallet(self, user_id, orders, queue, tracked_orders): """ to_insert = [] for ticker, partial_orders in queue.queues.items(): - if ticker.ticker == 'CRSR': - print("CHECL") - # self._logger.debug(f"Processing ticker {ticker.ticker} - {ticker}") if not ticker: self._logger.error("Ticker not found!") continue @@ -124,19 +121,16 @@ def calc_wallet(self, user_id, orders, queue, tracked_orders): continue # Calculate current benefits taking into account sells - current_benefits = 0 current_benefits_eur = 0 total_sell = 0 total_sell_eur = 0 for order in [w for w in tracked_orders if w.sell_trade.ticker == ticker]: - current_benefits += order.benefits current_benefits_eur += order.benefits_in_eur total_sell += order.sell_trade.price * order.amount total_sell_eur += order.sell_trade.price * order.amount * order.sell_trade.currency_rate fees += order.sell_trade.fees # self._logger.debug(f"Benefits: {current_benefits}. Fees: {fees}") - current_benefits += fees break_even = (total_cost_eur - current_benefits_eur) / shares w = Wallet( diff --git a/backend/wallet_processor/entities/crypto.py b/backend/wallet_processor/entities/crypto.py index c720059..50440c9 100644 --- a/backend/wallet_processor/entities/crypto.py +++ b/backend/wallet_processor/entities/crypto.py @@ -151,7 +151,6 @@ def calc_wallet(self, user_id, orders, queue, tracked_orders): """ Calculate current open orders """ - # calculating wallet, using Wallet and OpenOrders to_insert = [] for ticker, partial_orders in queue.queues.items(): self._logger.debug(f"Start processing ticker {ticker}") diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 649e45b..17b7999 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -13,8 +13,9 @@