feat: include version. Fix currency missing afecting Clicktrade

.gitattributes

@@ -0,0 +1 @@
+*.vue linguist-detectable=false

README.md

@@ -50,6 +50,7 @@ docker-compose run migrate
 7. Add stock to a watchlist
 
 ## Development
+Technology stack: Python, Vue, Redis and Timescaledb.
 
 ### Structure
     .

VERSION

@@ -0,0 +1 @@
+0.0.1

api/app.py

@@ -13,6 +13,9 @@
 # import config and models from main path
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')))
 
+with open("../VERSION") as f:
+    VERSION = f.read()
+
 from config import settings
 from models.sql import create_db_connection
 
@@ -81,6 +84,10 @@ def initialize_app(flask_app):
 
     blueprint = Blueprint('api', __name__, url_prefix='/api')
 
+    @app.route("/api/version")
+    def version():
+        return VERSION
+
     if settings.DEMO_MODE:
         log.info("Setting DEMO MODE!")
 

backend/finance_reader/entities/brokers/clicktrade.py

@@ -106,10 +106,10 @@ def get_bookings(self, trade_id):
 
     def check_if_tradeable(self, isin):
         url = "https://fssoclicktrader.clicktrade.es/openapi/ref/v1/instruments/?$top=201&$skip=0&includeNonTradable=true&AssetTypes=Stock%2CEtf%2CEtc%2CEtn%2CFund%2CRights%2CCompanyWarrant%2CStockIndex&keywords={}&OrderBy=Popularity&ClientKey={}"
-        response4 = self._client.get(url.format(isin, self.client_key))
-        if response4.status_code != 200:
+        response = self._client.get(url.format(isin, self.client_key))
+        if response.status_code != 200:
             self._logger.error("Error, unable to search isin {}!!!".format(isin))
-        return response4.json()
+        return response.json()['Data']
 
     def read_transactions(self, start_date):
         url = "https://fssoclicktrader.clicktrade.es/openapi/cs/v1/reports/trades/{}?fromDate={}&toDate={}"
@@ -142,7 +142,7 @@ def read_transactions(self, start_date):
 
             fee = 0
             currency_rate = 0
-            currency = 0
+            currency = None
             exchange_fee = 0
             for q in detail.get('Data', []):
                 if q['BkAmountType'] == "Exchange Fee":
@@ -158,14 +158,30 @@ def read_transactions(self, start_date):
 
             products = self.check_if_tradeable(isin)
             status = Ticker.Status.ACTIVE
-            if not len(products['Data']):
+            if not len(products):
                 status = Ticker.Status.INACTIVE
 
+            if len(products) > 1:
+                self._logger.warning(f"Check multiple tradeable products for {isin}")
+                products = [p for p in products if p['Identifier'] == d.get('Uic')]
+                if not len(products):
+                    self._logger.warning(f"No products found for {isin}")
+                    continue
+            product = products[0]
+
+            new_symbol, new_exchange_mic = product['Symbol'].upper().split(":")
+            if new_symbol != symbol:
+                self._logger.warning(f"Different symbol: {new_symbol} vs {symbol}. Saving {new_symbol}")
+
+            if d['InstrumentDescription'] != product['Description']:
+                self._logger.warning(f"Different description: {d['InstrumentDescription']} vs {product['Description']}")
+
             ticker = Ticker()
             ticker.isin = isin
-            ticker.ticker = symbol
-            ticker.name = d['InstrumentDescription']
+            ticker.ticker = new_symbol
+            ticker.name = product['Description']
             ticker.active = status
+            ticker.currency = product['CurrencyCode']
             ticker.exchange = exchange_mic
 
             t = Transaction()
@@ -179,7 +195,7 @@ def read_transactions(self, start_date):
             t.fee = fee
             t.exchange_fee = exchange_fee
             t.currency_rate = currency_rate
-            t.currency = currency if currency else None
+            t.currency = currency if currency else ticker.currency
 
             to_insert.append(t)
 

backend/finance_reader/handlers/broker_read.py

@@ -9,8 +9,7 @@
 
 class BrokerReader:
     def __init__(self):
-        self.exchange_mic = {}
-        self._get_iso_codes()
+        pass
 
     @staticmethod
     def _validate_data(data):
@@ -52,8 +51,7 @@ def process(self, data):
         if not entity_account:
             return
 
-        # start_date = self._get_start_date(account_id) or "01/01/2017"
-        start_date = "01/01/2017"
+        start_date = self._get_start_date(account_id) or "01/01/2017"
         logger.info(f"Reading transactions from {start_date}...")
         transactions = broker.read_transactions(start_date)
         logger.info("Found {} transactions in {}".format(len(transactions), broker_name))
@@ -98,60 +96,9 @@ def _parse_read(self, account_id, entity_account, trans):
 
         StockTransaction.bulk_insert(transactions)
 
-    def _get_iso_codes(self):
-        import csv
-        with open("finance_reader/utils/ISO10383_MIC_NewFormat.csv") as f:
-            reader = csv.reader(f, delimiter=',')
-            for idx, row in enumerate(reader):
-                if idx == 0:
-                    continue
-                self.exchange_mic[row[0]] = {
-                    "mic": row[1],
-                    "name": row[3],
-                    "acronym": row[7],
-                    "deleted": True if row[11] == 'DELETED' else False
-                }
-
-    def _search_yahoo_ticker(self, ticker):
-        # y = YahooClient()
-        logger.info(f"Searching {ticker.ticker}")
-        if ticker.ticker == 'SHIP':
-            print("HCEKC")
-#        yahoo_symbol = y.get_yahoo_symbol(ticker.ticker)
-        import requests
-        client = requests.Session()
-        client.headers.update({
-                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36',
-                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-                'Accept-Language': 'en-US,en;q=0.8,es;q=0.6,ca;q=0.4',
-                'Accept-Encoding': 'deflate'
-            })
-        r = client.get(f"https://query2.finance.yahoo.com/v1/finance/search?q={ticker.ticker}")
-        d = r.json()['quotes']
-        items = [c for c in d if c['typeDisp'] == 'Equity']
-        yahoo_symbol = None
-        for item in items:
-            if item['symbol'] == ticker.ticker:
-                if item['exchDisp'] == ticker.exchange or item['exchDisp'] == self.exchange_mic.get(ticker.exchange, {}).get('acronym'):
-                    yahoo_symbol = item['symbol']
-                    break
-                else:
-                    logger.warning(f"Check ticker exchange {ticker.ticker} - {ticker.exchange}")
-
-            if ticker.exchange and (item['exchange'] in ticker.exchange or item['exchange'] in self.exchange_mic.get(ticker.exchange, {}).get('acronym')):
-                logger.warning(f"Check ticker exchange {ticker.ticker} - {ticker.exchange}")
-                if ticker.ticker in item['symbol']:
-                    yahoo_symbol = item['symbol']
-                    break
-
-        if not yahoo_symbol:
-            logger.error("Unable to detect the Yahoo symbol")
-            return None
-        return yahoo_symbol
-
-    def _create_new_ticker(self, tickers, t):
+    @staticmethod
+    def _create_new_ticker(tickers, t):
         logger.info(f"Creating new ticker {t.ticker.ticker} - {t.ticker.isin}!")
-        # TODO: search in yahoo
 
         if t.ticker.active == Ticker.Status.ACTIVE:
             # check if some ticker with this ticker already exists, and set to status INACTIVE
@@ -165,7 +112,7 @@ def _create_new_ticker(self, tickers, t):
             except:
                 pass
 
-        yahoo_ticker = self._search_yahoo_ticker(t.ticker)
+        yahoo_ticker = YahooClient().get_ticker(t.ticker)
         logger.info(f"Ticker {t.ticker.ticker} - Yahoo ticker: {yahoo_ticker}!")
         ticker = Ticker(ticker=t.ticker.ticker,
                         isin=t.ticker.isin,

backend/finance_reader/utils/yahoo.py

@@ -1,10 +1,6 @@
 import requests
-import json
-from datetime import datetime
-from urllib.parse import urlparse, parse_qs
+import csv
 import requests.utils
-import binascii
-import pickle
 import logging
 
 
@@ -23,74 +19,47 @@ def __init__(self):
                 'Accept-Encoding': 'deflate'
             }
         )
-        try:
-            with open("finance_reader/utils/cookies.pickle") as f:
-                self.client.cookies = pickle.loads(binascii.a2b_base64(f.read()))
-        except:
-            self.generate_cookies()
 
-    def get_ticker_info(self, symbol):
-        endpoint = "https://query1.finance.yahoo.com/v7/finance/quote?lang=en-US&region=US&corsDomain=finance.yahoo.com"
-        flds = ("symbol", "shortName", "currency", "fullExchangeName", "exchange")
-
-        url = f"{endpoint}&fields={','.join(flds)}&symbols={symbol}"
-        r = self.client.get(url)
-        api_json = json.loads(r.text)
-
-        return api_json['quoteResponse']['result'][0]['shortName'], api_json['quoteResponse']['result'][0]['currency']
-
-    def generate_cookies(self):
-        logger.info("Generating cookies!")
-        url = "https://finance.yahoo.com"
-        r = self.client.get(url)
-
-        o = urlparse(r.url)
-        params = parse_qs(o.query)
-        session_id = params['sessionId'][0]
-        url = f"https://consent.yahoo.com/v2/collectConsent?sessionId={session_id}"
-
-        o = urlparse(r.history[1].url)
-        params = parse_qs(o.query)
-        csrf_token = params['gcrumb'][0]
-        data = {
-            "csrfToken": csrf_token,
-            "sessionId": session_id,
-            "originalDoneUrl": "https://finance.yahoo.com/?guccounter=1",
-            "namespace": "yahoo",
-            "agree": "agree"
-        }
-
-        self.client.post(url, data=data)
-
-        logger.info("Saving cookies to pickle file")
-        cookies = requests.utils.dict_from_cookiejar(self.client.cookies)
-        parsed_cookies = binascii.b2a_base64(pickle.dumps(self.client.cookies)).decode()
-
-        with open("finance_reader/utils/cookies.pickle", 'w') as f:
-            f.write(parsed_cookies)
-
-    def get_yahoo_symbol(self, symbol):
-        url = "http://d.yimg.com/autoc.finance.yahoo.com/autoc?query={}&region=1&lang=en".format(symbol)
-        url = "https://finance.yahoo.com/_finance_doubledown/api/resource/searchassist;searchTerm={}?device=console&returnMeta=true".format(symbol)
-
-        r = self.client.get(url)
-        if r.status_code != 200:
-            logger.error(f"Status code: {r.status_code}. Error: {r.text}")
-            return None
-
-        try:
-            result = r.json()
-        except:
-            logger.error("Unable to read response as json")
+        self.exchange_mic = {}
+        self._get_iso_codes()
+
+    def _get_iso_codes(self):
+        with open("finance_reader/utils/ISO10383_MIC_NewFormat.csv") as f:
+            reader = csv.reader(f, delimiter=',')
+            for idx, row in enumerate(reader):
+                if idx == 0:
+                    continue
+                self.exchange_mic[row[0]] = {
+                    "mic": row[1],
+                    "name": row[3],
+                    "acronym": row[7],
+                    "deleted": True if row[11] == 'DELETED' else False
+                }
+
+    def get_ticker(self, ticker):
+        r = self.client.get(f"https://query2.finance.yahoo.com/v1/finance/search?q={ticker.ticker}")
+        d = r.json()['quotes']
+        items = [c for c in d if c['typeDisp'] == 'Equity']
+        yahoo_symbol = None
+        for item in items:
+            if item['symbol'] == ticker.ticker:
+                if item['exchDisp'] == ticker.exchange or item['exchDisp'] == self.exchange_mic.get(ticker.exchange,
+                                                                                                    {}).get('acronym'):
+                    yahoo_symbol = item['symbol']
+                    break
+                else:
+                    logger.warning(f"Check ticker exchange {ticker.ticker} - {ticker.exchange}")
+
+            if ticker.exchange and (
+                    item['exchange'] in ticker.exchange or item['exchange'] in self.exchange_mic.get(ticker.exchange,
+                                                                                                     {}).get(
+                    'acronym')):
+                logger.warning(f"Check ticker exchange {ticker.ticker} - {ticker.exchange}")
+                if ticker.ticker in item['symbol']:
+                    yahoo_symbol = item['symbol']
+                    break
+
+        if not yahoo_symbol:
+            logger.error("Unable to detect the Yahoo symbol")
             return None
-
-        if len(result['data']['items']) == 1:
-            return result['data']['items'][0]['symbol']
-
-        # search by name
-        for x in result['data']['items']:
-            # if x['name']
-            return x['symbol']
-            # if x['symbol'] == symbol:
-
-        return None
+        return yahoo_symbol

backend/wallet_processor/entities/broker.py

@@ -86,9 +86,6 @@ def calc_wallet(self, user_id, orders, queue, tracked_orders):
         """
         to_insert = []
         for ticker, partial_orders in queue.queues.items():
-            if ticker.ticker == 'CRSR':
-                print("CHECL")
-            # self._logger.debug(f"Processing ticker {ticker.ticker} - {ticker}")
             if not ticker:
                 self._logger.error("Ticker not found!")
                 continue
@@ -124,19 +121,16 @@ def calc_wallet(self, user_id, orders, queue, tracked_orders):
                 continue
 
             # Calculate current benefits taking into account sells
-            current_benefits = 0
             current_benefits_eur = 0
             total_sell = 0
             total_sell_eur = 0
             for order in [w for w in tracked_orders if w.sell_trade.ticker == ticker]:
-                current_benefits += order.benefits
                 current_benefits_eur += order.benefits_in_eur
                 total_sell += order.sell_trade.price * order.amount
                 total_sell_eur += order.sell_trade.price * order.amount * order.sell_trade.currency_rate
                 fees += order.sell_trade.fees
 
             # self._logger.debug(f"Benefits: {current_benefits}. Fees: {fees}")
-            current_benefits += fees
             break_even = (total_cost_eur - current_benefits_eur) / shares
 
             w = Wallet(

backend/wallet_processor/entities/crypto.py

@@ -151,7 +151,6 @@ def calc_wallet(self, user_id, orders, queue, tracked_orders):
         """
         Calculate current open orders
         """
-        # calculating wallet, using Wallet and OpenOrders
         to_insert = []
         for ticker, partial_orders in queue.queues.items():
             self._logger.debug(f"Start processing ticker {ticker}")

frontend/src/App.vue

@@ -13,8 +13,9 @@
 
 <script>
 // Loading some plugin css asynchronously
-import 'sweetalert2/dist/sweetalert2.css'
-import 'vue-notifyjs/themes/default.css'
+import 'sweetalert2/dist/sweetalert2.css';
+import 'vue-notifyjs/themes/default.css';
+import axios from "axios";
 
 export default {
   created: function () {
@@ -29,12 +30,16 @@ export default {
       //return error;
     })
   },
+  mounted() {
+    axios.get(process.env.VUE_APP_BACKEND_URL + "/version").then(function (d) {
+      localStorage.setItem("version", d.data);
+    });
+  },
   metaInfo() {
     return {
       title: "Queds",
       script: (function () {
-          return [];
-
+        return [];
       })(),
     };
   },