From b5c6c5998c6039c12aa5e396dc16798749ad8df5 Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Mon, 28 Feb 2022 00:40:47 -0800 Subject: [PATCH 01/10] adding initial build of treasury app --- treasury/treasury/app.py | 452 +++++++++++++++++++++++++++++++++ treasury/treasury/callbacks.py | 204 +++++++++++++++ treasury/treasury/models.py | 34 +++ 3 files changed, 690 insertions(+) create mode 100644 treasury/treasury/app.py create mode 100644 treasury/treasury/callbacks.py create mode 100644 treasury/treasury/models.py diff --git a/treasury/treasury/app.py b/treasury/treasury/app.py new file mode 100644 index 0000000..f16616d --- /dev/null +++ b/treasury/treasury/app.py @@ -0,0 +1,452 @@ +import argparse + +import dash +import dash_bootstrap_components as dbc +import pandas as pd +import plotly.graph_objects as go +from dash import dcc, html +from dash.dependencies import Input, Output, State +from data import get_gecko_prices, get_gecko_spot, lookup_balance +from models import get_mean_variance_forecasts + +# from callbacks import * + +# defaults for startup +START_DATE = '2021-12-01' +END_DATE = pd.to_datetime('today').strftime('%Y-%m-%d') +FORECAST_PERIODS = 90 +NUM_SIMS = 1000 +REF_TICKER = 'XEM' +THEME = dbc.themes.VAPOR +PRICE_DATA_LOC = 'price_data.csv' +TITLE = 'Symbol Treasury Analysis Tool v1.0' + +COLOR_DICT = { + 'XEM': '#67b9e8', + 'XYM': '#44004e', + 'BTC': '#f7931a', + 'ETH': '#373737' +} + +# TODO: figure out how to automate chart colors from stylesheet + + +def get_app(prices, lookback_prices, summary_df, accounts, asset_values, serve, base_path): + + app = dash.Dash(__name__, serve_locally=serve, url_base_pathname=base_path, external_stylesheets=[THEME]) + app.title = TITLE + + accounts = accounts.copy() + accounts['Address'] = accounts['Address'].apply(lambda x: html.A(f'{x[:10]}...', href=f'https://symbol.fyi/accounts/{x}')) + + app.layout = dbc.Container([ + dbc.Row([html.H1(TITLE)], justify='center'), + dbc.Row([ + dbc.Table.from_dataframe(summary_df, bordered=True, color='dark') + ], id='summary-table'), + dbc.Row([ + dbc.Col([ + dbc.Table.from_dataframe(accounts[['Name', 'Balance', 'Address']], bordered=True, color='dark', id='address-table'), + dbc.FormText( + 'Select the asset used to seed the simulation. Historical data from this asset will be used ' + + 'to fit a model for future price changes, which samples the possible future price paths.'), + dbc.InputGroup( + [ + dbc.InputGroupText('Reference Asset:'), + dbc.Select(id='ref-ticker', options=[{'label': ticker, 'value': ticker} for ticker in prices], value='XYM') + ], + className='mb-3', + ), + dbc.FormText('Choose how many days into the future you wish to forecast.'), + dbc.InputGroup( + [ + dbc.InputGroupText('Forecast Days:'), + dbc.Input(id='forecast-days', value=FORECAST_PERIODS, type='number', min=1, max=1000, step=1, debounce=True) + ], + className='mb-3', + ), + dbc.FormText( + 'Choose how many price simulations you wish to run. More simulations will take slightly ' + + 'longer to run, but will allow for better estimation of probabilities.'), + dbc.InputGroup( + [ + dbc.InputGroupText('Number of Simulations:'), + dbc.Input(id='num-sims', value=NUM_SIMS, type='number', min=1, step=1, debounce=True) + ], + className='mb-3', + ), + dbc.FormText( + 'Choose the date from which historical data will be collected. A longer data collection ' + + 'period will result in better estimation. Can also be used to perform hypothetical analysis of past scenarios.'), + dbc.InputGroup( + [ + dbc.InputGroupText('Data Start:'), + dbc.Input(id='start-date', value=START_DATE, type='text', debounce=True) + ], + className='mb-3', + ), + dbc.FormText( + 'Choose the end date for the historical data. The simulation will start from this date. ' + + 'Values in the future or too far in the past may result in errors.'), + dbc.InputGroup( + [ + dbc.InputGroupText('Data End:'), + dbc.Input(id='end-date', value=END_DATE, type='text', debounce=True) + ], + className='mb-3', + ), + dbc.FormText( + 'Pick a threshold percentage used to calculate the best and worst case estimates. A value ' + + 'of 95% means that the high and low bars shown will contain an (estimated) 95% of possible scenarios. ' + + 'Set to 100% to see the absolute minimum and maximum from the simulation.'), + dbc.InputGroup( + [ + dbc.InputGroupText('Risk Threshold:'), + dbc.Input(id='risk-threshold', value=0.95, type='number', min=0.0, max=1.0, debounce=True) + ], + className='mb-3', + ), + dbc.FormText( + 'Set a value that scales the trend seen in the historical data. For example, a value of 3 ' + + 'will cause the simulation to trend 3 times as strongly as the historical data, and a value of -1 will ' + + 'reverse the trend in the historical data.'), + dbc.InputGroup( + [ + dbc.InputGroupText('Trend Multiplier:'), + dbc.Input(id='trend-scale', value=1.0, type='number', debounce=True) + ], + className='mb-3', + ), + dbc.FormText( + 'Set a value that scales the volatility seen in the historical data. For example, value of ' + + '2 will cause the simulation to be twice as volatile as the historical data. Must be greater than zero.'), + dbc.InputGroup( + [ + dbc.InputGroupText('Volatility Multiplier:'), + dbc.Input(id='vol-scale', value=1.0, type='number', min=0.1, debounce=True) + ], + className='mb-3', + ), + ], + className='col-lg-4 col-md-12', + ), + dbc.Col([ + dbc.Spinner( + dcc.Graph( + id='forecast-graph', + style={'width': '100%', 'height': '60vh'}, + config={'scrollZoom': False}, + responsive=True)), + dbc.Spinner( + dcc.Graph( + id='price-graph', + style={'width': '100%', 'height': '60vh'}, + config={'scrollZoom': False}, + responsive=True)), + ], + className='p-3 col-lg-8 col-md-12 col-sm-12', + width=8, + ), + ], className='p-3'), + dcc.Store(id='ref-prices'), + dcc.Store(id='lookback-prices') + ], fluid=True) + + # TODO: spot updates should trigger every minute + + @app.callback( + Output('summary-table', 'children'), + Input('lookback-prices', 'data'), + Input('ref-ticker', 'value')) + def update_summary(lookback_prices, ref_ticker): + lookback_prices = pd.read_json(lookback_prices, orient='split') + summary_dict = { + 'Latest XYM Price': [f"${get_gecko_spot('XYM'):.4}"], + 'Latest XEM Price': [f"${get_gecko_spot('XEM'):.4}"], + 'Reference Trend (Daily)': [f'{lookback_prices[ref_ticker].pct_change().mean():.3%}'], + 'Reference Vol (Daily)': [f'{lookback_prices[ref_ticker].pct_change().std():.3%}'] + } + return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] + + @app.callback( + Output('ref-prices', 'data'), + Output('lookback-prices', 'data'), + Input('start-date', 'value'), + Input('end-date', 'value'), + State('ref-prices', 'data'), + State('lookback-prices', 'data')) + def update_prices( + start_date, + end_date, + ref_prices, + lookback_prices): + + # TODO: progress bar when pulling API data? + if ref_prices is None: + ref_prices = prices.copy() + else: + ref_prices = pd.read_json(ref_prices, orient='split') + price_len = len(ref_prices) + + start_date = pd.to_datetime(start_date).tz_localize('UTC') + end_date = pd.to_datetime(end_date).tz_localize('UTC') + + # collect data if we don't already have what we need + # notify the user with an alert when we have to pull prices? + # could probably break this out into its own callback and add a loading bar, etc. + if start_date < ref_prices.index[0]: + new_prices = [] + for asset in ref_prices.columns: + new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) + new_prices = pd.concat(new_prices, axis=1) + ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() + if end_date > ref_prices.index[-1]: + new_prices = [] + for asset in ref_prices.columns: + new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) + new_prices = pd.concat(new_prices, axis=1) + ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() + + if len(ref_prices) != price_len: + ref_prices.to_csv(PRICE_DATA_LOC) + + lookback_prices = ref_prices.loc[start_date:end_date] + + return ref_prices.to_json(date_format='iso', orient='split'), lookback_prices.to_json(date_format='iso', orient='split') + + @app.callback( + Output('forecast-graph', 'figure'), + Input('lookback-prices', 'data'), + Input('ref-ticker', 'value'), + Input('forecast-days', 'value'), + Input('num-sims', 'value'), + Input('trend-scale', 'value'), + Input('vol-scale', 'value'), + Input('risk-threshold', 'value'), + State('forecast-graph', 'figure')) + def update_forecast_chart( + lookback_prices, + ref_ticker, + forecast_window, + num_sims, + trend_scale, + vol_scale, + risk_threshold, + forecast_fig): + + lookback_prices = pd.read_json(lookback_prices, orient='split') + + # run models + forecasts = get_mean_variance_forecasts( + lookback_prices, + ref_ticker, + forecast_window=forecast_window, + trend_scale=trend_scale, + vol_scale=vol_scale, + num_sims=num_sims) + + # TODO: remove this global dependence on asset_values + # TODO: add labels to all sims that show the quantile value at that point + historical_value = lookback_prices.apply(lambda x: x*asset_values.get(x.name, 0)).sum(axis=1) + forecasts = forecasts * historical_value.iloc[-1] + forecast_quantiles = forecasts.quantile([(1-risk_threshold)/2, (1+risk_threshold)/2], axis=1).T + forecast_ranks = forecasts.rank(axis=1, pct=True) + historical_value = pd.concat([historical_value, forecasts.quantile(0.5, axis=1).T], axis=0) + historical_value.drop_duplicates(inplace=True) + + forecast_traces = [] + for sim in forecasts: + forecast_traces.append( + go.Scattergl( + x=forecasts.index.values, + y=forecasts[sim].values, + line=dict(width=1.5, color='#6f42c1'), + opacity=0.1, + customdata=forecast_ranks[sim], + hovertemplate=( + "Date: %{x}
" + + "Treasury Value: $%{y:,.6r}
" + + "Quantile: %{customdata:.4p}
" + + ""), + showlegend=False, + mode='lines')) + + for quantile in forecast_quantiles: + forecast_traces.append( + go.Scattergl( + x=forecast_quantiles.index.values, + y=forecast_quantiles[quantile].values, + line=dict( + width=3, + color='#e44d56' if quantile < 0.5 else '#33ce6e', + dash='solid'), + opacity=1.0, + name=f'{float(quantile):.2%} risk level', + showlegend=True, + legendrank=1 - quantile, + hovertemplate=( + f'{float(quantile):.2%} risk level
' + + "Date: %{x}
" + + "Treasury Value: $%{y:,.6r}
" + + ""), + mode='lines')) + + forecast_traces.append( + go.Scattergl( + x=historical_value.index.values, + y=historical_value.values, + line=dict(width=3, color='#1ea2f3'), + opacity=1, + name='Historical Value / Average', + hovertemplate=( + "Historical Value / Average
" + + "Date: %{x}
" + + "Treasury Value: $%{y:,.6r}
" + + ""), + showlegend=True, + legendrank=0, + mode='lines')) + + if forecast_fig is None: + forecast_fig = go.Figure( + data=forecast_traces, + layout=go.Layout( + plot_bgcolor='#140223', + paper_bgcolor='rgba(0,0,0,0)', + legend=dict( + bgcolor='#140223', + yanchor="top", + y=0.99, + xanchor="left", + x=0.01), + showlegend=True, + hovermode='closest', + font=dict(size=16, color='#32fbe2'), + margin=dict(b=20, l=5, r=5, t=40), + xaxis=dict( + linecolor='#46d4e6', + gridcolor='#46d4e6', + showgrid=True, + showticklabels=True), + yaxis=dict( + linecolor='#46d4e6', + gridcolor='#46d4e6', + showgrid=True, + zeroline=True, + zerolinecolor='#46d4e6', + showticklabels=True))) + forecast_fig.update_layout( + title="Historical Treasury Value With Future Simulations", + xaxis_title="Date", + yaxis_title="Value ($USD)", + legend_title="Quantiles") + else: + forecast_fig['data'] = forecast_traces + + return forecast_fig + + @app.callback( + Output('price-graph', 'figure'), + Input('lookback-prices', 'data'), + State('price-graph', 'figure')) + def update_price_chart(lookback_prices, price_fig): + + lookback_prices = pd.read_json(lookback_prices, orient='split') + lookback_returns = (lookback_prices.pct_change().fillna(0.0)+1).cumprod() - 1 + + price_traces = [] + for asset in prices.columns: + price_traces.append( + go.Scattergl( + x=lookback_returns.index.values, + y=lookback_returns[asset].values, + line=dict(width=2), # , color=COLOR_DICT.get(asset,'#ffffff')), + name=asset, + customdata=lookback_prices[asset], + hovertemplate=( + ""+asset+"
" + + "Date: %{x}
" + + "Pct. Return: %{y:.2%}
" + + "Price: $%{customdata:.4f}
" + + ""), + mode='lines')) + + if price_fig is None: + price_fig = go.Figure( + data=price_traces, + layout=go.Layout( + plot_bgcolor='#140223', + paper_bgcolor='rgba(0,0,0,0)', + legend=dict( + bgcolor='#140223', + yanchor="top", + y=0.99, + xanchor="left", + x=0.01), + showlegend=True, + hovermode='closest', + font=dict(size=16, color='#32fbe2'), + margin=dict(b=20, l=5, r=5, t=40), + xaxis=dict( + linecolor='#46d4e6', + gridcolor='#46d4e6', + showgrid=True, + showticklabels=True), + yaxis=dict( + linecolor='#46d4e6', + gridcolor='#46d4e6', + showgrid=True, + zeroline=True, + zerolinecolor='#46d4e6', + showticklabels=True))) + price_fig.update_layout( + title="Asset Price Change over Lookback Period", + xaxis_title="Date", + yaxis_title="% Change", + legend_title="Asset") + else: + price_fig['data'] = price_traces + + return price_fig + + return app + + +def main(): + parser = argparse.ArgumentParser(description='webapp that processes data files and renders fork information') + # parser.add_argument('--resources', help='directory containing resources', required=True) + parser.add_argument('--host', help='host ip, defaults to localhost', default='127.0.0.1') + parser.add_argument('--port', type=int, help='port for webserver', default=8080) + parser.add_argument('--proxy', help='proxy spec of the form ip:port::gateway to render urls', default=None) + parser.add_argument('--base_path', help='extension if server is not at root of url', default=None) + parser.add_argument('--serve', action='store_true', help='flag to indicate whether server will recieve external requests') + parser.add_argument('--price_data_loc', help='path to flat file storing collected data', default='../data/price_data.csv') + parser.add_argument('--accounts_loc', help='path to csv with account information', default='../data/accounts.csv') + parser.add_argument('--start_date', help='default start date', default='2021-12-01') + parser.add_argument('--end_date', help='default end date', default=None) + args = parser.parse_args() + + if args.end_date is None: + args.end_date = pd.to_datetime('today').strftime('%Y-%m-%d') + + # prep data + prices = pd.read_csv(PRICE_DATA_LOC, header=0, index_col=0, parse_dates=True) + lookback_prices = prices.loc[START_DATE:END_DATE] + + # TODO: account values should be in app state, prices should be stored locally + accounts = pd.read_csv(args.accounts_loc, header=0, index_col=None) + accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] + asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() + + summary_df = pd.DataFrame.from_records({ + 'Latest XYM Price': [f"${get_gecko_spot('XYM'):.4}"], + 'Latest XEM Price': [f"${get_gecko_spot('XEM'):.4}"], + 'Reference Trend (Daily)': [f'{prices[REF_TICKER].pct_change().mean():.3%}'], + 'Reference Vol (Daily)': [f'{prices[REF_TICKER].pct_change().std():.3%}']}) + + app = get_app(prices, lookback_prices, summary_df, accounts, asset_values, args.serve, args.base_path) + app.run_server(host=args.host, port=args.port, threaded=True, proxy=args.proxy, debug=True) + + +if __name__ == '__main__': + main() diff --git a/treasury/treasury/callbacks.py b/treasury/treasury/callbacks.py new file mode 100644 index 0000000..9874aec --- /dev/null +++ b/treasury/treasury/callbacks.py @@ -0,0 +1,204 @@ +# import plotly.graph_objects as go +# import tensorflow_probability as tfp + +# tfd = tfp.distributions + +# import dash_bootstrap_components as dbc + +# from data import * +# from models import * + + +# def update_summary(lookback_prices, ref_ticker): +# lookback_prices = pd.read_json(lookback_prices, orient='split') +# summary_dict={ +# 'Latest XYM Price': [f"${get_gecko_spot('XYM'):.4}"], +# 'Latest XEM Price': [f"${get_gecko_spot('XEM'):.4}"], +# 'Reference Trend (Daily)': [f'{lookback_prices[ref_ticker].pct_change().mean():.3%}'], +# 'Reference Vol (Daily)': [f'{lookback_prices[ref_ticker].pct_change().std():.3%}'] +# } +# return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] + + +# def update_prices(start_date, end_date, ref_prices, lookback_prices): +# # TODO: progress bar when pulling API data? +# if ref_prices is None: +# ref_prices = prices.copy() +# else: +# ref_prices = pd.read_json(ref_prices, orient='split') +# price_len = len(ref_prices) + +# start_date = pd.to_datetime(start_date).tz_localize('UTC') +# end_date = pd.to_datetime(end_date).tz_localize('UTC') + +# # collect data if we don't already have what we need +# # notify the user with an alert when we have to pull prices? +# # could probably break this out into its own callback and add a loading bar, etc. +# if start_date < ref_prices.index[0]: +# new_prices = [] +# for asset in ref_prices.columns: +# new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) +# new_prices = pd.concat(new_prices, axis=1) +# ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() +# if end_date > ref_prices.index[-1]: +# new_prices = [] +# for asset in ref_prices.columns: +# new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) +# new_prices = pd.concat(new_prices, axis=1) +# ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() + +# if len(ref_prices) != price_len: +# ref_prices.to_csv(PRICE_DATA_LOC) + +# lookback_prices = ref_prices.loc[start_date:end_date] + +# return ref_prices.to_json(date_format='iso', orient='split'), lookback_prices.to_json(date_format='iso', orient='split') + + +# def update_forecast_chart( +# lookback_prices, +# ref_ticker, +# forecast_window, +# num_sims, +# trend_scale, +# vol_scale, +# risk_threshold, +# forecast_fig): + +# lookback_prices = pd.read_json(lookback_prices, orient='split') + +# # run models +# forecasts = get_mean_variance_forecasts( +# lookback_prices, +# ref_ticker, +# forecast_window=forecast_window, +# trend_scale=trend_scale, +# vol_scale=vol_scale, +# num_sims=num_sims) + +# # TODO: remove this global dependence on asset_values +# # TODO: add labels to all sims that show the quantile value at that point +# historical_value = lookback_prices.apply(lambda x: x*asset_values.get(x.name,0)).sum(axis=1) +# forecasts = forecasts * historical_value.iloc[-1] +# forecast_quantiles = forecasts.quantile([(1-risk_threshold)/2,(1+risk_threshold)/2],axis=1).T +# forecast_ranks = forecasts.rank(axis=1,pct=True) +# historical_value = pd.concat([historical_value,forecasts.quantile(0.5,axis=1).T],axis=0) +# historical_value.drop_duplicates(inplace=True) + +# forecast_traces = [] +# for sim in forecasts: +# forecast_traces.append( +# go.Scattergl( +# x=forecasts.index.values, y=forecasts[sim].values, +# line = dict(width=1.5, color='#6f42c1'), +# opacity=0.1, +# customdata=forecast_ranks[sim], +# hovertemplate=( +# "Date: %{x}
" + +# "Treasury Value: $%{y:,.6r}
" + +# "Quantile: %{customdata:.4p}
" + +# ""), +# showlegend=False, +# mode='lines')) + +# for quantile in forecast_quantiles: +# forecast_traces.append( +# go.Scattergl( +# x=forecast_quantiles.index.values, y=forecast_quantiles[quantile].values, +# line = dict(width=3, color='#e44d56' if quantile < 0.5 else '#33ce6e', dash='solid'), +# opacity=1.0, +# name=f'{float(quantile):.2%} risk level', +# showlegend=True, +# legendrank=1-quantile, +# hovertemplate=( +# f'{float(quantile):.2%} risk level
'+ +# "Date: %{x}
" + +# "Treasury Value: $%{y:,.6r}
" + +# ""), +# mode='lines')) + +# forecast_traces.append( +# go.Scattergl( +# x = historical_value.index.values,y=historical_value.values, +# line = dict(width=3, color='#1ea2f3'), +# opacity=1, +# name='Historical Value / Average', +# hovertemplate=( +# "Historical Value / Average
" + +# "Date: %{x}
" + +# "Treasury Value: $%{y:,.6r}
" + +# ""), +# showlegend=True, +# legendrank=0, +# mode='lines')) + +# if forecast_fig is None: +# forecast_fig = go.Figure(data=forecast_traces, +# layout=go.Layout( +# plot_bgcolor='#140223', +# paper_bgcolor='rgba(0,0,0,0)', +# legend=dict(bgcolor='#140223',yanchor="top",y=0.99,xanchor="left",x=0.01), +# showlegend=True, +# hovermode='closest', +# font=dict(size=16,color='#32fbe2'), +# margin=dict(b=20,l=5,r=5,t=40), +# xaxis=dict(linecolor='#46d4e6',gridcolor='#46d4e6',showgrid=True, showticklabels=True), +# yaxis=dict(linecolor='#46d4e6',gridcolor='#46d4e6',showgrid=True, zeroline=True, zerolinecolor='#46d4e6', showticklabels=True)) +# ) +# forecast_fig.update_layout( +# title="Historical Treasury Value With Future Simulations", +# xaxis_title="Date", +# yaxis_title="Value ($USD)", +# legend_title="Quantiles", +# ) +# else: +# forecast_fig['data'] = forecast_traces + +# return forecast_fig + + +# def update_price_chart(lookback_prices,price_fig): + +# lookback_prices = pd.read_json(lookback_prices, orient='split') +# lookback_returns = (lookback_prices.pct_change().fillna(0.0)+1).cumprod() - 1 + +# price_traces = [] +# for asset in prices.columns: +# price_traces.append( +# go.Scattergl( +# x=lookback_returns.index.values, +# y=lookback_returns[asset].values, +# line=dict(width=2), +# name=asset, +# customdata=lookback_prices[asset], +# hovertemplate=( +# ""+asset+"
" + +# "Date: %{x}
" + +# "Pct. Return: %{y:.2%}
" + +# "Price: $%{customdata:.4f}
" + +# ""), +# mode='lines')) + +# if price_fig is None: +# price_fig = go.Figure(data=price_traces, +# layout=go.Layout( +# plot_bgcolor='#140223', +# paper_bgcolor='rgba(0,0,0,0)', +# legend=dict(bgcolor='#140223',yanchor="top",y=0.99,xanchor="left",x=0.01), +# showlegend=True, +# hovermode='closest', +# font=dict(size=16,color='#32fbe2'), +# margin=dict(b=20,l=5,r=5,t=40), +# xaxis=dict(linecolor='#46d4e6',gridcolor='#46d4e6',showgrid=True, showticklabels=True), +# yaxis=dict(linecolor='#46d4e6',gridcolor='#46d4e6',showgrid=True, zeroline=True, zerolinecolor='#46d4e6', showticklabels=True) +# )) +# price_fig.update_layout( +# title="Asset Price Change over Lookback Period", +# xaxis_title="Date", +# yaxis_title="% Change", +# legend_title="Asset", +# ) +# else: +# price_fig['data'] = price_traces + +# return price_fig \ No newline at end of file diff --git a/treasury/treasury/models.py b/treasury/treasury/models.py new file mode 100644 index 0000000..abff80e --- /dev/null +++ b/treasury/treasury/models.py @@ -0,0 +1,34 @@ +import arch +import numpy as np +import pandas as pd +import tensorflow as tf +import tensorflow_probability as tfp + +tfd = tfp.distributions + + +def get_garch_residuals(data, dist='skewt'): + model = arch.arch_model(100*data.dropna(), dist=dist) + res = model.fit(update_freq=1) + return res.std_resid + + +def get_garch_forecasts(data, last_obs, mean='AR', lags=2, dist='skewt', horizon=1, simulations=1000): + last_obs = pd.to_datetime(last_obs, utc=True) + split_date = data.loc[:last_obs].index[-2] + res = arch.arch_model(100*data.loc[:split_date].dropna(), mean=mean, lags=lags, dist=dist).fit(update_freq=0, last_obs=last_obs) + fres = arch.arch_model(100*data.loc[split_date:].dropna(), mean=mean, lags=lags, dist=dist).fit(update_freq=0) + # can potentially be improved by using the .fix() method on the arch_model class instead of .fit + return fres.forecast(res.params, horizon=horizon, start=last_obs, method='simulation', simulations=simulations) + # forecasts.simulations.values.squeeze().T[:, 2:] + + +def get_mean_variance_forecasts(prices, ref_ticker, forecast_window=60, trend_scale=1.0, vol_scale=1.0, num_sims=1000): + forecast_dates = pd.date_range(prices.index[-1], periods=forecast_window+1) + logret = np.log(prices[ref_ticker].pct_change().dropna()+1) + loc = logret.mean() + sigma = logret.std() + dist = tfd.Normal(loc=loc*trend_scale, scale=sigma*vol_scale) + data = dist.sample([forecast_window+1, num_sims]).numpy() + data[0] = 0.0 + return np.e**pd.DataFrame(data, index=forecast_dates).cumsum() From a1cfcaa261b25aa73a8e8be763364694d3fc849b Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Mon, 28 Feb 2022 00:41:39 -0800 Subject: [PATCH 02/10] added basic shell entry point and first stab at package config --- treasury/serve.sh | 3 +++ treasury/setup.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 treasury/serve.sh create mode 100644 treasury/setup.py diff --git a/treasury/serve.sh b/treasury/serve.sh new file mode 100644 index 0000000..97fc565 --- /dev/null +++ b/treasury/serve.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python3 -m treasury.app --proxy 'http://0.0.0.0:8080::https://magicmouth.monster/treasury/' --base_path '/treasury/' --serve --host '0.0.0.0' \ No newline at end of file diff --git a/treasury/setup.py b/treasury/setup.py new file mode 100644 index 0000000..2e403c9 --- /dev/null +++ b/treasury/setup.py @@ -0,0 +1,28 @@ +from setuptools import setup + +setup( + name='symbol-treasury-analysis', + version='1.0', + packages=['treasury'], + package_data={ + '': ['*.json', '*.csv'], + 'treasury': ['treasury/*'] + }, + # scripts=[ + # 'block/extractor/extract', + # 'block/harvester/get_block_stats', + # 'block/delegates/find_delegates' + # ], + install_requires=[ + 'requests', + 'dash', + 'jupyter-dash', + 'msgpack-python', + 'numpy', + 'pandas', + 'tensorflow', + 'tensorflow-probability', + 'tqdm', + 'networkx', + ] +) From 39524f85cacee38e51cf9546a725dd7421399f00 Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Mon, 28 Feb 2022 01:11:15 -0800 Subject: [PATCH 03/10] refactored callbacks into separate module --- treasury/treasury/app.py | 263 ++----------------- treasury/treasury/callbacks.py | 445 ++++++++++++++++++--------------- 2 files changed, 259 insertions(+), 449 deletions(-) diff --git a/treasury/treasury/app.py b/treasury/treasury/app.py index f16616d..a3e3721 100644 --- a/treasury/treasury/app.py +++ b/treasury/treasury/app.py @@ -3,13 +3,11 @@ import dash import dash_bootstrap_components as dbc import pandas as pd -import plotly.graph_objects as go from dash import dcc, html from dash.dependencies import Input, Output, State -from data import get_gecko_prices, get_gecko_spot, lookup_balance -from models import get_mean_variance_forecasts +from data import get_gecko_spot, lookup_balance -# from callbacks import * +from callbacks import update_summary, update_prices, update_price_chart, update_forecast_chart # defaults for startup START_DATE = '2021-12-01' @@ -18,7 +16,6 @@ NUM_SIMS = 1000 REF_TICKER = 'XEM' THEME = dbc.themes.VAPOR -PRICE_DATA_LOC = 'price_data.csv' TITLE = 'Symbol Treasury Analysis Tool v1.0' COLOR_DICT = { @@ -142,79 +139,33 @@ def get_app(prices, lookback_prices, summary_df, accounts, asset_values, serve, id='price-graph', style={'width': '100%', 'height': '60vh'}, config={'scrollZoom': False}, - responsive=True)), + responsive=True)), ], className='p-3 col-lg-8 col-md-12 col-sm-12', width=8, ), ], className='p-3'), - dcc.Store(id='ref-prices'), - dcc.Store(id='lookback-prices') + dcc.Store(id='ref-prices', data=prices.to_json(date_format='iso', orient='split')), + dcc.Store(id='lookback-prices', data=lookback_prices.to_json(date_format='iso', orient='split')), + dcc.Store(id='asset-values', data=asset_values), ], fluid=True) # TODO: spot updates should trigger every minute - @app.callback( + app.callback( Output('summary-table', 'children'), Input('lookback-prices', 'data'), - Input('ref-ticker', 'value')) - def update_summary(lookback_prices, ref_ticker): - lookback_prices = pd.read_json(lookback_prices, orient='split') - summary_dict = { - 'Latest XYM Price': [f"${get_gecko_spot('XYM'):.4}"], - 'Latest XEM Price': [f"${get_gecko_spot('XEM'):.4}"], - 'Reference Trend (Daily)': [f'{lookback_prices[ref_ticker].pct_change().mean():.3%}'], - 'Reference Vol (Daily)': [f'{lookback_prices[ref_ticker].pct_change().std():.3%}'] - } - return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] + Input('ref-ticker', 'value'))(update_summary) - @app.callback( + app.callback( Output('ref-prices', 'data'), Output('lookback-prices', 'data'), Input('start-date', 'value'), Input('end-date', 'value'), State('ref-prices', 'data'), - State('lookback-prices', 'data')) - def update_prices( - start_date, - end_date, - ref_prices, - lookback_prices): - - # TODO: progress bar when pulling API data? - if ref_prices is None: - ref_prices = prices.copy() - else: - ref_prices = pd.read_json(ref_prices, orient='split') - price_len = len(ref_prices) - - start_date = pd.to_datetime(start_date).tz_localize('UTC') - end_date = pd.to_datetime(end_date).tz_localize('UTC') - - # collect data if we don't already have what we need - # notify the user with an alert when we have to pull prices? - # could probably break this out into its own callback and add a loading bar, etc. - if start_date < ref_prices.index[0]: - new_prices = [] - for asset in ref_prices.columns: - new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) - new_prices = pd.concat(new_prices, axis=1) - ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() - if end_date > ref_prices.index[-1]: - new_prices = [] - for asset in ref_prices.columns: - new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) - new_prices = pd.concat(new_prices, axis=1) - ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() - - if len(ref_prices) != price_len: - ref_prices.to_csv(PRICE_DATA_LOC) + State('lookback-prices', 'data'))(update_prices) - lookback_prices = ref_prices.loc[start_date:end_date] - - return ref_prices.to_json(date_format='iso', orient='split'), lookback_prices.to_json(date_format='iso', orient='split') - - @app.callback( + app.callback( Output('forecast-graph', 'figure'), Input('lookback-prices', 'data'), Input('ref-ticker', 'value'), @@ -223,191 +174,13 @@ def update_prices( Input('trend-scale', 'value'), Input('vol-scale', 'value'), Input('risk-threshold', 'value'), - State('forecast-graph', 'figure')) - def update_forecast_chart( - lookback_prices, - ref_ticker, - forecast_window, - num_sims, - trend_scale, - vol_scale, - risk_threshold, - forecast_fig): - - lookback_prices = pd.read_json(lookback_prices, orient='split') - - # run models - forecasts = get_mean_variance_forecasts( - lookback_prices, - ref_ticker, - forecast_window=forecast_window, - trend_scale=trend_scale, - vol_scale=vol_scale, - num_sims=num_sims) + State('forecast-graph', 'figure'), + State('asset-values', 'data'))(update_forecast_chart) - # TODO: remove this global dependence on asset_values - # TODO: add labels to all sims that show the quantile value at that point - historical_value = lookback_prices.apply(lambda x: x*asset_values.get(x.name, 0)).sum(axis=1) - forecasts = forecasts * historical_value.iloc[-1] - forecast_quantiles = forecasts.quantile([(1-risk_threshold)/2, (1+risk_threshold)/2], axis=1).T - forecast_ranks = forecasts.rank(axis=1, pct=True) - historical_value = pd.concat([historical_value, forecasts.quantile(0.5, axis=1).T], axis=0) - historical_value.drop_duplicates(inplace=True) - - forecast_traces = [] - for sim in forecasts: - forecast_traces.append( - go.Scattergl( - x=forecasts.index.values, - y=forecasts[sim].values, - line=dict(width=1.5, color='#6f42c1'), - opacity=0.1, - customdata=forecast_ranks[sim], - hovertemplate=( - "Date: %{x}
" + - "Treasury Value: $%{y:,.6r}
" + - "Quantile: %{customdata:.4p}
" + - ""), - showlegend=False, - mode='lines')) - - for quantile in forecast_quantiles: - forecast_traces.append( - go.Scattergl( - x=forecast_quantiles.index.values, - y=forecast_quantiles[quantile].values, - line=dict( - width=3, - color='#e44d56' if quantile < 0.5 else '#33ce6e', - dash='solid'), - opacity=1.0, - name=f'{float(quantile):.2%} risk level', - showlegend=True, - legendrank=1 - quantile, - hovertemplate=( - f'{float(quantile):.2%} risk level
' + - "Date: %{x}
" + - "Treasury Value: $%{y:,.6r}
" + - ""), - mode='lines')) - - forecast_traces.append( - go.Scattergl( - x=historical_value.index.values, - y=historical_value.values, - line=dict(width=3, color='#1ea2f3'), - opacity=1, - name='Historical Value / Average', - hovertemplate=( - "Historical Value / Average
" + - "Date: %{x}
" + - "Treasury Value: $%{y:,.6r}
" + - ""), - showlegend=True, - legendrank=0, - mode='lines')) - - if forecast_fig is None: - forecast_fig = go.Figure( - data=forecast_traces, - layout=go.Layout( - plot_bgcolor='#140223', - paper_bgcolor='rgba(0,0,0,0)', - legend=dict( - bgcolor='#140223', - yanchor="top", - y=0.99, - xanchor="left", - x=0.01), - showlegend=True, - hovermode='closest', - font=dict(size=16, color='#32fbe2'), - margin=dict(b=20, l=5, r=5, t=40), - xaxis=dict( - linecolor='#46d4e6', - gridcolor='#46d4e6', - showgrid=True, - showticklabels=True), - yaxis=dict( - linecolor='#46d4e6', - gridcolor='#46d4e6', - showgrid=True, - zeroline=True, - zerolinecolor='#46d4e6', - showticklabels=True))) - forecast_fig.update_layout( - title="Historical Treasury Value With Future Simulations", - xaxis_title="Date", - yaxis_title="Value ($USD)", - legend_title="Quantiles") - else: - forecast_fig['data'] = forecast_traces - - return forecast_fig - - @app.callback( + app.callback( Output('price-graph', 'figure'), Input('lookback-prices', 'data'), - State('price-graph', 'figure')) - def update_price_chart(lookback_prices, price_fig): - - lookback_prices = pd.read_json(lookback_prices, orient='split') - lookback_returns = (lookback_prices.pct_change().fillna(0.0)+1).cumprod() - 1 - - price_traces = [] - for asset in prices.columns: - price_traces.append( - go.Scattergl( - x=lookback_returns.index.values, - y=lookback_returns[asset].values, - line=dict(width=2), # , color=COLOR_DICT.get(asset,'#ffffff')), - name=asset, - customdata=lookback_prices[asset], - hovertemplate=( - ""+asset+"
" + - "Date: %{x}
" + - "Pct. Return: %{y:.2%}
" + - "Price: $%{customdata:.4f}
" + - ""), - mode='lines')) - - if price_fig is None: - price_fig = go.Figure( - data=price_traces, - layout=go.Layout( - plot_bgcolor='#140223', - paper_bgcolor='rgba(0,0,0,0)', - legend=dict( - bgcolor='#140223', - yanchor="top", - y=0.99, - xanchor="left", - x=0.01), - showlegend=True, - hovermode='closest', - font=dict(size=16, color='#32fbe2'), - margin=dict(b=20, l=5, r=5, t=40), - xaxis=dict( - linecolor='#46d4e6', - gridcolor='#46d4e6', - showgrid=True, - showticklabels=True), - yaxis=dict( - linecolor='#46d4e6', - gridcolor='#46d4e6', - showgrid=True, - zeroline=True, - zerolinecolor='#46d4e6', - showticklabels=True))) - price_fig.update_layout( - title="Asset Price Change over Lookback Period", - xaxis_title="Date", - yaxis_title="% Change", - legend_title="Asset") - else: - price_fig['data'] = price_traces - - return price_fig + State('price-graph', 'figure'))(update_price_chart) return app @@ -430,7 +203,7 @@ def main(): args.end_date = pd.to_datetime('today').strftime('%Y-%m-%d') # prep data - prices = pd.read_csv(PRICE_DATA_LOC, header=0, index_col=0, parse_dates=True) + prices = pd.read_csv(args.price_data_loc, header=0, index_col=0, parse_dates=True) lookback_prices = prices.loc[START_DATE:END_DATE] # TODO: account values should be in app state, prices should be stored locally @@ -439,8 +212,8 @@ def main(): asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() summary_df = pd.DataFrame.from_records({ - 'Latest XYM Price': [f"${get_gecko_spot('XYM'):.4}"], - 'Latest XEM Price': [f"${get_gecko_spot('XEM'):.4}"], + 'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'], + 'Latest XEM Price': [f'${get_gecko_spot("XEM"):.4}'], 'Reference Trend (Daily)': [f'{prices[REF_TICKER].pct_change().mean():.3%}'], 'Reference Vol (Daily)': [f'{prices[REF_TICKER].pct_change().std():.3%}']}) diff --git a/treasury/treasury/callbacks.py b/treasury/treasury/callbacks.py index 9874aec..e395b75 100644 --- a/treasury/treasury/callbacks.py +++ b/treasury/treasury/callbacks.py @@ -1,204 +1,241 @@ -# import plotly.graph_objects as go -# import tensorflow_probability as tfp - -# tfd = tfp.distributions - -# import dash_bootstrap_components as dbc - -# from data import * -# from models import * - - -# def update_summary(lookback_prices, ref_ticker): -# lookback_prices = pd.read_json(lookback_prices, orient='split') -# summary_dict={ -# 'Latest XYM Price': [f"${get_gecko_spot('XYM'):.4}"], -# 'Latest XEM Price': [f"${get_gecko_spot('XEM'):.4}"], -# 'Reference Trend (Daily)': [f'{lookback_prices[ref_ticker].pct_change().mean():.3%}'], -# 'Reference Vol (Daily)': [f'{lookback_prices[ref_ticker].pct_change().std():.3%}'] -# } -# return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] - - -# def update_prices(start_date, end_date, ref_prices, lookback_prices): -# # TODO: progress bar when pulling API data? -# if ref_prices is None: -# ref_prices = prices.copy() -# else: -# ref_prices = pd.read_json(ref_prices, orient='split') -# price_len = len(ref_prices) - -# start_date = pd.to_datetime(start_date).tz_localize('UTC') -# end_date = pd.to_datetime(end_date).tz_localize('UTC') - -# # collect data if we don't already have what we need -# # notify the user with an alert when we have to pull prices? -# # could probably break this out into its own callback and add a loading bar, etc. -# if start_date < ref_prices.index[0]: -# new_prices = [] -# for asset in ref_prices.columns: -# new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) -# new_prices = pd.concat(new_prices, axis=1) -# ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() -# if end_date > ref_prices.index[-1]: -# new_prices = [] -# for asset in ref_prices.columns: -# new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) -# new_prices = pd.concat(new_prices, axis=1) -# ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() - -# if len(ref_prices) != price_len: -# ref_prices.to_csv(PRICE_DATA_LOC) - -# lookback_prices = ref_prices.loc[start_date:end_date] - -# return ref_prices.to_json(date_format='iso', orient='split'), lookback_prices.to_json(date_format='iso', orient='split') - - -# def update_forecast_chart( -# lookback_prices, -# ref_ticker, -# forecast_window, -# num_sims, -# trend_scale, -# vol_scale, -# risk_threshold, -# forecast_fig): - -# lookback_prices = pd.read_json(lookback_prices, orient='split') - -# # run models -# forecasts = get_mean_variance_forecasts( -# lookback_prices, -# ref_ticker, -# forecast_window=forecast_window, -# trend_scale=trend_scale, -# vol_scale=vol_scale, -# num_sims=num_sims) - -# # TODO: remove this global dependence on asset_values -# # TODO: add labels to all sims that show the quantile value at that point -# historical_value = lookback_prices.apply(lambda x: x*asset_values.get(x.name,0)).sum(axis=1) -# forecasts = forecasts * historical_value.iloc[-1] -# forecast_quantiles = forecasts.quantile([(1-risk_threshold)/2,(1+risk_threshold)/2],axis=1).T -# forecast_ranks = forecasts.rank(axis=1,pct=True) -# historical_value = pd.concat([historical_value,forecasts.quantile(0.5,axis=1).T],axis=0) -# historical_value.drop_duplicates(inplace=True) - -# forecast_traces = [] -# for sim in forecasts: -# forecast_traces.append( -# go.Scattergl( -# x=forecasts.index.values, y=forecasts[sim].values, -# line = dict(width=1.5, color='#6f42c1'), -# opacity=0.1, -# customdata=forecast_ranks[sim], -# hovertemplate=( -# "Date: %{x}
" + -# "Treasury Value: $%{y:,.6r}
" + -# "Quantile: %{customdata:.4p}
" + -# ""), -# showlegend=False, -# mode='lines')) - -# for quantile in forecast_quantiles: -# forecast_traces.append( -# go.Scattergl( -# x=forecast_quantiles.index.values, y=forecast_quantiles[quantile].values, -# line = dict(width=3, color='#e44d56' if quantile < 0.5 else '#33ce6e', dash='solid'), -# opacity=1.0, -# name=f'{float(quantile):.2%} risk level', -# showlegend=True, -# legendrank=1-quantile, -# hovertemplate=( -# f'{float(quantile):.2%} risk level
'+ -# "Date: %{x}
" + -# "Treasury Value: $%{y:,.6r}
" + -# ""), -# mode='lines')) - -# forecast_traces.append( -# go.Scattergl( -# x = historical_value.index.values,y=historical_value.values, -# line = dict(width=3, color='#1ea2f3'), -# opacity=1, -# name='Historical Value / Average', -# hovertemplate=( -# "Historical Value / Average
" + -# "Date: %{x}
" + -# "Treasury Value: $%{y:,.6r}
" + -# ""), -# showlegend=True, -# legendrank=0, -# mode='lines')) - -# if forecast_fig is None: -# forecast_fig = go.Figure(data=forecast_traces, -# layout=go.Layout( -# plot_bgcolor='#140223', -# paper_bgcolor='rgba(0,0,0,0)', -# legend=dict(bgcolor='#140223',yanchor="top",y=0.99,xanchor="left",x=0.01), -# showlegend=True, -# hovermode='closest', -# font=dict(size=16,color='#32fbe2'), -# margin=dict(b=20,l=5,r=5,t=40), -# xaxis=dict(linecolor='#46d4e6',gridcolor='#46d4e6',showgrid=True, showticklabels=True), -# yaxis=dict(linecolor='#46d4e6',gridcolor='#46d4e6',showgrid=True, zeroline=True, zerolinecolor='#46d4e6', showticklabels=True)) -# ) -# forecast_fig.update_layout( -# title="Historical Treasury Value With Future Simulations", -# xaxis_title="Date", -# yaxis_title="Value ($USD)", -# legend_title="Quantiles", -# ) -# else: -# forecast_fig['data'] = forecast_traces - -# return forecast_fig - - -# def update_price_chart(lookback_prices,price_fig): - -# lookback_prices = pd.read_json(lookback_prices, orient='split') -# lookback_returns = (lookback_prices.pct_change().fillna(0.0)+1).cumprod() - 1 - -# price_traces = [] -# for asset in prices.columns: -# price_traces.append( -# go.Scattergl( -# x=lookback_returns.index.values, -# y=lookback_returns[asset].values, -# line=dict(width=2), -# name=asset, -# customdata=lookback_prices[asset], -# hovertemplate=( -# ""+asset+"
" + -# "Date: %{x}
" + -# "Pct. Return: %{y:.2%}
" + -# "Price: $%{customdata:.4f}
" + -# ""), -# mode='lines')) - -# if price_fig is None: -# price_fig = go.Figure(data=price_traces, -# layout=go.Layout( -# plot_bgcolor='#140223', -# paper_bgcolor='rgba(0,0,0,0)', -# legend=dict(bgcolor='#140223',yanchor="top",y=0.99,xanchor="left",x=0.01), -# showlegend=True, -# hovermode='closest', -# font=dict(size=16,color='#32fbe2'), -# margin=dict(b=20,l=5,r=5,t=40), -# xaxis=dict(linecolor='#46d4e6',gridcolor='#46d4e6',showgrid=True, showticklabels=True), -# yaxis=dict(linecolor='#46d4e6',gridcolor='#46d4e6',showgrid=True, zeroline=True, zerolinecolor='#46d4e6', showticklabels=True) -# )) -# price_fig.update_layout( -# title="Asset Price Change over Lookback Period", -# xaxis_title="Date", -# yaxis_title="% Change", -# legend_title="Asset", -# ) -# else: -# price_fig['data'] = price_traces - -# return price_fig \ No newline at end of file +import dash_bootstrap_components as dbc +import pandas as pd +import plotly.graph_objects as go +import tensorflow_probability as tfp + +from data import get_gecko_spot, get_gecko_prices +from models import get_mean_variance_forecasts + +tfd = tfp.distributions + + +def update_summary(lookback_prices, ref_ticker): + lookback_prices = pd.read_json(lookback_prices, orient='split') + summary_dict = { + 'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'], + 'Latest XEM Price': [f'${get_gecko_spot("XEM"):.4}'], + 'Reference Trend (Daily)': [f'{lookback_prices[ref_ticker].pct_change().mean():.3%}'], + 'Reference Vol (Daily)': [f'{lookback_prices[ref_ticker].pct_change().std():.3%}'] + } + return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] + + +def update_prices( + start_date, + end_date, + ref_prices, + lookback_prices): + + # TODO: progress bar when pulling API data? + ref_prices = pd.read_json(ref_prices, orient='split') + # price_len = len(ref_prices) + + start_date = pd.to_datetime(start_date).tz_localize('UTC') + end_date = pd.to_datetime(end_date).tz_localize('UTC') + + # collect data if we don't already have what we need + # notify the user with an alert when we have to pull prices? + # could probably break this out into its own callback and add a loading bar, etc. + if start_date < ref_prices.index[0]: + new_prices = [] + for asset in ref_prices.columns: + new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) + new_prices = pd.concat(new_prices, axis=1) + ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() + if end_date > ref_prices.index[-1]: + new_prices = [] + for asset in ref_prices.columns: + new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) + new_prices = pd.concat(new_prices, axis=1) + ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() + + # TODO: re-implement data persistence; avoid storing full data in app if possible + # if len(ref_prices) != price_len: + # ref_prices.to_csv(PRICE_DATA_LOC) + + lookback_prices = ref_prices.loc[start_date:end_date] + + return ref_prices.to_json(date_format='iso', orient='split'), lookback_prices.to_json(date_format='iso', orient='split') + + +def update_forecast_chart( + lookback_prices, + ref_ticker, + forecast_window, + num_sims, + trend_scale, + vol_scale, + risk_threshold, + forecast_fig, + asset_values): + + lookback_prices = pd.read_json(lookback_prices, orient='split') + + # run models + forecasts = get_mean_variance_forecasts( + lookback_prices, + ref_ticker, + forecast_window=forecast_window, + trend_scale=trend_scale, + vol_scale=vol_scale, + num_sims=num_sims) + + # TODO: add labels to all sims that show the quantile value at that point + historical_value = lookback_prices.apply(lambda x: x*asset_values.get(x.name, 0)).sum(axis=1) + forecasts = forecasts * historical_value.iloc[-1] + forecast_quantiles = forecasts.quantile([(1-risk_threshold)/2, (1+risk_threshold)/2], axis=1).T + forecast_ranks = forecasts.rank(axis=1, pct=True) + historical_value = pd.concat([historical_value, forecasts.quantile(0.5, axis=1).T], axis=0) + historical_value.drop_duplicates(inplace=True) + + forecast_traces = [] + for sim in forecasts: + forecast_traces.append( + go.Scattergl( + x=forecasts.index.values, + y=forecasts[sim].values, + line=dict(width=1.5, color='#6f42c1'), + opacity=0.1, + customdata=forecast_ranks[sim], + hovertemplate=( + 'Date: %{x}
' + + 'Treasury Value: $%{y:,.6r}
' + + 'Quantile: %{customdata:.4p}
' + + ''), + showlegend=False, + mode='lines')) + + for quantile in forecast_quantiles: + forecast_traces.append( + go.Scattergl( + x=forecast_quantiles.index.values, + y=forecast_quantiles[quantile].values, + line=dict( + width=3, + color='#e44d56' if quantile < 0.5 else '#33ce6e', + dash='solid'), + opacity=1.0, + name=f'{float(quantile):.2%} risk level', + showlegend=True, + legendrank=1 - quantile, + hovertemplate=( + f'{float(quantile):.2%} risk level
' + + 'Date: %{x}
' + + 'Treasury Value: $%{y:,.6r}
' + + ''), + mode='lines')) + + forecast_traces.append( + go.Scattergl( + x=historical_value.index.values, + y=historical_value.values, + line=dict(width=3, color='#1ea2f3'), + opacity=1, + name='Historical Value / Average', + hovertemplate=( + 'Historical Value / Average
' + + 'Date: %{x}
' + + 'Treasury Value: $%{y:,.6r}
' + + ''), + showlegend=True, + legendrank=0, + mode='lines')) + + if forecast_fig is None: + forecast_fig = go.Figure( + data=forecast_traces, + layout=go.Layout( + plot_bgcolor='#140223', + paper_bgcolor='rgba(0,0,0,0)', + legend=dict( + bgcolor='#140223', + yanchor='top', + y=0.99, + xanchor='left', + x=0.01), + showlegend=True, + hovermode='closest', + font=dict(size=16, color='#32fbe2'), + margin=dict(b=20, l=5, r=5, t=40), + xaxis=dict( + linecolor='#46d4e6', + gridcolor='#46d4e6', + showgrid=True, + showticklabels=True), + yaxis=dict( + linecolor='#46d4e6', + gridcolor='#46d4e6', + showgrid=True, + zeroline=True, + zerolinecolor='#46d4e6', + showticklabels=True))) + forecast_fig.update_layout( + title='Historical Treasury Value With Future Simulations', + xaxis_title='Date', + yaxis_title='Value ($USD)', + legend_title='Quantiles') + else: + forecast_fig['data'] = forecast_traces + + return forecast_fig + + +def update_price_chart(lookback_prices, price_fig): + + lookback_prices = pd.read_json(lookback_prices, orient='split') + lookback_returns = (lookback_prices.pct_change().fillna(0.0)+1).cumprod() - 1 + + price_traces = [] + for asset in lookback_prices.columns: + price_traces.append( + go.Scattergl( + x=lookback_returns.index.values, + y=lookback_returns[asset].values, + line=dict(width=2), # , color=COLOR_DICT.get(asset,'#ffffff')), + name=asset, + customdata=lookback_prices[asset], + hovertemplate=( + ''+asset+'
' + + 'Date: %{x}
' + + 'Pct. Return: %{y:.2%}
' + + 'Price: $%{customdata:.4f}
' + + ''), + mode='lines')) + + if price_fig is None: + price_fig = go.Figure( + data=price_traces, + layout=go.Layout( + plot_bgcolor='#140223', + paper_bgcolor='rgba(0,0,0,0)', + legend=dict( + bgcolor='#140223', + yanchor='top', + y=0.99, + xanchor='left', + x=0.01), + showlegend=True, + hovermode='closest', + font=dict(size=16, color='#32fbe2'), + margin=dict(b=20, l=5, r=5, t=40), + xaxis=dict( + linecolor='#46d4e6', + gridcolor='#46d4e6', + showgrid=True, + showticklabels=True), + yaxis=dict( + linecolor='#46d4e6', + gridcolor='#46d4e6', + showgrid=True, + zeroline=True, + zerolinecolor='#46d4e6', + showticklabels=True))) + price_fig.update_layout( + title='Asset Price Change over Lookback Period', + xaxis_title='Date', + yaxis_title='% Change', + legend_title='Asset') + else: + price_fig['data'] = price_traces + + return price_fig From 75c0ba47d3c5816636b595d08f7cab5b75d4df6d Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Tue, 1 Mar 2022 15:44:07 -0800 Subject: [PATCH 04/10] added data module for capturing live and historical data dynamically --- treasury/treasury/data.py | 142 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 treasury/treasury/data.py diff --git a/treasury/treasury/data.py b/treasury/treasury/data.py new file mode 100644 index 0000000..d69cef0 --- /dev/null +++ b/treasury/treasury/data.py @@ -0,0 +1,142 @@ +import time +import os +from datetime import date + +import pandas as pd +import requests +from tqdm import tqdm + +XYM_API_HOST = os.getenv('XYM_API_HOST', 'wolf.importance.jp') +XEM_API_HOST = os.getenv('XEM_API_HOST', 'alice5.nem.ninja') +CM_KEY = os.getenv('CM_KEY' '') + +GECKO_TICKER_MAP = { + 'ADA': 'cardano', + 'AVAX': 'avalanche-2', + 'BTC': 'bitcoin', + 'DOT': 'polkadot', + 'ETH': 'ethereum', + 'LTC': 'litecoin', + 'MATIC': 'matic-network', + 'SOL': 'solana', + 'TRX': 'tron', + 'XEM': 'nem', + 'XMR': 'monero', + 'XYM': 'symbol', +} + +XYM_MOSAIC_ID = '6BED913FA20223F8' +MAX_TRIES = 6 +RETRY_S = 15 + + +def lookup_balance(address, asset): + asset = asset.lower() + if asset in ['symbol', 'xym']: + return lookup_xym_balance(address) + elif asset in ['nem', 'xem']: + return lookup_xem_balance(address) + else: + raise ValueError(f"Asset not supported: {asset}") + + +def lookup_xym_balance(address): + + balance = 0 + json_account = requests.get('https://' + XYM_API_HOST + ':3001/accounts/' + address).json() + json_mosaics = json_account['account']['mosaics'] + for json_mosaic in json_mosaics: + if XYM_MOSAIC_ID == json_mosaic['id']: + balance = float(json_mosaic['amount']) / 1000000 + break + + return balance + + +def lookup_xem_balance(address): + + response = requests.get('http://' + XEM_API_HOST + ':7890/account/get?address=' + address).json() + balance = float(response['account']['balance']) / 1000000 + return balance + + +def get_cm_prices(ticker, datetime): + + response = requests.get( + 'https://api.coinmetrics.io/v4/timeseries/asset-metrics?assets=' + + ticker + + '&start_time=' + + datetime + + '&limit_per_asset=1&metrics=PriceUSD&api_key=' + CM_KEY).json() + ref_rate = response['data'][0]['PriceUSD'] + return ref_rate + + +def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, frequency='1d'): + if end_time is None: + end_time = date.today() + data = [] + url = ( + f'https://api.coinmetrics.io/v4/timeseries/asset-metrics?' + + f'assets={",".join(assets)}&' + + f'start_time={start_time}&' + + f'end_time={end_time}&' + + f'metrics={",".join(metrics)}&' + + f'frequency={frequency}&' + + f'pretty=true&api_key={CM_KEY}') + while True: + r = requests.get(url).json() + data.extend(r['data']) + if 'next_page_url' in r: + url = r['next_page_url'] + else: + break + return data + + +def fix_ticker(ticker): + if ticker in GECKO_TICKER_MAP: + ticker = GECKO_TICKER_MAP[ticker] + return ticker + + +def get_gecko_spot(ticker, currency='usd'): + ticker = fix_ticker(ticker) + tries = 1 + while tries <= MAX_TRIES: + try: + response = requests.get('https://api.coingecko.com/api/v3/simple/price?ids=' + ticker + '&vs_currencies=' + currency).json() + return response[ticker][currency] + except: + time.sleep(RETRY_S) + tries += 1 + return None + + +def get_gecko_price(ticker, date, currency='usd'): + ticker = fix_ticker(ticker) + tries = 1 + while tries <= MAX_TRIES: + try: + response = requests.get( + 'https://api.coingecko.com/api/v3/coins/' + + ticker + + '/history?date=' + + date + + '&localization=false').json() + if 'name' in response and 'market_data' not in response: + return None + return response['market_data']['current_price'][currency] + except: + print(f'Failed to get gecko price {ticker} : {date} on try {tries}; retrying in {RETRY_S}s') + time.sleep(RETRY_S) + tries += 1 + return None + + +def get_gecko_prices(ticker, start_date, end_date, currency='usd'): + dates = pd.date_range(start_date, end_date, freq='D') + prices = {'date': dates, ticker: []} + for datetime in tqdm(dates): + prices[ticker].append(get_gecko_price(ticker, datetime.strftime('%d-%m-%Y'), currency)) + return pd.DataFrame.from_records(prices).set_index('date') From 7cb1ccd82489a8e395931a21318fcdb61bcc4178 Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Fri, 25 Mar 2022 16:34:02 -0700 Subject: [PATCH 05/10] added functionality to store and download simulations --- treasury/treasury/app.py | 39 +++++++++++++++++++++++++++- treasury/treasury/callbacks.py | 46 ++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/treasury/treasury/app.py b/treasury/treasury/app.py index a3e3721..6dfbfef 100644 --- a/treasury/treasury/app.py +++ b/treasury/treasury/app.py @@ -7,7 +7,8 @@ from dash.dependencies import Input, Output, State from data import get_gecko_spot, lookup_balance -from callbacks import update_summary, update_prices, update_price_chart, update_forecast_chart +from callbacks import (update_summary, update_prices, update_price_chart, update_forecast_chart, download_full_prices, + download_full, download_small_prices, download_small) # defaults for startup START_DATE = '2021-12-01' @@ -124,6 +125,10 @@ def get_app(prices, lookback_prices, summary_df, accounts, asset_values, serve, ], className='mb-3', ), + dcc.Download(id='download-small-dataframe'), + dcc.Download(id='download-full-dataframe'), + dcc.Download(id='download-small-prices'), + dcc.Download(id='download-full-prices'), ], className='col-lg-4 col-md-12', ), @@ -148,10 +153,38 @@ def get_app(prices, lookback_prices, summary_df, accounts, asset_values, serve, dcc.Store(id='ref-prices', data=prices.to_json(date_format='iso', orient='split')), dcc.Store(id='lookback-prices', data=lookback_prices.to_json(date_format='iso', orient='split')), dcc.Store(id='asset-values', data=asset_values), + dcc.Store(id='full-prices'), + dcc.Store(id='small-prices'), + dcc.Store(id='full-sims'), + dcc.Store(id='small-sims'), ], fluid=True) # TODO: spot updates should trigger every minute + app.callback( + Output('download-full-prices', 'data'), + Input('price-button-full', 'n_clicks'), + State('full-prices', 'data'), + prevent_initial_call=True)(download_full_prices) + + app.callback( + Output('download-small-prices', 'data'), + Input('price-button-small', 'n_clicks'), + State('small-prices', 'data'), + prevent_initial_call=True)(download_small_prices) + + app.callback( + Output('download-full-dataframe', 'data'), + Input('download-button-full', 'n_clicks'), + State('full-sims', 'data'), + prevent_initial_call=True)(download_full) + + app.callback( + Output('download-small-dataframe', 'data'), + Input('download-button-small', 'n_clicks'), + State('small-sims', 'data'), + prevent_initial_call=True)(download_small) + app.callback( Output('summary-table', 'children'), Input('lookback-prices', 'data'), @@ -167,6 +200,10 @@ def get_app(prices, lookback_prices, summary_df, accounts, asset_values, serve, app.callback( Output('forecast-graph', 'figure'), + Output('full-sims', 'data'), + Output('small-sims', 'data'), + Output('full-prices', 'data'), + Output('small-prices', 'data'), Input('lookback-prices', 'data'), Input('ref-ticker', 'value'), Input('forecast-days', 'value'), diff --git a/treasury/treasury/callbacks.py b/treasury/treasury/callbacks.py index e395b75..18a5ddf 100644 --- a/treasury/treasury/callbacks.py +++ b/treasury/treasury/callbacks.py @@ -3,12 +3,33 @@ import plotly.graph_objects as go import tensorflow_probability as tfp +from dash import dcc from data import get_gecko_spot, get_gecko_prices from models import get_mean_variance_forecasts tfd = tfp.distributions +def download_full_prices(_, full_prices): + full_prices = pd.read_json(full_prices, orient='split') + return dcc.send_data_frame(full_prices.to_csv, 'simulated_prices.csv') + + +def download_small_prices(_, small_prices): + small_prices = pd.read_json(small_prices, orient='split') + return dcc.send_data_frame(small_prices.to_csv, 'high_low_mid_prices.csv') + + +def download_full(_, full_sims): + full_sims = pd.read_json(full_sims, orient='split') + return dcc.send_data_frame(full_sims.to_csv, 'full_simulations.csv') + + +def download_small(_, small_sims): + small_sims = pd.read_json(small_sims, orient='split') + return dcc.send_data_frame(small_sims.to_csv, 'high_low_mid_simulations.csv') + + def update_summary(lookback_prices, ref_ticker): lookback_prices = pd.read_json(lookback_prices, orient='split') summary_dict = { @@ -49,7 +70,7 @@ def update_prices( new_prices = pd.concat(new_prices, axis=1) ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() - # TODO: re-implement data persistence; avoid storing full data in app if possible + # TODO: re-implement data persistence; avoid storing full data in app if possible or set environment vars for location # if len(ref_prices) != price_len: # ref_prices.to_csv(PRICE_DATA_LOC) @@ -80,6 +101,12 @@ def update_forecast_chart( vol_scale=vol_scale, num_sims=num_sims) + historical_prices = lookback_prices[ref_ticker].copy() + forecast_prices = forecasts * lookback_prices[ref_ticker].iloc[-1] + price_quantiles = forecast_prices.quantile([(1-risk_threshold)/2, (1+risk_threshold)/2], axis=1).T + historical_prices = pd.concat([historical_prices, forecast_prices.quantile(0.5, axis=1).T], axis=0) + historical_prices.drop_duplicates(inplace=True) + # TODO: add labels to all sims that show the quantile value at that point historical_value = lookback_prices.apply(lambda x: x*asset_values.get(x.name, 0)).sum(axis=1) forecasts = forecasts * historical_value.iloc[-1] @@ -177,7 +204,22 @@ def update_forecast_chart( else: forecast_fig['data'] = forecast_traces - return forecast_fig + historical_value.name = 'Historical Value / Median Forecast' + forecast_quantiles.columns = [f'{float(quantile):.2%} risk level' for quantile in forecast_quantiles.columns] + historical_value = pd.concat([historical_value, forecast_quantiles], axis=1) + forecasts.columns = [f'Balance Sim {col}' for col in forecasts.columns] + + historical_prices.name = f'Historical {ref_ticker} Price / Median Forecast' + price_quantiles.columns = [f'{float(quantile):.2%} {ref_ticker} Forecast' for quantile in price_quantiles.columns] + historical_prices = pd.concat([historical_prices, price_quantiles], axis=1) + forecast_prices.columns = [f'{ref_ticker} Price Sim {col}' for col in forecast_prices.columns] + + return ( + forecast_fig, + forecasts.to_json(date_format='iso', orient='split'), + historical_value.to_json(date_format='iso', orient='split'), + forecast_prices.to_json(date_format='iso', orient='split'), + historical_prices.to_json(date_format='iso', orient='split')) def update_price_chart(lookback_prices, price_fig): From ab48d282f3c1e27b8f254d0e253f446d6fd94b18 Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Sun, 27 Mar 2022 01:34:18 -0700 Subject: [PATCH 06/10] resturctured app and callbacks to manage state and inject price data dependency correctly --- treasury/treasury/app.py | 48 +++++++++-------------- treasury/treasury/callbacks.py | 69 +++++++++++++++++++--------------- treasury/treasury/data.py | 29 +++++++------- treasury/treasury/models.py | 1 - 4 files changed, 69 insertions(+), 78 deletions(-) diff --git a/treasury/treasury/app.py b/treasury/treasury/app.py index 6dfbfef..df396ab 100644 --- a/treasury/treasury/app.py +++ b/treasury/treasury/app.py @@ -7,7 +7,7 @@ from dash.dependencies import Input, Output, State from data import get_gecko_spot, lookup_balance -from callbacks import (update_summary, update_prices, update_price_chart, update_forecast_chart, download_full_prices, +from callbacks import (update_summary, get_update_prices, update_price_chart, update_forecast_chart, download_full_prices, download_full, download_small_prices, download_small) # defaults for startup @@ -19,24 +19,27 @@ THEME = dbc.themes.VAPOR TITLE = 'Symbol Treasury Analysis Tool v1.0' -COLOR_DICT = { - 'XEM': '#67b9e8', - 'XYM': '#44004e', - 'BTC': '#f7931a', - 'ETH': '#373737' -} -# TODO: figure out how to automate chart colors from stylesheet - - -def get_app(prices, lookback_prices, summary_df, accounts, asset_values, serve, base_path): +def get_app(price_data_loc, accounts_loc, serve, base_path): app = dash.Dash(__name__, serve_locally=serve, url_base_pathname=base_path, external_stylesheets=[THEME]) app.title = TITLE - accounts = accounts.copy() + # prep data + prices = pd.read_csv(price_data_loc, header=0, index_col=0, parse_dates=True) + lookback_prices = prices.loc[START_DATE:END_DATE] + + accounts = pd.read_csv(accounts_loc, header=0, index_col=None) + accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] + asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() accounts['Address'] = accounts['Address'].apply(lambda x: html.A(f'{x[:10]}...', href=f'https://symbol.fyi/accounts/{x}')) + summary_df = pd.DataFrame.from_records({ + 'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'], + 'Latest XEM Price': [f'${get_gecko_spot("XEM"):.4}'], + 'Reference Trend (Daily)': [f'{prices[REF_TICKER].pct_change().mean():.3%}'], + 'Reference Vol (Daily)': [f'{prices[REF_TICKER].pct_change().std():.3%}']}) + app.layout = dbc.Container([ dbc.Row([html.H1(TITLE)], justify='center'), dbc.Row([ @@ -159,8 +162,6 @@ def get_app(prices, lookback_prices, summary_df, accounts, asset_values, serve, dcc.Store(id='small-sims'), ], fluid=True) - # TODO: spot updates should trigger every minute - app.callback( Output('download-full-prices', 'data'), Input('price-button-full', 'n_clicks'), @@ -196,7 +197,7 @@ def get_app(prices, lookback_prices, summary_df, accounts, asset_values, serve, Input('start-date', 'value'), Input('end-date', 'value'), State('ref-prices', 'data'), - State('lookback-prices', 'data'))(update_prices) + State('lookback-prices', 'data'))(get_update_prices(price_data_loc)) app.callback( Output('forecast-graph', 'figure'), @@ -239,22 +240,7 @@ def main(): if args.end_date is None: args.end_date = pd.to_datetime('today').strftime('%Y-%m-%d') - # prep data - prices = pd.read_csv(args.price_data_loc, header=0, index_col=0, parse_dates=True) - lookback_prices = prices.loc[START_DATE:END_DATE] - - # TODO: account values should be in app state, prices should be stored locally - accounts = pd.read_csv(args.accounts_loc, header=0, index_col=None) - accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] - asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() - - summary_df = pd.DataFrame.from_records({ - 'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'], - 'Latest XEM Price': [f'${get_gecko_spot("XEM"):.4}'], - 'Reference Trend (Daily)': [f'{prices[REF_TICKER].pct_change().mean():.3%}'], - 'Reference Vol (Daily)': [f'{prices[REF_TICKER].pct_change().std():.3%}']}) - - app = get_app(prices, lookback_prices, summary_df, accounts, asset_values, args.serve, args.base_path) + app = get_app(args.price_data_loc, args.accounts_loc, args.serve, args.base_path) app.run_server(host=args.host, port=args.port, threaded=True, proxy=args.proxy, debug=True) diff --git a/treasury/treasury/callbacks.py b/treasury/treasury/callbacks.py index 18a5ddf..26d0e34 100644 --- a/treasury/treasury/callbacks.py +++ b/treasury/treasury/callbacks.py @@ -11,26 +11,31 @@ def download_full_prices(_, full_prices): + """Callback to feed the full price simulation download feature""" full_prices = pd.read_json(full_prices, orient='split') return dcc.send_data_frame(full_prices.to_csv, 'simulated_prices.csv') def download_small_prices(_, small_prices): + """Callback to feed the price simulation boundary download feature""" small_prices = pd.read_json(small_prices, orient='split') return dcc.send_data_frame(small_prices.to_csv, 'high_low_mid_prices.csv') def download_full(_, full_sims): + """Callback to feed the full simulation download feature""" full_sims = pd.read_json(full_sims, orient='split') return dcc.send_data_frame(full_sims.to_csv, 'full_simulations.csv') def download_small(_, small_sims): + """Callback to feed the simulation boundary download feature""" small_sims = pd.read_json(small_sims, orient='split') return dcc.send_data_frame(small_sims.to_csv, 'high_low_mid_simulations.csv') def update_summary(lookback_prices, ref_ticker): + """Callback that produces the headline summary table""" lookback_prices = pd.read_json(lookback_prices, orient='split') summary_dict = { 'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'], @@ -41,42 +46,43 @@ def update_summary(lookback_prices, ref_ticker): return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] -def update_prices( - start_date, - end_date, - ref_prices, - lookback_prices): +def get_update_prices(price_data_loc): + """Wrapper to inject location dependency into price data callback""" - # TODO: progress bar when pulling API data? - ref_prices = pd.read_json(ref_prices, orient='split') - # price_len = len(ref_prices) + def update_prices( + start_date, + end_date, + ref_prices, + lookback_prices): + """Callback that slices price data and fetches new bars from coingecko as needed""" - start_date = pd.to_datetime(start_date).tz_localize('UTC') - end_date = pd.to_datetime(end_date).tz_localize('UTC') + ref_prices = pd.read_json(ref_prices, orient='split') + price_len = len(ref_prices) - # collect data if we don't already have what we need - # notify the user with an alert when we have to pull prices? - # could probably break this out into its own callback and add a loading bar, etc. - if start_date < ref_prices.index[0]: - new_prices = [] - for asset in ref_prices.columns: - new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) - new_prices = pd.concat(new_prices, axis=1) - ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() - if end_date > ref_prices.index[-1]: - new_prices = [] - for asset in ref_prices.columns: - new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) - new_prices = pd.concat(new_prices, axis=1) - ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() + start_date = pd.to_datetime(start_date).tz_localize('UTC') + end_date = pd.to_datetime(end_date).tz_localize('UTC') - # TODO: re-implement data persistence; avoid storing full data in app if possible or set environment vars for location - # if len(ref_prices) != price_len: - # ref_prices.to_csv(PRICE_DATA_LOC) + if start_date < ref_prices.index[0]: + new_prices = [] + for asset in ref_prices.columns: + new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) + new_prices = pd.concat(new_prices, axis=1) + ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() + if end_date > ref_prices.index[-1]: + new_prices = [] + for asset in ref_prices.columns: + new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) + new_prices = pd.concat(new_prices, axis=1) + ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() - lookback_prices = ref_prices.loc[start_date:end_date] + if len(ref_prices) != price_len: + ref_prices.to_csv(price_data_loc) - return ref_prices.to_json(date_format='iso', orient='split'), lookback_prices.to_json(date_format='iso', orient='split') + lookback_prices = ref_prices.loc[start_date:end_date] + + return ref_prices.to_json(date_format='iso', orient='split'), lookback_prices.to_json(date_format='iso', orient='split') + + return update_prices def update_forecast_chart( @@ -89,6 +95,7 @@ def update_forecast_chart( risk_threshold, forecast_fig, asset_values): + """Callback that runs forecasting algorithm, builds forecast chart, and returns simulations for export""" lookback_prices = pd.read_json(lookback_prices, orient='split') @@ -107,7 +114,6 @@ def update_forecast_chart( historical_prices = pd.concat([historical_prices, forecast_prices.quantile(0.5, axis=1).T], axis=0) historical_prices.drop_duplicates(inplace=True) - # TODO: add labels to all sims that show the quantile value at that point historical_value = lookback_prices.apply(lambda x: x*asset_values.get(x.name, 0)).sum(axis=1) forecasts = forecasts * historical_value.iloc[-1] forecast_quantiles = forecasts.quantile([(1-risk_threshold)/2, (1+risk_threshold)/2], axis=1).T @@ -223,6 +229,7 @@ def update_forecast_chart( def update_price_chart(lookback_prices, price_fig): + """Callback that builds and styles a chart containing asset returns""" lookback_prices = pd.read_json(lookback_prices, orient='split') lookback_returns = (lookback_prices.pct_change().fillna(0.0)+1).cumprod() - 1 diff --git a/treasury/treasury/data.py b/treasury/treasury/data.py index d69cef0..bb06d2c 100644 --- a/treasury/treasury/data.py +++ b/treasury/treasury/data.py @@ -1,6 +1,6 @@ import time import os -from datetime import date +import datetime import pandas as pd import requests @@ -37,7 +37,7 @@ def lookup_balance(address, asset): elif asset in ['nem', 'xem']: return lookup_xem_balance(address) else: - raise ValueError(f"Asset not supported: {asset}") + raise ValueError(f'Asset not supported: {asset}') def lookup_xym_balance(address): @@ -54,19 +54,18 @@ def lookup_xym_balance(address): def lookup_xem_balance(address): - response = requests.get('http://' + XEM_API_HOST + ':7890/account/get?address=' + address).json() balance = float(response['account']['balance']) / 1000000 return balance -def get_cm_prices(ticker, datetime): +def get_cm_prices(ticker, date_time): response = requests.get( 'https://api.coinmetrics.io/v4/timeseries/asset-metrics?assets=' + ticker + '&start_time=' + - datetime + + date_time + '&limit_per_asset=1&metrics=PriceUSD&api_key=' + CM_KEY).json() ref_rate = response['data'][0]['PriceUSD'] return ref_rate @@ -74,10 +73,10 @@ def get_cm_prices(ticker, datetime): def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, frequency='1d'): if end_time is None: - end_time = date.today() + end_time = datetime.date.today() data = [] url = ( - f'https://api.coinmetrics.io/v4/timeseries/asset-metrics?' + + 'https://api.coinmetrics.io/v4/timeseries/asset-metrics?' + f'assets={",".join(assets)}&' + f'start_time={start_time}&' + f'end_time={end_time}&' + @@ -85,10 +84,10 @@ def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, freq f'frequency={frequency}&' + f'pretty=true&api_key={CM_KEY}') while True: - r = requests.get(url).json() - data.extend(r['data']) - if 'next_page_url' in r: - url = r['next_page_url'] + response = requests.get(url).json() + data.extend(response['data']) + if 'next_page_url' in response: + url = response['next_page_url'] else: break return data @@ -107,7 +106,7 @@ def get_gecko_spot(ticker, currency='usd'): try: response = requests.get('https://api.coingecko.com/api/v3/simple/price?ids=' + ticker + '&vs_currencies=' + currency).json() return response[ticker][currency] - except: + except (KeyError, requests.exceptions.RequestException) as _: time.sleep(RETRY_S) tries += 1 return None @@ -127,7 +126,7 @@ def get_gecko_price(ticker, date, currency='usd'): if 'name' in response and 'market_data' not in response: return None return response['market_data']['current_price'][currency] - except: + except (KeyError, requests.exceptions.RequestException) as _: print(f'Failed to get gecko price {ticker} : {date} on try {tries}; retrying in {RETRY_S}s') time.sleep(RETRY_S) tries += 1 @@ -137,6 +136,6 @@ def get_gecko_price(ticker, date, currency='usd'): def get_gecko_prices(ticker, start_date, end_date, currency='usd'): dates = pd.date_range(start_date, end_date, freq='D') prices = {'date': dates, ticker: []} - for datetime in tqdm(dates): - prices[ticker].append(get_gecko_price(ticker, datetime.strftime('%d-%m-%Y'), currency)) + for date_time in tqdm(dates): + prices[ticker].append(get_gecko_price(ticker, date_time.strftime('%d-%m-%Y'), currency)) return pd.DataFrame.from_records(prices).set_index('date') diff --git a/treasury/treasury/models.py b/treasury/treasury/models.py index abff80e..28c88f7 100644 --- a/treasury/treasury/models.py +++ b/treasury/treasury/models.py @@ -1,7 +1,6 @@ import arch import numpy as np import pandas as pd -import tensorflow as tf import tensorflow_probability as tfp tfd = tfp.distributions From 1ed9b9ae1a2b17b73a54e55783ebab99e33f4c2a Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Mon, 9 May 2022 23:25:10 -0700 Subject: [PATCH 07/10] finished core treasury analysis refactor, built basic package structure --- treasury/serve.sh | 2 +- treasury/setup.py | 9 +------ treasury/treasury/app.py | 43 +++++++++++++++++++++------------- treasury/treasury/callbacks.py | 27 ++++++++++++++++++--- treasury/treasury/data.py | 15 ++++++------ treasury/treasury/models.py | 6 +++-- 6 files changed, 64 insertions(+), 38 deletions(-) mode change 100644 => 100755 treasury/serve.sh diff --git a/treasury/serve.sh b/treasury/serve.sh old mode 100644 new mode 100755 index 97fc565..cd56c86 --- a/treasury/serve.sh +++ b/treasury/serve.sh @@ -1,3 +1,3 @@ #!/bin/bash -python3 -m treasury.app --proxy 'http://0.0.0.0:8080::https://magicmouth.monster/treasury/' --base_path '/treasury/' --serve --host '0.0.0.0' \ No newline at end of file +python3 -m treasury.app --account_data_loc './data/accounts.csv' --price_data_loc './data/price_data.csv' --proxy 'http://0.0.0.0:8080::https://magicmouth.monster/treasury/' --base_path '/treasury/' --serve --host '0.0.0.0' \ No newline at end of file diff --git a/treasury/setup.py b/treasury/setup.py index 2e403c9..603e43d 100644 --- a/treasury/setup.py +++ b/treasury/setup.py @@ -8,21 +8,14 @@ '': ['*.json', '*.csv'], 'treasury': ['treasury/*'] }, - # scripts=[ - # 'block/extractor/extract', - # 'block/harvester/get_block_stats', - # 'block/delegates/find_delegates' - # ], install_requires=[ 'requests', 'dash', - 'jupyter-dash', - 'msgpack-python', + 'dash-bootstrap-components', 'numpy', 'pandas', 'tensorflow', 'tensorflow-probability', 'tqdm', - 'networkx', ] ) diff --git a/treasury/treasury/app.py b/treasury/treasury/app.py index df396ab..002198a 100644 --- a/treasury/treasury/app.py +++ b/treasury/treasury/app.py @@ -3,16 +3,13 @@ import dash import dash_bootstrap_components as dbc import pandas as pd +from callbacks import (download_full, download_full_prices, download_small, download_small_prices, get_update_balances, get_update_prices, + update_forecast_chart, update_price_chart, update_summary) from dash import dcc, html from dash.dependencies import Input, Output, State from data import get_gecko_spot, lookup_balance -from callbacks import (update_summary, get_update_prices, update_price_chart, update_forecast_chart, download_full_prices, - download_full, download_small_prices, download_small) - # defaults for startup -START_DATE = '2021-12-01' -END_DATE = pd.to_datetime('today').strftime('%Y-%m-%d') FORECAST_PERIODS = 90 NUM_SIMS = 1000 REF_TICKER = 'XEM' @@ -20,19 +17,18 @@ TITLE = 'Symbol Treasury Analysis Tool v1.0' -def get_app(price_data_loc, accounts_loc, serve, base_path): +def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_date, auto_update_delay_seconds=600): app = dash.Dash(__name__, serve_locally=serve, url_base_pathname=base_path, external_stylesheets=[THEME]) app.title = TITLE - # prep data + # preprocess data for fast load prices = pd.read_csv(price_data_loc, header=0, index_col=0, parse_dates=True) - lookback_prices = prices.loc[START_DATE:END_DATE] + lookback_prices = prices.loc[start_date:end_date] - accounts = pd.read_csv(accounts_loc, header=0, index_col=None) + accounts = pd.read_csv(account_data_loc, header=0, index_col=None) accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() - accounts['Address'] = accounts['Address'].apply(lambda x: html.A(f'{x[:10]}...', href=f'https://symbol.fyi/accounts/{x}')) summary_df = pd.DataFrame.from_records({ 'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'], @@ -47,7 +43,7 @@ def get_app(price_data_loc, accounts_loc, serve, base_path): ], id='summary-table'), dbc.Row([ dbc.Col([ - dbc.Table.from_dataframe(accounts[['Name', 'Balance', 'Address']], bordered=True, color='dark', id='address-table'), + dbc.Spinner(html.Div([], id='address-table')), dbc.FormText( 'Select the asset used to seed the simulation. Historical data from this asset will be used ' + 'to fit a model for future price changes, which samples the possible future price paths.'), @@ -82,7 +78,7 @@ def get_app(price_data_loc, accounts_loc, serve, base_path): dbc.InputGroup( [ dbc.InputGroupText('Data Start:'), - dbc.Input(id='start-date', value=START_DATE, type='text', debounce=True) + dbc.Input(id='start-date', value=start_date, type='text', debounce=True) ], className='mb-3', ), @@ -92,7 +88,7 @@ def get_app(price_data_loc, accounts_loc, serve, base_path): dbc.InputGroup( [ dbc.InputGroupText('Data End:'), - dbc.Input(id='end-date', value=END_DATE, type='text', debounce=True) + dbc.Input(id='end-date', value=end_date, type='text', debounce=True) ], className='mb-3', ), @@ -128,6 +124,12 @@ def get_app(price_data_loc, accounts_loc, serve, base_path): ], className='mb-3', ), + dbc.Row([ + dbc.Button('Download Sim Balances', id='download-button-full', color='primary', className='me-1'), + dbc.Button('Download High/Low/Mid Balances', id='download-button-small', color='secondary', className='me-1'), + dbc.Button('Download Simulated Ref Asset Prices', id='price-button-full', color='success', className='me-1'), + dbc.Button('Download High/Low/Mid Ref Asset Prices', id='price-button-small', color='warning', className='me-1'), + ]), dcc.Download(id='download-small-dataframe'), dcc.Download(id='download-full-dataframe'), dcc.Download(id='download-small-prices'), @@ -160,6 +162,10 @@ def get_app(price_data_loc, accounts_loc, serve, base_path): dcc.Store(id='small-prices'), dcc.Store(id='full-sims'), dcc.Store(id='small-sims'), + dcc.Interval( + id='auto-update-trigger', + interval=auto_update_delay_seconds*1000, # in milliseconds + n_intervals=0) ], fluid=True) app.callback( @@ -191,6 +197,11 @@ def get_app(price_data_loc, accounts_loc, serve, base_path): Input('lookback-prices', 'data'), Input('ref-ticker', 'value'))(update_summary) + app.callback( + Output('address-table', 'children'), + Output('asset-values', 'data'), + Input('auto-update-trigger', 'n_intervals'))(get_update_balances(account_data_loc)) + app.callback( Output('ref-prices', 'data'), Output('lookback-prices', 'data'), @@ -232,15 +243,15 @@ def main(): parser.add_argument('--base_path', help='extension if server is not at root of url', default=None) parser.add_argument('--serve', action='store_true', help='flag to indicate whether server will recieve external requests') parser.add_argument('--price_data_loc', help='path to flat file storing collected data', default='../data/price_data.csv') - parser.add_argument('--accounts_loc', help='path to csv with account information', default='../data/accounts.csv') + parser.add_argument('--account_data_loc', help='path to csv with account information', default='../data/accounts.csv') parser.add_argument('--start_date', help='default start date', default='2021-12-01') parser.add_argument('--end_date', help='default end date', default=None) args = parser.parse_args() if args.end_date is None: - args.end_date = pd.to_datetime('today').strftime('%Y-%m-%d') + args.end_date = (pd.to_datetime('today')-pd.Timedelta(1, unit='D')).strftime('%Y-%m-%d') - app = get_app(args.price_data_loc, args.accounts_loc, args.serve, args.base_path) + app = get_app(args.price_data_loc, args.account_data_loc, args.serve, args.base_path, args.start_date, args.end_date) app.run_server(host=args.host, port=args.port, threaded=True, proxy=args.proxy, debug=True) diff --git a/treasury/treasury/callbacks.py b/treasury/treasury/callbacks.py index 26d0e34..ee9b6ba 100644 --- a/treasury/treasury/callbacks.py +++ b/treasury/treasury/callbacks.py @@ -2,14 +2,19 @@ import pandas as pd import plotly.graph_objects as go import tensorflow_probability as tfp - -from dash import dcc -from data import get_gecko_spot, get_gecko_prices +from dash import dcc, html +from data import get_gecko_prices, get_gecko_spot, lookup_balance from models import get_mean_variance_forecasts tfd = tfp.distributions +EXPLORER_URL_MAP = { + 'XYM': 'https://symbol.fyi/accounts/', + 'XEM': 'https://explorer.nemtool.com/#/s_account?account=' +} + + def download_full_prices(_, full_prices): """Callback to feed the full price simulation download feature""" full_prices = pd.read_json(full_prices, orient='split') @@ -46,6 +51,22 @@ def update_summary(lookback_prices, ref_ticker): return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] +def get_update_balances(account_data_loc): + """Wrapper to inject location dependency into account balance callback""" + + def update_balances(_): + accounts = pd.read_csv(account_data_loc, header=0, index_col=None) + accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] + asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() + updated_addresses = [] + for _, row in accounts.iterrows(): + updated_addresses.append(html.A(f'{row.Address[:10]}...', href=f'{EXPLORER_URL_MAP[row.Asset]}{row.Address}')) + accounts['Address'] = updated_addresses + return [dbc.Table.from_dataframe(accounts[['Name', 'Balance', 'Address']], bordered=True, color='dark')], asset_values + + return update_balances + + def get_update_prices(price_data_loc): """Wrapper to inject location dependency into price data callback""" diff --git a/treasury/treasury/data.py b/treasury/treasury/data.py index bb06d2c..fa92258 100644 --- a/treasury/treasury/data.py +++ b/treasury/treasury/data.py @@ -1,14 +1,15 @@ -import time -import os import datetime +import os +import time import pandas as pd import requests from tqdm import tqdm XYM_API_HOST = os.getenv('XYM_API_HOST', 'wolf.importance.jp') -XEM_API_HOST = os.getenv('XEM_API_HOST', 'alice5.nem.ninja') -CM_KEY = os.getenv('CM_KEY' '') +# XEM_API_HOST = os.getenv('XEM_API_HOST', 'alice5.nem.ninja') +XEM_API_HOST = os.getenv('XEM_API_HOST', 'bigalice3.nem.ninja') +CM_KEY = os.getenv('CM_KEY', '') GECKO_TICKER_MAP = { 'ADA': 'cardano', @@ -34,7 +35,7 @@ def lookup_balance(address, asset): asset = asset.lower() if asset in ['symbol', 'xym']: return lookup_xym_balance(address) - elif asset in ['nem', 'xem']: + if asset in ['nem', 'xem']: return lookup_xem_balance(address) else: raise ValueError(f'Asset not supported: {asset}') @@ -94,9 +95,7 @@ def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, freq def fix_ticker(ticker): - if ticker in GECKO_TICKER_MAP: - ticker = GECKO_TICKER_MAP[ticker] - return ticker + return GECKO_TICKER_MAP.get(ticker, ticker) def get_gecko_spot(ticker, currency='usd'): diff --git a/treasury/treasury/models.py b/treasury/treasury/models.py index 28c88f7..4d1f902 100644 --- a/treasury/treasury/models.py +++ b/treasury/treasury/models.py @@ -1,10 +1,14 @@ import arch import numpy as np import pandas as pd +import tensorflow as tf import tensorflow_probability as tfp tfd = tfp.distributions +for gpu in tf.config.experimental.list_physical_devices('GPU'): + tf.config.experimental.set_memory_growth(gpu, True) + def get_garch_residuals(data, dist='skewt'): model = arch.arch_model(100*data.dropna(), dist=dist) @@ -17,9 +21,7 @@ def get_garch_forecasts(data, last_obs, mean='AR', lags=2, dist='skewt', horizon split_date = data.loc[:last_obs].index[-2] res = arch.arch_model(100*data.loc[:split_date].dropna(), mean=mean, lags=lags, dist=dist).fit(update_freq=0, last_obs=last_obs) fres = arch.arch_model(100*data.loc[split_date:].dropna(), mean=mean, lags=lags, dist=dist).fit(update_freq=0) - # can potentially be improved by using the .fix() method on the arch_model class instead of .fit return fres.forecast(res.params, horizon=horizon, start=last_obs, method='simulation', simulations=simulations) - # forecasts.simulations.values.squeeze().T[:, 2:] def get_mean_variance_forecasts(prices, ref_ticker, forecast_window=60, trend_scale=1.0, vol_scale=1.0, num_sims=1000): From d41eacdaf11061df5a4d663ee35fdd1fd41315c2 Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Mon, 9 May 2022 23:41:51 -0700 Subject: [PATCH 08/10] refactored models to avoid heavy dependencies; refactored serve script to make generic --- treasury/serve.sh | 2 +- treasury/setup.py | 2 -- treasury/treasury/models.py | 25 +------------------------ 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/treasury/serve.sh b/treasury/serve.sh index cd56c86..c2af419 100755 --- a/treasury/serve.sh +++ b/treasury/serve.sh @@ -1,3 +1,3 @@ #!/bin/bash -python3 -m treasury.app --account_data_loc './data/accounts.csv' --price_data_loc './data/price_data.csv' --proxy 'http://0.0.0.0:8080::https://magicmouth.monster/treasury/' --base_path '/treasury/' --serve --host '0.0.0.0' \ No newline at end of file +python3 -m treasury.app --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0' diff --git a/treasury/setup.py b/treasury/setup.py index 603e43d..3c6249a 100644 --- a/treasury/setup.py +++ b/treasury/setup.py @@ -14,8 +14,6 @@ 'dash-bootstrap-components', 'numpy', 'pandas', - 'tensorflow', - 'tensorflow-probability', 'tqdm', ] ) diff --git a/treasury/treasury/models.py b/treasury/treasury/models.py index 4d1f902..e1be784 100644 --- a/treasury/treasury/models.py +++ b/treasury/treasury/models.py @@ -1,27 +1,5 @@ -import arch import numpy as np import pandas as pd -import tensorflow as tf -import tensorflow_probability as tfp - -tfd = tfp.distributions - -for gpu in tf.config.experimental.list_physical_devices('GPU'): - tf.config.experimental.set_memory_growth(gpu, True) - - -def get_garch_residuals(data, dist='skewt'): - model = arch.arch_model(100*data.dropna(), dist=dist) - res = model.fit(update_freq=1) - return res.std_resid - - -def get_garch_forecasts(data, last_obs, mean='AR', lags=2, dist='skewt', horizon=1, simulations=1000): - last_obs = pd.to_datetime(last_obs, utc=True) - split_date = data.loc[:last_obs].index[-2] - res = arch.arch_model(100*data.loc[:split_date].dropna(), mean=mean, lags=lags, dist=dist).fit(update_freq=0, last_obs=last_obs) - fres = arch.arch_model(100*data.loc[split_date:].dropna(), mean=mean, lags=lags, dist=dist).fit(update_freq=0) - return fres.forecast(res.params, horizon=horizon, start=last_obs, method='simulation', simulations=simulations) def get_mean_variance_forecasts(prices, ref_ticker, forecast_window=60, trend_scale=1.0, vol_scale=1.0, num_sims=1000): @@ -29,7 +7,6 @@ def get_mean_variance_forecasts(prices, ref_ticker, forecast_window=60, trend_sc logret = np.log(prices[ref_ticker].pct_change().dropna()+1) loc = logret.mean() sigma = logret.std() - dist = tfd.Normal(loc=loc*trend_scale, scale=sigma*vol_scale) - data = dist.sample([forecast_window+1, num_sims]).numpy() + data = np.random.normal(loc=loc*trend_scale, scale=sigma*vol_scale, size=[forecast_window+1, num_sims]) data[0] = 0.0 return np.e**pd.DataFrame(data, index=forecast_dates).cumsum() From a6d7ebb76386e33408ef977751d36d3d81a5c24d Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Mon, 9 May 2022 23:42:36 -0700 Subject: [PATCH 09/10] normalized arg names, fixed requirements, removed random comments --- requirements.txt | 7 ++++++- treasury/treasury/__init__.py | 0 treasury/treasury/app.py | 20 ++++++++++---------- treasury/treasury/callbacks.py | 9 +++------ treasury/treasury/data.py | 1 - 5 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 treasury/treasury/__init__.py diff --git a/requirements.txt b/requirements.txt index 13745d9..65f05a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,9 @@ +dash==2.0.0 +dash-bootstrap-components==1.0.3 +numpy==1.21.2 +pandas==1.3.3 PyYAML==5.4.1 requests==2.26.0 symbol-sdk-core-python==2.0.1 -zenlog==1.1 +tqdm==4.62.3 +zenlog==1.1 \ No newline at end of file diff --git a/treasury/treasury/__init__.py b/treasury/treasury/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/treasury/treasury/app.py b/treasury/treasury/app.py index 002198a..907a4a5 100644 --- a/treasury/treasury/app.py +++ b/treasury/treasury/app.py @@ -3,11 +3,12 @@ import dash import dash_bootstrap_components as dbc import pandas as pd -from callbacks import (download_full, download_full_prices, download_small, download_small_prices, get_update_balances, get_update_prices, - update_forecast_chart, update_price_chart, update_summary) from dash import dcc, html from dash.dependencies import Input, Output, State -from data import get_gecko_spot, lookup_balance + +from treasury.callbacks import (download_full, download_full_prices, download_small, download_small_prices, get_update_balances, + get_update_prices, update_forecast_chart, update_price_chart, update_summary) +from treasury.data import get_gecko_spot, lookup_balance # defaults for startup FORECAST_PERIODS = 90 @@ -164,7 +165,7 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ dcc.Store(id='small-sims'), dcc.Interval( id='auto-update-trigger', - interval=auto_update_delay_seconds*1000, # in milliseconds + interval=auto_update_delay_seconds*1000, n_intervals=0) ], fluid=True) @@ -236,16 +237,15 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ def main(): parser = argparse.ArgumentParser(description='webapp that processes data files and renders fork information') - # parser.add_argument('--resources', help='directory containing resources', required=True) parser.add_argument('--host', help='host ip, defaults to localhost', default='127.0.0.1') parser.add_argument('--port', type=int, help='port for webserver', default=8080) parser.add_argument('--proxy', help='proxy spec of the form ip:port::gateway to render urls', default=None) - parser.add_argument('--base_path', help='extension if server is not at root of url', default=None) + parser.add_argument('--base-path', help='extension if server is not at root of url', default=None) parser.add_argument('--serve', action='store_true', help='flag to indicate whether server will recieve external requests') - parser.add_argument('--price_data_loc', help='path to flat file storing collected data', default='../data/price_data.csv') - parser.add_argument('--account_data_loc', help='path to csv with account information', default='../data/accounts.csv') - parser.add_argument('--start_date', help='default start date', default='2021-12-01') - parser.add_argument('--end_date', help='default end date', default=None) + parser.add_argument('--price-data-loc', help='path to flat file storing collected data', default='../data/price_data.csv') + parser.add_argument('--account-data-loc', help='path to csv with account information', default='../data/accounts.csv') + parser.add_argument('--start-date', help='default start date', default='2021-12-01') + parser.add_argument('--end-date', help='default end date', default=None) args = parser.parse_args() if args.end_date is None: diff --git a/treasury/treasury/callbacks.py b/treasury/treasury/callbacks.py index ee9b6ba..4de9d02 100644 --- a/treasury/treasury/callbacks.py +++ b/treasury/treasury/callbacks.py @@ -1,13 +1,10 @@ import dash_bootstrap_components as dbc import pandas as pd import plotly.graph_objects as go -import tensorflow_probability as tfp from dash import dcc, html -from data import get_gecko_prices, get_gecko_spot, lookup_balance -from models import get_mean_variance_forecasts - -tfd = tfp.distributions +from treasury.data import get_gecko_prices, get_gecko_spot, lookup_balance +from treasury.models import get_mean_variance_forecasts EXPLORER_URL_MAP = { 'XYM': 'https://symbol.fyi/accounts/', @@ -261,7 +258,7 @@ def update_price_chart(lookback_prices, price_fig): go.Scattergl( x=lookback_returns.index.values, y=lookback_returns[asset].values, - line=dict(width=2), # , color=COLOR_DICT.get(asset,'#ffffff')), + line=dict(width=2), name=asset, customdata=lookback_prices[asset], hovertemplate=( diff --git a/treasury/treasury/data.py b/treasury/treasury/data.py index fa92258..ab1155e 100644 --- a/treasury/treasury/data.py +++ b/treasury/treasury/data.py @@ -7,7 +7,6 @@ from tqdm import tqdm XYM_API_HOST = os.getenv('XYM_API_HOST', 'wolf.importance.jp') -# XEM_API_HOST = os.getenv('XEM_API_HOST', 'alice5.nem.ninja') XEM_API_HOST = os.getenv('XEM_API_HOST', 'bigalice3.nem.ninja') CM_KEY = os.getenv('CM_KEY', '') From df44c3e2420705c86bf896f97586710ef0fcfca8 Mon Sep 17 00:00:00 2001 From: 0x6861746366574 Date: Tue, 10 May 2022 22:10:02 -0700 Subject: [PATCH 10/10] refactored to put state in treasury_config.json, updated README and shell script accordingly --- README.md | 26 ++++++++++++- treasury/data/accounts.csv | 6 +++ treasury/serve.sh | 2 +- treasury/treasury/app.py | 70 +++++++++++++++++++++++++--------- treasury/treasury/callbacks.py | 27 +++++++------ treasury/treasury/data.py | 51 ++++++++++--------------- treasury/treasury_config.json | 30 +++++++++++++++ 7 files changed, 152 insertions(+), 60 deletions(-) create mode 100644 treasury/data/accounts.csv create mode 100644 treasury/treasury_config.json diff --git a/README.md b/README.md index c17e327..c3b2583 100644 --- a/README.md +++ b/README.md @@ -188,4 +188,28 @@ Example: Retrieve all accounts with balances greater than 50M from the network d ```sh python -m network.richlist_symbol --resources templates/symbol.mainnet.yaml --min-balance 50000000 --output 50M.csv -```` +``` + +## treasury + +### serve + +_runs a self-contained webapp for monitoring account balances, scraping price data, and visualizing future account values in aggregate_ + +Default location for storage is `treasury/data` but any alternative location can be provided. Configuration is located at `treasury/treasury_config.json`. **All config fields other than cm_key are required.** + +Price data download is relatively lightweight for default configuration. If no data is present, the app will attempt to collect prices for all assets defined in configuration on first load. Price data is cached to disk as collected, so data for any particular asset/date combination should be download no more than once. + +Server can be invoked with defaults via: + +```sh +treasury/serve.sh +``` + +or manually with: + +```sh +python -m treasury.app --config './treasury_config.json' --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0' +``` + +With default settings the app will listen for requests at [http:\\\\localhost:8080](http:\\\\localhost:8080) diff --git a/treasury/data/accounts.csv b/treasury/data/accounts.csv new file mode 100644 index 0000000..57ed85d --- /dev/null +++ b/treasury/data/accounts.csv @@ -0,0 +1,6 @@ +Asset,Name,Address +XYM,XYM Treasury,NCHEST3QRQS4JZGOO64TH7NFJ2A63YA7TPM5PXI +XYM,XYM Sink (Mosaic),NAVORTEX3IPBAUWQBBI3I3BDIOS4AVHPZLCFC7Y +XYM,XYM Sink (Fees),NCVORTEX4XD5IQASZQEHDWUXT33XBOTBMKFDCLI +XEM,XEM Treasury,NCHESTYVD2P6P646AMY7WSNG73PCPZDUQNSD6JAK +XEM,XEM Rewards,NCPAYOUTH2BGEGT3Q7K75PV27QKMVNN2IZRVZWMD \ No newline at end of file diff --git a/treasury/serve.sh b/treasury/serve.sh index c2af419..b695de6 100755 --- a/treasury/serve.sh +++ b/treasury/serve.sh @@ -1,3 +1,3 @@ #!/bin/bash -python3 -m treasury.app --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0' +python3 -m treasury.app --config './treasury_config.json' --account-data-loc './data/accounts.csv' --price-data-loc './data/price_data.csv' --serve --host '0.0.0.0' diff --git a/treasury/treasury/app.py b/treasury/treasury/app.py index 907a4a5..de9f02b 100644 --- a/treasury/treasury/app.py +++ b/treasury/treasury/app.py @@ -1,4 +1,5 @@ import argparse +import json import dash import dash_bootstrap_components as dbc @@ -8,34 +9,49 @@ from treasury.callbacks import (download_full, download_full_prices, download_small, download_small_prices, get_update_balances, get_update_prices, update_forecast_chart, update_price_chart, update_summary) -from treasury.data import get_gecko_spot, lookup_balance +from treasury.data import get_gecko_spot, get_gecko_prices, lookup_balance -# defaults for startup -FORECAST_PERIODS = 90 -NUM_SIMS = 1000 -REF_TICKER = 'XEM' THEME = dbc.themes.VAPOR TITLE = 'Symbol Treasury Analysis Tool v1.0' -def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_date, auto_update_delay_seconds=600): +def get_app(price_data_loc, account_data_loc, config, serve, base_path, start_date, end_date, auto_update_delay_seconds=600): app = dash.Dash(__name__, serve_locally=serve, url_base_pathname=base_path, external_stylesheets=[THEME]) app.title = TITLE # preprocess data for fast load - prices = pd.read_csv(price_data_loc, header=0, index_col=0, parse_dates=True) + try: + prices = pd.read_csv(price_data_loc, header=0, index_col=0, parse_dates=True) + except FileNotFoundError: + print('No price data found, pulling fresh data for assets in config (this may take a while)') + if len(config['assets']) > 0: + prices = [] + for asset in config['assets']: + prices.append(get_gecko_prices( + asset, + start_date, + end_date, + config['max_api_tries'], + config['retry_delay_seconds'])) + prices = pd.concat(prices, axis=1).sort_index(axis=0).sort_index(axis=1) + print(f'Prices acquired successfully; writing to {price_data_loc}') + prices.to_csv(price_data_loc) + else: + print('No assets found in config; aborting!') + raise + lookback_prices = prices.loc[start_date:end_date] accounts = pd.read_csv(account_data_loc, header=0, index_col=None) - accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] + accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset, config['api_hosts'])) for row in accounts.itertuples()] asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() summary_df = pd.DataFrame.from_records({ 'Latest XYM Price': [f'${get_gecko_spot("XYM"):.4}'], 'Latest XEM Price': [f'${get_gecko_spot("XEM"):.4}'], - 'Reference Trend (Daily)': [f'{prices[REF_TICKER].pct_change().mean():.3%}'], - 'Reference Vol (Daily)': [f'{prices[REF_TICKER].pct_change().std():.3%}']}) + 'Reference Trend (Daily)': [f'{prices[config["default_ref_ticker"]].pct_change().mean():.3%}'], + 'Reference Vol (Daily)': [f'{prices[config["default_ref_ticker"]].pct_change().std():.3%}']}) app.layout = dbc.Container([ dbc.Row([html.H1(TITLE)], justify='center'), @@ -51,7 +67,10 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ dbc.InputGroup( [ dbc.InputGroupText('Reference Asset:'), - dbc.Select(id='ref-ticker', options=[{'label': ticker, 'value': ticker} for ticker in prices], value='XYM') + dbc.Select( + id='ref-ticker', + options=[{'label': ticker, 'value': ticker} for ticker in prices], + value=config['default_ref_ticker']) ], className='mb-3', ), @@ -59,7 +78,14 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ dbc.InputGroup( [ dbc.InputGroupText('Forecast Days:'), - dbc.Input(id='forecast-days', value=FORECAST_PERIODS, type='number', min=1, max=1000, step=1, debounce=True) + dbc.Input( + id='forecast-days', + value=config['default_forecast_periods'], + type='number', + min=1, + max=1000, + step=1, + debounce=True) ], className='mb-3', ), @@ -69,7 +95,7 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ dbc.InputGroup( [ dbc.InputGroupText('Number of Simulations:'), - dbc.Input(id='num-sims', value=NUM_SIMS, type='number', min=1, step=1, debounce=True) + dbc.Input(id='num-sims', value=config['default_num_sims'], type='number', min=1, step=1, debounce=True) ], className='mb-3', ), @@ -201,7 +227,8 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ app.callback( Output('address-table', 'children'), Output('asset-values', 'data'), - Input('auto-update-trigger', 'n_intervals'))(get_update_balances(account_data_loc)) + Input('auto-update-trigger', 'n_intervals'))( + get_update_balances(account_data_loc, config['api_hosts'], config['explorer_url_map'])) app.callback( Output('ref-prices', 'data'), @@ -209,7 +236,7 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ Input('start-date', 'value'), Input('end-date', 'value'), State('ref-prices', 'data'), - State('lookback-prices', 'data'))(get_update_prices(price_data_loc)) + State('lookback-prices', 'data'))(get_update_prices(price_data_loc, config['max_api_tries'], config['retry_delay_seconds'])) app.callback( Output('forecast-graph', 'figure'), @@ -236,7 +263,8 @@ def get_app(price_data_loc, account_data_loc, serve, base_path, start_date, end_ def main(): - parser = argparse.ArgumentParser(description='webapp that processes data files and renders fork information') + parser = argparse.ArgumentParser(description='webapp that monitors treasury balances and crypto asset prices') + parser.add_argument('--config', '-c', help='configuration file location', default='../treasury_config.json') parser.add_argument('--host', help='host ip, defaults to localhost', default='127.0.0.1') parser.add_argument('--port', type=int, help='port for webserver', default=8080) parser.add_argument('--proxy', help='proxy spec of the form ip:port::gateway to render urls', default=None) @@ -251,7 +279,15 @@ def main(): if args.end_date is None: args.end_date = (pd.to_datetime('today')-pd.Timedelta(1, unit='D')).strftime('%Y-%m-%d') - app = get_app(args.price_data_loc, args.account_data_loc, args.serve, args.base_path, args.start_date, args.end_date) + try: + with open(args.config) as config_file: + args.config = json.load(config_file) + except FileNotFoundError: + print(f'No configuration file found at {args.config}') + print('Configuration is required to run the app!') + raise + + app = get_app(args.price_data_loc, args.account_data_loc, args.config, args.serve, args.base_path, args.start_date, args.end_date) app.run_server(host=args.host, port=args.port, threaded=True, proxy=args.proxy, debug=True) diff --git a/treasury/treasury/callbacks.py b/treasury/treasury/callbacks.py index 4de9d02..3c1489b 100644 --- a/treasury/treasury/callbacks.py +++ b/treasury/treasury/callbacks.py @@ -6,11 +6,6 @@ from treasury.data import get_gecko_prices, get_gecko_spot, lookup_balance from treasury.models import get_mean_variance_forecasts -EXPLORER_URL_MAP = { - 'XYM': 'https://symbol.fyi/accounts/', - 'XEM': 'https://explorer.nemtool.com/#/s_account?account=' -} - def download_full_prices(_, full_prices): """Callback to feed the full price simulation download feature""" @@ -48,23 +43,23 @@ def update_summary(lookback_prices, ref_ticker): return [dbc.Table.from_dataframe(pd.DataFrame.from_records(summary_dict), bordered=True, color='dark')] -def get_update_balances(account_data_loc): +def get_update_balances(account_data_loc, api_hosts, explorer_url_map): """Wrapper to inject location dependency into account balance callback""" def update_balances(_): accounts = pd.read_csv(account_data_loc, header=0, index_col=None) - accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset)) for row in accounts.itertuples()] + accounts['Balance'] = [int(lookup_balance(row.Address, row.Asset, api_hosts)) for row in accounts.itertuples()] asset_values = accounts.groupby('Asset')['Balance'].sum().to_dict() updated_addresses = [] for _, row in accounts.iterrows(): - updated_addresses.append(html.A(f'{row.Address[:10]}...', href=f'{EXPLORER_URL_MAP[row.Asset]}{row.Address}')) + updated_addresses.append(html.A(f'{row.Address[:10]}...', href=f'{explorer_url_map[row.Asset]}{row.Address}')) accounts['Address'] = updated_addresses return [dbc.Table.from_dataframe(accounts[['Name', 'Balance', 'Address']], bordered=True, color='dark')], asset_values return update_balances -def get_update_prices(price_data_loc): +def get_update_prices(price_data_loc, max_api_tries, retry_delay_seconds): """Wrapper to inject location dependency into price data callback""" def update_prices( @@ -83,13 +78,23 @@ def update_prices( if start_date < ref_prices.index[0]: new_prices = [] for asset in ref_prices.columns: - new_prices.append(get_gecko_prices(asset, start_date, ref_prices.index[0]-pd.Timedelta(days=1))) + new_prices.append(get_gecko_prices( + asset, + start_date, + ref_prices.index[0]-pd.Timedelta(days=1), + max_api_tries, + retry_delay_seconds)) new_prices = pd.concat(new_prices, axis=1) ref_prices = pd.concat([new_prices, ref_prices], axis=0).sort_index().drop_duplicates() if end_date > ref_prices.index[-1]: new_prices = [] for asset in ref_prices.columns: - new_prices.append(get_gecko_prices(asset, ref_prices.index[-1]+pd.Timedelta(days=1), end_date)) + new_prices.append(get_gecko_prices( + asset, + ref_prices.index[-1]+pd.Timedelta(days=1), + end_date, + max_api_tries, + retry_delay_seconds)) new_prices = pd.concat(new_prices, axis=1) ref_prices = pd.concat([ref_prices, new_prices], axis=0).sort_index().drop_duplicates() diff --git a/treasury/treasury/data.py b/treasury/treasury/data.py index ab1155e..18a2dbd 100644 --- a/treasury/treasury/data.py +++ b/treasury/treasury/data.py @@ -1,15 +1,10 @@ import datetime -import os import time import pandas as pd import requests from tqdm import tqdm -XYM_API_HOST = os.getenv('XYM_API_HOST', 'wolf.importance.jp') -XEM_API_HOST = os.getenv('XEM_API_HOST', 'bigalice3.nem.ninja') -CM_KEY = os.getenv('CM_KEY', '') - GECKO_TICKER_MAP = { 'ADA': 'cardano', 'AVAX': 'avalanche-2', @@ -26,24 +21,20 @@ } XYM_MOSAIC_ID = '6BED913FA20223F8' -MAX_TRIES = 6 -RETRY_S = 15 -def lookup_balance(address, asset): +def lookup_balance(address, asset, api_hosts): asset = asset.lower() if asset in ['symbol', 'xym']: - return lookup_xym_balance(address) + return lookup_xym_balance(address, api_hosts['XYM']) if asset in ['nem', 'xem']: - return lookup_xem_balance(address) - else: - raise ValueError(f'Asset not supported: {asset}') - + return lookup_xem_balance(address, api_hosts['XEM']) + raise ValueError(f'Asset not supported for balance lookup: {asset}') -def lookup_xym_balance(address): +def lookup_xym_balance(address, xym_api_host): balance = 0 - json_account = requests.get('https://' + XYM_API_HOST + ':3001/accounts/' + address).json() + json_account = requests.get('https://' + xym_api_host + ':3001/accounts/' + address).json() json_mosaics = json_account['account']['mosaics'] for json_mosaic in json_mosaics: if XYM_MOSAIC_ID == json_mosaic['id']: @@ -53,25 +44,25 @@ def lookup_xym_balance(address): return balance -def lookup_xem_balance(address): - response = requests.get('http://' + XEM_API_HOST + ':7890/account/get?address=' + address).json() +def lookup_xem_balance(address, xem_api_host): + response = requests.get('http://' + xem_api_host + ':7890/account/get?address=' + address).json() balance = float(response['account']['balance']) / 1000000 return balance -def get_cm_prices(ticker, date_time): +def get_cm_prices(ticker, date_time, cm_key): response = requests.get( 'https://api.coinmetrics.io/v4/timeseries/asset-metrics?assets=' + ticker + '&start_time=' + date_time + - '&limit_per_asset=1&metrics=PriceUSD&api_key=' + CM_KEY).json() + '&limit_per_asset=1&metrics=PriceUSD&api_key=' + cm_key).json() ref_rate = response['data'][0]['PriceUSD'] return ref_rate -def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, frequency='1d'): +def get_cm_metrics(assets, metrics, cm_key, start_time='2016-01-01', end_time=None, frequency='1d'): if end_time is None: end_time = datetime.date.today() data = [] @@ -82,7 +73,7 @@ def get_cm_metrics(assets, metrics, start_time='2016-01-01', end_time=None, freq f'end_time={end_time}&' + f'metrics={",".join(metrics)}&' + f'frequency={frequency}&' + - f'pretty=true&api_key={CM_KEY}') + f'pretty=true&api_key={cm_key}') while True: response = requests.get(url).json() data.extend(response['data']) @@ -97,23 +88,23 @@ def fix_ticker(ticker): return GECKO_TICKER_MAP.get(ticker, ticker) -def get_gecko_spot(ticker, currency='usd'): +def get_gecko_spot(ticker, max_api_tries=6, retry_delay_seconds=15, currency='usd'): ticker = fix_ticker(ticker) tries = 1 - while tries <= MAX_TRIES: + while tries <= max_api_tries: try: response = requests.get('https://api.coingecko.com/api/v3/simple/price?ids=' + ticker + '&vs_currencies=' + currency).json() return response[ticker][currency] except (KeyError, requests.exceptions.RequestException) as _: - time.sleep(RETRY_S) + time.sleep(retry_delay_seconds) tries += 1 return None -def get_gecko_price(ticker, date, currency='usd'): +def get_gecko_price(ticker, date, max_api_tries=6, retry_delay_seconds=15, currency='usd'): ticker = fix_ticker(ticker) tries = 1 - while tries <= MAX_TRIES: + while tries <= max_api_tries: try: response = requests.get( 'https://api.coingecko.com/api/v3/coins/' + @@ -125,15 +116,15 @@ def get_gecko_price(ticker, date, currency='usd'): return None return response['market_data']['current_price'][currency] except (KeyError, requests.exceptions.RequestException) as _: - print(f'Failed to get gecko price {ticker} : {date} on try {tries}; retrying in {RETRY_S}s') - time.sleep(RETRY_S) + print(f'Failed to get gecko price {ticker} : {date} on try {tries}; retrying in {retry_delay_seconds}s') + time.sleep(retry_delay_seconds) tries += 1 return None -def get_gecko_prices(ticker, start_date, end_date, currency='usd'): +def get_gecko_prices(ticker, start_date, end_date, max_api_tries=6, retry_delay_seconds=15, currency='usd'): dates = pd.date_range(start_date, end_date, freq='D') prices = {'date': dates, ticker: []} for date_time in tqdm(dates): - prices[ticker].append(get_gecko_price(ticker, date_time.strftime('%d-%m-%Y'), currency)) + prices[ticker].append(get_gecko_price(ticker, date_time.strftime('%d-%m-%Y'), max_api_tries, retry_delay_seconds, currency)) return pd.DataFrame.from_records(prices).set_index('date') diff --git a/treasury/treasury_config.json b/treasury/treasury_config.json new file mode 100644 index 0000000..e049d88 --- /dev/null +++ b/treasury/treasury_config.json @@ -0,0 +1,30 @@ +{ + "assets": [ + "ADA", + "AVAX", + "BTC", + "DOT", + "ETH", + "LTC", + "MATIC", + "SOL", + "TRX", + "XEM", + "XMR", + "XYM" + ], + "api_hosts" : { + "XYM" : "wolf.importance.jp", + "XEM" : "bigalice3.nem.ninja" + }, + "cm_key" : "", + "default_forecast_periods" : 90, + "default_num_sims" : 1000, + "default_ref_ticker" : "XEM", + "explorer_url_map" : { + "XYM": "https://symbol.fyi/accounts/", + "XEM": "https://explorer.nemtool.com/#/s_account?account=" + }, + "max_api_tries" : 6, + "retry_delay_seconds" : 15 +} \ No newline at end of file