Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Python 3 support and unit tests #539

Open
wants to merge 16 commits into
base: master
Choose a base branch
from
41 changes: 24 additions & 17 deletions lendingbot.py
Original file line number Diff line number Diff line change
@@ -5,8 +5,13 @@
import time
import traceback
from decimal import Decimal
from httplib import BadStatusLine
from urllib2 import URLError
try:
from httplib import BadStatusLine
from urllib2 import URLError
except ImportError:
# Python 3
from http.client import BadStatusLine
from urllib.error import URLError

import modules.Configuration as Config
import modules.Data as Data
@@ -82,7 +87,7 @@
# load plugins
PluginsManager.init(Config, api, log, notify_conf)

print 'Welcome to ' + Config.get("BOT", "label", "Lending Bot") + ' on ' + exchange
print("Welcome to {0} on {1}".format(Config.get("BOT", "label", "Lending Bot"), exchange))

try:
while True:
@@ -102,37 +107,39 @@
# allow existing the main bot loop
raise
except Exception as ex:
if not hasattr(ex, 'message'):
ex.message = str(ex)
log.log_error(ex.message)
log.persistStatus()
if 'Invalid API key' in ex.message:
print "!!! Troubleshooting !!!"
print "Are your API keys correct? No quotation. Just plain keys."
print("!!! Troubleshooting !!!")
print("Are your API keys correct? No quotation. Just plain keys.")
exit(1)
elif 'Nonce must be greater' in ex.message:
print "!!! Troubleshooting !!!"
print "Are you reusing the API key in multiple applications? Use a unique key for every application."
print("!!! Troubleshooting !!!")
print("Are you reusing the API key in multiple applications? Use a unique key for every application.")
exit(1)
elif 'Permission denied' in ex.message:
print "!!! Troubleshooting !!!"
print "Are you using IP filter on the key? Maybe your IP changed?"
print("!!! Troubleshooting !!!")
print("Are you using IP filter on the key? Maybe your IP changed?")
exit(1)
elif 'timed out' in ex.message:
print "Timed out, will retry in " + str(Lending.get_sleep_time()) + "sec"
print("Timed out, will retry in {0} sec".format(Lending.get_sleep_time()))
elif isinstance(ex, BadStatusLine):
print "Caught BadStatusLine exception from Poloniex, ignoring."
print("Caught BadStatusLine exception from Poloniex, ignoring.")
elif 'Error 429' in ex.message:
additional_sleep = max(130.0-Lending.get_sleep_time(), 0)
additional_sleep = max(130.0 - Lending.get_sleep_time(), 0)
sum_sleep = additional_sleep + Lending.get_sleep_time()
log.log_error('IP has been banned due to many requests. Sleeping for {} seconds'.format(sum_sleep))
time.sleep(additional_sleep)
# Ignore all 5xx errors (server error) as we can't do anything about it (https://httpstatuses.com/)
elif isinstance(ex, URLError):
print "Caught {0} from exchange, ignoring.".format(ex.message)
print("Caught {0} from exchange, ignoring.".format(ex.message))
elif isinstance(ex, ApiError):
print "Caught {0} reading from exchange API, ignoring.".format(ex.message)
print("Caught {0} reading from exchange API, ignoring.".format(ex.message))
else:
print traceback.format_exc()
print "Unhandled error, please open a Github issue so we can fix it!"
print(traceback.format_exc())
print("Unhandled error, please open a Github issue so we can fix it!")
if notify_conf['notify_caught_exception']:
log.notify("{0}\n-------\n{1}".format(ex, traceback.format_exc()), notify_conf)
sys.stdout.flush()
@@ -144,5 +151,5 @@
WebServer.stop_web_server()
PluginsManager.on_bot_exit()
log.log('bye')
print 'bye'
print('bye')
os._exit(0) # Ad-hoc solution in place of 'exit(0)' TODO: Find out why non-daemon thread(s) are hanging on exit
15 changes: 9 additions & 6 deletions modules/Bitfinex.py
Original file line number Diff line number Diff line change
@@ -20,8 +20,8 @@ def __init__(self, cfg, log):
self.log = log
self.lock = threading.RLock()
self.req_per_min = 60
self.req_period = 15 # seconds
self.req_per_period = int(self.req_per_min / ( 60.0 / self.req_period))
self.req_period = 15 # seconds
self.req_per_period = int(self.req_per_min / (60.0 / self.req_period))
self.req_time_log = RingBuffer(self.req_per_period)
self.url = 'https://api.bitfinex.com'
self.key = self.cfg.get("API", "apikey", None)
@@ -51,7 +51,7 @@ def limit_request_rate(self):
time_since_oldest_req = now - self.req_time_log[0]
# check if oldest request is more than self.req_period ago
if time_since_oldest_req < self.req_period:
# print self.req_time_log.get()
# print(self.req_time_log.get())
# uncomment to debug
# print("Waiting {0} sec, {1} to keep api request rate".format(self.req_period - time_since_oldest_req,
# threading.current_thread()))
@@ -61,7 +61,7 @@ def limit_request_rate(self):
return
# uncomment to debug
# else:
# print self.req_time_log.get()
# print(self.req_time_log.get())
# print("Not Waiting {0}".format(threading.current_thread()))
# print("Req:{0} Oldest req:{1} Diff:{2} sec".format(now, self.req_time_log[0], time_since_oldest_req))
# append current request time to the log, pushing out the 60th request time before it
@@ -103,6 +103,9 @@ def _request(self, method, request, payload=None, verify=True):

return r.json()

except ApiError as ex:
ex.message = "{0} Requesting {1}".format(str(ex), self.url + request)
raise ex
except Exception as ex:
ex.message = ex.message if ex.message else str(ex)
ex.message = "{0} Requesting {1}".format(ex.message, self.url + request)
@@ -259,7 +262,7 @@ def create_loan_offer(self, currency, amount, duration, auto_renew, lending_rate
payload = {
"currency": currency,
"amount": str(amount),
"rate": str(round(float(lending_rate),10) * 36500),
"rate": str(round(float(lending_rate),10) * 36500),
"period": int(duration),
"direction": "lend"
}
@@ -346,7 +349,7 @@ def return_lending_history(self, start, stop, limit=500):
"amount": "0.0",
"duration": "0.0",
"interest": str(amount / 0.85),
"fee": str(amount-amount / 0.85),
"fee": str(amount - amount / 0.85),
"earned": str(amount),
"open": Bitfinex2Poloniex.convertTimestamp(entry['timestamp']),
"close": Bitfinex2Poloniex.convertTimestamp(entry['timestamp'])
2 changes: 1 addition & 1 deletion modules/Bitfinex2Poloniex.py
Original file line number Diff line number Diff line change
@@ -28,7 +28,7 @@ def convertOpenLoanOffers(bfxOffers):
if offer['direction'] == 'lend' and float(offer['remaining_amount']) > 0:
plxOffers[offer['currency']].append({
"id": offer['id'],
"rate": str(float(offer['rate'])/36500),
"rate": str(float(offer['rate']) / 36500),
"amount": offer['remaining_amount'],
"duration": offer['period'],
"autoRenew": 0,
43 changes: 26 additions & 17 deletions modules/Configuration.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
# coding=utf-8
from ConfigParser import SafeConfigParser
try:
from ConfigParser import SafeConfigParser
except ImportError:
# Python 3
from configparser import SafeConfigParser
import json
import os
from decimal import Decimal
from builtins import input

config = SafeConfigParser()
Data = None
@@ -19,9 +24,9 @@ def init(file_location, data=None):
# Copy default config file if not found
try:
shutil.copy('default.cfg.example', file_location)
print '\ndefault.cfg.example has been copied to ' + file_location + '\n' \
'Edit it with your API key and custom settings.\n'
raw_input("Press Enter to acknowledge and exit...")
print("\ndefault.cfg.example has been copied to ".format(file_location))
print("Edit it with your API key and custom settings.\n")
input("Press Enter to acknowledge and exit...")
exit(1)
except Exception as ex:
ex.message = ex.message if ex.message else str(ex)
@@ -33,14 +38,16 @@ def init(file_location, data=None):
def has_option(category, option):
try:
return True if os.environ["{0}_{1}".format(category, option)] else _
except (KeyError, NameError): # KeyError for no env var, NameError for _ (empty var) and then to continue
except (KeyError, NameError): # KeyError for no env var, NameError for _ (empty var) and then to continue
return config.has_option(category, option)


def getboolean(category, option, default_value=False):
if has_option(category, option):
try:
return bool(os.environ["{0}_{1}".format(category, option)])
v = os.environ["{0}_{1}".format(category, option)]
return v.lower() in ['true', '1', 't', 'y', 'yes']

except KeyError:
return config.getboolean(category, option)
else:
@@ -55,22 +62,24 @@ def get(category, option, default_value=False, lower_limit=False, upper_limit=Fa
value = config.get(category, option)
try:
if lower_limit and float(value) < float(lower_limit):
print "WARN: [%s]-%s's value: '%s' is below the minimum limit: %s, which will be used instead." % \
(category, option, value, lower_limit)
print("WARN: [{0}]-{1}'s value: '{2}' is below the minimum limit: {3}, which will be used instead."
.format(category, option, value, lower_limit))
value = lower_limit
if upper_limit and float(value) > float(upper_limit):
print "WARN: [%s]-%s's value: '%s' is above the maximum limit: %s, which will be used instead." % \
(category, option, value, upper_limit)
print("WARN: [{0}]-{1}'s value: '{2}' is above the maximum limit: {3}, which will be used instead."
.format(category, option, value, upper_limit))
value = upper_limit
return value
except ValueError:
if default_value is None:
print "ERROR: [%s]-%s is not allowed to be left empty. Please check your config." % (category, option)
print("ERROR: [{0}]-{1} is not allowed to be left empty. Please check your config."
.format(category, option))
exit(1)
return default_value
else:
if default_value is None:
print "ERROR: [%s]-%s is not allowed to be left unset. Please check your config." % (category, option)
print("ERROR: [{0}]-{1} is not allowed to be left unset. Please check your config."
.format(category, option))
exit(1)
return default_value

@@ -162,8 +171,8 @@ def get_gap_mode(category, option):
full_list = ['raw', 'rawbtc', 'relative']
value = get(category, 'gapmode', False).lower().strip(" ")
if value not in full_list:
print "ERROR: Invalid entry '%s' for [%s]-gapMode. Please check your config. Allowed values are: %s" % \
(value, category, ", ".join(full_list))
print("ERROR: Invalid entry '{0}' for [{1}]-gapMode. Please check your config. Allowed values are: {2}"
.format(value, category, ", ".join(full_list)))
exit(1)
return value.lower()
else:
@@ -193,8 +202,8 @@ def get_notification_config():
notify_conf = {'enable_notifications': config.has_section('notifications')}

# For boolean parameters
for conf in ['notify_tx_coins', 'notify_xday_threshold', 'notify_new_loans', 'notify_caught_exception', 'email', 'slack', 'telegram',
'pushbullet', 'irc']:
for conf in ['notify_tx_coins', 'notify_xday_threshold', 'notify_new_loans', 'notify_caught_exception', 'email',
'slack', 'telegram', 'pushbullet', 'irc']:
notify_conf[conf] = getboolean('notifications', conf)

# For string-based parameters
@@ -239,5 +248,5 @@ def get_notification_config():
def get_plugins_config():
active_plugins = []
if config.has_option("BOT", "plugins"):
active_plugins = map(str.strip, config.get("BOT", "plugins").split(','))
active_plugins = list(map(str.strip, config.get("BOT", "plugins").split(',')))
return active_plugins
2 changes: 2 additions & 0 deletions modules/ConsoleUtils.py
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@
import platform
import subprocess


def get_terminal_size():
""" getTerminalSize()
- get width and height of console
@@ -45,6 +46,7 @@ def _get_terminal_size_windows():
except:
pass


def _get_terminal_size_tput():
# get terminal width
# src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window
17 changes: 10 additions & 7 deletions modules/Data.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import datetime
from decimal import Decimal
from urllib import urlopen
try:
from urllib import urlopen
except ImportError:
# Python 3
from urllib.request import urlopen

import json

api = None
@@ -27,15 +32,15 @@ def get_max_duration(end_date, context):
return ""
try:
now_time = datetime.date.today()
config_date = map(int, end_date.split(','))
config_date = list(map(int, end_date.split(',')))
end_time = datetime.date(*config_date) # format YEAR,MONTH,DAY all ints, also used splat operator
diff_days = (end_time - now_time).days
if context == "order":
return diff_days # Order needs int
if context == "status":
return " - Days Remaining: " + str(diff_days) # Status needs string
except Exception as ex:
ex.message = ex.message if ex.message else str(ex)
ex.message = ex.message if hasattr(ex, 'message') and ex.message else str(ex)
print("ERROR: There is something wrong with your endDate option. Error: {0}".format(ex.message))
exit(1)

@@ -45,10 +50,8 @@ def get_total_lent():
total_lent = {}
rate_lent = {}
for item in crypto_lent["provided"]:
item_str = item["amount"].encode("utf-8")
item_float = Decimal(item_str)
item_rate_str = item["rate"].encode("utf-8")
item_rate_float = Decimal(item_rate_str)
item_float = Decimal(item["amount"])
item_rate_float = Decimal(item["rate"])
if item["currency"] in total_lent:
crypto_lent_sum = total_lent[item["currency"]] + item_float
crypto_lent_rate = rate_lent[item["currency"]] + (item_rate_float * item_float)
4 changes: 2 additions & 2 deletions modules/ExchangeApi.py
Original file line number Diff line number Diff line change
@@ -3,13 +3,13 @@
"""

import abc
import six
import calendar
import time


@six.add_metaclass(abc.ABCMeta)
class ExchangeApi(object):
__metaclass__ = abc.ABCMeta

def __str__(self):
return self.__class__.__name__.upper()

20 changes: 11 additions & 9 deletions modules/Lending.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# coding=utf-8
from decimal import Decimal
from six import iteritems
import sched
import time
import threading
@@ -58,7 +59,7 @@ def init(cfg, api1, log1, data, maxtolend, dry_run1, analysis, notify_conf1):
global sleep_time, sleep_time_active, sleep_time_inactive, min_daily_rate, max_daily_rate, spread_lend, \
gap_bottom_default, gap_top_default, xday_threshold, xday_spread, xdays, min_loan_size, end_date, coin_cfg, \
min_loan_sizes, dry_run, transferable_currencies, keep_stuck_orders, hide_coins, scheduler, gap_mode_default, \
exchange, analysis_method, currencies_to_analyse
exchange, analysis_method, currencies_to_analyse

exchange = Config.get_exchange()

@@ -131,17 +132,18 @@ def notify_new_loans(sleep_time):
if loans_provided:
# function to return a set of ids from the api result
# get_id_set = lambda loans: set([x['id'] for x in loans])
def get_id_set(loans): return set([x['id'] for x in loans])
def get_id_set(loans):
return set([x['id'] for x in loans])
loans_amount = {}
loans_info = {}
for loan_id in get_id_set(new_provided) - get_id_set(loans_provided):
loan = [x for x in new_provided if x['id'] == loan_id][0]
# combine loans with the same rate
k = 'c'+loan['currency']+'r'+loan['rate']+'d'+str(loan['duration'])
k = 'c' + loan['currency'] + 'r' + loan['rate'] + 'd' + str(loan['duration'])
loans_amount[k] = float(loan['amount']) + (loans_amount[k] if k in loans_amount else 0)
loans_info[k] = loan
# send notifications with the grouped info
for k, amount in loans_amount.iteritems():
for k, amount in iteritems(loans_amount):
loan = loans_info[k]
t = "{0} {1} loan filled for {2} days at a rate of {3:.4f}%"
text = t.format(amount, loan['currency'], loan['duration'], float(loan['rate']) * 100)
@@ -175,7 +177,7 @@ def create_lend_offer(currency, amt, rate):
if Config.has_option('BOT', 'endDate'):
days_remaining = int(Data.get_max_duration(end_date, "order"))
if int(days_remaining) <= 2:
print "endDate reached. Bot can no longer lend.\nExiting..."
print("endDate reached. Bot can no longer lend.\nExiting...")
log.log("The end date has almost been reached and the bot can no longer lend. Exiting.")
log.refreshStatus(Data.stringify_total_lent(*Data.get_total_lent()), Data.get_max_duration(
end_date, "status"))
@@ -218,7 +220,7 @@ def cancel_all():
ex.message = ex.message if ex.message else str(ex)
log.log("Error canceling loan offer: {0}".format(ex.message))
else:
print "Not enough " + CUR + " to lend if bot canceled open orders. Not cancelling."
print("Not enough {0} to lend if bot canceled open orders. Not cancelling.".format(CUR))


def lend_all():
@@ -378,12 +380,12 @@ def get_gap_mode_rates(cur, cur_active_bal, cur_total_balance, ticker):
top_rate = get_gap_rate(cur, gap_top, order_book, cur_total_balance)
else:
if use_gap_cfg:
print "WARN: Invalid setting for gapMode for [%s], using defaults..." % cur
print("WARN: Invalid setting for gapMode for [{0}], using defaults...".format(cur))
coin_cfg[cur]['gapmode'] = "rawbtc"
coin_cfg[cur]['gapbottom'] = 10
coin_cfg[cur]['gaptop'] = 100
else:
print "WARN: Invalid setting for gapMode, using defaults..."
print("WARN: Invalid setting for gapMode, using defaults...")
gap_mode_default = "relative"
gap_bottom_default = 10
gap_top_default = 200
@@ -457,5 +459,5 @@ def transfer_balances():
log.log(log.digestApiMsg(msg))
log.notify(log.digestApiMsg(msg), notify_conf)
if coin not in exchange_balances:
print "WARN: Incorrect coin entered for transferCurrencies: " + coin
print("WARN: Incorrect coin entered for transferCurrencies: ".format(coin))
transferable_currencies.remove(coin)
16 changes: 11 additions & 5 deletions modules/Logger.py
Original file line number Diff line number Diff line change
@@ -6,10 +6,16 @@
import sys
import time

import ConsoleUtils
import modules.Configuration as Config
from RingBuffer import RingBuffer
from Notify import send_notification
from builtins import str
try:
import ConsoleUtils
from Notify import send_notification
from RingBuffer import RingBuffer
except ModuleNotFoundError:
from . import ConsoleUtils
from .Notify import send_notification
from .RingBuffer import RingBuffer


class ConsoleOutput(object):
@@ -61,7 +67,7 @@ def printline(self, line):
def writeJsonFile(self):
with io.open(self.jsonOutputFile, 'w', encoding='utf-8') as f:
self.jsonOutput["log"] = self.jsonOutputLog.get()
f.write(unicode(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True)))
f.write(str(json.dumps(self.jsonOutput, ensure_ascii=False, sort_keys=True)))
f.close()

def addSectionLog(self, section, key, value):
@@ -110,7 +116,7 @@ def log_error(self, msg):
log_message = "{0} Error {1}".format(self.timestamp(), msg)
self.output.printline(log_message)
if isinstance(self.output, JsonOutput):
print log_message
print(log_message)
self.refreshStatus()

def offer(self, amt, cur, rate, days, msg):
7 changes: 4 additions & 3 deletions modules/MarketAnalysis.py
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@
import pandas as pd
import sqlite3 as sqlite
from sqlite3 import Error
from builtins import range

# Bot libs
import modules.Configuration as Config
@@ -161,7 +162,7 @@ def update_market_thread(self, cur, levels=None):
except Exception as ex:
self.print_traceback(ex, "Error in returning data from exchange")
market_data = []
for i in xrange(levels):
for i in range(levels):
market_data.append(str(raw_data[i]['rate']))
market_data.append(str(raw_data[i]['amount']))
market_data.append('0') # Percentile field not being filled yet.
@@ -171,7 +172,7 @@ def insert_into_db(self, db_con, market_data, levels=None):
if levels is None:
levels = self.recorded_levels
insert_sql = "INSERT INTO loans ("
for level in xrange(levels):
for level in range(levels):
insert_sql += "rate{0}, amnt{0}, ".format(level)
insert_sql += "percentile) VALUES ({0});".format(','.join(market_data)) # percentile = 0
with db_con:
@@ -405,7 +406,7 @@ def create_rate_table(self, db_con, levels):
cursor = db_con.cursor()
create_table_sql = "CREATE TABLE IF NOT EXISTS loans (id INTEGER PRIMARY KEY AUTOINCREMENT," + \
"unixtime integer(4) not null default (strftime('%s','now')),"
for level in xrange(levels):
for level in range(levels):
create_table_sql += "rate{0} FLOAT, ".format(level)
create_table_sql += "amnt{0} FLOAT, ".format(level)
create_table_sql += "percentile FLOAT);"
27 changes: 18 additions & 9 deletions modules/Notify.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
# coding=utf-8
import urllib
import urllib2
import json
import smtplib
from six import iteritems
try:
# Python 3
from urllib.parse import urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
unicode = str
except ImportError:
# Python 2
from urllib import urlencode
from urllib2 import urlopen, Request, HTTPError
try:
from irc import client
IRC_LOADED = True
@@ -16,7 +25,7 @@
# Slack post data needs to be encoded in UTF-8
def encoded_dict(in_dict):
out_dict = {}
for k, v in in_dict.iteritems():
for k, v in iteritems(in_dict):
if isinstance(v, unicode):
v = v.encode('utf8')
elif isinstance(v, str):
@@ -41,9 +50,9 @@ def check_urlib_response(response, platform):
def post_to_slack(msg, channels, token):
for channel in channels:
post_data = {'text': msg, 'channel': channel, 'token': token}
enc_post_data = urllib.urlencode(encoded_dict(post_data))
enc_post_data = urlencode(encoded_dict(post_data))
url = 'https://{}/api/{}'.format('slack.com', 'chat.postMessage')
response = urllib2.urlopen(url, enc_post_data)
response = urlopen(url, enc_post_data)
check_urlib_response(response, 'slack')


@@ -52,9 +61,9 @@ def post_to_telegram(msg, chat_ids, bot_id):
post_data = {"chat_id": chat_id, "text": msg}
url = "https://api.telegram.org/bot" + bot_id + "/sendMessage"
try:
response = urllib2.urlopen(url, urllib.urlencode(post_data))
response = urlopen(url, urlencode(post_data).encode('utf8'))
check_urlib_response(response, 'telegram')
except urllib2.HTTPError as e:
except HTTPError as e:
msg = "Your bot id is probably configured incorrectly"
raise NotificationException("{0}\n{1}".format(e, msg))

@@ -89,8 +98,8 @@ def send_email(msg, email_login_address, email_login_password, email_smtp_server
def post_to_pushbullet(msg, token, deviceid):
post_data = {'body': msg, 'device_iden': deviceid, 'title': 'Poloniex Bot', 'type': 'note'}
opener = urllib2.build_opener()
req = urllib2.Request('https://api.pushbullet.com/v2/pushes', data=json.dumps(post_data),
headers={'Content-Type': 'application/json', 'Access-Token': token})
req = Request('https://api.pushbullet.com/v2/pushes', data=json.dumps(post_data),
headers={'Content-Type': 'application/json', 'Access-Token': token})
try:
opener.open(req)
except Exception as e:
52 changes: 34 additions & 18 deletions modules/Poloniex.py
Original file line number Diff line number Diff line change
@@ -4,14 +4,25 @@
import json
import socket
import time
import urllib
import urllib2
import threading
import modules.Configuration as Config

from modules.RingBuffer import RingBuffer
from modules.ExchangeApi import ExchangeApi
from modules.ExchangeApi import ApiError
from builtins import range

try:
# Python 3
from urllib.parse import urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
PYVER = 3
except ImportError:
# Python 2
from urllib import urlencode
from urllib2 import urlopen, Request, HTTPError
PYVER = 2


def post_process(before):
@@ -20,10 +31,11 @@ def post_process(before):
# Add timestamps if there isnt one but is a datetime
if 'return' in after:
if isinstance(after['return'], list):
for x in xrange(0, len(after['return'])):
for x in range(0, len(after['return'])):
if isinstance(after['return'][x], dict):
if 'datetime' in after['return'][x] and 'timestamp' not in after['return'][x]:
after['return'][x]['timestamp'] = float(ExchangeApi.create_time_stamp(after['return'][x]['datetime']))
after['return'][x]['timestamp'] \
= float(ExchangeApi.create_time_stamp(after['return'][x]['datetime']))

return after

@@ -35,6 +47,8 @@ def __init__(self, cfg, log):
self.log = log
self.APIKey = self.cfg.get("API", "apikey", None)
self.Secret = self.cfg.get("API", "secret", None)
if PYVER == 3:
self.Secret = bytes(self.Secret, 'latin-1')
self.req_per_sec = 6
self.req_time_log = RingBuffer(self.req_per_sec)
self.lock = threading.RLock()
@@ -48,17 +62,17 @@ def limit_request_rate(self):
time_since_oldest_req = now - self.req_time_log[0]
# check if oldest request is more than 1sec ago
if time_since_oldest_req < 1:
# print self.req_time_log.get()
# print(self.req_time_log.get())
# uncomment to debug
# print "Waiting %s sec to keep api request rate" % str(1 - time_since_oldest_req)
# print "Req: %d 6th Req: %d Diff: %f sec" %(now, self.req_time_log[0], time_since_oldest_req)
# print("Waiting %s sec to keep api request rate" % str(1 - time_since_oldest_req))
# print("Req: %d 6th Req: %d Diff: %f sec" %(now, self.req_time_log[0], time_since_oldest_req))
self.req_time_log.append(now + 1 - time_since_oldest_req)
time.sleep(1 - time_since_oldest_req)
return
# uncomment to debug
# else:
# print self.req_time_log.get()
# print "Req: %d 6th Req: %d Diff: %f sec" % (now, self.req_time_log[0], time_since_oldest_req)
# print(self.req_time_log.get())
# print("Req: %d 6th Req: %d Diff: %f sec" % (now, self.req_time_log[0], time_since_oldest_req))
# append current request time to the log, pushing out the 6th request time before it
self.req_time_log.append(now)

@@ -78,14 +92,14 @@ def _read_response(resp):

try:
if command == "returnTicker" or command == "return24hVolume":
ret = urllib2.urlopen(urllib2.Request('https://poloniex.com/public?command=' + command))
ret = urlopen(Request('https://poloniex.com/public?command=' + command))
return _read_response(ret)
elif command == "returnOrderBook":
ret = urllib2.urlopen(urllib2.Request(
ret = urlopen(Request(
'https://poloniex.com/public?command=' + command + '&currencyPair=' + str(req['currencyPair'])))
return _read_response(ret)
elif command == "returnMarketTradeHistory":
ret = urllib2.urlopen(urllib2.Request(
ret = urlopen(Request(
'https://poloniex.com/public?command=' + "returnTradeHistory" + '&currencyPair=' + str(
req['currencyPair'])))
return _read_response(ret)
@@ -94,23 +108,25 @@ def _read_response(resp):
+ '&currency=' + str(req['currency']))
if req['limit'] > 0:
req_url += ('&limit=' + str(req['limit']))
ret = urllib2.urlopen(urllib2.Request(req_url))
ret = urlopen(Request(req_url))
return _read_response(ret)
else:
req['command'] = command
req['nonce'] = int(time.time() * 1000)
post_data = urllib.urlencode(req)
post_data = urlencode(req)
if PYVER == 3:
post_data = bytes(post_data, 'latin-1')

sign = hmac.new(self.Secret, post_data, hashlib.sha512).hexdigest()
headers = {
'Sign': sign,
'Key': self.APIKey
}

ret = urllib2.urlopen(urllib2.Request('https://poloniex.com/tradingApi', post_data, headers))
ret = urlopen(Request('https://poloniex.com/tradingApi', post_data, headers))
json_ret = _read_response(ret)
return post_process(json_ret)
except urllib2.HTTPError as ex:
except HTTPError as ex:
raw_polo_response = ex.read()
try:
data = json.loads(raw_polo_response)
@@ -122,11 +138,11 @@ def _read_response(resp):
': The web server reported a bad gateway or gateway timeout error.'
else:
polo_error_msg = raw_polo_response
ex.message = ex.message if ex.message else str(ex)
ex.message = ex.message if hasattr(ex, 'message') and ex.message else str(ex)
ex.message = "{0} Requesting {1}. Poloniex reports: '{2}'".format(ex.message, command, polo_error_msg)
raise ex
except Exception as ex:
ex.message = ex.message if ex.message else str(ex)
ex.message = ex.message if hasattr(ex, 'message') and ex.message else str(ex)
ex.message = "{0} Requesting {1}".format(ex.message, command)
raise

66 changes: 34 additions & 32 deletions modules/RingBuffer.py
Original file line number Diff line number Diff line change
@@ -2,41 +2,43 @@
# also known as ring buffer, pops the oldest data item
# to make room for newest data item when max size is reached
# uses the double ended queue available in Python24

from collections import deque



class RingBuffer(deque):
"""
inherits deque, pops the oldest data to make room
for the newest data when size is reached
"""
def __init__(self, size):
deque.__init__(self)
self.size = size

def full_append(self, item):
deque.append(self, item)
# full, pop the oldest item, left most item
self.popleft()

def append(self, item):
deque.append(self, item)
# max size reached, append becomes full_append
if len(self) == self.size:
self.append = self.full_append

def get(self):
"""returns a list of size items (newest items)"""
return list(self)

"""
inherits deque, pops the oldest data to make room
for the newest data when size is reached
"""
def __init__(self, size):
deque.__init__(self)
self.size = size

def full_append(self, item):
deque.append(self, item)
# full, pop the oldest item, left most item
self.popleft()

def append(self, item):
deque.append(self, item)
# max size reached, append becomes full_append
if len(self) == self.size:
self.append = self.full_append

def get(self):
"""returns a list of size items (newest items)"""
return list(self)


# testing
if __name__ == '__main__':
size = 5
ring = RingBuffer(size)
for x in range(9):
ring.append(x)
print ring.get() # test
size = 5
ring = RingBuffer(size)
for x in range(9):
ring.append(x)
print(ring.get()) # test

"""
notice that the left most item is popped to make room
result =
@@ -49,4 +51,4 @@ def get(self):
[2, 3, 4, 5, 6]
[3, 4, 5, 6, 7]
[4, 5, 6, 7, 8]
"""
"""
9 changes: 7 additions & 2 deletions modules/WebServer.py
Original file line number Diff line number Diff line change
@@ -43,9 +43,14 @@ def start_web_server():
'''
Start the web server
'''
import SimpleHTTPServer
import SocketServer
import socket
try:
import SimpleHTTPServer
import SocketServer
except ImportError:
# Python 3 (this isn't a nice way to do it, but the nicer way involves installing future from pip for py2)
import http.server as SimpleHTTPServer
import socketserver as SocketServer

try:
port = int(web_server_port)
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -3,3 +3,4 @@ pandas
hypothesis
requests
pytz
future
123 changes: 123 additions & 0 deletions tests/test_Configuration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
import pytest
import tempfile
from six import iteritems
from decimal import Decimal

# Hack to get relative imports - probably need to fix the dir structure instead but we need this at the minute for
# pytest to work
import os, sys, inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)


def add_to_env(category, option, value):
os.environ['{0}_{1}'.format(category, option)] = value


def rm_from_env(category, option):
os.environ.pop('{0}_{1}'.format(category, option))


def write_to_cfg(filename, category, val_dict):
with open(filename, 'a') as out_file:
out_file.write('\n')
out_file.write('[{0}]\n'.format(category))
for option, value in iteritems(val_dict):
out_file.write('{0} = {1}\n'.format(option, value))


def write_skeleton_exchange(filename, exchange):
write_to_cfg(filename, 'API', {'exchange': exchange})
write_to_cfg(filename, exchange.upper(), {'all_currencies': 'XMR'})


@pytest.fixture(autouse=True)
def env_vars():
var_set_1 = "ENVVAR,BOOL_T,true"
var_set_2 = "ENVVAR,BOOL_F,false"
var_set_3 = "ENVVAR,NUM,60"
var_list = [var_set_1, var_set_2, var_set_3]
for var_set in var_list:
c, o, v = var_set.split(',')
add_to_env(c, o, v)
yield var_list # Teardown after yield
for var_set in var_list:
c, o, v = var_set.split(',')
rm_from_env(c, o)


@pytest.fixture()
def config():
import modules.Configuration as Config
cfg = {"BOOL_T": "true",
"BOOL_F": "false",
"NUM": "60"}
f = tempfile.NamedTemporaryFile(delete=False)
write_to_cfg(f.name, 'CFG', cfg)
Config.filename = f.name
Config.init(Config.filename)
yield Config # Teardown after yield
del Config
os.remove(f.name)


class TestClass(object):
def test_has_option(self, config):
assert(not config.has_option('fail', 'fail'))
assert(config.has_option('ENVVAR', 'BOOL_T'))
assert(config.has_option('CFG', 'BOOL_T'))

def test_getboolean(self, config):
assert(not config.getboolean('fail', 'fail'))
assert(config.getboolean('ENVVAR', 'BOOL_T'))
assert(config.getboolean('ENVVAR', 'BOOL_F') is False)
assert(config.getboolean('CFG', 'BOOL_T'))
assert(config.getboolean('CFG', 'BOOL_F') is False)
with pytest.raises(ValueError):
config.getboolean('ENVVAR', 'NUM')
config.getboolean('CFG', 'NUM')
assert(config.getboolean('some', 'default', True))
assert(config.getboolean('some', 'default') is False)

def test_get(self, config):
assert(config.get('ENVVAR', 'NUM') == '60')
assert(config.get('ENVVAR', 'NUM', False, 61) == 61)
assert(config.get('ENVVAR', 'NUM', False, 1, 59) == 59)
assert(config.get('ENVVAR', 'NO_NUM', 100) == 100)
with pytest.raises(SystemExit):
assert(config.get('ENVVAR', 'NO_NUM', None))

def test_get_exchange_poloniex(self, config):
write_skeleton_exchange(config.filename, 'Poloniex')
config.init(config.filename)
assert(config.get_exchange() == 'POLONIEX')

def test_get_exchange_bitfinex(self, config):
write_skeleton_exchange(config.filename, 'Bitfinex')
config.init(config.filename)
assert(config.get_exchange() == 'BITFINEX')

def test_get_coin_cfg_new(self, config):
write_skeleton_exchange(config.filename, 'Bitfinex')
cfg = {'minloansize': 0.01,
'mindailyrate': 0.18,
'maxactiveamount': 1,
'maxtolend': 0,
'maxpercenttolend': 0,
'maxtolendrate': 0}
write_to_cfg(config.filename, 'XMR', cfg)
config.init(config.filename)
result = {'XMR': {'minrate': Decimal('0.0018'), 'maxactive': Decimal('1'), 'maxtolend': Decimal('0'),
'maxpercenttolend': Decimal('0'), 'maxtolendrate': Decimal('0'), 'gapmode': False,
'gapbottom': Decimal('0'), 'gaptop': Decimal('0')}}
assert(config.get_coin_cfg() == result)

def test_get_coin_cfg_old(self, config):
write_to_cfg(config.filename, 'BOT', {'coinconfig': '["BTC:0.18:1:0:0:0","DASH:0.6:1:0:0:0"]'})
config.init(config.filename)
result = {'BTC': {'minrate': Decimal('0.0018'), 'maxactive': Decimal('1'), 'maxtolend': Decimal('0'),
'maxpercenttolend': Decimal('0'), 'maxtolendrate': Decimal('0')},
'DASH': {'minrate': Decimal('0.006'), 'maxactive': Decimal('1'), 'maxtolend': Decimal('0'),
'maxpercenttolend': Decimal('0'), 'maxtolendrate': Decimal('0')}}
assert(config.get_coin_cfg() == result)
12 changes: 7 additions & 5 deletions tests/test_PoloniexAPI.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import time
from builtins import range

# Hack to get relative imports - probably need to fix the dir structure instead but we need this at the minute for
# pytest to work
@@ -25,7 +26,7 @@
# thread1.start()
# except Exception as e:
# assert False, 'api_query ' + str(i + 1) + ':' + e.message
#
#
#
# # Test fast api calls
# def test_multiple_calls():
@@ -36,13 +37,14 @@ def api_rate_limit(n, start):
api.limit_request_rate()
# verify that the (N % 6) th request is delayed by (N / 6) sec from the start time
if n != 0 and n % 6 == 0:
print 'limit request ' + str(n) + ' ' + str(start) + ' ' + str(time.time()) + '\n'
print('limit request ' + str(n) + ' ' + str(start) + ' ' + str(time.time()) + '\n')
assert time.time() - start >= int(n / 6), "rate limit failed"


# Test rate limiter
def test_rate_limiter():
start = time.time()
for i in xrange(20):
thread1 = threading.Thread(target=api_rate_limit, args=(i, start))
thread1.start()
for i in range(20):
thread = threading.Thread(target=api_rate_limit, args=(i, start))
thread.start()
thread.join()