diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a36fe865..ea8845e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,11 +6,11 @@ repos: - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - - repo: http://gitlab.com/PyCQA/flake8 - rev: 3.9.0 + - repo: https://gitlab.com/PyCQA/flake8 + rev: 3.9.2 hooks: - id: flake8 - - repo: http://github.com/PyCQA/isort + - repo: https://github.com/PyCQA/isort rev: 5.7.0 hooks: - id: isort diff --git a/algobot/__main__.py b/algobot/__main__.py index 3e65e7f0..a37c6052 100644 --- a/algobot/__main__.py +++ b/algobot/__main__.py @@ -26,8 +26,8 @@ setup_graph_plots, setup_graphs, update_backtest_graph_limits, update_main_graphs) -from algobot.helpers import (ROOT_DIR, create_folder, create_folder_if_needed, - get_caller_string, open_file_or_folder) +from algobot.helpers import (PATHS, create_folder_if_needed, get_caller_string, + open_file_or_folder) from algobot.interface.about import About from algobot.interface.config_utils.state_utils import load_state, save_state from algobot.interface.config_utils.strategy_utils import get_strategies @@ -47,7 +47,7 @@ from algobot.traders.simulationtrader import SimulationTrader app = QApplication(sys.argv) -mainUi = os.path.join(ROOT_DIR, 'UI', 'algobot.ui') +mainUi = os.path.join(PATHS.get_ui_dir(), 'algobot.ui') class Interface(QMainWindow): @@ -252,21 +252,23 @@ def export_optimizer(self, file_type: str): """ if self.optimizer: if len(self.optimizer.optimizerRows) > 0: - optimizerFolderPath = create_folder('Optimizer Results') - innerPath = os.path.join(optimizerFolderPath, self.optimizer.symbol) - create_folder_if_needed(innerPath, optimizerFolderPath) - defaultFileName = self.optimizer.get_default_result_file_name('optimizer', ext=file_type.lower()) - defaultPath = os.path.join(innerPath, defaultFileName) - filePath, _ = QFileDialog.getSaveFileName(self, 'Save Optimizer', defaultPath, - f'{file_type} (*.{file_type.lower()})') - if not filePath: + optimizer_folder_path = PATHS.get_optimizer_results_dir() + create_folder_if_needed(optimizer_folder_path) + inner_path = os.path.join(optimizer_folder_path, self.optimizer.symbol) + create_folder_if_needed(inner_path) + default_file_name = self.optimizer.get_default_result_file_name(optimizer_folder_path, + 'optimizer', ext=file_type.lower()) + default_path = os.path.join(inner_path, default_file_name) + file_path, _ = QFileDialog.getSaveFileName(self, 'Save Optimizer', default_path, + f'{file_type} (*.{file_type.lower()})') + if not file_path: create_popup(self, "Export cancelled.") else: - self.optimizer.export_optimizer_rows(filePath, file_type) - create_popup(self, f'Exported successfully to {filePath}.') + self.optimizer.export_optimizer_rows(file_path, file_type) + create_popup(self, f'Exported successfully to {file_path}.') if open_from_msg_box(text='Do you want to open the optimization report?', title='Optimizer Report'): - open_file_or_folder(filePath) + open_file_or_folder(file_path) else: create_popup(self, "No table rows found.") @@ -331,7 +333,6 @@ def initiate_optimizer(self): self.threads[OPTIMIZER] = optimizerThread.OptimizerThread(gui=self, logger=self.logger, combos=combos) worker = self.threads[OPTIMIZER] - worker.signals.started.connect(lambda: self.set_optimizer_buttons(running=True, clear=True)) worker.signals.restore.connect(lambda: self.set_optimizer_buttons(running=False, clear=False)) worker.signals.error.connect(lambda x: create_popup(self, x)) if self.configuration.enabledOptimizerNotification.isChecked(): @@ -407,18 +408,19 @@ def end_backtest(self): """ Ends backtest and prompts user if they want to see the results. """ - backtestFolderPath = create_folder('Backtest Results') - innerPath = os.path.join(backtestFolderPath, self.backtester.symbol) - create_folder_if_needed(innerPath, backtestFolderPath) - defaultFile = os.path.join(innerPath, self.backtester.get_default_result_file_name()) - fileName, _ = QFileDialog.getSaveFileName(self, 'Save Result', defaultFile, 'TXT (*.txt)') - fileName = fileName.strip() - fileName = fileName if fileName != '' else None + backtest_folder_path = PATHS.get_backtest_results_dir() + create_folder_if_needed(backtest_folder_path) + inner_path = os.path.join(backtest_folder_path, self.backtester.symbol) + create_folder_if_needed(inner_path) + default_file = os.path.join(inner_path, self.backtester.get_default_result_file_name(backtest_folder_path)) + file_name, _ = QFileDialog.getSaveFileName(self, 'Save Result', default_file, 'TXT (*.txt)') + file_name = file_name.strip() + file_name = file_name if file_name != '' else None - if not fileName: + if not file_name: self.add_to_backtest_monitor('Ended backtest.') else: - path = self.backtester.write_results(resultFile=fileName) + path = self.backtester.write_results(resultFile=file_name) self.add_to_backtest_monitor(f'Ended backtest and saved results to {path}.') if open_from_msg_box(text=f"Backtest results have been saved to {path}.", title="Backtest Results"): @@ -1293,12 +1295,13 @@ def export_trades(self, caller): trade.append(item.text()) trades.append(trade) - path = create_folder("Trade History") + trade_history_dir = PATHS.get_trade_history_dir() + create_folder_if_needed(trade_history_dir) if caller == LIVE: - defaultFile = os.path.join(path, 'live_trades.csv') + defaultFile = os.path.join(trade_history_dir, 'live_trades.csv') else: - defaultFile = os.path.join(path, 'simulation_trades.csv') + defaultFile = os.path.join(trade_history_dir, 'simulation_trades.csv') path, _ = QFileDialog.getSaveFileName(self, 'Export Trades', defaultFile, 'CSV (*.csv)') @@ -1317,8 +1320,9 @@ def import_trades(self, caller): """ table = self.interfaceDictionary[caller]['mainInterface']['historyTable'] label = self.interfaceDictionary[caller]['mainInterface']['historyLabel'] - path = create_folder("Trade History") - path, _ = QFileDialog.getOpenFileName(self, 'Import Trades', path, "CSV (*.csv)") + trade_history_dir = PATHS.get_trade_history_dir() + create_folder_if_needed(trade_history_dir) + path, _ = QFileDialog.getOpenFileName(self, 'Import Trades', trade_history_dir, "CSV (*.csv)") try: with open(path, 'r') as f: diff --git a/algobot/data.py b/algobot/data.py index 1a4625e1..2345b79a 100644 --- a/algobot/data.py +++ b/algobot/data.py @@ -9,7 +9,8 @@ from binance.client import Client from binance.helpers import interval_to_milliseconds -from algobot.helpers import (ROOT_DIR, get_logger, get_normalized_data, +from algobot import helpers +from algobot.helpers import (PATHS, get_logger, get_normalized_data, get_ups_and_downs) from algobot.typing_hints import DATA_TYPE @@ -142,12 +143,12 @@ def get_database_file(self) -> str: Retrieves database file path. :return: Database file path. """ - database_folder = os.path.join(ROOT_DIR, 'Databases') + database_folder = PATHS.get_database_dir() if not os.path.exists(database_folder): - os.mkdir(database_folder) + os.makedirs(database_folder) - filePath = os.path.join(database_folder, f'{self.symbol}.db') - return filePath + file_path = os.path.join(database_folder, f'{self.symbol}.db') + return file_path def create_table(self): """ @@ -521,19 +522,20 @@ def get_interval_minutes(self) -> int: else: raise ValueError("Invalid interval.", 4) - def create_folders_and_change_path(self, folderName: str): + def create_folders_and_change_path(self, folder_name: str): """ Creates appropriate folders for data storage then changes current working directory to it. - :param folderName: Folder to create. + :param folder_name: Folder to create. """ - os.chdir(ROOT_DIR) - if not os.path.exists(folderName): # Create CSV folder if it doesn't exist - os.mkdir(folderName) - os.chdir(folderName) # Go inside the folder. + if not os.path.exists(folder_name): + helpers.create_folder_if_needed(folder_name) - if not os.path.exists(self.symbol): # Create symbol folder inside CSV folder if it doesn't exist. + os.chdir(folder_name) + + if not os.path.exists(self.symbol): os.mkdir(self.symbol) - os.chdir(self.symbol) # Go inside the folder. + + os.chdir(self.symbol) def write_csv_data(self, totalData: list, fileName: str, armyTime: bool = True) -> str: """ @@ -544,7 +546,7 @@ def write_csv_data(self, totalData: list, fileName: str, armyTime: bool = True) :return: Absolute path to CSV file. """ currentPath = os.getcwd() - self.create_folders_and_change_path(folderName="CSV") + self.create_folders_and_change_path(PATHS.get_csv_dir()) with open(fileName, 'w') as f: f.write("Date_UTC, Open, High, Low, Close, Volume, Quote_Asset_Volume, Number_of_Trades, " diff --git a/algobot/helpers.py b/algobot/helpers.py index 70ba6121..99ec1034 100644 --- a/algobot/helpers.py +++ b/algobot/helpers.py @@ -6,11 +6,13 @@ import random import re import subprocess +import tempfile import time from datetime import datetime from typing import Dict, List, Tuple, Union import requests +from appdirs import AppDirs from dateutil import parser import algobot @@ -19,7 +21,10 @@ BASE_DIR = os.path.dirname(__file__) ROOT_DIR = os.path.dirname(BASE_DIR) -LOG_FOLDER = 'Logs' + +APP_NAME = "algobot" +APP_AUTHOR = "ZENALC" + SHORT_INTERVAL_MAP = { '1m': '1 Minute', @@ -40,6 +45,89 @@ LONG_INTERVAL_MAP = {v: k for k, v in SHORT_INTERVAL_MAP.items()} +class AppDirTemp(AppDirs): + def __init__(self): + self.root = tempfile.mkdtemp() + + @property + def user_data_dir(self): + return os.path.join(self.root, "UserData") + + @property + def user_log_dir(self): + return os.path.join(self.root, "UserLog") + + @property + def site_data_dir(self): + return os.path.join(self.root, "SiteData") + + @property + def user_config_dir(self): + return os.path.join(self.root, "UserConfig") + + @property + def site_config_dir(self): + return os.path.join(self.root, "SiteConfig") + + @property + def user_cache_dir(self): + return os.path.join(self.root, "UserCache") + + @property + def user_state_dir(self): + return os.path.join(self.root, "UserState") + + +class Paths: + """ Encapsulates all the path information for the app to store its configuration. """ + def __init__(self, root_dir: str, app_dirs: AppDirs): + self.root_dir = root_dir + self.app_dirs = app_dirs + + def get_ui_dir(self) -> str: + return os.path.join(self.root_dir, 'UI') + + def get_log_dir(self) -> str: + return os.path.join(self.app_dirs.user_log_dir, 'Logs') + + def get_database_dir(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'Databases') + + def get_state_path(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'state.json') + + def get_optimizer_results_dir(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'Optimizer Results') + + def get_backtest_results_dir(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'Backtest Results') + + def get_trade_history_dir(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'Trade History') + + def get_volatility_results_dir(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'Volatility Results') + + def get_csv_dir(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'CSV') + + def get_configuration_dir(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'Configuration') + + def get_credentials_dir(self) -> str: + return os.path.join(self.app_dirs.user_data_dir, 'Credentials') + + +def _get_app_dirs() -> AppDirs: + if os.getenv("ALGOBOT_TESTING"): + return AppDirTemp() + + return AppDirs(APP_NAME, APP_AUTHOR) + + +PATHS = Paths(ROOT_DIR, _get_app_dirs()) + + def get_latest_version() -> str: """ Gets the latest Algobot version from GitHub. @@ -90,65 +178,53 @@ def open_folder(folder: str): """ This will open a folder even if it doesn't exist. It'll create one if it doesn't exist. """ - targetPath = create_folder(folder) - open_file_or_folder(targetPath) - - -def create_folder(folder: str): - """ - This will create a folder if needed in the root directory. - """ - targetPath = os.path.join(ROOT_DIR, folder) - create_folder_if_needed(targetPath) - - return targetPath + create_folder_if_needed(folder) + open_file_or_folder(folder) -def create_folder_if_needed(targetPath: str, basePath: str = ROOT_DIR) -> bool: +def create_folder_if_needed(target_path: str) -> bool: """ This function will create the appropriate folders in the root folder if needed. - :param targetPath: Target path to have exist. - :param basePath: Base path to start from. By default, it'll be the root directory. + :param target_path: Target path to have exist. :return: Boolean whether folder was created or not. """ - if not os.path.exists(targetPath): - folder = os.path.basename(targetPath) - os.mkdir(os.path.join(basePath, folder)) + if not os.path.exists(target_path): + os.makedirs(target_path, exist_ok=True) return True return False -def open_file_or_folder(targetPath: str): +def open_file_or_folder(target_path: str): """ Opens a file or folder based on targetPath. - :param targetPath: File or folder to open with system defaults. + :param target_path: File or folder to open with system defaults. """ if platform.system() == "Windows": - os.startfile(targetPath) + os.startfile(target_path) elif platform.system() == "Darwin": - subprocess.Popen(["open", targetPath]) + subprocess.Popen(["open", target_path]) else: - subprocess.Popen(["xdg-open", targetPath]) + subprocess.Popen(["xdg-open", target_path]) -def setup_and_return_log_path(fileName: str) -> str: +def setup_and_return_log_path(filename: str) -> str: """ Creates folders (if needed) and returns default log path. - :param fileName: Log filename to be created. + :param filename: Log filename to be created. :return: Absolute path to log file. """ - LOG_DIR = os.path.join(ROOT_DIR, LOG_FOLDER) - if not os.path.exists(LOG_DIR): - os.mkdir(LOG_DIR) + log_dir = PATHS.get_log_dir() + if not os.path.exists(log_dir): + os.makedirs(log_dir) - todayDate = datetime.today().strftime('%Y-%m-%d') - LOG_DATE_FOLDER = os.path.join(LOG_DIR, todayDate) - if not os.path.exists(LOG_DATE_FOLDER): - os.mkdir(LOG_DATE_FOLDER) + today_date = datetime.today().strftime('%Y-%m-%d') + log_date_folder = os.path.join(log_dir, today_date) + if not os.path.exists(log_date_folder): + os.mkdir(log_date_folder) - logFileName = f'{datetime.now().strftime("%H-%M-%S")}-{fileName}.log' - fullPath = os.path.join(LOG_DATE_FOLDER, logFileName) - return fullPath + log_file_name = f'{datetime.now().strftime("%H-%M-%S")}-{filename}.log' + full_path = os.path.join(log_date_folder, log_file_name) + return full_path def get_logger(log_file: str, logger_name: str) -> logging.Logger: @@ -164,7 +240,7 @@ def get_logger(log_file: str, logger_name: str) -> logging.Logger: log_level = logging.DEBUG logger.setLevel(log_level) formatter = logging.Formatter('%(message)s') - handler = logging.FileHandler(filename=setup_and_return_log_path(fileName=log_file), delay=True) + handler = logging.FileHandler(filename=setup_and_return_log_path(filename=log_file), delay=True) handler.setFormatter(formatter) logger.addHandler(handler) diff --git a/algobot/interface/config_utils/credential_utils.py b/algobot/interface/config_utils/credential_utils.py index b09c041d..b8e8c1e8 100644 --- a/algobot/interface/config_utils/credential_utils.py +++ b/algobot/interface/config_utils/credential_utils.py @@ -33,7 +33,7 @@ def save_credentials(config_obj): Function that saves credentials to base path in a JSON format. Obviously not very secure, but temp fix. :param config_obj: Configuration QDialog object (from configuration.py) """ - targetFolder = os.path.join(helpers.ROOT_DIR, config_obj.credentialsFolder) + targetFolder = config_obj.credentialsFolder helpers.create_folder_if_needed(targetFolder) apiKey = config_obj.binanceApiKey.text() @@ -60,7 +60,7 @@ def load_credentials(config_obj, auto: bool = True): :param auto: Boolean regarding whether bot called this function or not. If bot called it, silently try to load credentials. If a user called it, however, open a file dialog to ask for the file path to credentials. """ - targetFolder = os.path.join(helpers.ROOT_DIR, config_obj.credentialsFolder) + targetFolder = config_obj.credentialsFolder if helpers.create_folder_if_needed(targetFolder): config_obj.credentialResult.setText('No credentials found.') return diff --git a/algobot/interface/config_utils/data_utils.py b/algobot/interface/config_utils/data_utils.py index acf2d8bc..8a93680e 100644 --- a/algobot/interface/config_utils/data_utils.py +++ b/algobot/interface/config_utils/data_utils.py @@ -3,8 +3,8 @@ """ from PyQt5.QtWidgets import QFileDialog -from algobot import helpers from algobot.enums import BACKTEST +from algobot.helpers import PATHS, convert_long_interval, load_from_csv from algobot.interface.config_utils.calendar_utils import setup_calendar from algobot.threads import downloadThread @@ -19,12 +19,12 @@ def import_data(config_obj, caller: int = BACKTEST): action = 'backtest' if caller == BACKTEST else 'optimization' inner_dict['infoLabel'].setText("Importing data...") - filePath, _ = QFileDialog.getOpenFileName(config_obj, 'Open file', helpers.ROOT_DIR, "CSV (*.csv)") + filePath, _ = QFileDialog.getOpenFileName(config_obj, 'Open file', PATHS.get_csv_dir(), "CSV (*.csv)") if filePath == '': inner_dict['infoLabel'].setText("Data not imported.") inner_dict['downloadProgress'].setValue(0) else: - inner_dict['data'] = helpers.load_from_csv(filePath, descending=False) + inner_dict['data'] = load_from_csv(filePath, descending=False) inner_dict['dataType'] = "Imported" inner_dict['dataInterval'] = inner_dict['dataIntervalComboBox'].currentText() inner_dict['infoLabel'].setText("Imported data successfully.") @@ -44,7 +44,7 @@ def download_data(config_obj, caller: int = BACKTEST): set_download_progress(config_obj, progress=0, message="Attempting to download...", caller=caller, enableStop=False) symbol = config_obj.optimizer_backtest_dict[caller]['tickers'].text() - interval = helpers.convert_long_interval(config_obj.optimizer_backtest_dict[caller]['intervals'].currentText()) + interval = convert_long_interval(config_obj.optimizer_backtest_dict[caller]['intervals'].currentText()) thread = downloadThread.DownloadThread(symbol=symbol, interval=interval, caller=caller, logger=config_obj.logger) thread.signals.progress.connect(lambda progress, msg: set_download_progress(config_obj=config_obj, message=msg, diff --git a/algobot/interface/config_utils/user_config_utils.py b/algobot/interface/config_utils/user_config_utils.py index 8395c8d4..af24e7d0 100644 --- a/algobot/interface/config_utils/user_config_utils.py +++ b/algobot/interface/config_utils/user_config_utils.py @@ -23,11 +23,10 @@ def create_appropriate_config_folders(config_obj, folder: str) -> str: :param folder: Folder to create inside configuration folder. :return: Absolute path to new folder. """ - basePath = os.path.join(helpers.ROOT_DIR, config_obj.configFolder) - helpers.create_folder_if_needed(basePath) + basePath = config_obj.configFolder targetPath = os.path.join(basePath, folder) - helpers.create_folder_if_needed(targetPath, basePath=basePath) + helpers.create_folder_if_needed(targetPath) return targetPath diff --git a/algobot/interface/configuration.py b/algobot/interface/configuration.py index 83924216..3cfbbda4 100644 --- a/algobot/interface/configuration.py +++ b/algobot/interface/configuration.py @@ -7,9 +7,9 @@ QLabel, QLayout, QMainWindow, QSpinBox, QTabWidget) -import algobot.helpers as helpers from algobot.enums import BACKTEST, LIVE, OPTIMIZER, SIMULATION, STOP, TRAILING from algobot.graph_helpers import create_infinite_line +from algobot.helpers import PATHS from algobot.interface.config_utils.credential_utils import load_credentials from algobot.interface.config_utils.slot_utils import load_slots from algobot.interface.config_utils.strategy_utils import ( @@ -22,7 +22,7 @@ from algobot.strategies import * # noqa: F403, F401 from algobot.strategies.strategy import Strategy -configurationUi = os.path.join(helpers.ROOT_DIR, 'UI', 'configuration.ui') +configurationUi = os.path.join(PATHS.get_ui_dir(), 'configuration.ui') class Configuration(QDialog): @@ -82,9 +82,9 @@ def __init__(self, parent: QMainWindow, logger: Logger = None): self.chatPass = False # Folders and files - self.credentialsFolder = "Credentials" - self.configFolder = 'Configuration' - self.stateFilePath = os.path.join(helpers.ROOT_DIR, 'state.json') + self.credentialsFolder = PATHS.get_credentials_dir() + self.configFolder = PATHS.get_configuration_dir() + self.stateFilePath = PATHS.get_state_path() self.categoryTabs = [ self.mainConfigurationTabWidget, diff --git a/algobot/interface/other_commands.py b/algobot/interface/other_commands.py index 94023d8f..aae7fbab 100644 --- a/algobot/interface/other_commands.py +++ b/algobot/interface/other_commands.py @@ -9,13 +9,15 @@ from PyQt5.QtWidgets import QDialog, QLineEdit, QMainWindow, QMessageBox import algobot -import algobot.helpers as helpers +from algobot.helpers import (PATHS, convert_long_interval, + create_folder_if_needed, get_logger, + open_file_or_folder) from algobot.interface.utils import create_popup, open_from_msg_box from algobot.threads.downloadThread import DownloadThread from algobot.threads.volatilitySnooperThread import VolatilitySnooperThread from algobot.threads.workerThread import Worker -otherCommandsUi = os.path.join(helpers.ROOT_DIR, 'UI', 'otherCommands.ui') +otherCommandsUi = os.path.join(PATHS.get_ui_dir(), 'otherCommands.ui') class OtherCommands(QDialog): @@ -56,33 +58,32 @@ def load_slots(self): self.stopVolatilityButton.clicked.connect(lambda: self.stop_volatility_snooper()) # Purge buttons. - self.purgeLogsButton.clicked.connect(lambda: self.purge('Logs')) - self.purgeDatabasesButton.clicked.connect(lambda: self.purge('Databases')) - self.purgeBacktestResultsButton.clicked.connect(lambda: self.purge('Backtest Results')) - self.purgeConfigurationFilesButton.clicked.connect(lambda: self.purge('Configuration')) - self.purgeCredentialsButton.clicked.connect(lambda: self.purge('Credentials')) - self.purgeCSVFilesButton.clicked.connect(lambda: self.purge('CSV')) + self.purgeLogsButton.clicked.connect(lambda: self.purge(PATHS.get_log_dir())) + self.purgeDatabasesButton.clicked.connect(lambda: self.purge(PATHS.get_database_dir())) + self.purgeBacktestResultsButton.clicked.connect(lambda: self.purge(PATHS.get_backtest_results_dir())) + self.purgeConfigurationFilesButton.clicked.connect(lambda: self.purge(PATHS.get_configuration_dir())) + self.purgeCredentialsButton.clicked.connect(lambda: self.purge(PATHS.get_credentials_dir())) + self.purgeCSVFilesButton.clicked.connect(lambda: self.purge(PATHS.get_csv_dir())) def purge(self, directory: str): """ Deletes directory provided. """ - path = os.path.join(helpers.ROOT_DIR, directory) - if not os.path.exists(path): + if not os.path.exists(directory): create_popup(self, f"No {directory.lower()} files detected.") return message = f'Are you sure you want to delete your {directory.lower()} files? You might not be able to undo ' \ - f'this operation. \n\nThe following path will be deleted: \n{path}' + f'this operation. \n\nThe following path will be deleted: \n{directory}' qm = QMessageBox ret = qm.question(self, 'Warning', message, qm.Yes | qm.No) - if ret == qm.Yes and os.path.exists(path): - shutil.rmtree(path) + if ret == qm.Yes and os.path.exists(directory): + shutil.rmtree(directory) self.infoLabel.setText(f'{directory.capitalize()} files have been successfully deleted.') - if directory == 'Logs': # Reinitialize log folder if old logs were purged. - self.parent.logger = helpers.get_logger(log_file='algobot', logger_name='algobot') + if directory.endswith('Logs'): # Reinitialize log folder if old logs were purged. + self.parent.logger = get_logger(log_file='algobot', logger_name='algobot') def start_date_thread(self): """ @@ -103,7 +104,7 @@ def get_start_date_for_csv(self) -> List[QDate]: Find start date by instantiating a Data object and fetching the Binance API. """ symbol = self.csvGenerationTicker.text() - interval = helpers.convert_long_interval(self.csvGenerationDataInterval.currentText()) + interval = convert_long_interval(self.csvGenerationDataInterval.currentText()) ts = algobot.BINANCE_CLIENT._get_earliest_valid_timestamp(symbol, interval) startDate = datetime.fromtimestamp(int(ts) / 1000, tz=timezone.utc) @@ -137,7 +138,7 @@ def initiate_csv_generation(self): symbol = self.csvGenerationTicker.text() descending = self.descendingDateRadio.isChecked() armyTime = self.armyDateRadio.isChecked() - interval = helpers.convert_long_interval(self.csvGenerationDataInterval.currentText()) + interval = convert_long_interval(self.csvGenerationDataInterval.currentText()) selectedDate = self.startDateCalendar.selectedDate().toPyDate() startDate = None if selectedDate == self.currentDateList[0] else selectedDate @@ -175,7 +176,7 @@ def end_csv_generation(self, savedPath: str): self.generateCSVButton.setEnabled(True) if open_from_msg_box(text=f"Successfully saved CSV data to {savedPath}.", title="Data saved successfully."): - helpers.open_file_or_folder(savedPath) + open_file_or_folder(savedPath) def modify_csv_ui(self, running: bool, reset: bool = False): self.generateCSVButton.setEnabled(not running) @@ -195,7 +196,8 @@ def stop_csv_generation(self): def end_snoop_generate_volatility_report(self, volatility_dict, output_type): self.volatilityStatus.setText("Finished snooping. Generating report...") self.volatilityProgressBar.setValue(100) - folder_path = helpers.create_folder("Volatility Results") + folder_path = PATHS.get_volatility_results_dir() + create_folder_if_needed(folder_path) file_name = f'Volatility_Results_{datetime.now().strftime("%m_%d_%Y_%H_%M_%S")}.{output_type.lower()}' file_path = os.path.join(folder_path, file_name) @@ -210,7 +212,7 @@ def end_snoop_generate_volatility_report(self, volatility_dict, output_type): self.volatilityStatus.setText(f"Generated report at {file_path}.") if open_from_msg_box(text='Do you want to open the volatility report?', title='Volatility Report'): - helpers.open_file_or_folder(file_path) + open_file_or_folder(file_path) def stop_volatility_snooper(self): if self.volatilityThread: diff --git a/algobot/interface/statistics.py b/algobot/interface/statistics.py index 164e180d..9897f8d8 100644 --- a/algobot/interface/statistics.py +++ b/algobot/interface/statistics.py @@ -5,9 +5,9 @@ from PyQt5.QtWidgets import (QDialog, QFormLayout, QLabel, QMainWindow, QTabWidget) -from algobot.helpers import ROOT_DIR, get_label_string +from algobot.helpers import PATHS, get_label_string -statisticsUi = os.path.join(ROOT_DIR, 'UI', 'statistics.ui') +statisticsUi = os.path.join(PATHS.get_ui_dir(), 'statistics.ui') class Statistics(QDialog): diff --git a/algobot/slots.py b/algobot/slots.py index 2162c05f..6cd1ae1e 100644 --- a/algobot/slots.py +++ b/algobot/slots.py @@ -1,7 +1,7 @@ import webbrowser from algobot.enums import BACKTEST, LIVE, SIMULATION -from algobot.helpers import open_folder +from algobot.helpers import PATHS, open_folder from algobot.interface.utils import clear_table, show_and_bring_window_to_front from algobot.themes import (set_bear_mode, set_bloomberg_mode, set_bull_mode, set_dark_mode, set_light_mode) @@ -54,14 +54,18 @@ def create_action_slots(gui): gui.aboutAlgobotAction.triggered.connect(lambda: show_and_bring_window_to_front(gui.about)) gui.liveStatisticsAction.triggered.connect(lambda: gui.show_statistics(0)) gui.simulationStatisticsAction.triggered.connect(lambda: gui.show_statistics(1)) - gui.openBacktestResultsFolderAction.triggered.connect(lambda: open_folder("Backtest Results")) - gui.openOptimizerResultsFolderAction.triggered.connect(lambda: open_folder('Optimizer Results')) - gui.openVolatilityResultsFolderAction.triggered.connect(lambda: open_folder('Volatility Results')) - gui.openLogFolderAction.triggered.connect(lambda: open_folder("Logs")) - gui.openCsvFolderAction.triggered.connect(lambda: open_folder('CSV')) - gui.openDatabasesFolderAction.triggered.connect(lambda: open_folder('Databases')) - gui.openCredentialsFolderAction.triggered.connect(lambda: open_folder('Credentials')) - gui.openConfigurationsFolderAction.triggered.connect(lambda: open_folder('Configuration')) + gui.openBacktestResultsFolderAction.triggered.connect(lambda: open_folder(PATHS.get_backtest_results_dir())) + gui.openOptimizerResultsFolderAction.triggered.connect( + lambda: open_folder(PATHS.get_optimizer_results_dir()) + ) + gui.openVolatilityResultsFolderAction.triggered.connect( + lambda: open_folder(PATHS.get_volatility_results_dir()) + ) + gui.openLogFolderAction.triggered.connect(lambda: open_folder(PATHS.get_log_dir())) + gui.openCsvFolderAction.triggered.connect(lambda: open_folder(PATHS.get_csv_dir())) + gui.openDatabasesFolderAction.triggered.connect(lambda: open_folder(PATHS.get_database_dir())) + gui.openCredentialsFolderAction.triggered.connect(lambda: open_folder(PATHS.get_credentials_dir())) + gui.openConfigurationsFolderAction.triggered.connect(lambda: open_folder(PATHS.get_configuration_dir())) gui.sourceCodeAction.triggered.connect(lambda: webbrowser.open("https://github.com/ZENALC/algobot")) gui.tradingViewLiveAction.triggered.connect(lambda: gui.open_trading_view(LIVE)) gui.tradingViewSimulationAction.triggered.connect(lambda: gui.open_trading_view(SIMULATION)) @@ -124,7 +128,7 @@ def create_backtest_slots(gui): gui.runBacktestButton.clicked.connect(gui.initiate_backtest) gui.endBacktestButton.clicked.connect(gui.end_backtest_thread) gui.clearBacktestTableButton.clicked.connect(lambda: clear_table(gui.backtestTable)) - gui.viewBacktestsButton.clicked.connect(lambda: open_folder("Backtest Results")) + gui.viewBacktestsButton.clicked.connect(lambda: open_folder(PATHS.get_backtest_results_dir())) gui.backtestResetCursorButton.clicked.connect(gui.reset_backtest_cursor) diff --git a/algobot/traders/backtester.py b/algobot/traders/backtester.py index a319ce5b..39885597 100644 --- a/algobot/traders/backtester.py +++ b/algobot/traders/backtester.py @@ -13,8 +13,7 @@ from algobot.enums import (BACKTEST, BEARISH, BULLISH, ENTER_LONG, ENTER_SHORT, EXIT_LONG, EXIT_SHORT, LONG, OPTIMIZER, SHORT) -from algobot.helpers import (LOG_FOLDER, ROOT_DIR, - convert_all_dates_to_datetime, +from algobot.helpers import (PATHS, convert_all_dates_to_datetime, convert_small_interval, get_interval_minutes, get_ups_and_downs, parse_strategy_name) from algobot.interface.config_utils.strategy_utils import \ @@ -200,7 +199,7 @@ def generate_error_message(error: Exception, strategy: Strategy) -> str: f' different parameters, rewriting your strategy, or taking a look at ' \ f'your strategy code again. The strategy that caused this crash is: ' \ f'{strategy.name}. You can find more details about the crash in the ' \ - f'logs file at {os.path.join(ROOT_DIR, LOG_FOLDER)}.' + f'logs file at {PATHS.get_log_dir()}.' return msg def strategy_loop(self, strategyData, thread) -> Union[None, str]: @@ -756,20 +755,19 @@ def print_trades(self, stdout=sys.__stdout__): sys.stdout = previous_stdout # revert stdout back to normal - def get_default_result_file_name(self, name: str = 'backtest', ext: str = 'txt'): + def get_default_result_file_name(self, results_folder: str, name: str = 'backtest', ext: str = 'txt'): """ Returns a default backtest/optimizer result file name. :return: String filename. """ - resultsFolder = os.path.join(ROOT_DIR, f'{name.capitalize()} Results') symbol = 'Imported' if not self.symbol else self.symbol dateString = datetime.now().strftime("%Y-%m-%d_%H-%M") resultFile = f'{symbol}_{name}_results_{"_".join(self.interval.lower().split())}-{dateString}.{ext}' - if not os.path.exists(resultsFolder): + if not os.path.exists(results_folder): return resultFile - innerFolder = os.path.join(resultsFolder, self.symbol) + innerFolder = os.path.join(results_folder, self.symbol) if not os.path.exists(innerFolder): return resultFile diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..bbc62c4d --- /dev/null +++ b/conftest.py @@ -0,0 +1,10 @@ +import os +import shutil + +os.environ["ALGOBOT_TESTING"] = "1" + + +def pytest_unconfigure(config): + from algobot.helpers import PATHS, AppDirTemp + if isinstance(PATHS.app_dirs, AppDirTemp): + shutil.rmtree(PATHS.app_dirs.root) diff --git a/requirements-test.txt b/requirements-test.txt index 6b23c06c..40118b38 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ -r requirements.txt pytest==6.2.4 pytest-socket==0.4.0 -flake8==3.9.0 -pre-commit==2.11.1 +flake8==3.9.2 +pre-commit==2.13.0 +freezegun==1.1.0 diff --git a/requirements.txt b/requirements.txt index f9896ad7..8b6c381e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ aiohttp==3.7.4.post0 +appdirs==1.4.4 APScheduler==3.6.3 async-timeout==3.0.1 attrs==21.2.0 diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 8d228158..75e00645 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,13 +1,17 @@ +import os +import tempfile import time import pytest +from freezegun import freeze_time from algobot.enums import BACKTEST, LIVE, OPTIMIZER, SIMULATION from algobot.helpers import (convert_long_interval, convert_small_interval, - get_caller_string, get_data_from_parameter, - get_elapsed_time, get_label_string, - get_normalized_data, get_ups_and_downs, - parse_strategy_name) + create_folder_if_needed, get_caller_string, + get_data_from_parameter, get_elapsed_time, + get_label_string, get_normalized_data, + get_ups_and_downs, parse_strategy_name, + setup_and_return_log_path) @pytest.mark.parametrize( @@ -108,8 +112,8 @@ def test_get_normalized_data(data, date_in_utc, expected): 'data, parameter, expected', [ ( - [{'high': 5}, {'high': 4}, {'high': 8}, {'high': 6}, {'high': 9}, {'high': 10}], 'high', - ([0, 0, 4, 0, 3, 1], [0, 1, 0, 2, 0, 0]) + [{'high': 5}, {'high': 4}, {'high': 8}, {'high': 6}, {'high': 9}, {'high': 10}], 'high', + ([0, 0, 4, 0, 3, 1], [0, 1, 0, 2, 0, 0]) ) ] ) @@ -140,3 +144,17 @@ def test_get_elapsed_time(elapsed, expected): ) def test_parse_strategy_name(name, expected): assert parse_strategy_name(name) == expected, f"Expected parsed strategy to be: {expected}." + + +def test_create_folder_if_needed(): + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, 'my-dir') + assert create_folder_if_needed(path) + assert not create_folder_if_needed(path) + + +@freeze_time("2021-01-14") +def test_setup_and_return_log_path(): + log_name = setup_and_return_log_path("my-awesome-log") + assert log_name.endswith("Logs/2021-01-14/00-00-00-my-awesome-log.log") + assert os.path.exists(log_name[:len(log_name) - 27])