Skip to content

Commit

Permalink
fixed importdata 运行失败
Browse files Browse the repository at this point in the history
  • Loading branch information
@fasiondog
fasiondog committed Jan 13, 2022
1 parent 646abc7 commit b6cd547
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 43 deletions.
31 changes: 16 additions & 15 deletions hikyuu/gui/data/UsePytdxImportToH5Thread.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class UsePytdxImportToH5Thread(QThread):
def __init__(self, parent, config):
super(self.__class__, self).__init__()
self.parent = parent
self.log_queue = parent.mp_log_q if parent is not None else None
self.config = config
self.msg_name = 'HDF5_IMPORT'

Expand Down Expand Up @@ -79,7 +80,7 @@ def init_task(self):

self.tasks = []
if self.config.getboolean('weight', 'enable', fallback=False):
self.tasks.append(ImportWeightToSqliteTask(self.parent.mp_log_q, self.queue, self.config, dest_dir))
self.tasks.append(ImportWeightToSqliteTask(self.log_queue, self.queue, self.config, dest_dir))

#if self.config.getboolean('finance', 'enable', fallback=False):
# self.tasks.append(ImportHistoryFinanceTask(self.queue, dest_dir))
Expand Down Expand Up @@ -128,14 +129,14 @@ def init_task(self):
trans_max_days = (today - trans_start_date).days + 1
self.tasks.append(
ImportPytdxTransToH5(
self.parent.mp_log_q, self.queue, sqlite_file_name, 'SH', self.quotations, use_hosts[cur_host][0],
self.log_queue, self.queue, sqlite_file_name, 'SH', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir, trans_max_days
)
)
cur_host += 1
self.tasks.append(
ImportPytdxTransToH5(
self.parent.mp_log_q, self.queue, sqlite_file_name, 'SZ', self.quotations, use_hosts[cur_host][0],
self.log_queue, self.queue, sqlite_file_name, 'SZ', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir, trans_max_days
)
)
Expand All @@ -145,16 +146,16 @@ def init_task(self):
start_date = datetime.datetime.strptime(config['ktype']['min_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.parent.mp_log_q, self.queue, self.config, 'SH', '1MIN', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
self.log_queue, self.queue, self.config, 'SH', '1MIN', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.parent.mp_log_q, self.queue, self.config, 'SZ', '1MIN', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
self.log_queue, self.queue, self.config, 'SZ', '1MIN', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
Expand All @@ -166,14 +167,14 @@ def init_task(self):
time_max_days = (today - time_start_date).days + 1
self.tasks.append(
ImportPytdxTimeToH5(
self.parent.mp_log_q, self.queue, sqlite_file_name, 'SH', self.quotations, use_hosts[cur_host][0],
self.log_queue, self.queue, sqlite_file_name, 'SH', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir, time_max_days
)
)
cur_host += 1
self.tasks.append(
ImportPytdxTimeToH5(
self.parent.mp_log_q, self.queue, sqlite_file_name, 'SZ', self.quotations, use_hosts[cur_host][0],
self.log_queue, self.queue, sqlite_file_name, 'SZ', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir, time_max_days
)
)
Expand All @@ -183,16 +184,16 @@ def init_task(self):
start_date = datetime.datetime.strptime(config['ktype']['min5_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.parent.mp_log_q, self.queue, self.config, 'SH', '5MIN', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
self.log_queue, self.queue, self.config, 'SH', '5MIN', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.parent.mp_log_q, self.queue, self.config, 'SZ', '5MIN', self.quotations,
use_hosts[cur_host][0], use_hosts[cur_host][1], dest_dir,
self.log_queue, self.queue, self.config, 'SZ', '5MIN', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
Expand All @@ -202,15 +203,15 @@ def init_task(self):
start_date = datetime.datetime.strptime(config['ktype']['day_start_date'], '%Y-%m-%d').date()
self.tasks.append(
ImportPytdxToH5(
self.parent.mp_log_q, self.queue, self.config, 'SH', 'DAY', self.quotations, use_hosts[cur_host][0],
self.log_queue, self.queue, self.config, 'SH', 'DAY', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
)
cur_host += 1
self.tasks.append(
ImportPytdxToH5(
self.parent.mp_log_q, self.queue, self.config, 'SZ', 'DAY', self.quotations, use_hosts[cur_host][0],
self.log_queue, self.queue, self.config, 'SZ', 'DAY', self.quotations, use_hosts[cur_host][0],
use_hosts[cur_host][1], dest_dir,
start_date.year * 100000000 + start_date.month * 1000000 + start_date.day * 10000
)
Expand Down
27 changes: 8 additions & 19 deletions hikyuu/gui/data/UseTdxImportToH5Thread.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def __init__(self, parent, config):
super(self.__class__, self).__init__()
self.logger = logging.getLogger(self.__class__.__name__)
self.parent = parent
self.log_queue = parent.mq_log_q if parent is not None else None
self.config = config
self.msg_name = 'HDF5_IMPORT'

Expand All @@ -66,39 +67,27 @@ def __init__(self, parent, config):
self.queue = Queue()
self.tasks = []
if self.config.getboolean('weight', 'enable', fallback=False):
self.tasks.append(ImportWeightToSqliteTask(self.parent.mp_log_q, self.queue, self.config, dest_dir))
self.tasks.append(ImportWeightToSqliteTask(self.log_queue, self.queue, self.config, dest_dir))
if self.config.getboolean('ktype', 'day', fallback=False):
self.tasks.append(
ImportTdxToH5Task(
self.parent.mp_log_q, self.queue, config, 'SH', 'DAY', self.quotations, src_dir, dest_dir
)
ImportTdxToH5Task(self.log_queue, self.queue, config, 'SH', 'DAY', self.quotations, src_dir, dest_dir)
)
self.tasks.append(
ImportTdxToH5Task(
self.parent.mp_log_q, self.queue, config, 'SZ', 'DAY', self.quotations, src_dir, dest_dir
)
ImportTdxToH5Task(self.log_queue, self.queue, config, 'SZ', 'DAY', self.quotations, src_dir, dest_dir)
)
if self.config.getboolean('ktype', 'min5', fallback=False):
self.tasks.append(
ImportTdxToH5Task(
self.parent.mp_log_q, self.queue, config, 'SH', '5MIN', self.quotations, src_dir, dest_dir
)
ImportTdxToH5Task(self.log_queue, self.queue, config, 'SH', '5MIN', self.quotations, src_dir, dest_dir)
)
self.tasks.append(
ImportTdxToH5Task(
self.parent.mp_log_q, self.queue, config, 'SZ', '5MIN', self.quotations, src_dir, dest_dir
)
ImportTdxToH5Task(self.log_queue, self.queue, config, 'SZ', '5MIN', self.quotations, src_dir, dest_dir)
)
if self.config.getboolean('ktype', 'min', fallback=False):
self.tasks.append(
ImportTdxToH5Task(
self.parent.mp_log_q, self.queue, config, 'SH', '1MIN', self.quotations, src_dir, dest_dir
)
ImportTdxToH5Task(self.log_queue, self.queue, config, 'SH', '1MIN', self.quotations, src_dir, dest_dir)
)
self.tasks.append(
ImportTdxToH5Task(
self.parent.mp_log_q, self.queue, config, 'SZ', '1MIN', self.quotations, src_dir, dest_dir
)
ImportTdxToH5Task(self.log_queue, self.queue, config, 'SZ', '1MIN', self.quotations, src_dir, dest_dir)
)

def __del__(self):
Expand Down
19 changes: 10 additions & 9 deletions hikyuu/gui/importdata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,24 @@ def getCurrentConfig(self):
#读取保存的配置文件信息，如果不存在，则使用默认配置
this_dir = self.getUserConfigDir()
import_config = ConfigParser()
import_config.read(this_dir + '/importdata-gui.ini', encoding = 'utf-8')
import_config.read(this_dir + '/importdata-gui.ini', encoding='utf-8')
return import_config

def initThreads(self):
self.escape_time_thread = None
self.hdf5_import_thread = None
self.mysql_import_thread = None
self.import_running = False
self.progress = {'DAY': 0, '1MIN': 0, '5MIN': 0, 'TRANS':0, 'TIME':0}
self.info_type = {'DAY': '日线数据', '1MIN': '一分钟线', '5MIN': '五分钟线', 'TRANS':'历史分笔', 'TIME':'分时数据'}
self.progress = {'DAY': 0, '1MIN': 0, '5MIN': 0, 'TRANS': 0, 'TIME': 0}
self.info_type = {'DAY': '日线数据', '1MIN': '一分钟线', '5MIN': '五分钟线', 'TRANS': '历史分笔', 'TIME': '分时数据'}
self.escape_time = 0.0
self.details = []

def print_progress(self, ktype, progress):
if progress != self.progress[ktype]:
print('import progress: {}% - {} - 已耗时 {:>.2f} 分钟'.format(progress, self.info_type[ktype], self.escape_time))
print(
'import progress: {}% - {} - 已耗时 {:>.2f} 分钟'.format(progress, self.info_type[ktype], self.escape_time)
)
self.progress[ktype] = progress

def on_message_from_thread(self, msg):
Expand All @@ -53,7 +55,7 @@ def on_message_from_thread(self, msg):
msg_name, msg_task_name = msg[:2]
if msg_name == 'ESCAPE_TIME':
self.escape_time = msg_task_name / 60

elif msg_name == 'HDF5_IMPORT':
if msg_task_name == 'INFO':
print(msg[2])
Expand Down Expand Up @@ -108,7 +110,6 @@ def on_message_from_thread(self, msg):
else:
print('权息{}'.format(msg[2]))


def start_import_data(self):
config = self.getCurrentConfig()
dest_dir = config.get('hdf5', 'dir')
Expand All @@ -130,9 +131,9 @@ def start_import_data(self):
QCoreApplication.processEvents()

if config.getboolean('tdx', 'enable'):
self.hdf5_import_thread = UseTdxImportToH5Thread(config)
self.hdf5_import_thread = UseTdxImportToH5Thread(None, config)
else:
self.hdf5_import_thread = UsePytdxImportToH5Thread(config)
self.hdf5_import_thread = UsePytdxImportToH5Thread(None, config)

self.hdf5_import_thread.message.connect(self.on_message_from_thread)
self.hdf5_import_thread.start()
Expand All @@ -150,5 +151,5 @@ def main():
sys.exit(app.exec())


if __name__ == '__main__':
if __name__ == '__main__':
main()
2 changes: 2 additions & 0 deletions hikyuu/util/mylog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,8 @@ def capture_multiprocess_all_logger(queue, level):
@param multiprocessing.Queue queue 指定的 mp Queue
@param level 日志输出等级
"""
if queue is None:
return
qh = logging.handlers.QueueHandler(queue)
level = get_default_logger().level
for name in logging.Logger.manager.loggerDict.keys():
Expand Down

0 comments on commit b6cd547

Please sign in to comment.