폴더 구조 변경 및 output 생성 유무 설정 추가

README.md

@@ -112,6 +112,14 @@ v3, v4 학습데이터는 시장 데이터와 종목 데이터를 합하여 사
 - `python -m cProfile -o profile.pstats main.py ...`
 - `python profile.py`
 
+# Tips
+
+- Windows Power Shell에서 로그 Tail 하는 방법
+
+```
+cat D:\dev\rltrader\output\train_000240_a2c_lstm\train_000240_a2c_lstm.log -Wait -Tail 100
+```
+
 # Troubleshooting
 
 ## TF 1.15에서 다음 에러가 나면 Python 3.6으로 맞춰준다.

main.py

@@ -4,6 +4,8 @@
 import argparse
 import json
 
+os.environ['RLTRADER_BASE'] = os.path.dirname(os.path.abspath(__file__))
+
 from quantylab.rltrader import settings
 from quantylab.rltrader import utils
 from quantylab.rltrader import data_manager
@@ -15,12 +17,12 @@
     parser.add_argument('--ver', choices=['v1', 'v2', 'v3', 'v4'], default='v2')
     parser.add_argument('--name', default=utils.get_time_str())
     parser.add_argument('--stock_code', nargs='+')
-    parser.add_argument('--rl_method', choices=['dqn', 'pg', 'ac', 'a2c', 'a3c', 'monkey'])
+    parser.add_argument('--rl_method', choices=['dqn', 'pg', 'ac', 'a2c', 'a3c', 'monkey'], default='a2c')
     parser.add_argument('--net', choices=['dnn', 'lstm', 'cnn', 'monkey'], default='dnn')
     parser.add_argument('--backend', choices=['pytorch', 'tensorflow', 'plaidml'], default='pytorch')
     parser.add_argument('--start_date', default='20200101')
     parser.add_argument('--end_date', default='20201231')
-    parser.add_argument('--lr', type=float, default=0.0001)
+    parser.add_argument('--lr', type=float, default=0.001)
     parser.add_argument('--discount_factor', type=float, default=0.7)
     parser.add_argument('--balance', type=int, default=100000000)
     args = parser.parse_args()
@@ -32,7 +34,7 @@
     value_network_name = f'{args.name}_{args.rl_method}_{args.net}_value.mdl'
     policy_network_name = f'{args.name}_{args.rl_method}_{args.net}_policy.mdl'
     start_epsilon = 1 if args.mode in ['train', 'update'] else 0
-    num_epoches = 1000 if args.mode in ['train', 'update'] else 1
+    num_epoches = 100 if args.mode in ['train', 'update'] else 1
     num_steps = 5 if args.net in ['lstm', 'cnn'] else 1
 
     # Backend 설정

requirements_common.txt

@@ -3,3 +3,4 @@ pandas
 matplotlib
 mplfinance
 tqdm
+sklearn

setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+description-file=README.md
+license_files=LICENSE

setup.py

@@ -2,10 +2,21 @@
 
 setup(
     name='rltrader',
-    version='3.0',
+    version='3.1',
     description='Quantylab Reinforcement Learning for Stock Trading',
     author='Quantylab',
     author_email='[email protected]',
     url='https://github.com/quantylab/rltrader',
-    packages=find_namespace_packages(include=['quantylab.*']),
+    packages=find_namespace_packages(where='src', include=['quantylab.*']),
+    package_dir={'': 'src'},
+    install_requires=[
+        'numpy',
+        'pandas',
+        'matplotlib',
+        'mplfinance',
+        'tqdm',
+        'sklearn',
+        'tensorflow==2.7.0',
+        'torch==1.10.1',
+    ]
 )

quantylab/rltrader/agent.py → src/quantylab/rltrader/agent.py

@@ -4,7 +4,7 @@
 
 class Agent:
     # 에이전트 상태가 구성하는 값 개수
-    # 주식 보유 비율, 현재 손익, 평균 매수 단가 대비 등락률
+    # 주식 보유 비율, 손익률, 주당 매수 단가 대비 주가 등락률
     STATE_DIM = 3
 
     # 매매 수수료 및 세금
@@ -42,7 +42,7 @@ def __init__(self, environment, initial_balance, min_trading_price, max_trading_
 
         # Agent 클래스의 상태
         self.ratio_hold = 0  # 주식 보유 비율
-        self.profitloss = 0  # 현재 손익
+        self.profitloss = 0  # 손익률
         self.avg_buy_price = 0  # 주당 매수 단가
 
     def reset(self):

quantylab/rltrader/data_manager.py → src/quantylab/rltrader/data_manager.py

@@ -212,7 +212,7 @@ def load_data(code, date_from, date_to, ver='v2'):
     else:
         raise Exception('Invalid version.')
 
-    return chart_data, training_data
+    return chart_data, training_data.values
 
 
 def load_data_v3_v4(code, date_from, date_to, ver):
@@ -243,18 +243,37 @@ def load_data_v3_v4(code, date_from, date_to, ver):
     # 날짜 오름차순 정렬
     df = df.sort_values(by='date').reset_index(drop=True)
 
+    # NaN 처리
+    df = df.fillna(method='ffill').fillna(method='bfill').reset_index(drop=True)
+    df = df.fillna(0)
+
     # 기간 필터링
     df['date'] = df['date'].str.replace('-', '')
     df = df[(df['date'] >= date_from) & (df['date'] <= date_to)]
-    df = df.fillna(method='ffill').reset_index(drop=True)
+    df = df.reset_index(drop=True)
 
     # 데이터 조정
-    df.loc[:, ['per', 'pbr', 'roe']] = df[['per', 'pbr', 'roe']].apply(lambda x: x / 100)
+    if ver == 'v3':
+        df.loc[:, ['per', 'pbr', 'roe']] = df[['per', 'pbr', 'roe']].apply(lambda x: x / 100)
 
     # 차트 데이터 분리
     chart_data = df[COLUMNS_CHART_DATA]
 
     # 학습 데이터 분리
-    training_data = df[columns]
+    training_data = df[columns].values
+
+    # 스케일링
+    if ver == 'v4':
+        from sklearn.preprocessing import RobustScaler
+        from joblib import dump, load
+        scaler_path = os.path.join(settings.BASE_DIR, 'scalers', f'scaler_{ver}.joblib')
+        scaler = None
+        if not os.path.exists(scaler_path):
+            scaler = RobustScaler()
+            scaler.fit(training_data)
+            dump(scaler, scaler_path)
+        else:
+            scaler = load(scaler_path)
+        training_data = scaler.transform(training_data)
 
     return chart_data, training_data

quantylab/rltrader/learners.py → src/quantylab/rltrader/learners.py

@@ -29,7 +29,7 @@ def __init__(self, rl_method='rl', stock_code=None,
                 discount_factor=0.9, num_epoches=1000,
                 balance=100000000, start_epsilon=1,
                 value_network=None, policy_network=None,
-                output_path='', reuse_models=True):
+                output_path='', reuse_models=True, gen_output=True):
         # 인자 확인
         assert min_trading_price > 0
         assert max_trading_price > 0
@@ -80,6 +80,7 @@ def __init__(self, rl_method='rl', stock_code=None,
         self.batch_size = 0
         # 로그 등 출력 경로
         self.output_path = output_path
+        self.gen_output = gen_output
 
     def init_value_network(self, shared_network=None, activation='linear', loss='mse'):
         if self.net == 'dnn':
@@ -158,7 +159,7 @@ def build_sample(self):
         self.environment.observe()
         if len(self.training_data) > self.training_data_idx + 1:
             self.training_data_idx += 1
-            self.sample = self.training_data.iloc[self.training_data_idx].tolist()
+            self.sample = self.training_data[self.training_data_idx, :].tolist()
             self.sample.extend(self.agent.get_states())
             return self.sample
         return None
@@ -221,12 +222,13 @@ def run(self, learning=True):
         self.visualizer.prepare(self.environment.chart_data, info)
 
         # 가시화 결과 저장할 폴더 준비
-        self.epoch_summary_dir = os.path.join(self.output_path, f'epoch_summary_{self.stock_code}')
-        if not os.path.isdir(self.epoch_summary_dir):
-            os.makedirs(self.epoch_summary_dir)
-        else:
-            for f in os.listdir(self.epoch_summary_dir):
-                os.remove(os.path.join(self.epoch_summary_dir, f))
+        if self.gen_output:
+            self.epoch_summary_dir = os.path.join(self.output_path, f'epoch_summary_{self.stock_code}')
+            if not os.path.isdir(self.epoch_summary_dir):
+                os.makedirs(self.epoch_summary_dir)
+            else:
+                for f in os.listdir(self.epoch_summary_dir):
+                    os.remove(os.path.join(self.epoch_summary_dir, f))
 
         # 학습에 대한 정보 초기화
         max_portfolio_value = 0
@@ -308,8 +310,9 @@ def run(self, learning=True):
                 f'Loss:{self.loss:.6f} ET:{elapsed_time_epoch:.4f}')
 
             # 에포크 관련 정보 가시화
-            if self.num_epoches == 1 or (epoch + 1) % int(self.num_epoches / 10) == 0:
-                self.visualize(epoch_str, self.num_epoches, epsilon)
+            if self.gen_output:
+                if self.num_epoches == 1 or (epoch + 1) % max(int(self.num_epoches / 10), 1) == 0:
+                    self.visualize(epoch_str, self.num_epoches, epsilon)
 
             # 학습 관련 정보 갱신
             max_portfolio_value = max(
@@ -355,16 +358,16 @@ def predict(self):
             pred_value = None
             pred_policy = None
             if self.value_network is not None:
-                pred_value = self.value_network.predict(list(q_sample))
+                pred_value = self.value_network.predict(list(q_sample)).tolist()
             if self.policy_network is not None:
-                pred_policy = self.policy_network.predict(list(q_sample))
+                pred_policy = self.policy_network.predict(list(q_sample)).tolist()
 
             # 신경망에 의한 행동 결정
-            action, confidence, _ = self.agent.decide_action(pred_value, pred_policy, 0)
-            result.append((self.environment.observation[0], int(action), float(confidence)))
-
-        with open(os.path.join(self.output_path, f'pred_{self.stock_code}.json'), 'w') as f:
-            print(json.dumps(result), file=f)
+            result.append((self.environment.observation[0], pred_value, pred_policy))
+
+        if self.gen_output:
+            with open(os.path.join(self.output_path, f'pred_{self.stock_code}.json'), 'w') as f:
+                print(json.dumps(result), file=f)
         return result
 
 

quantylab/rltrader/networks/__init__.py → src/quantylab/rltrader/networks/__init__.py

@@ -1,8 +1,10 @@
 import os
 
 if os.environ.get('RLTRADER_BACKEND', 'pytorch') == 'pytorch':
+    print('Enabling PyTorch...')
     from quantylab.rltrader.networks.networks_pytorch import Network, DNN, LSTMNetwork, CNN
 else:
+    print('Enabling TensorFlow...')
     from quantylab.rltrader.networks.networks_keras import Network, DNN, LSTMNetwork, CNN
 
 __all__ = [

quantylab/rltrader/settings.py → src/quantylab/rltrader/settings.py

@@ -9,7 +9,7 @@
 
 # 경로 설정
 BASE_DIR = os.environ.get('RLTRADER_BASE', 
-    os.path.abspath(os.path.join(__file__, os.path.pardir, os.path.pardir, os.path.pardir)))
+    os.path.abspath(os.path.join(__file__, os.path.pardir, os.path.pardir, os.path.pardir, os.path.pardir)))
 
 
 # 로케일 설정