diff --git a/tests/shared_objects/test_utilities.py b/tests/shared_objects/test_utilities.py index 8751e1ac..43dc420c 100644 --- a/tests/shared_objects/test_utilities.py +++ b/tests/shared_objects/test_utilities.py @@ -8,6 +8,7 @@ from vali_objects.enums.order_type_enum import OrderType from vali_objects.vali_dataclasses.perf_ledger import PerfCheckpoint, PerfLedger from vali_objects.vali_config import ValiConfig +from vali_objects.vali_dataclasses.perf_ledger import TP_ID_PORTFOLIO def get_time_in_range(percent, start, end): @@ -125,7 +126,8 @@ def generate_ledger( ) ) - return ledger_generator(checkpoints=checkpoint_list) + base_ledger = ledger_generator(checkpoints=checkpoint_list) + return {TP_ID_PORTFOLIO: base_ledger, TradePair.BTCUSD.trade_pair_id: base_ledger} def ledger_generator( diff --git a/tests/vali_tests/test_challengeperiod_integration.py b/tests/vali_tests/test_challengeperiod_integration.py index f0b16d07..583330d0 100644 --- a/tests/vali_tests/test_challengeperiod_integration.py +++ b/tests/vali_tests/test_challengeperiod_integration.py @@ -3,7 +3,7 @@ from vali_objects.enums.order_type_enum import OrderType from vali_objects.vali_dataclasses.order import Order -from vali_objects.vali_dataclasses.perf_ledger import PerfLedgerData +from vali_objects.vali_dataclasses.perf_ledger import PerfLedgerData, TP_ID_PORTFOLIO from tests.shared_objects.mock_classes import ( MockMetagraph, MockChallengePeriodManager, MockPositionManager, MockPerfLedgerManager, MockCacheController ) @@ -319,7 +319,7 @@ def test_refresh_elimination_disk(self): # Check one of the failing miners, to see if they are screened failing_miner = self.FAILING_MINER_NAMES[0] failing_screen, _ = self.challengeperiod_manager.screen_failing_criteria( - ledger_element=self.LEDGERS[failing_miner] + ledger_element=self.LEDGERS[failing_miner][TP_ID_PORTFOLIO] ) self.assertEqual(failing_screen, True) @@ -327,7 +327,7 @@ def test_refresh_elimination_disk(self): # Now inspect all the hotkeys challenge_success, challenge_eliminations = self.challengeperiod_manager.inspect( positions=self.POSITIONS, - ledger=self.LEDGERS, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in self.LEDGERS.items()}, success_hotkeys=self.SUCCESS_MINER_NAMES, inspection_hotkeys=self.challengeperiod_manager.challengeperiod_testing, current_time=self.OUTSIDE_OF_CHALLENGE diff --git a/tests/vali_tests/test_challengeperiod_unit.py b/tests/vali_tests/test_challengeperiod_unit.py index 35973a5b..2adf0f1f 100644 --- a/tests/vali_tests/test_challengeperiod_unit.py +++ b/tests/vali_tests/test_challengeperiod_unit.py @@ -18,7 +18,7 @@ from vali_objects.scoring.scoring import Scoring from vali_objects.vali_dataclasses.order import Order from vali_objects.enums.order_type_enum import OrderType - +from vali_objects.vali_dataclasses.perf_ledger import TP_ID_PORTFOLIO class TestChallengePeriodUnit(TestBase): @@ -136,15 +136,15 @@ def test_screen_drawdown(self): base_positions[i].return_at_close = 1.1 # Drawdown is high - 50% drawdown on the first period - base_ledger.cps[0].mdd = 0.5 + base_ledger[TP_ID_PORTFOLIO].cps[0].mdd = 0.5 # Drawdown criteria - max_drawdown = LedgerUtils.recent_drawdown(base_ledger.cps, restricted=False) + max_drawdown = LedgerUtils.recent_drawdown(base_ledger[TP_ID_PORTFOLIO].cps, restricted=False) max_drawdown_percentage = LedgerUtils.drawdown_percentage(max_drawdown) self.assertGreater(max_drawdown_percentage, ValiConfig.DRAWDOWN_MAXVALUE_PERCENTAGE) # Check that the miner is successfully screened as failing - screening_logic, _ = self.challengeperiod_manager.screen_failing_criteria(ledger_element=base_ledger) + screening_logic, _ = self.challengeperiod_manager.screen_failing_criteria(ledger_element=base_ledger[TP_ID_PORTFOLIO]) self.assertTrue(screening_logic) @@ -163,7 +163,7 @@ def test_failing_remaining_time(self): # Check that the miner is screened as failing passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=[], inspection_hotkeys={"miner": current_time}, current_time=current_time, @@ -190,7 +190,7 @@ def test_failing_no_remaining_time(self): # Check that the miner is screened as failing passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=[], inspection_hotkeys=inspection_hotkeys, current_time=current_time, @@ -218,7 +218,7 @@ def test_passing_remaining_time(self): # Check that the miner is screened as failing passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=[], inspection_hotkeys=inspection_hotkeys, current_time=current_time, @@ -245,7 +245,7 @@ def test_passing_no_remaining_time(self): # Check that the miner is screened as failing passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=[], inspection_hotkeys=inspection_hotkeys, current_time=current_time, @@ -271,7 +271,7 @@ def test_lingering_no_positions(self): # Check that the miner is screened as testing still passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=[], inspection_hotkeys=inspection_hotkeys, current_time=current_time @@ -289,7 +289,7 @@ def test_recently_re_registered_miner(self): base_ledger = deepcopy(self.DEFAULT_LEDGER) base_position = deepcopy(self.DEFAULT_POSITION) - base_position.orders[0].processed_ms = base_ledger.start_time_ms + 1 + base_position.orders[0].processed_ms = base_ledger[TP_ID_PORTFOLIO].start_time_ms + 1 base_positions = [base_position] inspection_positions = {"miner": base_positions} @@ -300,7 +300,7 @@ def test_recently_re_registered_miner(self): # Check that the miner is screened as testing still passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=self.SUCCESS_MINER_NAMES, inspection_hotkeys=inspection_hotkeys, current_time=current_time, @@ -325,7 +325,7 @@ def test_lingering_with_positions(self): # Check that the miner is screened as testing still passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=self.SUCCESS_MINER_NAMES, inspection_hotkeys=inspection_hotkeys, current_time=current_time, @@ -350,7 +350,7 @@ def test_just_above_threshold(self): # Check that the miner is screened as failing passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=[], inspection_hotkeys={"miner": current_time}, current_time=current_time, @@ -376,7 +376,7 @@ def test_just_below_threshold(self): # Check that the miner is screened as failing passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=[], inspection_hotkeys={"miner": current_time}, current_time=current_time, @@ -415,7 +415,7 @@ def test_at_threshold(self): # Check that the miner is screened as failing passing, failing = self.challengeperiod_manager.inspect( positions=inspection_positions, - ledger=inspection_ledger, + ledger={hk: v[TP_ID_PORTFOLIO] for hk, v in inspection_ledger.items()}, success_hotkeys=[], inspection_hotkeys={"miner": current_time}, current_time=current_time, diff --git a/tests/vali_tests/test_concentration.py b/tests/vali_tests/test_concentration.py index deaa7916..0ce68e56 100644 --- a/tests/vali_tests/test_concentration.py +++ b/tests/vali_tests/test_concentration.py @@ -3,7 +3,7 @@ from vali_objects.vali_config import TradePair from vali_objects.utils.ledger_utils import LedgerUtils - +from vali_objects.vali_dataclasses.perf_ledger import TP_ID_PORTFOLIO from vali_objects.utils.functional_utils import FunctionalUtils from tests.shared_objects.test_utilities import generate_ledger, ledger_generator, position_generator @@ -62,17 +62,18 @@ def test_daily_log_returns_concentration(self): # Daily returns with a single element sample_ledger = generate_ledger(gain=0.1, loss=0.0, nterms=1) - log_returns = LedgerUtils.daily_return_log(sample_ledger.cps) + log_returns = LedgerUtils.daily_return_log(sample_ledger[TP_ID_PORTFOLIO].cps) concentration_penalty = FunctionalUtils.concentration(log_returns) self.assertEqual(concentration_penalty, 1) # Daily returns with multiple elements sample_ledger = generate_ledger(gain=0.1, loss=0.0, nterms=50) sample_concentrated = copy.deepcopy(sample_ledger) - sample_concentrated.cps[-1].gain = 0.5 + for tp_id, dat in sample_concentrated.items(): + sample_concentrated[tp_id].cps[-1].gain = 0.5 - log_returns = LedgerUtils.daily_return_log(sample_ledger.cps) - log_returns_concentrated = LedgerUtils.daily_return_log(sample_concentrated.cps) + log_returns = LedgerUtils.daily_return_log(sample_ledger[TP_ID_PORTFOLIO].cps) + log_returns_concentrated = LedgerUtils.daily_return_log(sample_concentrated[TP_ID_PORTFOLIO].cps) concentration_penalty = FunctionalUtils.concentration(log_returns) concentration_penalty_concentrated = FunctionalUtils.concentration(log_returns_concentrated) diff --git a/tests/vali_tests/test_ledger_penalty.py b/tests/vali_tests/test_ledger_penalty.py index faee26e9..f52dabfd 100644 --- a/tests/vali_tests/test_ledger_penalty.py +++ b/tests/vali_tests/test_ledger_penalty.py @@ -1,6 +1,7 @@ import copy from copy import deepcopy from tests.vali_tests.base_objects.test_base import TestBase +from vali_objects.vali_dataclasses.perf_ledger import TP_ID_PORTFOLIO from vali_objects.utils.ledger_utils import LedgerUtils from vali_objects.vali_config import ValiConfig @@ -45,10 +46,10 @@ def test_daily_consistency(self): ) l4 = copy.deepcopy(l3) - l1_cps = l1.cps - l2_cps = l2.cps - l3_cps = l3.cps - l4_cps = l4.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps + l2_cps = l2[TP_ID_PORTFOLIO].cps + l3_cps = l3[TP_ID_PORTFOLIO].cps + l4_cps = l4[TP_ID_PORTFOLIO].cps # Inconsistent growth period - one interval l2_cps[len(l2_cps) // 2].gain = 1.0 @@ -65,7 +66,7 @@ def test_daily_consistency(self): # Many days, but all in one week l5 = deepcopy(self.DEFAULT_LEDGER) - l5_cps = l5.cps + l5_cps = l5[TP_ID_PORTFOLIO].cps # Set all gains to 1.0 for these days n_days_in_biweekly = biweekly_window // daily_window @@ -113,10 +114,10 @@ def test_biweekly_consistency(self): ) l4 = copy.deepcopy(l3) - l1_cps = l1.cps - l2_cps = l2.cps - l3_cps = l3.cps - l4_cps = l4.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps + l2_cps = l2[TP_ID_PORTFOLIO].cps + l3_cps = l3[TP_ID_PORTFOLIO].cps + l4_cps = l4[TP_ID_PORTFOLIO].cps # Inconsistent growth period - one interval l2_cps[len(l2_cps) // 2].gain = 1.0 @@ -132,7 +133,7 @@ def test_biweekly_consistency(self): # Many days, but all in one week l5 = generate_ledger(0.1) - l5_cps = l5.cps + l5_cps = l5[TP_ID_PORTFOLIO].cps # Set all gains to 1.0 for these days for i in range(0, biweekly_window, daily_window): @@ -180,7 +181,7 @@ def test_drawdown_abnormality(self): # Drawdown abnormality should work for only a few checkpoints self.assertLess(LedgerUtils.drawdown_abnormality(checkpoints), 1.0) l1 = generate_ledger(0.1, mdd=0.98) - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps # All recent drawdowns equal to 0 should return low abnormality for i in range(1, ValiConfig.RETURN_SHORT_LOOKBACK_LEDGER_WINDOWS + 1): @@ -190,27 +191,27 @@ def test_drawdown_abnormality(self): # All equal drawdown l2 = generate_ledger(0.1, mdd=0.98) - self.assertAlmostEqual(LedgerUtils.drawdown_abnormality(l2.cps), 1.0, places=2) # Essentially no penalty + self.assertAlmostEqual(LedgerUtils.drawdown_abnormality(l2[TP_ID_PORTFOLIO].cps), 1.0, places=2) # Essentially no penalty # Large abnormality near maximum drawdown threshold should have a score of 0 (i.e., high penalty) l3 = generate_ledger(0.1, mdd=0.99) - l3_cps = l3.cps + l3_cps = l3[TP_ID_PORTFOLIO].cps l3_cps[-1].mdd = 0.91 - self.assertAlmostEqual(LedgerUtils.drawdown_abnormality(l3.cps), 0.0, places=6) + self.assertAlmostEqual(LedgerUtils.drawdown_abnormality(l3[TP_ID_PORTFOLIO].cps), 0.0, places=6) def test_max_drawdown_threshold(self): l1 = generate_ledger(0.1, mdd=0.99) # 1% drawdown - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps l2 = copy.deepcopy(l1) - l2_cps = l2.cps + l2_cps = l2[TP_ID_PORTFOLIO].cps l2_cps[-1].mdd = 0.8 # 20% drawdown only on the most recent checkpoint l3 = copy.deepcopy(l1) - l3_cps = l3.cps + l3_cps = l3[TP_ID_PORTFOLIO].cps l3_cps[0].mdd = 0.8 # 20% drawdown only on the first checkpoint l4 = generate_ledger(0.1, mdd=0.8) # 20% drawdown - l4_cps = l4.cps + l4_cps = l4[TP_ID_PORTFOLIO].cps self.assertEqual(LedgerUtils.max_drawdown_threshold_penalty(l1_cps), 1.0) self.assertEqual(LedgerUtils.max_drawdown_threshold_penalty(l2_cps), 0.0) diff --git a/tests/vali_tests/test_ledger_utils.py b/tests/vali_tests/test_ledger_utils.py index 3036dddd..3438adc2 100644 --- a/tests/vali_tests/test_ledger_utils.py +++ b/tests/vali_tests/test_ledger_utils.py @@ -3,7 +3,7 @@ from vali_objects.utils.ledger_utils import LedgerUtils from vali_objects.vali_config import ValiConfig - +from vali_objects.vali_dataclasses.perf_ledger import TP_ID_PORTFOLIO from tests.shared_objects.test_utilities import generate_ledger, checkpoint_generator import random @@ -25,7 +25,7 @@ def test_daily_return_log(self): should bucket the checkpoint returns by full days """ self.assertEqual(LedgerUtils.daily_return_log([]), []) - checkpoints = self.DEFAULT_LEDGER.cps + checkpoints = self.DEFAULT_LEDGER[TP_ID_PORTFOLIO].cps # One checkpoint shouldn't be enough since full day is required self.assertEqual(len(LedgerUtils.daily_return_log([checkpoints[0]])), 0) @@ -44,7 +44,7 @@ def test_daily_return_log(self): self.assertEqual(len(LedgerUtils.daily_return_log(checkpoints)), 89) l1 = generate_ledger(0.1, start_time=10, end_time=ValiConfig.TARGET_LEDGER_WINDOW_MS, mdd=0.99) - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps self.assertEqual(len(LedgerUtils.daily_return_log(l1_cps)), 88) def test_daily_return(self): @@ -56,21 +56,21 @@ def test_daily_return(self): # No returns l1 = generate_ledger(0.1) - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps self.assertEqual(LedgerUtils.daily_return_percentage(l1_cps)[0], 0) # Simple returns >= log returns self.assertGreaterEqual(LedgerUtils.daily_return_percentage(l1_cps)[0], LedgerUtils.daily_return_log(l1_cps)[0] * 100) # Negative returns l1 = generate_ledger(0.1, gain=0.1, loss=-0.2) - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps self.assertLess(LedgerUtils.daily_return_percentage(l1_cps)[0], 0) # Simple returns >= log returns self.assertGreaterEqual(LedgerUtils.daily_return_percentage(l1_cps)[0], LedgerUtils.daily_return_log(l1_cps)[0] * 100) # Positive returns l1 = generate_ledger(0.1, gain=0.2, loss=-0.1) - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps self.assertGreater(LedgerUtils.daily_return_percentage(l1_cps)[0], 0) # Simple returns >= log returns self.assertGreaterEqual(LedgerUtils.daily_return_percentage(l1_cps)[0], LedgerUtils.daily_return_log(l1_cps)[0] * 100) @@ -78,7 +78,7 @@ def test_daily_return(self): # Want to test the individual functions inputs and outputs def test_recent_drawdown(self): l1 = generate_ledger(0.1, mdd=0.99) - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps self.assertEqual(LedgerUtils.recent_drawdown([]), 1) @@ -86,16 +86,16 @@ def test_recent_drawdown(self): self.assertEqual(LedgerUtils.recent_drawdown(l1_cps), 0.99) l2 = generate_ledger(0.1, mdd=0.95) - l2_cps = l2.cps + l2_cps = l2[TP_ID_PORTFOLIO].cps self.assertEqual(LedgerUtils.recent_drawdown(l2_cps), 0.95) l3 = generate_ledger(0.1, mdd=0.99) - l3_cps = l3.cps + l3_cps = l3[TP_ID_PORTFOLIO].cps l3_cps[-1].mdd = 0.5 self.assertEqual(LedgerUtils.recent_drawdown(l3_cps), 0.5) l4 = generate_ledger(0.1, mdd=0.99) - l4_cps = l4.cps + l4_cps = l4[TP_ID_PORTFOLIO].cps l4_cps[0].mdd = 0.5 self.assertEqual(LedgerUtils.recent_drawdown(l4_cps), 0.99) @@ -183,24 +183,24 @@ def test_mdd_augmentation(self): # Test max_drawdown_threshold_penalty def test_max_drawdown_threshold_penalty(self): - checkpoints = self.DEFAULT_LEDGER.cps + checkpoints = self.DEFAULT_LEDGER[TP_ID_PORTFOLIO].cps self.assertEqual(LedgerUtils.max_drawdown_threshold_penalty([]), 0) self.assertEqual(LedgerUtils.max_drawdown_threshold_penalty(checkpoints), 1) l1 = copy.deepcopy(self.DEFAULT_LEDGER) - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps l1_cps[-1].mdd = 0.8 self.assertEqual(LedgerUtils.max_drawdown_threshold_penalty(l1_cps), 0) # Test approximate_drawdown def test_approximate_drawdown(self): - checkpoints = self.DEFAULT_LEDGER.cps + checkpoints = self.DEFAULT_LEDGER[TP_ID_PORTFOLIO].cps self.assertEqual(LedgerUtils.approximate_drawdown([]), 0) self.assertLessEqual(LedgerUtils.approximate_drawdown(checkpoints), 1) l1 = generate_ledger(0.1, mdd=0.99) # 1% drawdown - l1_cps = l1.cps + l1_cps = l1[TP_ID_PORTFOLIO].cps for i in range(0, len(l1_cps) - len(l1_cps)//4): l1_cps[i].mdd = (random.random() / 10) + 0.9 @@ -209,7 +209,7 @@ def test_approximate_drawdown(self): self.assertGreater(LedgerUtils.approximate_drawdown(l1_cps), 0.8) l2 = generate_ledger(0.1, mdd=0.99) # 1% drawdown - l2_cps = l2.cps + l2_cps = l2[TP_ID_PORTFOLIO].cps l2_cps[-1].mdd = 0.8 # 20% drawdown only on the most recent checkpoint self.assertLessEqual(LedgerUtils.approximate_drawdown(l2_cps), 0.99) self.assertGreater(LedgerUtils.approximate_drawdown(l2_cps), 0.8) @@ -248,11 +248,11 @@ def test_mean_drawdown(self): # Should be set 0 if drawdowns are somehow negative self.assertEqual(LedgerUtils.mean_drawdown(checkpoints), 0) - self.assertEqual(LedgerUtils.mean_drawdown(self.DEFAULT_LEDGER.cps), 0.99) + self.assertEqual(LedgerUtils.mean_drawdown(self.DEFAULT_LEDGER[TP_ID_PORTFOLIO].cps), 0.99) # Test risk_normalization def test_risk_normalization(self): - checkpoints = self.DEFAULT_LEDGER.cps + checkpoints = self.DEFAULT_LEDGER[TP_ID_PORTFOLIO].cps self.assertEqual(LedgerUtils.risk_normalization([]), 0) self.assertLessEqual(LedgerUtils.risk_normalization(checkpoints), 1) diff --git a/tests/vali_tests/test_perf_ledgers.py b/tests/vali_tests/test_perf_ledgers.py index e9a26020..28f84fc6 100644 --- a/tests/vali_tests/test_perf_ledgers.py +++ b/tests/vali_tests/test_perf_ledgers.py @@ -6,7 +6,7 @@ from vali_objects.enums.order_type_enum import OrderType from vali_objects.position import Position from vali_objects.vali_dataclasses.order import Order -from vali_objects.vali_dataclasses.perf_ledger import PerfLedgerManager, TP_ID_PORTFOLIO +from vali_objects.vali_dataclasses.perf_ledger import PerfLedgerManager class TestPerfLedgers(TestBase): diff --git a/tests/vali_tests/test_weights_unit.py b/tests/vali_tests/test_weights_unit.py index 85fca56f..71680f6a 100644 --- a/tests/vali_tests/test_weights_unit.py +++ b/tests/vali_tests/test_weights_unit.py @@ -12,7 +12,7 @@ from vali_objects.vali_config import ValiConfig from tests.shared_objects.test_utilities import generate_ledger - +from vali_objects.vali_dataclasses.perf_ledger import TP_ID_PORTFOLIO class TestWeights(TestBase): @@ -48,7 +48,7 @@ def test_transform_and_scale_results_defaults(self): # Test the default values scaled_transformed_list: list[tuple[str, float]] = Scoring.compute_results_checkpoint( - ledger, + {hk: v[TP_ID_PORTFOLIO] for hk, v in ledger.items()}, miner_positions, evaluation_time_ms=self.EVALUATION_TIME_MS )