History Cache Hotfix

bitmind/__init__.py

@@ -18,7 +18,7 @@
 # DEALINGS IN THE SOFTWARE.
 
 
-__version__ = "1.2.0"
+__version__ = "1.2.1"
 version_split = __version__.split(".")
 __spec_version__ = (
     (1000 * int(version_split[0]))

bitmind/base/validator.py

@@ -54,7 +54,10 @@ def add_args(cls, parser: argparse.ArgumentParser):
     def __init__(self, config=None):
         super().__init__(config=config)
 
-        self.performance_tracker = MinerPerformanceTracker()
+        self.history_cache_path = os.path.join(
+            self.config.neuron.full_path, "miner_performance_tracker.pkl")
+
+        self.load_miner_history()
 
         # Save a copy of the hotkeys to local memory.
         self.hotkeys = copy.deepcopy(self.metagraph.hotkeys)
@@ -372,6 +375,24 @@ def update_scores(self, rewards: np.ndarray, uids: List[int]):
         self.scores: np.ndarray = alpha * scattered_rewards + (1 - alpha) * self.scores
         bt.logging.debug(f"Updated moving avg scores: {self.scores}")
 
+    def save_miner_history(self):
+        bt.logging.info(f"Saving miner performance history to {self.history_cache_path}")
+        joblib.dump(self.performance_tracker, self.history_cache_path)
+
+    def load_miner_history(self):
+        if os.path.exists(self.history_cache_path):
+            bt.logging.info(f"Loading miner performance history from {self.history_cache_path}")
+            self.performance_tracker = joblib.load(self.history_cache_path)
+            pred_history = self.performance_tracker.prediction_history
+            num_miners_history = len([
+                uid for uid in pred_history
+                if len([p for p in pred_history[uid] if p != -1]) > 0
+            ])
+            bt.logging.info(f"Loaded history for {num_miners_history} miners")
+        else:
+            bt.logging.info(f"No miner performance history found at {self.history_cache_path} - starting fresh!")
+            self.performance_tracker = MinerPerformanceTracker()
+
     def save_state(self):
         """Saves the state of the validator to a file."""
         bt.logging.info("Saving validator state.")
@@ -383,10 +404,7 @@ def save_state(self):
             scores=self.scores,
             hotkeys=self.hotkeys,
         )
-        joblib.dump(
-            self.performance_tracker,
-            os.path.join(self.config.neuron.full_path, "miner_performance_tracker.pkl")
-        )
+        self.save_miner_history()
 
     def load_state(self):
         """Loads the state of the validator from a file."""
@@ -397,8 +415,4 @@ def load_state(self):
         self.step = state["step"]
         self.scores = state["scores"]
         self.hotkeys = state["hotkeys"]
-
-        tracker_path = os.path.join(
-            self.config.neuron.full_path, "miner_performance_tracker.pkl")
-        if os.path.exists(tracker_path):
-            self.performance_tracker = joblib.load(tracker_path)
+        self.load_miner_history()

bitmind/utils/mock.py

@@ -89,12 +89,15 @@ def __init__(self, config):
         ]
         self.synthetic_image_generator = MockSyntheticImageGenerator(
             prompt_type='annotation', use_random_diffuser=True, diffuser_name=None)
-
+        self.scores = np.zeros(self.metagraph.n, dtype=np.float32)
         self._fake_prob = config.fake_prob
 
     def update_scores(self, rewards, miner_uids):
         pass
 
+    def save_miner_history(self):
+        pass
+
 
 class MockSubtensor(bt.MockSubtensor):
     def __init__(self, netuid, n=16, wallet=None, network="mock"):

bitmind/validator/forward.py

@@ -123,17 +123,6 @@ async def forward(self):
         timeout=9
     )
 
-    # update logging data
-    wandb_data['data_aug_params'] = data_aug_params
-    wandb_data['label'] = label
-    wandb_data['miner_uids'] = list(miner_uids)
-    wandb_data['miner_hotkeys'] = list([axon.hotkey for axon in axons])
-    wandb_data['predictions'] = responses
-    wandb_data['correct'] = [
-        np.round(y_hat) == y
-        for y_hat, y in zip(responses, [label] * len(responses))
-    ]
-
     rewards, metrics = get_rewards(
         label=label,
         responses=responses,
@@ -152,6 +141,19 @@ async def forward(self):
     # Update the scores based on the rewards.
     self.update_scores(rewards, miner_uids)
 
+    # update logging data
+    wandb_data['data_aug_params'] = data_aug_params
+    wandb_data['label'] = label
+    wandb_data['miner_uids'] = list(miner_uids)
+    wandb_data['miner_hotkeys'] = list([axon.hotkey for axon in axons])
+    wandb_data['predictions'] = responses
+    wandb_data['correct'] = [
+        np.round(y_hat) == y
+        for y_hat, y in zip(responses, [label] * len(responses))
+    ]
+    wandb_data['rewards'] = list(rewards)
+    wandb_data['scores'] = list(self.scores)
+
     metric_names = list(metrics[0].keys())
     for metric_name in metric_names:
         wandb_data[f'miner_{metric_name}'] = [m[metric_name] for m in metrics]
@@ -160,6 +162,9 @@ async def forward(self):
     if not self.config.wandb.off:
         wandb.log(wandb_data)
 
+    # ensure state is saved after each challenge
+    self.save_miner_history()
+
     # Track miners who have responded
     self.last_responding_miner_uids = []
     for i, pred in enumerate(responses):