From abb747dddc084fa7b7d67f31f2db8651b7041d5b Mon Sep 17 00:00:00 2001 From: Matthew Powelson Date: Tue, 2 Jul 2024 13:27:59 -0600 Subject: [PATCH] Initialize event storage when loading checkpoint in lightning module --- tools/lightning_train_net.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/lightning_train_net.py b/tools/lightning_train_net.py index 87cfe84feb..1239bf2a84 100644 --- a/tools/lightning_train_net.py +++ b/tools/lightning_train_net.py @@ -56,6 +56,9 @@ def on_save_checkpoint(self, checkpoint: Dict[str, Any]) -> None: def on_load_checkpoint(self, checkpointed_state: Dict[str, Any]) -> None: self.start_iter = checkpointed_state["iteration"] + if self.storage is None: + self.storage = EventStorage(0) + self.storage.__enter__() self.storage.iter = self.start_iter def setup(self, stage: str): @@ -83,6 +86,7 @@ def training_step(self, batch, batch_idx): self.storage.__enter__() self.iteration_timer.trainer = weakref.proxy(self) self.iteration_timer.before_step() + if self.writers is None: self.writers = ( default_writers(self.cfg.OUTPUT_DIR, self.max_iter) if comm.is_main_process()