diff --git a/torchbenchmark/canary_models/stable_diffusion_xl/install.py b/torchbenchmark/canary_models/stable_diffusion_xl/install.py
index da59a68fe4..bba7de8eb4 100644
--- a/torchbenchmark/canary_models/stable_diffusion_xl/install.py
+++ b/torchbenchmark/canary_models/stable_diffusion_xl/install.py
@@ -15,10 +15,10 @@ def load_model_checkpoint():
 
 
 if __name__ == "__main__":
+    install_diffusers()
     if not "HUGGING_FACE_HUB_TOKEN" in os.environ:
         warnings.warn(
             "Make sure to set `HUGGINGFACE_HUB_TOKEN` so you can download weights"
         )
     else:
-        install_diffusers()
         load_model_checkpoint()