Fix bug when loading 4bit checkpoint quantized in INC (#1447)

huggingface · Oct 29, 2024 · 95ea921 · 95ea921
1 parent 7e0deeb
commit 95ea921
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/examples/text-generation/utils.py b/examples/text-generation/utils.py
@@ -269,6 +269,9 @@ def setup_model(args, model_dtype, model_kwargs, logger):
             original_model=org_model,
             **model_kwargs,
         )
+        # TODO: This will be removed in v1.19 Synapse release
+        # the loaded model should have the same dtype as original_model
+        model = model.to(model_kwargs["torch_dtype"])
     else:
         if args.assistant_model is not None:
             assistant_model = AutoModelForCausalLM.from_pretrained(