diff --git a/edsnlp/pipes/trainable/embeddings/transformer/transformer.py b/edsnlp/pipes/trainable/embeddings/transformer/transformer.py
index bac4f7aac..b39f3a703 100644
--- a/edsnlp/pipes/trainable/embeddings/transformer/transformer.py
+++ b/edsnlp/pipes/trainable/embeddings/transformer/transformer.py
@@ -160,7 +160,9 @@ def __init__(
                 )
             )
             # and add a new entry to the model's embeddings
-            self.transformer.resize_token_embeddings(len(self.tokenizer))
+            self.transformer.resize_token_embeddings(
+                max(self.tokenizer.vocab.values()) + 1
+            )
 
     def to_disk(self, path, *, exclude: Optional[Set[str]]):
         repr_id = object.__repr__(self)