added value error for when there is no spm_model

facebookresearch · Sep 21, 2023 · c2f66cd · c2f66cd
1 parent 2ac3362
commit c2f66cd
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/laser_encoders/models.py b/laser_encoders/models.py
@@ -95,6 +95,10 @@ def __call__(self, sentences):
         if self.spm_model:
             sentences = self.tokenizer(sentences)
             return self.encode_sentences(sentences)
+        else:
+            raise ValueError(
+                "Either initialize the encoder with an spm_model or pre-tokenize and use the encode_sentences method."
+            )
 
     def _process_batch(self, batch):
         tokens = batch.tokens

diff --git a/laser_encoders/test_laser_tokenizer.py b/laser_encoders/test_laser_tokenizer.py
@@ -173,5 +173,5 @@ def test_sentence_encoder(
     sentence_embedding = sentence_encoder.encode_sentences([tokenized_text])
 
     assert isinstance(sentence_embedding, np.ndarray)
-    # assert sentence_embedding.shape == (1, 1024)
+    assert sentence_embedding.shape == (1, 1024)
     assert np.allclose(expected_array, sentence_embedding[:, :10], atol=1e-3)