Skip to content

Commit

Permalink
fix: update spacy patch
Browse files Browse the repository at this point in the history
  • Loading branch information
percevalw committed Dec 13, 2023
1 parent d14d4d0 commit 0102518
Showing 1 changed file with 11 additions and 8 deletions.
19 changes: 11 additions & 8 deletions edsnlp/patch_spacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ def __init__(
max_length: int = 10**6,
meta: Dict[str, Any] = {},
create_tokenizer: Optional[Callable[["Language"], Callable[[str], Doc]]] = None,
create_vectors: Optional[Callable[["Vocab"], Any]] = None,
batch_size: int = 1000,
**kwargs,
) -> None:
Expand Down Expand Up @@ -158,21 +159,23 @@ def __init__(
if vocab is True:
vectors_name = meta.get("vectors", {}).get("name")
vocab = create_vocab(self.lang, self.Defaults, vectors_name=vectors_name)
if (self.lang and vocab.lang) and (self.lang != vocab.lang):
raise ValueError(Errors.E150.format(nlp=self.lang, vocab=vocab.lang))
if not create_vectors and "@vectors" in self._config["nlp"]["vectors"]:
vectors_cfg = {"vectors": self._config["nlp"]["vectors"]}
create_vectors = registry.resolve(vectors_cfg)["vectors"]

Check warning on line 164 in edsnlp/patch_spacy.py

View check run for this annotation

Codecov / codecov/patch

edsnlp/patch_spacy.py#L163-L164

Added lines #L163 - L164 were not covered by tests
vocab.vectors = create_vectors(vocab)
else:
if (self.lang and vocab.lang) and (self.lang != vocab.lang):

Check warning on line 167 in edsnlp/patch_spacy.py

View check run for this annotation

Codecov / codecov/patch

edsnlp/patch_spacy.py#L167

Added line #L167 was not covered by tests
raise ValueError(Errors.E150.format(nlp=self.lang, vocab=vocab.lang))
self.vocab: Vocab = vocab
if self.lang is None:
self.lang = self.vocab.lang
self._components: List[Tuple[str, Callable[[Doc], Doc]]] = []
self._disabled: Set[str] = set()
self.max_length = max_length
# Create the default tokenizer from the default config
create_tokenizer = (
create_tokenizer
or registry.resolve({"tokenizer": self._config["nlp"]["tokenizer"]})[
"tokenizer"
]
)
if not create_tokenizer:
tokenizer_cfg = {"tokenizer": self._config["nlp"]["tokenizer"]}
create_tokenizer = registry.resolve(tokenizer_cfg)["tokenizer"]

Check warning on line 178 in edsnlp/patch_spacy.py

View check run for this annotation

Codecov / codecov/patch

edsnlp/patch_spacy.py#L177-L178

Added lines #L177 - L178 were not covered by tests
self.tokenizer = create_tokenizer(self)
self.batch_size = batch_size
self.default_error_handler = raise_error
Expand Down

0 comments on commit 0102518

Please sign in to comment.