Try using non-transformer spacy model

CouncilDataProject · Dec 5, 2023 · 623871b · 623871b
1 parent 06f618a
commit 623871b
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 5 deletions.
diff --git a/cdp_backend/sr_models/whisper.py b/cdp_backend/sr_models/whisper.py
@@ -9,7 +9,6 @@
 from typing import TYPE_CHECKING, Any
 
 import spacy
-import spacy_transformers  # noqa: F401
 from faster_whisper import WhisperModel as FasterWhisper
 from pydub import AudioSegment
 from spacy.cli.download import download as download_spacy_model
@@ -30,6 +29,7 @@
 
 ###############################################################################
 
+DEFAULT_SPACY_MODEL = "en_core_web_lg"
 spacy.prefer_gpu()
 
 ###############################################################################
@@ -48,7 +48,7 @@ class WhisperModel(SRModel):
     @staticmethod
     def _load_spacy_model() -> "Language":
         nlp = spacy.load(
-            "en_core_web_trf",
+            DEFAULT_SPACY_MODEL,
             # Only keep the parser
             # We are only using this for sentence parsing
             disable=[
@@ -127,7 +127,7 @@ def __init__(
         try:
             self.nlp = self._load_spacy_model()
         except Exception:
-            download_spacy_model("en_core_web_trf")
+            download_spacy_model(DEFAULT_SPACY_MODEL)
             self.nlp = self._load_spacy_model()
 
     def transcribe(

diff --git a/pyproject.toml b/pyproject.toml
@@ -57,7 +57,6 @@ Documentation = "https://CouncilDataProject.github.io/cdp-backend"
 # https://peps.python.org/pep-0621/#dependencies-optional-dependencies
 [project.optional-dependencies]
 pipeline = [
-  "curated-transformers>=0.1.1",
   "dask[distributed]>=2021.7.0",
   "faster-whisper==0.3.0",
   "ffmpeg-python>=0.2.0",
@@ -74,7 +73,6 @@ pipeline = [
   "pypdf>=2.0",
   "rapidfuzz>=2.0",
   "spacy>=3.4",
-  "spacy-transformers>=1.1",
   "tika==2.6.0",
   "torch>=1.10,<2",
   "tqdm>=4.62",