feat: add lowercasing functionality to merge_tokens_into_text and col…

…lect_svo_triples

id_svo_extractor/utils.py

@@ -1,17 +1,17 @@
 from spacy.tokens import Doc, Token
 
 
-def merge_tokens_into_text(tokens: list[Token]) -> str:
-    text = "".join(token.text_with_ws for token in tokens)
-    return text.rstrip()
+def merge_tokens_into_text(tokens: list[Token], lower: bool = False) -> str:
+    text = "".join(token.text_with_ws for token in tokens).rstrip()
+    return text.lower() if lower else text
 
 
-def collect_svo_triples(doc: Doc) -> list[tuple[str, str, str]]:
+def collect_svo_triples(doc: Doc, lower: bool = False) -> list[tuple[str, str, str]]:
     svo_triples = []
     for sentence in doc.sents:
         for s, v, o in sentence._.svo_triples:
-            s = merge_tokens_into_text(s)
-            v = merge_tokens_into_text(v)
-            o = merge_tokens_into_text(o)
+            s = merge_tokens_into_text(s, lower)
+            v = merge_tokens_into_text(v, lower)
+            o = merge_tokens_into_text(o, lower)
             svo_triples.append((s, v, o))
     return svo_triples

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "id-svo-extractor"
-version = "0.2.0"
+version = "0.3.0"
 requires-python = ">=3.10"
 description = "id-svo-extractor: Extract SVO triples from Indonesian text."
 readme = "README.md"