Enhancement : 전화번호, 이메일, 회사명 추출 기능 강화

utils/make_context.py

@@ -103,6 +103,7 @@ def get_serialization_string(api_output)->DefaultDict:
 
     serialized_text = detect_phone(serialized_text)
     serialized_text = detect_email(serialized_text)
+    serialized_text = detect_company(serialized_text)
     spacing = Spacing()
 
     for key in serialized_text.keys():
@@ -111,4 +112,5 @@ def get_serialization_string(api_output)->DefaultDict:
             spacing_text = spacing(spacing_text)        
             serialized_text[key] = spacing_text
 
+    serialized_text = remove_empty_list(serialized_text)
     return serialized_text 

utils/preprocess_info.py

@@ -1,9 +1,11 @@
+from collections import defaultdict
 from typing import DefaultDict, List
 import itertools
 import re
+from xxlimited import new
 
 def preprocess_text(text:str)->str:
-    special = ["#", "$", "%", "&", "*", "(", ")", ":", "-"]
+    special = ["#", "$", "%", "&", "*", "(", ")", ":", "-", "<", ">", "|"]
 
     for token in special:
         text = text.replace(token, "")
@@ -25,10 +27,9 @@ def remove_duplicate(serialized:DefaultDict, target_list:List[int]):
 
     for idx, (key,val) in enumerate(serialized.items()):
         for remove_idx in idx_list[idx]:
-            val[remove_idx] = ''
+            val[remove_idx] = None
         serialized[key] = list(filter(None, val))
     return serialized
-
 
 def detect_phone(serialized:DefaultDict)->DefaultDict:
     stack = []
@@ -50,21 +51,35 @@ def detect_phone(serialized:DefaultDict)->DefaultDict:
                 stack.clear()
 
     serialized = remove_duplicate(serialized, phone_list)
+    phone_list = list(filter(lambda x : x.startswith("010"), phone_list))
     serialized["phone"].extend(phone_list)
 
     return serialized
 
+def preprocess_domain(text:str)->str:
+    domains = ["co", "com", "kr", "ne", "net", "or"]
+
+    text = text.replace(".", "").strip()
+
+    for domain in domains:
+        if text == domain:
+            return f".{domain}" 
+
+    return text 
+
 def detect_email(serialized:DefaultDict)->DefaultDict:
     stack = []
     email_list = []
     p = re.compile('^[a-zA-Z0-9+-_.]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$')
 
     for val in itertools.chain.from_iterable(serialized.values()):
-        stack.append(val)
         if p.match(val):
             email_list.append(val)
             break
 
+        stack.append(val)
+        val = preprocess_domain(val)
+
         if '@' in stack[-1]:
             stack.clear()
             stack.append(val)
@@ -82,6 +97,30 @@ def detect_email(serialized:DefaultDict)->DefaultDict:
 
     return serialized
 
+def detect_company(serialized:DefaultDict)->DefaultDict:
+    company_related_sector = [0,1,2]
+    company_list = []
+
+
+    for key in company_related_sector:
+        for text in serialized[key]:
+            text = preprocess_text(text)
+            company_list.append(text.upper())
+
+    serialized = remove_duplicate(serialized, company_list)
+    company = " ".join(company_list)
+    serialized["company"].append(company)
+
+    return serialized
 
+def remove_empty_list(serialized:DefaultDict)->DefaultDict:
+    new_serialized = defaultdict(str)
 
+    for k,v in serialized.items():
+        if v != '' and type(v) == str:
+            new_serialized[k] = v
+
+        if type(v) == list and v:
+            new_serialized[k] = v[0]
 
+    return new_serialized