Merge pull request #4 from MGMCN/add-university

add university information
MGMCN · Mar 14, 2024 · cde0373 · cde0373
2 parents 4553d03 + e841a17
commit cde0373
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/crawler.py b/crawler.py
@@ -81,6 +81,7 @@ def clean_text(element):
                     return element.get_text(strip=True).replace("\n", "").replace("\t", "")
                 return ""
 
+            university = clean_text(soup.find('p', class_='me-md-3'))
             location = clean_text(soup.find('p', string=lambda x: x and '勤務地' in x))
             research_field = clean_text(soup.find('p', string=lambda x: x and '研究分野' in x))
             start_date = clean_text(soup.find('p', string=lambda x: x and '公開開始日' in x))
@@ -115,6 +116,7 @@ def clean_text(element):
                 if parent:
                     qualification += clean_text(parent)
 
+            # print(f'University: {university}')
             # print(f'Location: {location}')
             # print(f'Research Field: {research_field}')
             # print(f'Start Date: {start_date}')
@@ -124,6 +126,7 @@ def clean_text(element):
             # print(f'Qualification: {qualification}')
 
             return {
+                "university": university,
                 "location": location,
                 "research_field": research_field,
                 "start_date": start_date,
@@ -134,6 +137,7 @@ def clean_text(element):
             }
         else:
             return {
+                "university": None,
                 "location": None,
                 "research_field": None,
                 "start_date": None,