Merge pull request #78 from aapatre/restyled/pull-77

Restyle Fix bug where script doesn't scrape beyond first page
aapatre · Nov 6, 2020 · d346696 · d346696
2 parents 3adcb99 + 7a0c332
commit d346696
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/core/tutorialbar.py b/core/tutorialbar.py
@@ -15,7 +15,6 @@ class TutorialBarScraper:
     def __init__(self):
         self.current_page = 0
         self.last_page = None
-        self.all_courses_link = f"{self.DOMAIN}/all-courses/page/{self.current_page}/"
         self.links_per_page = 12
 
     def run(self) -> List:
@@ -26,7 +25,9 @@ def run(self) -> List:
         """
         self.current_page += 1
         print("Please Wait: Getting the course list from tutorialbar.com...")
-        course_links = self.get_course_links(self.all_courses_link)
+        course_links = self.get_course_links(
+            f"{self.DOMAIN}/all-courses/page/{self.current_page}/"
+        )
 
         print(f"Page: {self.current_page} of {self.last_page} scraped")
         udemy_links = self.gather_udemy_course_links(course_links)
@@ -74,8 +75,7 @@ def get_udemy_course_link(url: str) -> str:
         """
         response = requests.get(url=url)
         soup = BeautifulSoup(response.content, "html.parser")
-        udemy_link = soup.find(
-            "span", class_="rh_button_wrapper").find("a").get("href")
+        udemy_link = soup.find("span", class_="rh_button_wrapper").find("a").get("href")
         return udemy_link
 
     def gather_udemy_course_links(self, courses: List[str]) -> List: