Fix: update CA to brute scrape

i-dot-ai · Aug 23, 2024 · aafd926 · aafd926
1 parent 3127f8f
commit aafd926
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/caddy_scraper/caddy_scraper.py b/caddy_scraper/caddy_scraper.py
@@ -128,6 +128,7 @@ def recursive_crawler(self) -> List[str]:
             links (List[str]): a list of links found on the pages.
         """
         bs_transformer = BeautifulSoupTransformer()
+        cookie_dict = None
         if "advisernet" in self.base_url:
             cookie_dict = {
                 "Cookie": f".CitizensAdviceLogin={os.getenv(

diff --git a/caddy_scraper/scrape_config.json b/caddy_scraper/scrape_config.json
@@ -2,14 +2,15 @@
     {
         "base_url": "https://www.citizensadvice.org.uk/", 
         "sitemap_url": "https://www.citizensadvice.org.uk/sitemap.xml", 
-        "crawling_method": "sitemap", 
+        "crawling_method": "brute", 
+        "scrape_depth": 4,
         "output_dir": "citizensadvice_scrape"
     }, 
     {
         "base_url": "https://www.citizensadvice.org.uk/advisernet", 
-        "crawling_method": "brute", 
-        "output_dir": "advisernet_scrape", 
-        "scrape_depth": 4
+        "crawling_method": "brute",  
+        "scrape_depth": 4,
+        "output_dir": "advisernet_scrape"
     }, 
     {
         "base_url": "https://www.gov.uk/",